diff --git a/.changeset/chat-session-run-ttl.md b/.changeset/chat-session-run-ttl.md new file mode 100644 index 00000000000..8169b800631 --- /dev/null +++ b/.changeset/chat-session-run-ttl.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"@trigger.dev/sdk": patch +--- + +Chat server sessions can now set a `ttl` on the runs they trigger, so a run that is never picked up expires instead of waiting indefinitely. diff --git a/.changeset/fluffy-pans-argue.md b/.changeset/fluffy-pans-argue.md new file mode 100644 index 00000000000..5bbe81a0fa7 --- /dev/null +++ b/.changeset/fluffy-pans-argue.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +Fix chat transport discarding the next turn after stopping generation. `skipToTurnComplete` is now reset when a new message or action is sent, so a message sent after `stopGeneration` streams normally instead of leaving the chat stuck in a streaming state. diff --git a/.changeset/local-bundle-deploy.md b/.changeset/local-bundle-deploy.md new file mode 100644 index 00000000000..cb9cfdfee17 --- /dev/null +++ b/.changeset/local-bundle-deploy.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Add an experimental `--local-bundle` deploy flag that runs the install and bundling steps on your machine and uploads only the build output; the image is still built remotely. Useful when your project's install step needs tooling or credentials that only exist locally. diff --git a/.changeset/violet-buses-tease.md b/.changeset/violet-buses-tease.md new file mode 100644 index 00000000000..191cff77683 --- /dev/null +++ b/.changeset/violet-buses-tease.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Send the CLI version header on all API requests so deployments are attributable to a CLI version diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000..9fb39b1c505 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,40 @@ +name: CodeQL + +on: + push: + branches: [main] + pull_request: + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + if: github.repository == 'triggerdotdev/trigger.dev' + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write # Upload SARIF to GitHub Security tab + strategy: + fail-fast: false + matrix: + language: [actions, javascript-typescript] + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + with: + category: /language:${{ matrix.language }} diff --git a/.server-changes/admin-dashboard-enabled-flag.md b/.server-changes/admin-dashboard-enabled-flag.md new file mode 100644 index 00000000000..7fe33ef3ec2 --- /dev/null +++ b/.server-changes/admin-dashboard-enabled-flag.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Self-hosted instances can now disable the admin dashboard and user impersonation entirely. See the self-hosting docs for the new setting. diff --git a/.server-changes/deployment-logs-follow-scroll.md b/.server-changes/deployment-logs-follow-scroll.md new file mode 100644 index 00000000000..d5fcf23ce8d --- /dev/null +++ b/.server-changes/deployment-logs-follow-scroll.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Deployment logs no longer jump to the bottom while you are reading earlier output. Scroll up to pause auto-scroll, and scroll back down or use the new scroll-to-bottom button in the log header to resume following. diff --git a/.server-changes/env-var-inputs-autocomplete-off.md b/.server-changes/env-var-inputs-autocomplete-off.md new file mode 100644 index 00000000000..a17457cbd42 --- /dev/null +++ b/.server-changes/env-var-inputs-autocomplete-off.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Stop the browser offering to autofill or save environment variable values as saved credentials. diff --git a/.server-changes/runs-list-query-limit-error.md b/.server-changes/runs-list-query-limit-error.md new file mode 100644 index 00000000000..1bd16673ef8 --- /dev/null +++ b/.server-changes/runs-list-query-limit-error.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +When a runs list or runs.list API request spans too much data to complete, it now returns a clear, actionable error asking you to narrow the time range, instead of failing with a generic error. diff --git a/.server-changes/runs-list-read-isolation.md b/.server-changes/runs-list-read-isolation.md new file mode 100644 index 00000000000..5c411635735 --- /dev/null +++ b/.server-changes/runs-list-read-isolation.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Improved the performance and reliability of the runs list and the runs.list API, especially for large projects and filtered views. diff --git a/.server-changes/vercel-staging-gating.md b/.server-changes/vercel-staging-gating.md new file mode 100644 index 00000000000..c0000314efd --- /dev/null +++ b/.server-changes/vercel-staging-gating.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +The Staging branch setting now shows an upgrade prompt on plans that don't include a Staging environment, instead of looking editable and then silently doing nothing when saved. diff --git a/apps/webapp/app/components/admin/flagChangeList.ts b/apps/webapp/app/components/admin/flagChangeList.ts new file mode 100644 index 00000000000..512d3758bc7 --- /dev/null +++ b/apps/webapp/app/components/admin/flagChangeList.ts @@ -0,0 +1,56 @@ +import { derivedFlagsClearedWith } from "~/v3/featureFlags"; + +export type FlagChange = + | { key: string; type: "added"; newVal: string } + | { key: string; type: "removed"; oldVal: string } + | { key: string; type: "changed"; oldVal: string; newVal: string }; + +/** + * What a global flag save will do, for the confirm dialog. + * + * A graced primary that is unset also clears its stamps. Those keys are locked, so the caller + * filters them out of `initialValues` — the cascade therefore reads `storedValues`, which is the + * unfiltered set the loader returned. Reading `initialValues` finds nothing and understates the + * deletion, which is the defect this parameter exists to prevent. + */ +export function buildFlagChangeList(params: { + editableKeys: readonly string[]; + lockedKeys: readonly string[]; + initialValues: Record; + storedValues: Record; + newValues: Record; +}): FlagChange[] { + const { editableKeys, initialValues, storedValues, newValues } = params; + + return editableKeys.flatMap((key) => { + const wasSet = key in initialValues; + const isSet = key in newValues; + const oldVal = initialValues[key]; + const newVal = newValues[key]; + + if (!wasSet && !isSet) return []; + if (wasSet && isSet && stableValue(oldVal) === stableValue(newVal)) return []; + + if (!wasSet && isSet) { + return [{ key, type: "added", newVal: String(newVal) }]; + } + + if (wasSet && !isSet) { + // Only an unset clears the stamps. A change re-stamps instead. + const cascaded = derivedFlagsClearedWith(key) + .filter((derived) => derived in storedValues) + .map((derived) => ({ + key: derived, + type: "removed", + oldVal: String(storedValues[derived]), + })); + return [{ key, type: "removed", oldVal: String(oldVal) }, ...cascaded]; + } + + return [{ key, type: "changed", oldVal: String(oldVal), newVal: String(newVal) }]; + }); +} + +function stableValue(value: unknown): string { + return JSON.stringify(value ?? null); +} diff --git a/apps/webapp/app/components/integrations/VercelBuildSettings.tsx b/apps/webapp/app/components/integrations/VercelBuildSettings.tsx index e3be9a4f90e..56bc785368c 100644 --- a/apps/webapp/app/components/integrations/VercelBuildSettings.tsx +++ b/apps/webapp/app/components/integrations/VercelBuildSettings.tsx @@ -49,7 +49,6 @@ type BuildSettingsFieldsProps = { * the pin status is unknown — distinct from "not set". */ currentTriggerVersionFetchFailed?: boolean; /** Hide the section-level master toggles for "Pull env vars" and "Discover new env vars". */ - hideSectionToggles?: boolean; showAtomicDeployments?: boolean; layout?: "settings" | "card"; }; @@ -68,7 +67,6 @@ export function BuildSettingsFields({ onAutoPromoteChange, currentTriggerVersion, currentTriggerVersionFetchFailed, - hideSectionToggles, showAtomicDeployments = true, layout = "card", }: BuildSettingsFieldsProps) { @@ -222,7 +220,7 @@ export function BuildSettingsFields({
- {!hideSectionToggles && availableEnvSlugs.length > 1 && ( + {availableEnvSlugs.length > 1 && (
- {!hideSectionToggles && availableEnvSlugs.length > 1 && ( + {availableEnvSlugs.length > 1 && ( void; vercelManageAccessUrl?: string; }) { const { capture, startSessionRecording } = usePostHogTracking(); - const navigation = useNavigation(); const fetcher = useTypedFetcher(); const envMappingFetcher = useFetcher(); const completeOnboardingFetcher = useFetcher(); - const { Form: _CompleteOnboardingForm } = completeOnboardingFetcher; const [searchParams] = useSearchParams(); const origin = searchParams.get("origin"); const fromMarketplaceContext = origin === "marketplace"; @@ -130,7 +127,6 @@ export function VercelOnboardingModal({ () => onboardingData?.availableProjects ?? [], [onboardingData?.availableProjects] ); - const _hasProjectSelected = onboardingData?.hasProjectSelected ?? false; const customEnvironments = useMemo( () => onboardingData?.customEnvironments ?? [], [onboardingData?.customEnvironments] @@ -224,10 +220,6 @@ export function VercelOnboardingModal({ environmentId: string; displayName: string; } | null>(null); - const _availableEnvSlugsForOnboarding = getAvailableEnvSlugs( - hasStagingEnvironment, - hasPreviewEnvironment - ); const availableEnvSlugsForOnboardingBuildSettings = getAvailableEnvSlugsForBuildSettings( hasStagingEnvironment, hasPreviewEnvironment @@ -375,7 +367,6 @@ export function VercelOnboardingModal({ } break; - case "installing": case "project-selection": case "env-mapping": case "env-var-sync": @@ -459,8 +450,6 @@ export function VercelOnboardingModal({ const overlappingEnvVarsCount = enabledEnvVars.filter((v) => existingVars[v.key]).length; - const _isSubmitting = navigation.state === "submitting" || navigation.state === "loading"; - const actionUrl = vercelResourcePath(organizationSlug, projectSlug, environmentSlug); const handleToggleEnvVar = useCallback((key: string, enabled: boolean) => { @@ -634,19 +623,6 @@ export function VercelOnboardingModal({ gitHubAppInstallations.length, ]); - const _handleFinishOnboarding = useCallback( - (e: React.FormEvent) => { - e.preventDefault(); - const form = e.currentTarget; - const formData = new FormData(form); - completeOnboardingFetcher.submit(formData, { - method: "post", - action: actionUrl, - }); - }, - [completeOnboardingFetcher, actionUrl] - ); - useEffect(() => { if ( completeOnboardingFetcher.data && @@ -698,13 +674,6 @@ export function VercelOnboardingModal({ } }, [state, onClose, trackOnboarding, isGitHubConnectedForOnboarding]); - useEffect(() => { - if (state === "installing") { - const installUrl = vercelAppInstallPath(organizationSlug, projectSlug); - window.location.href = installUrl; - } - }, [state, organizationSlug, projectSlug]); - useEffect(() => { if ( envMappingFetcher.data && @@ -749,7 +718,6 @@ export function VercelOnboardingModal({ state === "loading-projects" || state === "loading-env-mapping" || state === "loading-env-vars" || - state === "installing" || (state === "idle" && !onboardingData); if (isLoadingState) { @@ -758,9 +726,7 @@ export function VercelOnboardingModal({ open={isOpen} onOpenChange={(open) => { if (!open && !fromMarketplaceContext) { - if ((state as string) !== "completed") { - trackOnboarding("vercel onboarding abandoned"); - } + trackOnboarding("vercel onboarding abandoned"); onClose(); } }} @@ -772,9 +738,30 @@ export function VercelOnboardingModal({ Set up Vercel Integration
-
- -
+ {onboardingDataUnavailable ? ( +
+ + We couldn't load your Vercel projects. The integration may have been removed or lost + access to this organization on Vercel. + +
+ {onDataReload && ( + + )} + {vercelManageAccessUrl && ( + + Manage access on Vercel + + )} +
+
+ ) : ( +
+ +
+ )} ); diff --git a/apps/webapp/app/components/runs/v3/RunsListErrorState.tsx b/apps/webapp/app/components/runs/v3/RunsListErrorState.tsx new file mode 100644 index 00000000000..85f75e00602 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/RunsListErrorState.tsx @@ -0,0 +1,28 @@ +import { Callout } from "~/components/primitives/Callout"; + +/** + * Error state for a runs list that failed to load. Shown as the `errorElement` of the deferred + * runs-list data. The most common recoverable cause is a query that was too expensive over a broad + * time range (see `RunsListQueryError`), so the copy guides narrowing the range; a refresh covers + * transient failures. The precise reason is not shown because Remix scrubs thrown error messages in + * production. + */ +export function RunsListErrorState() { + return ( +
+ + We couldn't load these runs. If you're filtering over a broad time range, try narrowing it, + then refresh to try again. + +
+ ); +} + +/** + * Renders nothing. Used as the `errorElement` for secondary awaits of the same runs-list promise + * (e.g. the pagination controls), so a rejection is handled locally there and does not bubble to + * the route error boundary. The primary awaits render {@link RunsListErrorState}. + */ +export function RunsListErrorStateNoop() { + return null; +} diff --git a/apps/webapp/app/components/runs/v3/deploymentLogsCache.test.ts b/apps/webapp/app/components/runs/v3/deploymentLogsCache.test.ts new file mode 100644 index 00000000000..3253b11411b --- /dev/null +++ b/apps/webapp/app/components/runs/v3/deploymentLogsCache.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { DeploymentLogsCache, type DeploymentLogEntry } from "./deploymentLogsCache"; + +function lines(count: number): DeploymentLogEntry[] { + return Array.from({ length: count }, (_, i) => ({ + message: `line ${i}`, + timestamp: new Date(0), + level: "info" as const, + })); +} + +describe("DeploymentLogsCache", () => { + it("returns undefined for unknown keys", () => { + const cache = new DeploymentLogsCache(2, 100); + expect(cache.get("missing")).toBeUndefined(); + }); + + it("stores and returns entries", () => { + const cache = new DeploymentLogsCache(2, 100); + const value = { logs: lines(3), nextSeqNum: 3, finalized: true, complete: true }; + cache.set("a", value); + expect(cache.get("a")).toBe(value); + expect(cache.size).toBe(1); + expect(cache.lineCount).toBe(3); + }); + + it("evicts the least recently used deployment past the entry limit", () => { + const cache = new DeploymentLogsCache(2, 100); + cache.set("a", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + cache.set("b", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + cache.get("a"); + cache.set("c", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + + expect(cache.get("b")).toBeUndefined(); + expect(cache.get("a")).toBeDefined(); + expect(cache.get("c")).toBeDefined(); + expect(cache.size).toBe(2); + }); + + it("evicts oldest deployments past the total line budget", () => { + const cache = new DeploymentLogsCache(10, 10); + cache.set("a", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.set("b", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.set("c", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("b")).toBeDefined(); + expect(cache.get("c")).toBeDefined(); + expect(cache.lineCount).toBe(8); + }); + + it("always keeps the entry just set, even when it alone exceeds the budget", () => { + const cache = new DeploymentLogsCache(10, 10); + cache.set("a", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.set("big", { logs: lines(50), nextSeqNum: 50, finalized: true, complete: true }); + + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("big")?.logs).toHaveLength(50); + expect(cache.size).toBe(1); + expect(cache.lineCount).toBe(50); + }); + + it("treats replacing a key as a recent use", () => { + const cache = new DeploymentLogsCache(2, 100); + cache.set("a", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + cache.set("b", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + cache.set("a", { logs: lines(2), nextSeqNum: 2, finalized: true, complete: true }); + cache.set("c", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + + expect(cache.get("b")).toBeUndefined(); + expect(cache.get("a")?.logs).toHaveLength(2); + expect(cache.get("c")).toBeDefined(); + }); + + it("keeps recently read deployments when evicting for the line budget", () => { + const cache = new DeploymentLogsCache(10, 10); + cache.set("a", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.set("b", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.get("a"); + cache.set("c", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + + expect(cache.get("b")).toBeUndefined(); + expect(cache.get("a")).toBeDefined(); + expect(cache.get("c")).toBeDefined(); + expect(cache.lineCount).toBe(8); + }); + + it("replaces an existing key without double counting lines", () => { + const cache = new DeploymentLogsCache(10, 100); + cache.set("a", { logs: lines(4), nextSeqNum: 4, finalized: false, complete: false }); + cache.set("a", { logs: lines(6), nextSeqNum: 6, finalized: true, complete: true }); + + expect(cache.size).toBe(1); + expect(cache.lineCount).toBe(6); + expect(cache.get("a")?.complete).toBe(true); + }); +}); diff --git a/apps/webapp/app/components/runs/v3/deploymentLogsCache.ts b/apps/webapp/app/components/runs/v3/deploymentLogsCache.ts new file mode 100644 index 00000000000..0cb3977437d --- /dev/null +++ b/apps/webapp/app/components/runs/v3/deploymentLogsCache.ts @@ -0,0 +1,57 @@ +export type DeploymentLogEntry = { + message: string; + timestamp: Date; + level: "info" | "error" | "warn" | "debug"; +}; + +export type CachedDeploymentLogs = { + logs: readonly DeploymentLogEntry[]; + nextSeqNum: number; + finalized: boolean; + complete: boolean; +}; + +export class DeploymentLogsCache { + private entries = new Map(); + private totalLines = 0; + + constructor( + private readonly maxDeployments: number, + private readonly maxTotalLines: number + ) {} + + get(key: string): CachedDeploymentLogs | undefined { + const entry = this.entries.get(key); + if (!entry) return undefined; + this.entries.delete(key); + this.entries.set(key, entry); + return entry; + } + + set(key: string, value: CachedDeploymentLogs) { + const existing = this.entries.get(key); + if (existing) { + this.totalLines -= existing.logs.length; + this.entries.delete(key); + } + this.entries.set(key, value); + this.totalLines += value.logs.length; + + for (const [oldestKey, oldest] of this.entries) { + if (oldestKey === key) break; + if (this.entries.size <= this.maxDeployments && this.totalLines <= this.maxTotalLines) break; + this.entries.delete(oldestKey); + this.totalLines -= oldest.logs.length; + } + } + + get size() { + return this.entries.size; + } + + get lineCount() { + return this.totalLines; + } +} + +export const deploymentLogsCache = new DeploymentLogsCache(20, 20_000); diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index a69c83cd375..c63e0aa2d4f 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -31,15 +31,18 @@ import { assertSplitRealtimeInterlock, } from "./v3/runOpsMigration/splitMode.server"; import { computeRunOpsSplitReadEnabled } from "./v3/runOpsMigration/runOpsSplitReadGate"; -import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; -import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; +import { resolveRunOpsPoolKnobs } from "./v3/runOpsPoolKnobs.server"; +import { buildRunOpsShardTable } from "./v3/runOpsShardTable"; import { + resolveShardResilience, controlPlaneTransactionResilience, registerTransactionResilience, resilienceForClient, runOpsLegacyTransactionResilience, runOpsTransactionResilience, } from "./v3/transactionResilience.server"; +import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; +import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; import type { Span } from "@opentelemetry/api"; import { context, trace } from "@opentelemetry/api"; import { queryPerformanceMonitor } from "./utils/queryPerformanceMonitor.server"; @@ -275,10 +278,19 @@ export const webhookReplica: WebhookReplicaDatabase = singleton("webhookReplica" type RunOpsClients = { writer: PrismaClient; replica: PrismaReplicaClient }; type NewRunOpsClients = { writer: RunOpsPrismaClient; replica: RunOpsPrismaClient }; +type ShardTopologyDescriptor = { + key: string; + url?: string; + replicaUrl?: string; + aliasOf?: "new"; +}; export type RunOpsTopology = { newRunOps: NewRunOpsClients; legacyRunOps: RunOpsClients; controlPlane: RunOpsClients; + // One client pair per gen-2 shard descriptor. Empty unless RUN_OPS_SHARDS is configured. An + // aliasOf:"new" descriptor maps to the newRunOps pair BY REFERENCE (no new pool). + shards: Map; }; export type SelectRunOpsTopologyConfig = { splitEnabled: boolean; @@ -288,6 +300,7 @@ export type SelectRunOpsTopologyConfig = { newReplicaUrl?: string; // When true, legacy reuses the control-plane client instead of opening its own pool. Defaults to false. legacySharesControlPlane?: boolean; + shards?: ShardTopologyDescriptor[]; }; export type RunOpsClientBuilders = { controlPlane: RunOpsClients; @@ -297,6 +310,10 @@ export type RunOpsClientBuilders = { // RunOpsPrismaClient double-cast needed): the legacy DB carries the full control-plane schema. buildLegacyWriter: (url: string, clientType: string) => PrismaClient; buildLegacyReplica: (url: string, clientType: string) => PrismaReplicaClient; + // Receive the whole descriptor so the singleton can resolve per-shard knobs and resilience by key. + // Optional so the existing test literals (which build no shards) need no change. + buildShardWriter?: (shard: ShardTopologyDescriptor) => RunOpsPrismaClient; + buildShardReplica?: (shard: ShardTopologyDescriptor) => RunOpsPrismaClient; }; // Pure run-ops client selector. No env, no isSplitEnabled() — those @@ -315,11 +332,11 @@ export function selectRunOpsTopology( }; if (!config.splitEnabled) { - return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane }; + return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane, shards: new Map() }; } if (!config.legacyUrl || !config.newUrl) { - return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane }; + return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane, shards: new Map() }; } // Same-DB legacy reuses the control-plane pool; only build a separate pool once the DSNs diverge. @@ -338,12 +355,28 @@ export function selectRunOpsTopology( const newReplica: RunOpsPrismaClient = config.newReplicaUrl ? builders.buildNewReplica(config.newReplicaUrl, "run-ops-replica") : newWriter; + const newRunOps: NewRunOpsClients = { writer: newWriter, replica: newReplica }; + + const shards = new Map(); + for (const shard of config.shards ?? []) { + if (shard.aliasOf === "new") { + // Aliased: share the new store's clients by reference. No builder, no new pool — the soak path. + shards.set(shard.key, newRunOps); + continue; + } + if (!shard.url || !builders.buildShardWriter || !builders.buildShardReplica) { + throw new Error( + `selectRunOpsTopology: shard "${shard.key}" needs a url and shard builders when not aliased` + ); + } + const shardWriter = builders.buildShardWriter(shard); + const shardReplica: RunOpsPrismaClient = shard.replicaUrl + ? builders.buildShardReplica(shard) + : shardWriter; + shards.set(shard.key, { writer: shardWriter, replica: shardReplica }); + } - return { - newRunOps: { writer: newWriter, replica: newReplica }, - legacyRunOps, - controlPlane, - }; + return { newRunOps, legacyRunOps, controlPlane, shards }; } // The env-bound run-ops topology singleton. The split decision uses @@ -376,6 +409,17 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { ); } + const newPoolKnobs = resolveRunOpsPoolKnobs("new"); + const shardDescriptorsByKey = new Map(env.RUN_OPS_SHARDS.map((d) => [d.key, d])); + + // Boot table: emit ONLY when shards are configured, so the inert (RUN_OPS_SHARDS unset) merge adds + // no new log output. The fingerprint is an address, not an identity claim (see runOpsAddressFingerprint). + if (env.RUN_OPS_SHARDS.length > 0) { + logger.info("run-ops shard topology (fingerprint is an address, NOT an identity claim)", { + shards: buildRunOpsShardTable(env.RUN_OPS_SHARDS), + }); + } + return selectRunOpsTopology( { splitEnabled, @@ -384,6 +428,12 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { newUrl, newReplicaUrl: env.RUN_OPS_DATABASE_READ_REPLICA_URL, legacySharesControlPlane, + shards: env.RUN_OPS_SHARDS.map((d) => ({ + key: d.key, + url: d.url, + replicaUrl: d.replicaUrl, + aliasOf: d.aliasOf, + })), }, { controlPlane: { writer: prisma, replica: $replica }, @@ -392,10 +442,14 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { captureInfraErrorsRunOps( tagDatasourceRunOps( "run-ops-writer", - buildRunOpsWriterClient({ + buildRunOpsClient({ url, clientType, - useDriverAdapter: env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", + role: "writer", + connectionLimit: newPoolKnobs.connectionLimit, + poolTimeout: newPoolKnobs.writerPoolTimeout, + connectTimeout: newPoolKnobs.writerConnectionTimeout, + useDriverAdapter: newPoolKnobs.writerDriverAdapter, }) ) ), @@ -409,10 +463,14 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { captureInfraErrorsRunOps( tagDatasourceRunOps( "run-ops-replica", - buildRunOpsReplicaClient({ + buildRunOpsClient({ url, clientType, - useDriverAdapter: env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + role: "replica", + connectionLimit: newPoolKnobs.replicaConnectionLimit, + poolTimeout: newPoolKnobs.replicaPoolTimeout, + connectTimeout: newPoolKnobs.replicaConnectionTimeout, + useDriverAdapter: newPoolKnobs.replicaDriverAdapter, }) ) ) @@ -450,6 +508,50 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { ) ) ), + // A gen-2 shard is a dedicated run-ops DB, so it mirrors buildNewWriter/buildNewReplica: same + // client class, same wrapper stack, its OWN resilience budget, and the "new"-role pool knobs + // merged with the descriptor's per-shard overrides. Shards share the run-ops datasource tag. + buildShardWriter: (shard) => { + const descriptor = shardDescriptorsByKey.get(shard.key); + const knobs = resolveRunOpsPoolKnobs("new", descriptor?.knobs); + return registerTransactionResilience( + captureInfraErrorsRunOps( + tagDatasourceRunOps( + "run-ops-writer", + buildRunOpsClient({ + url: shard.url!, + clientType: `run-ops-shard-${shard.key}-writer`, + role: "writer", + connectionLimit: knobs.connectionLimit, + poolTimeout: knobs.writerPoolTimeout, + connectTimeout: knobs.writerConnectionTimeout, + useDriverAdapter: knobs.writerDriverAdapter, + }) + ) + ), + resolveShardResilience(shard.key, descriptor?.knobs) + ); + }, + buildShardReplica: (shard) => { + const descriptor = shardDescriptorsByKey.get(shard.key); + const knobs = resolveRunOpsPoolKnobs("new", descriptor?.knobs); + return markReadReplicaClient( + captureInfraErrorsRunOps( + tagDatasourceRunOps( + "run-ops-replica", + buildRunOpsClient({ + url: shard.replicaUrl!, + clientType: `run-ops-shard-${shard.key}-replica`, + role: "replica", + connectionLimit: knobs.replicaConnectionLimit, + poolTimeout: knobs.replicaPoolTimeout, + connectTimeout: knobs.replicaConnectionTimeout, + useDriverAdapter: knobs.replicaDriverAdapter, + }) + ) + ) + ); + }, } ); }); @@ -475,12 +577,38 @@ export const runOpsLegacyPrismaClient: RunOpsPrismaClient = runOpsTopology.legac export const runOpsLegacyReplicaClient: RunOpsPrismaClient = runOpsTopology.legacyRunOps .replica as unknown as RunOpsPrismaClient; +// Gen-2 shard handles for the run-store boundary. Empty unless RUN_OPS_SHARDS is configured. +// `aliasOf` carries the descriptor's declared alias so the router can dedup an aliased shard (which +// shares its target's database) out of every fan-out sum — the DECLARATION, not client identity. +const runOpsShardAliasByKey = new Map(env.RUN_OPS_SHARDS.map((d) => [d.key, d.aliasOf])); +export const runOpsShardHandles: Array<{ + key: string; + writer: RunOpsPrismaClient; + replica: RunOpsPrismaClient; + aliasOf?: string; +}> = [...runOpsTopology.shards.entries()].map(([key, clients]) => ({ + key, + writer: clients.writer, + replica: clients.replica, + aliasOf: runOpsShardAliasByKey.get(key), +})); + export const runOpsSplitReadEnabled: boolean = computeRunOpsSplitReadEnabled({ newReplica: runOpsNewReplicaClient, controlPlaneWriter: prisma, controlPlaneReplica: $replica, hasNewUrl: !!env.RUN_OPS_DATABASE_URL, hasLegacyUrl: !!env.RUN_OPS_LEGACY_DATABASE_URL, + // Observability only: a non-distinct shard handle warns and never changes the gen-1 verdict. + // Empty unless RUN_OPS_SHARDS is configured. + shardHandles: runOpsShardHandles.map((handle) => ({ + key: handle.key, + writer: handle.writer, + replica: handle.replica, + // The DECLARED field, not client identity: an aliased shard shares its target's client by + // reference, so identity comparison cannot tell the two apart. + aliasOf: env.RUN_OPS_SHARDS.find((d) => d.key === handle.key)?.aliasOf, + })), logger, }); @@ -924,64 +1052,64 @@ export function buildReplicaClient({ return replicaClient; } -function buildRunOpsWriterClient({ +// One factory for the run-ops writer and replica clients, backed by the dedicated RunOpsPrismaClient +// (a separately generated Prisma package). Parameterized by role and the resolved pool knobs, so a +// gen-1 new store and every gen-2 shard share this single builder. The control-plane builders +// (buildWriterClient/buildReplicaClient) are a DIFFERENT path and are untouched — this reuses only +// the shared low-level helpers (buildPrismaConnectionUrl, buildDriverAdapterPool). +function buildRunOpsClient({ url, clientType, + role, + connectionLimit, + poolTimeout, + connectTimeout, useDriverAdapter = false, }: { url: string; clientType: string; + role: "writer" | "replica"; + connectionLimit: number; + poolTimeout: number; + connectTimeout: number; useDriverAdapter?: boolean; }): RunOpsPrismaClient { - const databaseUrl = buildPrismaConnectionUrl(url, { - connectionLimit: env.DATABASE_CONNECTION_LIMIT.toString(), - poolTimeout: (env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT).toString(), - connectTimeout: ( - env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT - ).toString(), + const isWriter = role === "writer"; + const setupLabel = isWriter ? "run-ops prisma client" : "run-ops read replica connection"; + const connectedLabel = isWriter + ? "run-ops prisma client connected" + : "run-ops read replica connected"; + + const connectionUrl = buildPrismaConnectionUrl(url, { + connectionLimit: connectionLimit.toString(), + poolTimeout: poolTimeout.toString(), + connectTimeout: connectTimeout.toString(), applicationName: env.SERVICE_NAME, }); console.log( - `🔌 setting up run-ops prisma client to ${redactUrlSecrets(databaseUrl)}${ + `🔌 setting up ${setupLabel} to ${redactUrlSecrets(connectionUrl)}${ useDriverAdapter ? " (pg driver adapter)" : "" }` ); + const log = [ + { emit: "event", level: "error" }, + { emit: "event", level: "info" }, + { emit: "event", level: "warn" }, + ...((process.env.VERBOSE_PRISMA_LOGS === "1" || + process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined + ? [{ emit: "event", level: "query" }] + : []) as { emit: "event"; level: "query" }[]), + ] as const; + const driverPool = useDriverAdapter - ? buildDriverAdapterPool( - url, - clientType, - env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.DATABASE_CONNECTION_LIMIT - ) + ? buildDriverAdapterPool(url, clientType, poolTimeout, connectionLimit) : undefined; const client = driverPool - ? new RunOpsPrismaClient({ - adapter: driverPool.adapter, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }) - : new RunOpsPrismaClient({ - datasources: { db: { url: databaseUrl.href } }, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }); + ? new RunOpsPrismaClient({ adapter: driverPool.adapter, log: [...log] }) + : new RunOpsPrismaClient({ datasources: { db: { url: connectionUrl.href } }, log: [...log] }); registerDatabaseMetricsSource( driverPool @@ -999,117 +1127,27 @@ function buildRunOpsWriterClient({ client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); client.$on("error", (log) => - logger.error("RunOpsPrismaClient error", { clientType, event: log, ignoreError: true }) - ); - } - - client.$on("query", (log) => queryPerformanceMonitor.onQuery("writer", log)); - - const connectPromise = client.$connect(); - if (env.NODE_ENV === "test") { - connectPromise.catch((error) => { - logger.warn("Failed to eagerly connect run-ops prisma client (writer)", { error }); - }); - } - - console.log(`🔌 run-ops prisma client connected`); - - return client; -} - -function buildRunOpsReplicaClient({ - url, - clientType, - useDriverAdapter = false, -}: { - url: string; - clientType: string; - useDriverAdapter?: boolean; -}): RunOpsPrismaClient { - const replicaUrl = buildPrismaConnectionUrl(url, { - connectionLimit: ( - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ).toString(), - poolTimeout: ( - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT - ).toString(), - connectTimeout: ( - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT - ).toString(), - applicationName: env.SERVICE_NAME, - }); - - console.log( - `🔌 setting up run-ops read replica connection to ${redactUrlSecrets(replicaUrl)}${ - useDriverAdapter ? " (pg driver adapter)" : "" - }` - ); - - const driverPool = useDriverAdapter - ? buildDriverAdapterPool( - url, + // The writer bridges P2002 -> 422 at the store boundary, so its infra errors are logged once + // there (ignoreError). Replica errors are not on that write path, so they log normally. + logger.error("RunOpsPrismaClient error", { clientType, - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ) - : undefined; - - const client = driverPool - ? new RunOpsPrismaClient({ - adapter: driverPool.adapter, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], + event: log, + ...(isWriter ? { ignoreError: true } : {}), }) - : new RunOpsPrismaClient({ - datasources: { db: { url: replicaUrl.href } }, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }); - - registerDatabaseMetricsSource( - driverPool - ? { - clientType, - usesDriverAdapter: true, - client, - pool: driverPool.pool, - poolCounters: driverPool.poolCounters, - } - : { clientType, usesDriverAdapter: false, client } - ); - - if (process.env.PRISMA_LOG_TO_STDOUT !== "1") { - client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); - client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); - client.$on("error", (log) => - logger.error("RunOpsPrismaClient error", { clientType, event: log }) ); } - client.$on("query", (log) => queryPerformanceMonitor.onQuery("replica", log)); + client.$on("query", (log) => queryPerformanceMonitor.onQuery(role, log)); - const connectPromise = client.$connect(); - if (env.NODE_ENV === "test") { - connectPromise.catch((error) => { - logger.warn("Failed to eagerly connect run-ops prisma client (replica)", { error }); - }); - } + // Eager connect is a warm-up only — Prisma reconnects lazily on first query. ALWAYS catch the + // rejection (not just under NODE_ENV=test), so a shard/run-ops DB that is unreachable at boot + // logs a warning instead of surfacing as an unhandled promise rejection. One unreachable shard + // must not take down webapp startup. + client.$connect().catch((error) => { + logger.warn(`Failed to eagerly connect run-ops prisma client (${role})`, { error }); + }); - console.log(`🔌 run-ops read replica connected`); + console.log(`🔌 ${connectedLabel}`); return client; } diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c9179306124..2b1fba86980 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { MachinePresetName } from "@trigger.dev/core/v3"; import { BoolEnv } from "./utils/boolEnv"; import { isValidDatabaseUrl } from "./utils/db"; +import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; import { isValidRegex } from "./utils/regex"; import { isValidDuration } from "./services/realtime/duration.server"; @@ -310,6 +311,8 @@ const EnvironmentSchema = z RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"), RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER: z.string().default("0"), RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"), + // Gen-2 shard descriptors as a JSON array. Unset/"" -> [] (today). See runOpsShards.server.ts. + RUN_OPS_SHARDS: z.string().optional().transform(parseRunOpsShards), // Control-plane cache relax knobs. Unset -> defaults (DEFAULT_CP_CACHE_TTL_MS / _MAX_ENTRIES). CONTROL_PLANE_CACHE_TTL_MS: z.coerce.number().int().optional(), CONTROL_PLANE_CACHE_MAX_ENTRIES: z.coerce.number().int().optional(), @@ -332,6 +335,8 @@ const EnvironmentSchema = z .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.") .optional(), ADMIN_EMAILS: z.string().refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.").optional(), + // Instance-level kill switch for the admin dashboard and user impersonation. + ADMIN_DASHBOARD_ENABLED: BoolEnv.default(true), REMIX_APP_PORT: z.string().optional(), // Opt-in, dev-only: stream this process's logs over a local telnet/TCP socket on this port. // Read directly from process.env in server.ts (before this schema loads); declared here for discoverability. @@ -816,6 +821,19 @@ const EnvironmentSchema = z .number() .int() .default(60 * 1000 * 15), // 15 minutes + DEPLOYMENT_CONTEXT_ARTIFACT_SIZE_LIMIT_BYTES: z.coerce + .number() + .int() + .default(100 * 1024 * 1024), // 100MB + DEPLOYMENT_BUNDLE_ARTIFACT_SIZE_LIMIT_BYTES: z.coerce + .number() + .int() + .default(100 * 1024 * 1024), // 100MB + DEPLOYMENT_BUILD_ENV_VARS_SIZE_LIMIT_BYTES: z.coerce + .number() + .int() + .default(128 * 1024), // 128KB + DEPLOYMENT_BUILD_ENV_VARS_MAX_KEYS: z.coerce.number().int().default(400), // When enabled, reject deploys made by v3 CLI versions (i.e. payloads that // omit the `type` field). v4 CLI versions always send `type` ("MANAGED" or "V1"), @@ -922,6 +940,10 @@ const EnvironmentSchema = z DISABLE_HTTP_INSTRUMENTATION: BoolEnv.default(false), INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(), + + // Second trace exporter receiving only `deployment.*` spans; they still flow to the main one + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL: z.string().optional(), + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_URL: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_ENABLED: z.string().default("0"), @@ -2233,6 +2255,15 @@ const EnvironmentSchema = z .enum(["log", "error", "warn", "info", "debug"]) .default("info"), RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"), + RUNS_LIST_CLICKHOUSE_REQUEST_TIMEOUT_MS: z.coerce.number().int().positive().default(40_000), + RUNS_LIST_CLICKHOUSE_MAX_EXECUTION_TIME: z.coerce.number().int().positive().default(35), + RUNS_LIST_CLICKHOUSE_MAX_THREADS: z.coerce.number().int().positive().default(4), + RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE: z.coerce + .number() + .int() + .positive() + .default(1_073_741_824), + RUNS_LIST_CLICKHOUSE_READONLY: z.enum(["0", "1", "2"]).default("2"), /** * Dedicated ClickHouse service for queue metrics: the ingestion consumer's inserts and every * queue-metrics read (dashboards, queue pages, run inspector, health report) go through it, so @@ -2467,6 +2498,14 @@ const EnvironmentSchema = z }); } } + if (!validateShardListAgainstNewUrl(env.RUN_OPS_SHARDS, env.RUN_OPS_DATABASE_URL)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["RUN_OPS_SHARDS"], + message: + "RUN_OPS_SHARDS is non-empty but RUN_OPS_DATABASE_URL is unset; a shard requires the gen-1 new store", + }); + } }); export type Environment = z.infer; diff --git a/apps/webapp/app/hooks/useDeploymentLogs.ts b/apps/webapp/app/hooks/useDeploymentLogs.ts new file mode 100644 index 00000000000..fb581aa87ea --- /dev/null +++ b/apps/webapp/app/hooks/useDeploymentLogs.ts @@ -0,0 +1,169 @@ +import { S2, S2Error } from "@s2-dev/streamstore"; +import { DeploymentEventFromString } from "@trigger.dev/core/v3/schemas"; +import type { WorkerDeploymentStatus } from "@trigger.dev/database"; +import { useEffect, useState } from "react"; +import { + deploymentLogsCache, + type DeploymentLogEntry, +} from "~/components/runs/v3/deploymentLogsCache"; + +type DeploymentEventStream = { + s2: { + basin: string; + stream: string; + accessToken: string; + }; +}; + +const FINISHED_DEPLOYMENT_STATUSES = new Set([ + "DEPLOYED", + "FAILED", + "CANCELED", + "TIMED_OUT", +]); + +type UseDeploymentLogsOptions = { + eventStream: DeploymentEventStream | undefined; + status: WorkerDeploymentStatus; +}; + +export function useDeploymentLogs({ eventStream, status }: UseDeploymentLogsOptions) { + const [logs, setLogs] = useState([]); + const [isStreaming, setIsStreaming] = useState(true); + const [streamError, setStreamError] = useState(null); + + const basin = eventStream?.s2.basin; + const stream = eventStream?.s2.stream; + const accessToken = eventStream?.s2.accessToken; + + useEffect(() => { + if (!basin || !stream || !accessToken) return; + + const isFinished = FINISHED_DEPLOYMENT_STATUSES.has(status); + const cacheKey = `${basin}/${stream}`; + const cached = deploymentLogsCache.get(cacheKey); + + let entries = cached?.logs ?? []; + let nextSeqNum = cached?.nextSeqNum ?? 0; + let pending: DeploymentLogEntry[] = []; + let flushTimer: ReturnType | undefined; + let finalized = cached?.finalized ?? false; + + // oxlint-disable-next-line react/set-state-in-effect -- Seed from the cache when the selected deployment changes. + setLogs(entries); + setStreamError(null); + + if (cached?.complete) { + setIsStreaming(false); + return; + } + + setIsStreaming(true); + + const abortController = new AbortController(); + + const flush = () => { + clearTimeout(flushTimer); + flushTimer = undefined; + if (abortController.signal.aborted || pending.length === 0) return; + entries = entries.concat(pending); + pending = []; + setLogs(entries); + }; + + const push = (entry: DeploymentLogEntry) => { + pending.push(entry); + flushTimer ??= setTimeout(flush, 0); + }; + + const store = () => { + clearTimeout(flushTimer); + flushTimer = undefined; + if (pending.length > 0) { + entries = entries.concat(pending); + pending = []; + } + if (entries.length === 0 && nextSeqNum === 0 && !finalized) return; + deploymentLogsCache.set(cacheKey, { + logs: entries, + nextSeqNum, + finalized, + complete: finalized && isFinished, + }); + }; + + const streamLogs = async () => { + try { + const s2Stream = new S2({ accessToken }).basin(basin).stream(stream); + + do { + const readSession = await s2Stream.readSession( + { + start: { from: { seqNum: nextSeqNum }, clamp: true }, + stop: { waitSecs: 60 }, + }, + { signal: abortController.signal } + ); + + for await (const record of readSession) { + nextSeqNum = record.seqNum + 1; + + const decoded = record.body; + const result = DeploymentEventFromString.safeParse(decoded); + + if (!result.success) { + // fallback to the previous format in s2 logs for compatibility + const headers: Record = {}; + if (record.headers) { + for (const [name, value] of record.headers) { + headers[name] = value; + } + } + const level = + (headers["level"]?.toLowerCase() as DeploymentLogEntry["level"]) ?? "info"; + + push({ timestamp: new Date(record.timestamp), message: decoded, level }); + continue; + } + + const event = result.data; + if (event.type === "finalized") finalized = true; + if (event.type !== "log") continue; + + push({ + timestamp: new Date(record.timestamp), + message: event.data.message, + level: event.data.level, + }); + } + } while (!abortController.signal.aborted && !finalized && !isFinished); + } catch (error) { + if (abortController.signal.aborted) return; + + if (error instanceof S2Error && error.code === "stream_not_found") { + finalized = isFinished; + return; + } + if (error instanceof S2Error && error.code === "permission_denied") return; + + console.error("Failed to stream logs:", error); + setStreamError("Failed to stream logs"); + } finally { + if (!abortController.signal.aborted) { + flush(); + setIsStreaming(false); + store(); + } + } + }; + + streamLogs(); + + return () => { + abortController.abort(); + store(); + }; + }, [basin, stream, accessToken, status]); + + return { logs, isStreaming, streamError }; +} diff --git a/apps/webapp/app/hooks/useFollowScroll.ts b/apps/webapp/app/hooks/useFollowScroll.ts new file mode 100644 index 00000000000..4189f0377a7 --- /dev/null +++ b/apps/webapp/app/hooks/useFollowScroll.ts @@ -0,0 +1,100 @@ +import { useEffect, useLayoutEffect, useRef, useState, type RefObject } from "react"; + +const AT_BOTTOM_TOLERANCE_PX = 4; + +type FollowState = { + follow: boolean; + pinnedScrollTop: number | null; + lastScrollTop: number; + lastClientHeight: number; +}; + +export function useFollowScroll(containerRef: RefObject, content: unknown) { + const [isAtBottom, setIsAtBottom] = useState(true); + const stateRef = useRef({ + follow: true, + pinnedScrollTop: null, + lastScrollTop: 0, + lastClientHeight: 0, + }); + + useEffect(() => { + const container = containerRef.current; + if (!container) return; + + const state = stateRef.current; + state.lastScrollTop = container.scrollTop; + state.lastClientHeight = container.clientHeight; + + const onScroll = () => { + const { scrollTop, scrollHeight, clientHeight } = container; + const isEcho = scrollTop === state.pinnedScrollTop; + const movedUp = scrollTop < state.lastScrollTop && clientHeight <= state.lastClientHeight; + state.pinnedScrollTop = null; + state.lastScrollTop = scrollTop; + state.lastClientHeight = clientHeight; + if (isEcho) return; + + state.follow = !movedUp && scrollHeight - scrollTop - clientHeight <= AT_BOTTOM_TOLERANCE_PX; + setIsAtBottom(state.follow); + }; + + const onWheel = (event: WheelEvent) => { + if (event.deltaY >= 0 || !canScroll(container)) return; + state.follow = false; + setIsAtBottom(false); + }; + + container.addEventListener("scroll", onScroll, { passive: true }); + container.addEventListener("wheel", onWheel, { passive: true }); + return () => { + container.removeEventListener("scroll", onScroll); + container.removeEventListener("wheel", onWheel); + }; + }, [containerRef]); + + useLayoutEffect(() => { + const container = containerRef.current; + if (container && stateRef.current.follow) pinToBottom(container, stateRef.current); + }, [containerRef, isAtBottom, content]); + + useEffect(() => { + const container = containerRef.current; + if (!container) return; + + const observer = new ResizeObserver(() => { + if (stateRef.current.follow) pinToBottom(container, stateRef.current); + }); + observer.observe(container); + return () => observer.disconnect(); + }, [containerRef]); + + const scrollToBottom = () => { + const container = containerRef.current; + if (!container) return; + stateRef.current.follow = true; + setIsAtBottom(true); + pinToBottom(container, stateRef.current); + }; + + const scrollToTop = () => { + const container = containerRef.current; + if (!container || !canScroll(container)) return; + stateRef.current.follow = false; + setIsAtBottom(false); + container.scrollTop = 0; + }; + + return { isAtBottom, scrollToBottom, scrollToTop }; +} + +function pinToBottom(container: HTMLElement, state: FollowState) { + container.scrollTop = container.scrollHeight; + state.pinnedScrollTop = container.scrollTop; + state.lastScrollTop = container.scrollTop; + state.lastClientHeight = container.clientHeight; +} + +function canScroll(container: HTMLElement) { + return container.scrollHeight - container.clientHeight > AT_BOTTOM_TOLERANCE_PX; +} diff --git a/apps/webapp/app/hooks/useUser.ts b/apps/webapp/app/hooks/useUser.ts index 2eed91b9734..08aff433192 100644 --- a/apps/webapp/app/hooks/useUser.ts +++ b/apps/webapp/app/hooks/useUser.ts @@ -48,6 +48,12 @@ export function useHasAdminAccess(matches?: UIMatch[]): boolean { const user = useOptionalUser(matches); const isImpersonating = useIsImpersonating(matches); const isViewingAsUser = useIsViewingAsUser(matches); + const routeMatch = useTypedMatchesData({ + id: "root", + matches, + }); + + if (routeMatch?.adminDashboardEnabled === false) return false; return (Boolean(user?.admin) || isImpersonating) && !isViewingAsUser; } diff --git a/apps/webapp/app/models/admin.server.ts b/apps/webapp/app/models/admin.server.ts index e93844dbaed..513b79c2261 100644 --- a/apps/webapp/app/models/admin.server.ts +++ b/apps/webapp/app/models/admin.server.ts @@ -5,16 +5,25 @@ import type { SearchParams } from "~/routes/admin._index"; import { clearImpersonationId, commitImpersonationSession, - getImpersonationId, + getRawImpersonationId, setImpersonationId, } from "~/services/impersonation.server"; import { authenticator } from "~/services/auth.server"; import { requireUser } from "~/services/session.server"; import { extractClientIp } from "~/utils/extractClientIp.server"; import { impersonationDestinationPath } from "~/utils/pathBuilder"; +import { env } from "~/env.server"; const pageSize = 20; +// 404, not 403, so a disabled instance doesn't advertise the feature. +// Stopping an impersonation is deliberately never gated. +export function requireAdminDashboardEnabled(): void { + if (!env.ADMIN_DASHBOARD_ENABLED) { + throw new Response("Not Found", { status: 404 }); + } +} + export async function adminGetUsers(userId: string, { page, search }: SearchParams) { page = page || 1; @@ -217,6 +226,8 @@ export async function redirectWithImpersonation( currentUser?: { id: string; admin: boolean }, prismaClient: PrismaClientOrTransaction = prisma ) { + requireAdminDashboardEnabled(); + const user = currentUser ?? (await requireUser(request)); if (!user.admin) { throw new Error("Unauthorized"); @@ -332,7 +343,8 @@ export async function startImpersonation( export async function clearImpersonation(request: Request, path: string) { const authUser = await authenticator.isAuthenticated(request); - const targetId = await getImpersonationId(request); + // Raw read: stops must audit and clear even with ADMIN_DASHBOARD_ENABLED off. + const targetId = await getRawImpersonationId(request); if (targetId && authUser?.userId) { const xff = request.headers.get("x-forwarded-for"); diff --git a/apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts index 67ef45ebd27..abe04948083 100644 --- a/apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts @@ -1,5 +1,5 @@ import type { BatchTaskRunExecutionResult } from "@trigger.dev/core/v3"; -import { ownerEngine } from "@trigger.dev/core/v3/isomorphic"; +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import { $replica, type PrismaClientOrTransaction, @@ -13,6 +13,8 @@ import { runStore as defaultRunStore } from "~/v3/runStore.server"; import { BasePresenter } from "./basePresenter.server"; import { boundedIn } from "@trigger.dev/database"; +import { runOpsShardReplicas } from "~/v3/runOpsMigration/shardHandles.server"; +import { logger } from "~/services/logger.server"; /** * Run-ops read-through wiring. All optional; absent (or `splitEnabled` falsy) collapses `call` to * passthrough. `legacyReplica` is a READ REPLICA handle only — there is NO legacy-primary field. @@ -21,6 +23,8 @@ type ApiBatchResultsReadThroughDeps = { splitEnabled?: boolean; newClient?: PrismaReplicaClient; legacyReplica?: PrismaReplicaClient; + /** Gen-2 shard replicas by shard char; empty (RUN_OPS_SHARDS unset) keeps today's behaviour. */ + shardReplicas?: ReadonlyMap; isPastRetention?: (runId: string) => boolean; }; @@ -181,16 +185,57 @@ export class ApiBatchResultsPresenter extends BasePresenter { const taskRunIds = batchRun.items.map((item) => item.taskRunId); - const newRows = (await newClient.taskRun.findMany({ - where: { id: { in: boundedIn(taskRunIds) } }, - select: memberRunSelect, - })) as TaskRunWithAttempts[]; + // A gen-2 id is directly routable to its own shard, so it must not join the gen-1 read: + // it would miss there, and (being dedicated-family) never reach the legacy probe either. + const shardReplicas = this.readThrough?.shardReplicas ?? runOpsShardReplicas; + const genOneIds: string[] = []; + const idsByShard = new Map(); + for (const id of taskRunIds) { + const shardKey = resolveShard(id); + if (shardKey === "new" || shardKey === "legacy") { + genOneIds.push(id); + } else if (shardReplicas.has(shardKey)) { + const group = idsByShard.get(shardKey); + group ? group.push(id) : idsByShard.set(shardKey, [id]); + } else { + // Not routable and not a gen-1 shape. A gen-1 store is the wrong database, and a + // dedicated-family id never reaches the legacy probe, so falling back there would + // drop the member silently. Drop it loudly instead. + logger.error("ApiBatchResultsPresenter: gen-2 member on an unconfigured shard key", { + runId: id, + shardKey, + configured: [...shardReplicas.keys()], + }); + } + } + + const newRows = ( + genOneIds.length > 0 + ? ((await newClient.taskRun.findMany({ + where: { id: { in: boundedIn(genOneIds) } }, + select: memberRunSelect, + })) as TaskRunWithAttempts[]) + : [] + ).concat( + ( + await Promise.all( + [...idsByShard.entries()].map( + async ([shardKey, ids]) => + (await shardReplicas.get(shardKey)!.taskRun.findMany({ + where: { id: { in: boundedIn(ids) } }, + select: memberRunSelect, + })) as TaskRunWithAttempts[] + ) + ) + ).flat() + ); const runsById = new Map(newRows.map((run) => [run.id, run])); - // A run-ops id can only live on NEW, so only misses that AREN'T run-ops-shaped are candidates - // for the legacy probe — mirrors readThroughRun's per-id "NEW residency skips legacy" rule. - const legacyCandidateIds = taskRunIds.filter( - (id) => !runsById.has(id) && ownerEngine(id) !== "NEW" + // A dedicated-family id (gen-1 v1 or gen-2) can only live on its own store, so only + // misses that AREN'T dedicated-shaped are candidates for the legacy probe — mirrors + // readThroughRun's per-id "dedicated residency skips legacy" rule. + const legacyCandidateIds = genOneIds.filter( + (id) => !runsById.has(id) && resolveShard(id) === "legacy" ); if (legacyCandidateIds.length > 0) { const legacyRows = (await legacyReplica.taskRun.findMany({ diff --git a/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts b/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts index a04368b4255..6ddd6250b58 100644 --- a/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts @@ -26,7 +26,7 @@ export class CreateBulkActionPresenter extends BasePresenter { const clickhouse = await clickhouseFactory.getClickhouseForOrganization( organizationId, - "standard" + "runsList" ); const runsRepository = new RunsRepository({ clickhouse, diff --git a/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts b/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts index 38149abe5e7..c2015c20bb4 100644 --- a/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts @@ -87,7 +87,8 @@ export class ErrorGroupPresenter extends BasePresenter { constructor( private readonly replica: PrismaClientOrTransaction, private readonly logsClickhouse: ClickHouse, - private readonly clickhouse: ClickHouse + private readonly clickhouse: ClickHouse, + private readonly runsListClickhouse: ClickHouse ) { super(undefined, replica); } @@ -409,7 +410,7 @@ export class ErrorGroupPresenter extends BasePresenter { columns?: RunColumnsSelect; } ): Promise { - const runListPresenter = new NextRunListPresenter(this.replica, this.clickhouse); + const runListPresenter = new NextRunListPresenter(this.replica, this.runsListClickhouse); const result = await runListPresenter.call(organizationId, environmentId, { userId: options.userId, diff --git a/apps/webapp/app/presenters/v3/GitHubSettingsPresenter.server.ts b/apps/webapp/app/presenters/v3/GitHubSettingsPresenter.server.ts index 53bd034f249..162e44f8a88 100644 --- a/apps/webapp/app/presenters/v3/GitHubSettingsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/GitHubSettingsPresenter.server.ts @@ -19,6 +19,7 @@ export class GitHubSettingsPresenter extends BasePresenter { connectedRepository: undefined, installations: undefined, isPreviewEnvironmentEnabled: undefined, + isStagingEnvironmentEnabled: undefined, }); } @@ -114,7 +115,8 @@ export class GitHubSettingsPresenter extends BasePresenter { }, where: { projectId: projectId, - slug: "preview", + type: "PREVIEW", + parentEnvironmentId: null, }, }), (error) => ({ @@ -123,15 +125,42 @@ export class GitHubSettingsPresenter extends BasePresenter { }) ).map((previewEnvironment) => previewEnvironment !== null); + const isStagingEnvironmentEnabled = () => + fromPromise( + (this._replica as PrismaClient).runtimeEnvironment.findFirst({ + select: { + id: true, + }, + where: { + projectId: projectId, + type: "STAGING", + parentEnvironmentId: null, + }, + }), + (error) => ({ + type: "other" as const, + cause: error, + }) + ).map((stagingEnvironment) => stagingEnvironment !== null); + return ResultAsync.combine([ isPreviewEnvironmentEnabled(), + isStagingEnvironmentEnabled(), findConnectedGithubRepository(), listGithubAppInstallations(), - ]).map(([isPreviewEnvironmentEnabled, connectedGithubRepository, githubAppInstallations]) => ({ - enabled: true, - connectedRepository: connectedGithubRepository, - installations: githubAppInstallations, - isPreviewEnvironmentEnabled, - })); + ]).map( + ([ + isPreviewEnvironmentEnabled, + isStagingEnvironmentEnabled, + connectedGithubRepository, + githubAppInstallations, + ]) => ({ + enabled: true, + connectedRepository: connectedGithubRepository, + installations: githubAppInstallations, + isPreviewEnvironmentEnabled, + isStagingEnvironmentEnabled, + }) + ); } } diff --git a/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts b/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts index 10a46c01b3a..841c929d141 100644 --- a/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts @@ -182,6 +182,7 @@ export class VercelSettingsPresenter extends BasePresenter { where: { projectId, type: "STAGING", + parentEnvironmentId: null, }, }), (error) => ({ @@ -199,6 +200,7 @@ export class VercelSettingsPresenter extends BasePresenter { where: { projectId, type: "PREVIEW", + parentEnvironmentId: null, }, }), (error) => ({ diff --git a/apps/webapp/app/root.tsx b/apps/webapp/app/root.tsx index 3cb547db4c7..de230f16a67 100644 --- a/apps/webapp/app/root.tsx +++ b/apps/webapp/app/root.tsx @@ -21,7 +21,8 @@ import { env } from "./env.server"; import { featuresForRequest } from "./features.server"; import { usePostHog } from "./hooks/usePostHog"; import { resolveThemePreference, useSystemThemeSync } from "./hooks/useSystemThemeSync"; -import { getImpersonationState } from "./services/impersonation.server"; +import { clearImpersonation } from "./models/admin.server"; +import { getImpersonationState, getRawImpersonationId } from "./services/impersonation.server"; import { getUser } from "./services/session.server"; import { normalizeIconContrast, @@ -117,6 +118,13 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { // the `user.isViewingAsUser` the server computes could disagree, and the // client-side admin UI would hide itself on a session that is not // impersonating. + // Flag off: terminate lingering impersonation sessions (audit + clear) + // rather than leaving a cookie that would resurrect on a later re-enable. + if (!env.ADMIN_DASHBOARD_ENABLED && (await getRawImpersonationId(request))) { + const url = new URL(request.url); + throw await clearImpersonation(request, `${url.pathname}${url.search}`); + } + const { isViewingAsUser } = await getImpersonationState(request, user?.id); const headers = new Headers(); @@ -126,6 +134,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { { user, isViewingAsUser, + adminDashboardEnabled: env.ADMIN_DASHBOARD_ENABLED, toastMessage, posthogProjectKey, posthogUiHost, diff --git a/apps/webapp/app/routes/@.runs.$runParam.ts b/apps/webapp/app/routes/@.runs.$runParam.ts index ed5ca156f38..d8ff7fd49d1 100644 --- a/apps/webapp/app/routes/@.runs.$runParam.ts +++ b/apps/webapp/app/routes/@.runs.$runParam.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { prisma } from "~/db.server"; import { runStore } from "~/v3/runStore.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { requireAdminDashboardEnabled } from "~/models/admin.server"; import { redirectWithErrorMessage } from "~/models/message.server"; import { requireUser } from "~/services/session.server"; import { impersonate, rootPath, v3RunPath, v3RunSpanPath } from "~/utils/pathBuilder"; @@ -13,6 +14,8 @@ const ParamsSchema = z.object({ }); export async function loader({ params, request }: LoaderFunctionArgs) { + requireAdminDashboardEnabled(); + const user = await requireUser(request); const { runParam } = ParamsSchema.parse(params); diff --git a/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx b/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx index 2923a6fdeeb..0fc4f41d1a3 100644 --- a/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx +++ b/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx @@ -10,6 +10,7 @@ import { env } from "~/env.server"; import { clearImpersonation, findImpersonationTarget, + requireAdminDashboardEnabled, startImpersonation, } from "~/models/admin.server"; import { logger } from "~/services/logger.server"; @@ -26,6 +27,8 @@ import { isSameOriginNavigation } from "~/utils/sameOriginNavigation"; // here would drag server-only modules into the client build. export async function loader({ request, params }: LoaderFunctionArgs) { + requireAdminDashboardEnabled(); + const user = await requireUser(request); // If already impersonating, we need to clear the impersonation. Redirects are @@ -101,6 +104,8 @@ function refererOrigin(request: Request): string | undefined { } export async function action({ request, params }: ActionFunctionArgs) { + requireAdminDashboardEnabled(); + if (request.method.toLowerCase() !== "post") { return new Response("Method not allowed", { status: 405 }); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx index e0354185363..2fd312e3257 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx @@ -22,6 +22,11 @@ import { Paragraph } from "~/components/primitives/Paragraph"; import * as Property from "~/components/primitives/PropertyTable"; import { Spinner } from "~/components/primitives/Spinner"; import { TabButton, TabContainer } from "~/components/primitives/Tabs"; +import { + RunsListErrorState, + RunsListErrorStateNoop, +} from "~/components/runs/v3/RunsListErrorState"; +import { RunsListQueryError } from "~/services/runsRepository/runsRepository.server"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable"; import { SessionsTable } from "~/components/sessions/v1/SessionsTable"; @@ -92,10 +97,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const directionRaw = url.searchParams.get("direction") ?? undefined; const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; - const clickhouse = await clickhouseFactory.getClickhouseForOrganization( - project.organizationId, - "standard" - ); + const [clickhouse, runsListClickhouse] = await Promise.all([ + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "runsList"), + ]); const presenter = new AgentDetailPresenter($replica, clickhouse); const agent = await presenter.findAgent({ @@ -154,7 +159,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }) .catch(() => ({ data: [], statuses: [] }) satisfies AgentActivity); - const runList = new NextRunListPresenter($replica, clickhouse) + const runList = new NextRunListPresenter($replica, runsListClickhouse) .call(project.organizationId, environment.id, { userId, projectId: project.id, @@ -166,7 +171,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { direction, columns: getRunColumnsForSelect(request), }) - .catch(() => null); + .catch((error) => { + if (error instanceof RunsListQueryError) { + throw error; + } + return null; + }); const sessionList = new SessionListPresenter($replica, clickhouse) .call(project.organizationId, environment.id, { @@ -341,7 +351,7 @@ export default function Page() { <> - + }> {(list) => (list ? : null)} @@ -395,7 +405,7 @@ function AgentContentArea({ ) : ( }> - }> + }> {(list) => list ? ( { } }; -type LogEntry = { - message: string; - timestamp: Date; - level: "info" | "error" | "warn" | "debug"; -}; - function getTriggeredViaDisplay(triggeredVia: string | null | undefined): { icon: React.ReactNode; label: string; @@ -205,110 +202,10 @@ export default function Page() { const page = new URLSearchParams(location.search).get("page"); const logsDisabled = eventStream === undefined; - const [logs, setLogs] = useState([]); - const [isStreaming, setIsStreaming] = useState(true); - const [streamError, setStreamError] = useState(null); - const isPending = deployment.status === "PENDING"; - - useEffect(() => { - if (logsDisabled) return; - - const abortController = new AbortController(); - - // oxlint-disable-next-line react/set-state-in-effect -- This effect intentionally synchronizes route state after an external or lifecycle change. - setLogs([]); - setStreamError(null); - setIsStreaming(true); - - const streamLogs = async () => { - try { - const s2 = new S2({ accessToken: eventStream.s2.accessToken }); - const basin = s2.basin(eventStream.s2.basin); - const stream = basin.stream(eventStream.s2.stream); - - const readSession = await stream.readSession( - { - start: { from: { seqNum: 0 }, clamp: true }, - stop: { waitSecs: 60 }, - }, - { signal: abortController.signal } - ); - - for await (const record of readSession) { - const decoded = record.body; - const result = DeploymentEventFromString.safeParse(decoded); - - if (!result.success) { - // fallback to the previous format in s2 logs for compatibility - try { - const headers: Record = {}; - - if (record.headers) { - for (const [name, value] of record.headers) { - headers[name] = value; - } - } - const level = (headers["level"]?.toLowerCase() as LogEntry["level"]) ?? "info"; - - setLogs((prevLogs) => [ - ...prevLogs, - { - timestamp: new Date(record.timestamp), - message: decoded, - level, - }, - ]); - } catch (err) { - console.error("Failed to parse log record:", err); - } - - continue; - } - - const event = result.data; - if (event.type !== "log") { - continue; - } - - setLogs((prevLogs) => [ - ...prevLogs, - { - timestamp: new Date(record.timestamp), - message: event.data.message, - level: event.data.level, - }, - ]); - } - } catch (error) { - if (abortController.signal.aborted) return; - - const isNotFoundError = - error instanceof S2Error && - error.code && - ["permission_denied", "stream_not_found"].includes(error.code); - if (isNotFoundError) return; - - console.error("Failed to stream logs:", error); - setStreamError("Failed to stream logs"); - } finally { - if (!abortController.signal.aborted) { - setIsStreaming(false); - } - } - }; - - streamLogs(); - - return () => { - abortController.abort(); - }; - }, [ - eventStream?.s2?.basin, - eventStream?.s2?.stream, - eventStream?.s2?.accessToken, - isPending, - logsDisabled, - ]); + const { logs, isStreaming, streamError } = useDeploymentLogs({ + eventStream, + status: deployment.status, + }); return (
@@ -413,6 +310,7 @@ export default function Page() { Logs (null); + const { isAtBottom, scrollToBottom, scrollToTop } = useFollowScroll(logsContainerRef, logs); useEffect(() => { // oxlint-disable-next-line react/set-state-in-effect, react/no-deriving-state-in-effects -- Deployment status changes intentionally reset the user-controlled collapse state. setCollapsed(initialCollapsed); }, [initialCollapsed]); - // auto-scroll log container to bottom when new logs arrive - useEffect(() => { - if (logsContainerRef.current) { - logsContainerRef.current.scrollTop = logsContainerRef.current.scrollHeight; - } - }, [logs]); - const onCopyLogs = useCallback( (event: React.MouseEvent) => { event.preventDefault(); @@ -690,6 +582,27 @@ function LogsDisplay({
{logs.length > 0 && (
+ + + + {isAtBottom ? ( + + ) : ( + + )} + + + {isAtBottom ? "Scroll to top" : "Scroll to bottom"} + + + + onChange({ ...value, key: e.currentTarget.value })} autoFocus={index === 0} onPaste={onPaste} + autoComplete="off" /> {fields.key.errors}
@@ -689,6 +690,7 @@ function VariableField({ placeholder="Not set" value={value.value} onChange={(e) => onChange({ ...value, value: e.currentTarget.value })} + autoComplete="off" /> {fields.value.errors}
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx index 617dc577b80..1974e2051fb 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx @@ -865,6 +865,7 @@ function EditEnvironmentVariablePanel({ placeholder={variable.isSecret ? "Set new secret value" : "Not set"} defaultValue={variable.value} type={"text"} + autoComplete="off" /> {value.errors} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx index cbca618883d..96751e4c1a9 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx @@ -55,6 +55,7 @@ import { Spinner } from "~/components/primitives/Spinner"; import { useToast } from "~/components/primitives/Toast"; import TooltipPortal from "~/components/primitives/TooltipPortal"; import type { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters"; +import { RunsListErrorState } from "~/components/runs/v3/RunsListErrorState"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable"; import { $replica } from "~/db.server"; @@ -254,12 +255,18 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const directionRaw = url.searchParams.get("direction") ?? undefined; const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; - const [logsClickhouseClient, clickhouseClient] = await Promise.all([ + const [logsClickhouseClient, clickhouseClient, runsListClickhouseClient] = await Promise.all([ clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "logs"), clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "runsList"), ]); - const presenter = new ErrorGroupPresenter($replica, logsClickhouseClient, clickhouseClient); + const presenter = new ErrorGroupPresenter( + $replica, + logsClickhouseClient, + clickhouseClient, + runsListClickhouseClient + ); const detailPromise = presenter .call(project.organizationId, environment.id, { @@ -393,16 +400,7 @@ export default function Page() {
} > - - - Unable to load error details. Please refresh the page or try again in a moment. - - - } - > + }> {(result) => { if ("error" in result) { return ( diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx index 921e429700c..0abcd9444ad 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx @@ -74,7 +74,7 @@ import { import { throwNotFound } from "~/utils/httpErrors"; import { ListPagination } from "../../components/ListPagination"; import { CreateBulkActionInspector } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction"; -import { Callout } from "~/components/primitives/Callout"; +import { RunsListErrorState } from "~/components/runs/v3/RunsListErrorState"; import { isRunsListLoading, RUNS_BULK_INSPECTOR_OPEN_VALUE, @@ -208,17 +208,7 @@ export default function Page() { } > - - - Unable to load your task runs. Please refresh the page or try again in a - moment. - - - } - > + }> {(list) => { return ( { const { projectParam, organizationSlug } = params; + const canManageBuildSettings = ability.can("write", { type: "github" }); const canManageIntegrations = - ability.can("write", { type: "github" }) || ability.can("write", { type: "vercel" }); + canManageBuildSettings || ability.can("write", { type: "vercel" }); if (!canManageIntegrations) { throwPermissionDenied("With your current role, you can't manage integrations."); @@ -102,6 +103,7 @@ export const loader = dashboardLoader( githubAppEnabled: gitHubApp.enabled, buildSettings, vercelIntegrationEnabled: OrgIntegrationRepository.isVercelSupported, + canManageBuildSettings, }); } ); @@ -208,7 +210,7 @@ export const action = dashboardAction( ); export default function IntegrationsSettingsPage() { - const { githubAppEnabled, buildSettings, vercelIntegrationEnabled } = + const { githubAppEnabled, buildSettings, vercelIntegrationEnabled, canManageBuildSettings } = useTypedLoaderData(); const project = useProject(); const organization = useOrganization(); @@ -223,6 +225,8 @@ export default function IntegrationsSettingsPage() { const loadVercelOnboarding = vercelFetcher.load; const onboardingData = vercelFetcher.data?.onboardingData ?? null; const hasVercelFetcherData = vercelFetcher.data !== undefined; + const onboardingDataUnavailable = + hasVercelFetcherData && vercelFetcher.state === "idle" && onboardingData === null; const vercelOnboardingPath = `${vercelResourcePath( organization.slug, project.slug, @@ -375,24 +379,27 @@ export default function IntegrationsSettingsPage() { /> )} - - - - Applies to deployments triggered from GitHub, and CLI deployments run with the{" "} - - --native-build-server - {" "} - flag. - - } - /> - - )} + + + + Applies to deployments triggered from GitHub, and CLI deployments run with the{" "} + + --native-build-server + {" "} + flag. + + } + /> + + {/* Vercel Onboarding Modal */} @@ -407,6 +414,7 @@ export default function IntegrationsSettingsPage() { hasStagingEnvironment={vercelFetcher.data?.hasStagingEnvironment ?? false} hasPreviewEnvironment={vercelFetcher.data?.hasPreviewEnvironment ?? false} hasOrgIntegration={vercelFetcher.data?.hasOrgIntegration ?? false} + onboardingDataUnavailable={onboardingDataUnavailable} nextUrl={nextUrl ?? undefined} vercelManageAccessUrl={vercelFetcher.data?.vercelManageAccessUrl} onDataReload={(vercelEnvironmentId) => { @@ -424,7 +432,13 @@ export default function IntegrationsSettingsPage() { ); } -function BuildSettingsForm({ buildSettings }: { buildSettings: BuildSettings }) { +function BuildSettingsForm({ + buildSettings, + canManageBuildSettings = true, +}: { + buildSettings: BuildSettings; + canManageBuildSettings?: boolean; +}) { const lastSubmission = useActionData() as any; const navigation = useNavigation(); @@ -572,7 +586,12 @@ function BuildSettingsForm({ buildSettings }: { buildSettings: BuildSettings }) name="action" value="update-build-settings" variant="secondary/small" - disabled={isBuildSettingsLoading || !hasBuildSettingsChanges} + disabled={isBuildSettingsLoading || !hasBuildSettingsChanges || !canManageBuildSettings} + tooltip={ + canManageBuildSettings + ? undefined + : "You don't have permission to manage build settings" + } LeadingIcon={isBuildSettingsLoading ? SpinnerWhite : undefined} > Save diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx index b38eee1a82f..75822975307 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx @@ -60,6 +60,11 @@ import { import { TabButton, TabContainer } from "~/components/primitives/Tabs"; import { useToast } from "~/components/primitives/Toast"; import { EnabledStatus } from "~/components/runs/v3/EnabledStatus"; +import { + RunsListErrorState, + RunsListErrorStateNoop, +} from "~/components/runs/v3/RunsListErrorState"; +import { RunsListQueryError } from "~/services/runsRepository/runsRepository.server"; import type { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters"; import { ScheduleTypeIcon, scheduleTypeName } from "~/components/runs/v3/ScheduleType"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; @@ -143,10 +148,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const directionRaw = url.searchParams.get("direction") ?? undefined; const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; - const clickhouse = await clickhouseFactory.getClickhouseForOrganization( - project.organizationId, - "standard" - ); + const [clickhouse, runsListClickhouse] = await Promise.all([ + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "runsList"), + ]); const taskPresenter = new TaskDetailPresenter($replica, clickhouse); const task = await taskPresenter.findTask({ @@ -211,7 +216,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }) .catch(() => null); - const runList = new NextRunListPresenter($replica, clickhouse) + const runList = new NextRunListPresenter($replica, runsListClickhouse) .call(project.organizationId, environment.id, { userId, projectId: project.id, @@ -224,7 +229,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { includeHasAnyRuns: true, columns: getRunColumnsForSelect(request), }) - .catch(() => null); + .catch((error) => { + if (error instanceof RunsListQueryError) { + throw error; + } + return null; + }); return typeddefer({ task, @@ -375,14 +385,14 @@ export default function Page() { ) : null} - + }> {(list) => (list ? : null)}
}> - }> + }> {(list) => list ? ( { const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; const versions = url.searchParams.getAll("versions").filter((v) => v.length > 0); - const clickhouse = await clickhouseFactory.getClickhouseForOrganization( - project.organizationId, - "standard" - ); + const [clickhouse, runsListClickhouse] = await Promise.all([ + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "runsList"), + ]); const presenter = new TaskDetailPresenter($replica, clickhouse); const task = await presenter.findTask({ @@ -153,7 +158,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }) .catch(() => ({ data: [], statuses: [] }) satisfies TaskActivity); - const runList = new NextRunListPresenter($replica, clickhouse) + const runList = new NextRunListPresenter($replica, runsListClickhouse) .call(project.organizationId, environment.id, { userId, projectId: project.id, @@ -167,7 +172,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { includeHasAnyRuns: true, columns: getRunColumnsForSelect(request), }) - .catch(() => null); + .catch((error) => { + if (error instanceof RunsListQueryError) { + throw error; + } + return null; + }); return typeddefer({ task, @@ -271,14 +281,14 @@ export default function Page() { ) : null} - + }> {(list) => (list ? : null)}
}> - }> + }> {(list) => list ? ( { const runsDirectionRaw = url.searchParams.get("runsDirection") ?? undefined; const runsDirection = runsDirectionRaw ? DirectionSchema.parse(runsDirectionRaw) : undefined; - const clickhouse = await clickhouseFactory.getClickhouseForOrganization( - project.organizationId, - "standard" - ); + const [clickhouse, runsListClickhouse] = await Promise.all([ + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "runsList"), + ]); const presenter = new WebhookDetailPresenter($replica, clickhouse); const webhook = await presenter.findWebhook({ @@ -156,7 +161,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }) .catch(() => ({ data: [], statuses: [] }) satisfies WebhookActivity); - const runList = new NextRunListPresenter($replica, clickhouse) + const runList = new NextRunListPresenter($replica, runsListClickhouse) .call(project.organizationId, environment.id, { userId, projectId: project.id, @@ -167,7 +172,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { cursor: runsCursor, direction: runsDirection, }) - .catch(() => null); + .catch((error) => { + if (error instanceof RunsListQueryError) { + throw error; + } + return null; + }); const deliveriesList = presenter .listDeliveries({ @@ -329,7 +339,7 @@ export default function Page() { ) : ( - + }> {(list) => list ? ( ) : ( }> - }> + }> {(list) => list ? (
diff --git a/apps/webapp/app/routes/admin._index.tsx b/apps/webapp/app/routes/admin._index.tsx index 3005934d226..3741f499d38 100644 --- a/apps/webapp/app/routes/admin._index.tsx +++ b/apps/webapp/app/routes/admin._index.tsx @@ -2,6 +2,7 @@ import { MagnifyingGlassIcon } from "@heroicons/react/20/solid"; import { Form } from "@remix-run/react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; +import { env } from "~/env.server"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { CopyableText } from "~/components/primitives/CopyableText"; import { Input } from "~/components/primitives/Input"; @@ -36,7 +37,7 @@ export const loader = dashboardLoader( } const result = await adminGetUsers(user.id, searchParams.params.getAll()); - return typedjson(result); + return typedjson({ ...result, impersonationEnabled: env.ADMIN_DASHBOARD_ENABLED }); } ); @@ -57,7 +58,8 @@ export const action = dashboardAction( ); export default function AdminDashboardRoute() { - const { users, filters, page, pageCount } = useTypedLoaderData(); + const { users, filters, page, pageCount, impersonationEnabled } = + useTypedLoaderData(); return (
{user.admin ? "✅" : ""} -
- - -
+ {impersonationEnabled && ( +
+ + +
+ )} ); diff --git a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts index 8cd4f77873e..e9da02effd9 100644 --- a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts @@ -3,7 +3,12 @@ import { json } from "@remix-run/server-runtime"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; -import { applyGlobalMintKindFlip, makeSetMultipleFlags } from "~/v3/featureFlags.server"; +import { + applyGlobalGracedFlips, + makeSetMultipleFlags, + touchesGracedGroup, + withoutDerivedKeys, +} from "~/v3/featureFlags.server"; import { validatePartialFeatureFlags } from "~/v3/featureFlags"; export async function action({ request }: ActionFunctionArgs) { @@ -25,19 +30,16 @@ export async function action({ request }: ActionFunctionArgs) { ); } - // Derived grace-stamp fields are computed server-side; never trust them from the body. - const { - runOpsMintKindPrev: _ignoredPrev, - runOpsMintKindFlippedAt: _ignoredFlippedAt, - ...requestedFlags - } = validationResult.data; + // Both the strip and the branch derive from the graced-group table, so adding a group needs + // no edit here. Naming the keys inline is how a new group ends up writing its stamp straight + // from the request body, with no lock. + const requestedFlags = withoutDerivedKeys(validationResult.data) as Partial< + typeof validationResult.data + >; - // A global mint-kind flip stamps its grace window under a lock (applyGlobalMintKindFlip); - // any other flag save writes directly. - const updatedFlags = - requestedFlags.runOpsMintKind !== undefined - ? await applyGlobalMintKindFlip(prisma, requestedFlags, env.RUN_OPS_MINT_FLIP_GRACE_MS) - : await makeSetMultipleFlags(prisma)(requestedFlags); + const updatedFlags = touchesGracedGroup(requestedFlags) + ? await applyGlobalGracedFlips(prisma, requestedFlags, env.RUN_OPS_MINT_FLIP_GRACE_MS) + : await makeSetMultipleFlags(prisma)(requestedFlags); return json({ success: true, diff --git a/apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts b/apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts new file mode 100644 index 00000000000..42de308be87 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts @@ -0,0 +1,74 @@ +import type { ActionFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/server-runtime"; +import { tryCatch } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { $replica, prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { backfillVercelExternalIds } from "~/v3/services/vercelExternalIdBackfill.server"; + +const BodySchema = z.object({ + cursor: z.string().optional(), + limit: z.number().int().min(1).max(500).default(50), + recentPerEnvironment: z.number().int().min(0).max(200).default(10), + parallelism: z.number().int().min(1).max(20).default(5), + dryRun: z.boolean().default(true), +}); + +export async function action({ request }: ActionFunctionArgs) { + await requireAdminApiRequest(request); + + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method Not Allowed" }, { status: 405 }); + } + + const [bodyError, body] = await tryCatch(request.json()); + if (bodyError) { + return json({ error: bodyError.message }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(body); + if (!parsedBody.success) { + return json({ error: parsedBody.error.message }, { status: 400 }); + } + + const { cursor, limit, recentPerEnvironment, parallelism, dryRun } = parsedBody.data; + + logger.info("Vercel external id backfill starting", { + cursor, + limit, + recentPerEnvironment, + parallelism, + dryRun, + }); + + const [error, result] = await tryCatch( + backfillVercelExternalIds({ + prisma, + replica: $replica, + cursor, + limit, + recentPerEnvironment, + parallelism, + dryRun, + }) + ); + + if (error) { + logger.error("Vercel external id backfill failed", { cursor, error }); + return json({ error: error.message }, { status: 500 }); + } + + logger.info("Vercel external id backfill batch complete", { + dryRun, + cursor, + projectCount: result.projects, + environmentCount: result.environments.length, + summary: result.summary, + deployments: result.deployments, + next: result.next, + done: result.done, + }); + + return json(result); +} diff --git a/apps/webapp/app/routes/admin.data-stores.tsx b/apps/webapp/app/routes/admin.data-stores.tsx index af4a15dfcae..acbb4675c42 100644 --- a/apps/webapp/app/routes/admin.data-stores.tsx +++ b/apps/webapp/app/routes/admin.data-stores.tsx @@ -25,6 +25,7 @@ import { TableRow, } from "~/components/primitives/Table"; import { prisma } from "~/db.server"; +import { env } from "~/env.server"; import { requireUser } from "~/services/session.server"; import { ClickhouseConnectionSchema } from "~/services/clickhouse/clickhouseSecretSchemas.server"; import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server"; @@ -36,7 +37,7 @@ import { tryCatch } from "@trigger.dev/core/utils"; export const loader = async ({ request }: LoaderFunctionArgs) => { const user = await requireUser(request); - if (!user.admin) throw redirect("/"); + if (!user.admin || !env.ADMIN_DASHBOARD_ENABLED) throw redirect("/"); const dataStores = await prisma.organizationDataStore.findMany({ orderBy: { createdAt: "desc" }, @@ -72,7 +73,7 @@ const FormSchema = z.discriminatedUnion("_action", [AddSchema, UpdateSchema, Del export async function action({ request }: ActionFunctionArgs) { const user = await requireUser(request); - if (!user.admin) throw redirect("/"); + if (!user.admin || !env.ADMIN_DASHBOARD_ENABLED) throw redirect("/"); const formData = await request.formData(); diff --git a/apps/webapp/app/routes/admin.feature-flags.tsx b/apps/webapp/app/routes/admin.feature-flags.tsx index ef8caec4bd4..e987812f520 100644 --- a/apps/webapp/app/routes/admin.feature-flags.tsx +++ b/apps/webapp/app/routes/admin.feature-flags.tsx @@ -5,17 +5,19 @@ import { json } from "@remix-run/server-runtime"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { LockClosedIcon } from "@heroicons/react/20/solid"; -import { boundedIn, prisma } from "~/db.server"; +import { prisma } from "~/db.server"; import { env } from "~/env.server"; import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; import { FEATURE_FLAG, GLOBAL_LOCKED_FLAGS, + type FeatureFlagKey, type FlagControlType, getAllFlagControlTypes, + lockedFlagsInPayload, validatePartialFeatureFlags, } from "~/v3/featureFlags"; -import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; +import { flags as getGlobalFlags, replaceGlobalFeatureFlags } from "~/v3/featureFlags.server"; import { featuresForRequest } from "~/features.server"; import { Button } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; @@ -28,6 +30,7 @@ import { DialogFooter, } from "~/components/primitives/Dialog"; import { cn } from "~/utils/cn"; +import { buildFlagChangeList } from "~/components/admin/flagChangeList"; import { UNSET_VALUE, BooleanControl, @@ -38,6 +41,12 @@ import { type WorkerGroup, } from "~/components/admin/FlagControls"; +/** What the page posts to the action. See the note on payloadSchema. */ +type SaveFlagsBody = { + flags: Record; + unlockLockedFlags: boolean; +}; + export const loader = dashboardLoader( { authorization: { requireSuper: true } }, async ({ request }) => { @@ -87,7 +96,16 @@ export const action = dashboardAction( return json({ error: "Invalid JSON body" }, { status: 400 }); } - const payloadSchema = z.object({ flags: z.record(z.unknown()) }); + // The zod schema leaves unlockLockedFlags optional so a tab opened before this shipped still + // saves, defaulting to the safe answer. SaveFlagsBody keeps it required for our own client, so + // dropping it from the page is a compile error rather than a silently disabled unlock. + const payloadSchema = z.object({ + flags: z.record(z.unknown()), + // The page only submits the flags it is managing, so an omitted key is ambiguous for the + // locked flags: this says whether the admin unlocked them and is therefore authoritative + // over them too. + unlockLockedFlags: z.boolean().optional(), + }); const parsed = payloadSchema.safeParse(body); if (!parsed.success) { return json({ error: "Invalid payload" }, { status: 400 }); @@ -95,17 +113,12 @@ export const action = dashboardAction( const { isManagedCloud } = featuresForRequest(request); - // On managed cloud, reject if payload includes locked flags - if (isManagedCloud) { - const lockedInPayload = Object.keys(parsed.data.flags).filter((key) => - GLOBAL_LOCKED_FLAGS.includes(key) + const lockedInPayload = lockedFlagsInPayload(Object.keys(parsed.data.flags), isManagedCloud); + if (lockedInPayload.length > 0) { + return json( + { error: `Cannot modify locked flags: ${lockedInPayload.join(", ")}` }, + { status: 400 } ); - if (lockedInPayload.length > 0) { - return json( - { error: `Cannot modify locked flags: ${lockedInPayload.join(", ")}` }, - { status: 400 } - ); - } } const validationResult = validatePartialFeatureFlags(parsed.data.flags); @@ -116,39 +129,13 @@ export const action = dashboardAction( ); } - const validatedFlags = validationResult.data as Record; - const controlTypes = getAllFlagControlTypes(); - const catalogKeys = Object.keys(controlTypes); - - const keysToDelete: string[] = []; - const upsertOps: ReturnType[] = []; - - for (const key of catalogKeys) { - if (key in validatedFlags) { - upsertOps.push( - prisma.featureFlag.upsert({ - where: { key }, - create: { key, value: validatedFlags[key] as any }, - update: { value: validatedFlags[key] as any }, - }) - ); - } else { - // On cloud, never delete locked flags (they're not in the payload - // because the UI doesn't include them). Locally, delete everything - // the user didn't include - full control. - const isProtected = isManagedCloud && GLOBAL_LOCKED_FLAGS.includes(key); - if (!isProtected) { - keysToDelete.push(key); - } - } - } - - await prisma.$transaction([ - ...upsertOps, - ...(keysToDelete.length > 0 - ? [prisma.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } })] - : []), - ]); + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: validationResult.data as Record, + catalogKeys: Object.keys(getAllFlagControlTypes()) as FeatureFlagKey[], + isManagedCloud, + unlockLockedFlags: parsed.data.unlockLockedFlags ?? false, + graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS, + }); return json({ success: true }); } @@ -213,7 +200,8 @@ export default function AdminFeatureFlagsRoute() { }; const handleSave = () => { - saveFetcher.submit(JSON.stringify({ flags: values }), { + const body: SaveFlagsBody = { flags: values, unlockLockedFlags: unlocked }; + saveFetcher.submit(JSON.stringify(body), { method: "POST", encType: "application/json", }); @@ -411,6 +399,7 @@ export default function AdminFeatureFlagsRoute() { open={confirmOpen} onOpenChange={setConfirmOpen} initialValues={initialValues} + storedValues={allFlags} newValues={values} controlTypes={typedControlTypes} lockedKeys={unlocked ? [] : GLOBAL_LOCKED_FLAGS} @@ -477,6 +466,7 @@ function ConfirmDialog({ open, onOpenChange, initialValues, + storedValues, newValues, controlTypes, lockedKeys, @@ -487,6 +477,7 @@ function ConfirmDialog({ open: boolean; onOpenChange: (open: boolean) => void; initialValues: Record; + storedValues: Record; newValues: Record; controlTypes: Record; lockedKeys: readonly string[]; @@ -498,34 +489,12 @@ function ConfirmDialog({ .filter((key) => !lockedKeys.includes(key)) .sort(); - type Change = - | { key: string; type: "added"; newVal: string } - | { key: string; type: "removed"; oldVal: string } - | { key: string; type: "changed"; oldVal: string; newVal: string }; - - const changes = editableKeys.flatMap((key) => { - const wasSet = key in initialValues; - const isSet = key in newValues; - const oldVal = initialValues[key]; - const newVal = newValues[key]; - - if (!wasSet && !isSet) return []; - if (wasSet && isSet && stableStringify(oldVal) === stableStringify(newVal)) return []; - - if (!wasSet && isSet) { - return [{ key, type: "added" as const, newVal: String(newVal) }]; - } - if (wasSet && !isSet) { - return [{ key, type: "removed" as const, oldVal: String(oldVal) }]; - } - return [ - { - key, - type: "changed" as const, - oldVal: String(oldVal), - newVal: String(newVal), - }, - ]; + const changes = buildFlagChangeList({ + editableKeys, + lockedKeys, + initialValues, + storedValues, + newValues, }); return ( diff --git a/apps/webapp/app/routes/admin.impersonate.tsx b/apps/webapp/app/routes/admin.impersonate.tsx index 458ed5b2a7e..46f711b26e8 100644 --- a/apps/webapp/app/routes/admin.impersonate.tsx +++ b/apps/webapp/app/routes/admin.impersonate.tsx @@ -4,7 +4,7 @@ import { type LoaderFunctionArgs, } from "@remix-run/server-runtime"; import { z } from "zod"; -import { redirectWithImpersonation } from "~/models/admin.server"; +import { redirectWithImpersonation, requireAdminDashboardEnabled } from "~/models/admin.server"; import { requireUser } from "~/services/session.server"; import { validateAndConsumeImpersonationToken } from "~/services/impersonation.server"; import { logger } from "~/services/logger.server"; @@ -20,6 +20,8 @@ async function handleImpersonationRequest(request: Request, userId: string): Pro } export const loader = async ({ request }: LoaderFunctionArgs) => { + requireAdminDashboardEnabled(); + const url = new URL(request.url); const impersonateUserId = url.searchParams.get("impersonate"); const impersonationToken = url.searchParams.get("impersonationToken"); @@ -50,6 +52,8 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { }; export async function action({ request }: ActionFunctionArgs) { + requireAdminDashboardEnabled(); + if (request.method.toLowerCase() !== "post") { return new Response("Method not allowed", { status: 405 }); } diff --git a/apps/webapp/app/routes/admin.orgs.tsx b/apps/webapp/app/routes/admin.orgs.tsx index 51cd9552325..132ad860bfd 100644 --- a/apps/webapp/app/routes/admin.orgs.tsx +++ b/apps/webapp/app/routes/admin.orgs.tsx @@ -3,6 +3,7 @@ import { Form } from "@remix-run/react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { useState } from "react"; import { z } from "zod"; +import { env } from "~/env.server"; import { FeatureFlagsDialog } from "~/components/admin/FeatureFlagsDialog"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { CopyableText } from "~/components/primitives/CopyableText"; @@ -38,12 +39,13 @@ export const loader = dashboardLoader( } const result = await adminGetOrganizations(user.id, searchParams.params.getAll()); - return typedjson(result); + return typedjson({ ...result, impersonationEnabled: env.ADMIN_DASHBOARD_ENABLED }); } ); export default function AdminDashboardRoute() { - const { organizations, filters, page, pageCount } = useTypedLoaderData(); + const { organizations, filters, page, pageCount, impersonationEnabled } = + useTypedLoaderData(); const [flagsOrgId, setFlagsOrgId] = useState(null); const [flagsOpen, setFlagsOpen] = useState(false); @@ -127,17 +129,19 @@ export default function AdminDashboardRoute() { - - Impersonate - + {impersonationEnabled && ( + + Impersonate + + )}
diff --git a/apps/webapp/app/routes/api.v1.artifacts.ts b/apps/webapp/app/routes/api.v1.artifacts.ts index a706f9e04ef..12c2a10a9ea 100644 --- a/apps/webapp/app/routes/api.v1.artifacts.ts +++ b/apps/webapp/app/routes/api.v1.artifacts.ts @@ -64,6 +64,9 @@ export async function action({ request }: ActionFunctionArgs) { case "deployment_context": errorMessage = `Artifact size (${sizeMB} MB) exceeds the allowed limit of ${limitMB} MB. Make sure you are in the correct directory of your Trigger.dev project. Reach out to us if you are seeing this error consistently.`; break; + case "deployment_bundle": + errorMessage = `Bundle size (${sizeMB} MB) exceeds the allowed limit of ${limitMB} MB. Reach out to us if you are seeing this error consistently.`; + break; default: body.data.type satisfies never; errorMessage = `Artifact size (${sizeMB} MB) exceeds the allowed limit of ${limitMB} MB`; diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.build-env-vars.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.build-env-vars.ts new file mode 100644 index 00000000000..739046df13b --- /dev/null +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.build-env-vars.ts @@ -0,0 +1,98 @@ +import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { type GetDeploymentBuildEnvVarsResponseBody } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { decryptSecret, EncryptedSecretValueSchema } from "~/services/secrets/secretStore.server"; +import { FINAL_DEPLOYMENT_STATUSES } from "~/v3/services/failDeployment.server"; + +const ParamsSchema = z.object({ + deploymentId: z.string(), +}); + +// Secret material, deliberately separate from the main GET deployment endpoint. +export async function loader({ request, params }: LoaderFunctionArgs) { + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + try { + const authResult = await authenticateApiKeyWithScope(request, { + action: "read", + resource: { type: "deployments" }, + }); + + if (!authResult.ok) { + logger.info("Invalid or missing api key", { url: request.url }); + return json({ error: authResult.error }, { status: authResult.status }); + } + + const authenticatedEnv = authResult.authentication.environment; + + const { deploymentId } = parsedParams.data; + + const deployment = await prisma.workerDeployment.findFirst({ + where: { + friendlyId: deploymentId, + environmentId: authenticatedEnv.id, + }, + select: { + id: true, + status: true, + buildEnvVars: true, + }, + }); + + if (!deployment) { + return json({ error: "Deployment not found" }, { status: 404 }); + } + + logger.info("Build env vars read", { + deploymentId, + environmentId: authenticatedEnv.id, + projectId: authenticatedEnv.projectId, + status: deployment.status, + hasVars: deployment.buildEnvVars !== null, + }); + + // Never serve secrets for a build that is no longer active, even if a clear is still in flight + if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) { + return json({ variables: {} } satisfies GetDeploymentBuildEnvVarsResponseBody, { + status: 200, + }); + } + + if (!deployment.buildEnvVars) { + return json({ variables: {} } satisfies GetDeploymentBuildEnvVarsResponseBody, { + status: 200, + }); + } + + // Present-but-unreadable must fail loud: an empty record would let the build run without its secrets + const envelope = EncryptedSecretValueSchema.safeParse(deployment.buildEnvVars); + + if (!envelope.success) { + logger.error("Stored build env vars are not a valid encrypted envelope", { + deploymentId, + environmentId: authenticatedEnv.id, + }); + return json( + { error: "The stored build environment variables could not be read. Retry the deploy." }, + { status: 500 } + ); + } + + const decrypted = await decryptSecret(env.ENCRYPTION_KEY, envelope.data); + const variables = z.record(z.string()).parse(JSON.parse(decrypted)); + + return json({ variables } satisfies GetDeploymentBuildEnvVarsResponseBody, { status: 200 }); + } catch (error) { + if (error instanceof Response) throw error; + logger.error("Failed to load deployment build env vars", { error }); + return json({ error: "Internal Server Error" }, { status: 500 }); + } +} diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 5be291bae27..184fa996d97 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -42,7 +42,9 @@ export async function action({ request, params }: ActionFunctionArgs) { const service = new InitializeDeploymentService(); try { - const result = await service.call(authenticatedEnv, body.data); + const result = await service.call(authenticatedEnv, body.data, { + cliVersion: parseCliVersionHeader(request), + }); const { deployment, imageRef } = result; const responseBody: InitializeDeploymentResponseBody = { @@ -75,6 +77,14 @@ export async function action({ request, params }: ActionFunctionArgs) { } } +// Client-controlled and persisted, so cap what we accept +const CLI_VERSION_MAX_LENGTH = 128; + +function parseCliVersionHeader(request: Request): string | undefined { + const value = request.headers.get("x-trigger-cli-version"); + return value && value.length <= CLI_VERSION_MAX_LENGTH ? value : undefined; +} + export const loader = createLoaderApiRoute( { searchParams: ApiDeploymentListSearchParams, diff --git a/apps/webapp/app/routes/api.v1.plain.customer-cards.ts b/apps/webapp/app/routes/api.v1.plain.customer-cards.ts index bfb9988bee2..de7f96be6c6 100644 --- a/apps/webapp/app/routes/api.v1.plain.customer-cards.ts +++ b/apps/webapp/app/routes/api.v1.plain.customer-cards.ts @@ -156,7 +156,7 @@ export async function action({ request }: ActionFunctionArgs) { * Derived from which lookup actually matched, not from whether an external id was *sent* — an * id that misses and falls through to email must not unlock impersonation. */ - const canImpersonate = Boolean(byExternalId); + const canImpersonate = Boolean(byExternalId) && env.ADMIN_DASHBOARD_ENABLED; // No matching user: still answer every requested key, with no data so Plain hides the cards. if (!user) { diff --git a/apps/webapp/app/routes/api.v1.runs.ts b/apps/webapp/app/routes/api.v1.runs.ts index dca246a0c24..a9f2b8b4d2c 100644 --- a/apps/webapp/app/routes/api.v1.runs.ts +++ b/apps/webapp/app/routes/api.v1.runs.ts @@ -8,6 +8,7 @@ import { createLoaderApiRoute, everyResource, } from "~/services/routeBuilders/apiBuilder.server"; +import { RunsListQueryError } from "~/services/runsRepository/runsRepository.server"; export const loader = createLoaderApiRoute( { @@ -40,13 +41,23 @@ export const loader = createLoaderApiRoute( }, async ({ searchParams, authentication, apiVersion }) => { const presenter = new ApiRunListPresenter(); - const result = await presenter.call( - authentication.environment.project, - searchParams, - apiVersion, - authentication.environment - ); + try { + const result = await presenter.call( + authentication.environment.project, + searchParams, + apiVersion, + authentication.environment + ); - return json(result); + return json(result); + } catch (error) { + if (error instanceof RunsListQueryError) { + return json( + { error: error.message }, + { status: error.status, headers: { "x-should-retry": "false" } } + ); + } + throw error; + } } ); diff --git a/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts b/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts index ea1ebab0679..566ffc05876 100644 --- a/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts +++ b/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts @@ -38,7 +38,14 @@ const { action } = createActionApiRoute( }); if (!waitpoint) { - throw json({ error: "Waitpoint not found" }, { status: 404 }); + // Retryable: a miss here can be replica lag. resolveWaitpointThroughReadThrough + // deliberately does not read the legacy primary, so it relies on the caller retrying. + // A plain 404 is not retried by the SDK, which would turn a transient miss into a + // permanent failure. + throw json( + { error: "Waitpoint not found" }, + { status: 404, headers: { "x-should-retry": "true" } } + ); } const _result = await engine.blockRunWithWaitpoint({ @@ -55,6 +62,11 @@ const { action } = createActionApiRoute( { status: 200 } ); } catch (error) { + // A Response thrown inside the try is a deliberate status (the 404 above), not a + // failure. Re-throw it untouched, or every intentional 4xx here becomes a 500. + if (error instanceof Response) { + throw error; + } logger.error("Failed to wait for waitpoint", { runId, waitpointId, error }); throw json({ error: "Failed to wait for waitpoint token" }, { status: 500 }); } diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx index 4421b76daf1..fba94b80bba 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx @@ -779,6 +779,7 @@ function GitHubSettingsRows({ export function ConnectedGitHubRepoForm({ connectedGitHubRepo, previewEnvironmentEnabled, + stagingEnvironmentEnabled, organizationSlug, projectSlug, environmentSlug, @@ -788,6 +789,7 @@ export function ConnectedGitHubRepoForm({ }: { connectedGitHubRepo: ConnectedGitHubRepo; previewEnvironmentEnabled?: boolean; + stagingEnvironmentEnabled?: boolean; organizationSlug: string; projectSlug: string; environmentSlug: string; @@ -956,24 +958,42 @@ export function ConnectedGitHubRepoForm({ { - setGitSettingsValues((prev) => ({ - ...prev, - stagingBranch: e.target.value, - })); - }} - /> + stagingEnvironmentEnabled ? ( + { + setGitSettingsValues((prev) => ({ + ...prev, + stagingBranch: e.target.value, + })); + }} + /> + ) : ( + + Upgrade + + ) } > - + void; @@ -698,19 +695,6 @@ function VercelGitHubWarning() { ); } -function envSlugLabel(slug: EnvSlug): string { - switch (slug) { - case "prod": - return "Production"; - case "stg": - return "Staging"; - case "preview": - return "Preview"; - case "dev": - return "Development"; - } -} - function ConnectedVercelProjectForm({ connectedProject, hasStagingEnvironment, @@ -774,7 +758,7 @@ function ConnectedVercelProjectForm({ stagingEnvChanged || autoPromoteChanged; - const [configForm, _fields] = useForm({ + const [configForm] = useForm({ id: "update-vercel-config", lastResult: lastSubmission, shouldRevalidate: "onSubmit", @@ -833,26 +817,22 @@ function ConnectedVercelProjectForm({ const actionUrl = vercelResourcePath(organizationSlug, projectSlug, environmentSlug); - const availableEnvSlugs = getAvailableEnvSlugs(hasStagingEnvironment, hasPreviewEnvironment); const availableEnvSlugsForBuildSettings = getAvailableEnvSlugsForBuildSettings( hasStagingEnvironment, hasPreviewEnvironment ); + const hasVercelCustomEnvironments = customEnvironments.length > 0; + const disabledEnvSlugsForBuildSettings: Partial> | undefined = hasStagingEnvironment && !configValues.vercelStagingEnvironment - ? { stg: "Set a Vercel environment for Staging first." } + ? { + stg: hasVercelCustomEnvironments + ? "Set a Vercel environment for Staging first." + : "Add a custom environment to this project in Vercel to use Staging.", + } : undefined; - const _formatSelectedEnvs = ( - selected: EnvSlug[], - availableSlugs: EnvSlug[] = availableEnvSlugs - ): string => { - if (selected.length === 0) return "None selected"; - if (selected.length === availableSlugs.length) return "All environments"; - return selected.map(envSlugLabel).join(", "); - }; - return ( <> - {/* Staging environment mapping */} - {hasStagingEnvironment && customEnvironments && customEnvironments.length > 0 && ( + {hasStagingEnvironment && ( - { + if (!Array.isArray(value)) { + const env = customEnvironments?.find((e) => e.id === value); + setConfigValues((prev) => { + const next = { + ...prev, + vercelStagingEnvironment: env + ? { environmentId: env.id, displayName: env.slug } + : null, + }; + // When clearing the staging mapping, strip "stg" from build settings + if (!env) { + next.pullEnvVarsBeforeBuild = prev.pullEnvVarsBeforeBuild.filter( + (s) => s !== "stg" + ); + next.discoverEnvVars = prev.discoverEnvVars.filter((s) => s !== "stg"); + } + return next; + }); + } + }} + items={[{ id: "", slug: "None" }, ...customEnvironments]} + variant="secondary/small" + placeholder="Select environment" + dropdownIcon + text={ + configValues.vercelStagingEnvironment ? ( + + ) : ( + "None" + ) } - }} - items={[{ id: "", slug: "None" }, ...customEnvironments]} - variant="secondary/small" - placeholder="Select environment" - dropdownIcon - text={ - configValues.vercelStagingEnvironment ? ( - - ) : ( - "None" - ) - } - > - {[ - - None - , - ...customEnvironments.map((env) => ( - - - - )), - ]} - -
+ > + {[ + + None + , + ...customEnvironments.map((env) => ( + + + + )), + ]} + +
+ ) } /> )} @@ -1041,7 +1029,6 @@ function ConnectedVercelProjectForm({ } currentTriggerVersion={currentTriggerVersion} currentTriggerVersionFetchFailed={currentTriggerVersionFetchFailed} - hideSectionToggles layout="settings" /> @@ -1205,44 +1192,16 @@ function VercelSettingsPanel({ }) { const fetcher = useTypedFetcher(); const { load } = fetcher; - const _location = useLocation(); const data = fetcher.data; - const [hasError, _setHasError] = useState(false); const [hasFetched, setHasFetched] = useState(false); useEffect(() => { - if (!data?.authInvalid && !hasError && !data && !hasFetched) { + if (!data && !hasFetched) { load(vercelResourcePath(organizationSlug, projectSlug, environmentSlug)); // oxlint-disable-next-line react/set-state-in-effect -- This effect intentionally synchronizes route state after an external or lifecycle change. setHasFetched(true); } - }, [ - organizationSlug, - projectSlug, - environmentSlug, - data?.authInvalid, - hasError, - data, - hasFetched, - load, - ]); - - if (hasError) { - return ( -
-
- -
-

Failed to load Vercel settings

-

- There was an error loading the Vercel integration settings. Please refresh the page to - try again. -

-
-
-
- ); - } + }, [organizationSlug, projectSlug, environmentSlug, data, hasFetched, load]); if (fetcher.state === "loading" && !data) { return ( @@ -1258,40 +1217,30 @@ function VercelSettingsPanel({ } const showGitHubWarning = data.connectedProject && !data.isGitHubConnected; - const showAuthInvalid = data.authInvalid || data.onboardingData?.authInvalid; if (data.connectedProject) { return ( <> - {showAuthInvalid && ( - - )} {showGitHubWarning && } - {!showAuthInvalid && } - {!showAuthInvalid && ( - - )} + + ); } - if (showAuthInvalid) { + if (data.authInvalid) { return ( ): ResolveIdempotencyClientDeps { return { isSplitEnabled: async () => true, fallbackClient: FALLBACK, - newClient: NEW_CLIENT, - legacyClient: LEGACY_CLIENT, + clientFor: (key) => clientMap().get(key), resolveMintKind: async () => "runOpsId", + // Kept as an injected seam: the real resolveShard is total, so only an injected + // classifier can exercise the throw-to-fallback arm below. classify: (id) => { - if (id.length === 26 && id[25] === "1") return "NEW"; - if (id.length === 25) return "LEGACY"; + if (id.length === 26 && id[25] === "2") return id[24]!; + if (id.length === 26 && id[25] === "1") return "new"; + if (id.length === 25) return "legacy"; throw new Error(`unclassifiable: ${id.length}`); }, - isMigrated: undefined, ...over, }; } @@ -72,29 +83,51 @@ describe("resolveIdempotencyDedupClient", () => { expect(client).toBe(LEGACY_CLIENT); }); - it("routes a swept (migrated) cuid-parent child to the NEW client", async () => { - const cuidParent = RunId.toFriendlyId("c".repeat(25)); + it("falls back to the fallback client when a present parent id is unclassifiable", async () => { const client = await resolveIdempotencyDedupClient( - { environmentForMint: env, parentRunFriendlyId: cuidParent }, - makeDeps({ isMigrated: async () => true }) + { environmentForMint: env, parentRunFriendlyId: "run_not-a-valid-length" }, + makeDeps({}) ); - expect(client).toBe(NEW_CLIENT); + expect(client).toBe(FALLBACK); }); - it("routes a non-migrated cuid-parent child to the LEGACY client even when isMigrated is provided", async () => { - const cuidParent = RunId.toFriendlyId("d".repeat(25)); + it("routes a child to its OWN SHARD client when the parent is a gen-2 id", async () => { + const genTwoParent = RunId.toFriendlyId("e".repeat(24) + "a2"); const client = await resolveIdempotencyDedupClient( - { environmentForMint: env, parentRunFriendlyId: cuidParent }, - makeDeps({ isMigrated: async () => false }) + { environmentForMint: env, parentRunFriendlyId: genTwoParent }, + makeDeps({ resolveMintKind: async () => "cuid" }) // mint flag must NOT win for a child ); - expect(client).toBe(LEGACY_CLIENT); + expect(client).toBe(SHARD_A_CLIENT); }); - it("falls back to the fallback client when a present parent id is unclassifiable", async () => { + it("falls back and logs when a gen-2 parent names an unconfigured shard key", async () => { + const errors: unknown[] = []; + const genTwoParent = RunId.toFriendlyId("f".repeat(24) + "z2"); const client = await resolveIdempotencyDedupClient( - { environmentForMint: env, parentRunFriendlyId: "run_not-a-valid-length" }, - makeDeps({}) + { environmentForMint: env, parentRunFriendlyId: genTwoParent }, + makeDeps({ logger: { error: (_m, meta) => errors.push(meta) } }) ); expect(client).toBe(FALLBACK); + expect(errors).toHaveLength(1); + }); +}); + +describe("clientForShardKey", () => { + it("selects the same client the map holds for each reserved key and shard key", () => { + const clients = clientMap(); + const clientFor = (key: string) => clients.get(key); + expect(clientForShardKey("new", clientFor, FALLBACK)).toBe(NEW_CLIENT); + expect(clientForShardKey("legacy", clientFor, FALLBACK)).toBe(LEGACY_CLIENT); + expect(clientForShardKey("a", clientFor, FALLBACK)).toBe(SHARD_A_CLIENT); + }); + + it("returns the fallback and logs for a key the map does not hold", () => { + const errors: unknown[] = []; + const map = clientMap(); + const client = clientForShardKey("z", (key) => map.get(key), FALLBACK, { + error: (_m, meta) => errors.push(meta), + }); + expect(client).toBe(FALLBACK); + expect(errors).toHaveLength(1); }); }); diff --git a/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.ts b/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.ts index 86f1435654b..f2a731e61ca 100644 --- a/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.ts +++ b/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.ts @@ -1,22 +1,44 @@ -import { ownerEngine, RunId, type Residency } from "@trigger.dev/core/v3/isomorphic"; +import { resolveShard, RunId, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import type { PrismaClientOrTransaction } from "@trigger.dev/database"; type MintKind = "cuid" | "runOpsId"; +type Logger = { error: (message: string, meta?: Record) => void }; + export type ResolveIdempotencyClientDeps = { isSplitEnabled: () => Promise; fallbackClient: PrismaClientOrTransaction; - newClient: PrismaClientOrTransaction; - legacyClient: PrismaClientOrTransaction; + /** The store that owns a shard key: the reserved `legacy`/`new`, or a gen-2 shard. */ + clientFor: (shardKey: ShardKey) => PrismaClientOrTransaction | undefined; resolveMintKind: (environment: { organizationId: string; id: string; orgFeatureFlags?: unknown; }) => Promise; - classify?: (id: string) => Residency; - isMigrated?: (id: string) => Promise; + classify?: (id: string) => ShardKey; + logger?: Logger; }; +/** + * The one place an id becomes a client. `ShardKey` collapses to `string`, so the compiler + * cannot catch a wrong key here — an absent key takes an explicit logged branch to the + * fallback rather than a silent `?? legacy`. The configured set is not repeated in the log: + * boot already prints the shard table. + */ +export function clientForShardKey( + shardKey: ShardKey, + clientFor: (shardKey: ShardKey) => PrismaClientOrTransaction | undefined, + fallback: PrismaClientOrTransaction, + logger?: Logger +): PrismaClientOrTransaction { + const client = clientFor(shardKey); + if (client === undefined) { + logger?.error("idempotency: no client configured for shard key", { shardKey }); + return fallback; + } + return client; +} + export async function resolveIdempotencyDedupClient( args: { environmentForMint: { organizationId: string; id: string; orgFeatureFlags?: unknown }; @@ -28,9 +50,9 @@ export async function resolveIdempotencyDedupClient( return deps.fallbackClient; } - const classify = deps.classify ?? ownerEngine; - const clientFor = (residency: Residency): PrismaClientOrTransaction => - residency === "NEW" ? deps.newClient : deps.legacyClient; + const classify = deps.classify ?? resolveShard; + const clientFor = (shardKey: ShardKey): PrismaClientOrTransaction => + clientForShardKey(shardKey, deps.clientFor, deps.fallbackClient, deps.logger); if (args.parentRunFriendlyId) { let parentInternalId: string; @@ -39,18 +61,18 @@ export async function resolveIdempotencyDedupClient( } catch { return deps.fallbackClient; } - let residency: Residency; + let shardKey: ShardKey; try { - residency = classify(parentInternalId); + shardKey = classify(parentInternalId); } catch { return deps.fallbackClient; } - if (residency === "LEGACY" && deps.isMigrated && (await deps.isMigrated(parentInternalId))) { - return deps.newClient; - } - return clientFor(residency); + return clientFor(shardKey); } + // Mint kind, not an id: there is no shard to decode, so this keeps resolving to the + // gen-1 pair exactly as before. Which shard a gen-2 env mints into is the mint layer's + // decision, and this client is a read-your-writes signal rather than a correctness gate. const kind = await deps.resolveMintKind(args.environmentForMint); - return clientFor(kind === "runOpsId" ? "NEW" : "LEGACY"); + return clientFor(kind === "runOpsId" ? "new" : "legacy"); } diff --git a/apps/webapp/app/runEngine/concerns/resolveWaitpointThroughReadThrough.server.ts b/apps/webapp/app/runEngine/concerns/resolveWaitpointThroughReadThrough.server.ts index ec5adc13a6c..b1c7a2bd05d 100644 --- a/apps/webapp/app/runEngine/concerns/resolveWaitpointThroughReadThrough.server.ts +++ b/apps/webapp/app/runEngine/concerns/resolveWaitpointThroughReadThrough.server.ts @@ -1,3 +1,4 @@ +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import type { PrismaReplicaClient } from "~/db.server"; import { runOpsLegacyReplica as defaultLegacyReplica, @@ -5,12 +6,18 @@ import { runOpsNewReplica as defaultNewClient, runOpsSplitReadEnabled as defaultSplitReadEnabled, } from "~/db.server"; +import { + runOpsShardReplicas as defaultShardReplicas, + runOpsShardWriters as defaultShardWriters, +} from "~/v3/runOpsMigration/shardHandles.server"; import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server"; type ResolveWaitpointDeps = { newClient?: PrismaReplicaClient; legacyReplica?: PrismaReplicaClient; newPrimary?: PrismaReplicaClient; + shardReplicas?: ReadonlyMap; + shardWriters?: ReadonlyMap; splitEnabled?: boolean; isPastRetention?: (id: string) => boolean; }; @@ -21,6 +28,8 @@ export type ResolveWaitpointReadThroughDefaults = { newClient: PrismaReplicaClient; legacyReplica: PrismaReplicaClient; newPrimary: PrismaReplicaClient; + shardReplicas: ReadonlyMap; + shardWriters: ReadonlyMap; splitEnabled: boolean; }; @@ -28,6 +37,8 @@ const productionDefaults: ResolveWaitpointReadThroughDefaults = { newClient: defaultNewClient, legacyReplica: defaultLegacyReplica, newPrimary: defaultNewPrimary as unknown as PrismaReplicaClient, + shardReplicas: defaultShardReplicas, + shardWriters: defaultShardWriters as unknown as ReadonlyMap, splitEnabled: defaultSplitReadEnabled, }; @@ -43,7 +54,8 @@ export async function resolveWaitpointThroughReadThrough(opts: { const splitEnabled = opts.deps?.splitEnabled ?? defaults.splitEnabled; const result = await readThroughRun({ - runId: opts.waitpointId, + id: opts.waitpointId, + idKind: "waitpoint", environmentId: opts.environmentId, readNew: (client) => opts.read(client), readLegacy: (replica) => opts.read(replica), @@ -51,22 +63,31 @@ export async function resolveWaitpointThroughReadThrough(opts: { splitEnabled, newClient: opts.deps?.newClient ?? defaults.newClient, legacyReplica: opts.deps?.legacyReplica ?? defaults.legacyReplica, + shardReplicas: opts.deps?.shardReplicas ?? defaults.shardReplicas, isPastRetention: opts.deps?.isPastRetention, }, }); - if (result.source === "new" || result.source === "legacy-replica") { + if (result.found) { return result.value; } // past-retention is an intentional not-found: the token is gone. - if (result.source === "past-retention") { + if (result.reason === "past-retention") { return null; } // Read-your-writes fallback for a token completed immediately after mint, before it replicated: - // re-read from the run-ops PRIMARY only. We deliberately never read the control-plane/legacy + // re-read from the owning store's PRIMARY only. We deliberately never read the control-plane/legacy // primary here (that is the load the replica-only read-through exists to shed), so a legacy-resident // token that misses its replica stays a miss and the caller retries, rather than adding primary load. + const shardKey = resolveShard(opts.waitpointId); + if (shardKey !== "new" && shardKey !== "legacy") { + // A gen-2 token's primary is its OWN shard's writer. The gen-1 new writer is a different + // database, so reading it would miss and silently disable read-your-writes here. + const shardWriter = (opts.deps?.shardWriters ?? defaults.shardWriters).get(shardKey); + return shardWriter ? await opts.read(shardWriter) : null; + } + const fromNewPrimary = await opts.read(opts.deps?.newPrimary ?? defaults.newPrimary); if (fromNewPrimary != null) { return fromNewPrimary; diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts index 5a2b3b86eee..a29664a4502 100644 --- a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts @@ -1,4 +1,4 @@ -import { ClickHouse } from "@internal/clickhouse"; +import { ClickHouse, type ClickHouseSettings } from "@internal/clickhouse"; import { createHash } from "crypto"; import { ClickhouseEventRepository } from "~/v3/eventRepository/clickhouseEventRepository.server"; import { env } from "~/env.server"; @@ -292,6 +292,45 @@ function initializeRealtimeClickhouseClient(): ClickHouse { }); } +/** + * Server-side query protection for the runs-list read pool. Every setting here is PER-QUERY, so a + * pathological query only ever kills itself: a slow one hits `max_execution_time`, a memory-hungry + * one hits `max_memory_usage`, a thread-hungry one hits `max_threads`. Per-USER limits + * (`max_*_for_user`) are deliberately NOT used: everything connects as `default`, so a per-user cap + * would reject whichever query arrives once the shared budget is hit, punishing innocent tenants + * for a noisy one. The node itself is protected by the server-level `max_server_memory_usage`. + * Safe as client-level settings ONLY because this pool is read-only; on a mixed read+write pool a + * client-level `max_execution_time` would also kill slow inserts. `readonly=2` enforces read-only + * while still allowing these settings to apply (`readonly=1` rejects them). + */ +/** + * Client request timeout for the runs-list pool, forced above the server-side `max_execution_time` + * so the server cap is what stops a slow query and the client stays connected to receive that + * error. If the client timed out first, it would abort while ClickHouse kept executing, which is + * the abandoned-query behaviour this pool is trying to prevent. + */ +function getRunsListRequestTimeoutMs() { + return Math.max( + env.RUNS_LIST_CLICKHOUSE_REQUEST_TIMEOUT_MS, + (env.RUNS_LIST_CLICKHOUSE_MAX_EXECUTION_TIME + 5) * 1000 + ); +} + +function getRunsListClickhouseSettings(): ClickHouseSettings { + const settings: ClickHouseSettings = { + max_execution_time: env.RUNS_LIST_CLICKHOUSE_MAX_EXECUTION_TIME, + timeout_before_checking_execution_speed: 0, + max_threads: env.RUNS_LIST_CLICKHOUSE_MAX_THREADS, + max_memory_usage: env.RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE.toString(), + }; + + if (env.RUNS_LIST_CLICKHOUSE_READONLY !== "0") { + settings.readonly = env.RUNS_LIST_CLICKHOUSE_READONLY; + } + + return settings; +} + /** Runs list reads — dashboard + API (`RUNS_LIST_CLICKHOUSE_URL`); * falls back to the default client if unset. */ const defaultRunsListClickhouseClient = singleton( @@ -319,6 +358,8 @@ function initializeRunsListClickhouseClient(): ClickHouse { request: env.RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST === "1", }, maxOpenConnections: env.RUNS_LIST_CLICKHOUSE_MAX_OPEN_CONNECTIONS, + requestTimeoutMs: getRunsListRequestTimeoutMs(), + clickhouseSettings: getRunsListClickhouseSettings(), }); } @@ -550,10 +591,25 @@ function buildOrgClickhouseClient(url: string, clientType: ClientType): ClickHou }, maxOpenConnections: env.REALTIME_BACKEND_NATIVE_CLICKHOUSE_MAX_OPEN_CONNECTIONS, }); + case "runsList": + return new ClickHouse({ + url: parsed.toString(), + name, + keepAlive: { + enabled: env.RUNS_LIST_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.RUNS_LIST_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.RUNS_LIST_CLICKHOUSE_LOG_LEVEL, + compression: { + request: env.RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST === "1", + }, + maxOpenConnections: env.RUNS_LIST_CLICKHOUSE_MAX_OPEN_CONNECTIONS, + requestTimeoutMs: getRunsListRequestTimeoutMs(), + clickhouseSettings: getRunsListClickhouseSettings(), + }); case "standard": case "query": case "admin": - case "runsList": return new ClickHouse({ url: parsed.toString(), name, diff --git a/apps/webapp/app/services/dashboardAgent.server.ts b/apps/webapp/app/services/dashboardAgent.server.ts index 5ff460ae33f..7252ba2d178 100644 --- a/apps/webapp/app/services/dashboardAgent.server.ts +++ b/apps/webapp/app/services/dashboardAgent.server.ts @@ -86,10 +86,18 @@ export function isDashboardAgentConfigured(): boolean { return Boolean(env.DASHBOARD_AGENT_SECRET_KEY); } +// With no agent worker available a turn's run would sit queued indefinitely and +// could be dequeued much later with a stale token. Expire it instead — the turn +// is long dead by then on the client. +const DASHBOARD_AGENT_RUN_TTL = "2m"; + // Pins every agent session (and its continuation runs) to a deployed version // when DASHBOARD_AGENT_VERSION is set; unset runs on the env's current version. -export function dashboardAgentTriggerConfig(): { lockToVersion: string } | undefined { - return env.DASHBOARD_AGENT_VERSION ? { lockToVersion: env.DASHBOARD_AGENT_VERSION } : undefined; +export function dashboardAgentTriggerConfig(): { ttl: string; lockToVersion?: string } { + return { + ttl: DASHBOARD_AGENT_RUN_TTL, + ...(env.DASHBOARD_AGENT_VERSION ? { lockToVersion: env.DASHBOARD_AGENT_VERSION } : {}), + }; } export async function startDashboardAgentSession(params: { diff --git a/apps/webapp/app/services/impersonation.server.ts b/apps/webapp/app/services/impersonation.server.ts index aa850ba0468..bce88bfbc66 100644 --- a/apps/webapp/app/services/impersonation.server.ts +++ b/apps/webapp/app/services/impersonation.server.ts @@ -37,6 +37,14 @@ export function commitImpersonationSession(session: Session) { } export async function getImpersonationId(request: Request) { + if (!env.ADMIN_DASHBOARD_ENABLED) return undefined; + + return getRawImpersonationId(request); +} + +// Ignores ADMIN_DASHBOARD_ENABLED — only for terminating or auditing a session +// the gated reader no longer resolves, never for authorizing anything. +export async function getRawImpersonationId(request: Request) { const session = await getImpersonationSession(request); return session.get(IMPERSONATED_USER_ID_KEY) as string | undefined; @@ -74,6 +82,14 @@ export async function getImpersonationState( request: Request, resolvedUserId: string | undefined ): Promise { + if (!env.ADMIN_DASHBOARD_ENABLED) { + return resolveImpersonationState({ + impersonatedUserId: undefined, + viewingAsUser: undefined, + resolvedUserId, + }); + } + const session = await getImpersonationSession(request); return resolveImpersonationState({ diff --git a/apps/webapp/app/services/platform.v3.server.ts b/apps/webapp/app/services/platform.v3.server.ts index 01b1c7d4972..2b7e8fb3d4f 100644 --- a/apps/webapp/app/services/platform.v3.server.ts +++ b/apps/webapp/app/services/platform.v3.server.ts @@ -1095,6 +1095,7 @@ export async function enqueueBuild( options: { skipPromotion?: boolean; configFilePath?: string; + fromBundle?: boolean; } ) { if (!client) return undefined; @@ -1235,6 +1236,10 @@ export function isCloud(): boolean { return true; } + if (env.LOGIN_ORIGIN?.endsWith(".triggerlabs.dev")) { + return true; + } + if (process.env.CLOUD_ENV === "development" && process.env.NODE_ENV === "development") { return true; } diff --git a/apps/webapp/app/services/projectSettings.server.ts b/apps/webapp/app/services/projectSettings.server.ts index 3e265ee5e77..57b733ee73b 100644 --- a/apps/webapp/app/services/projectSettings.server.ts +++ b/apps/webapp/app/services/projectSettings.server.ts @@ -234,20 +234,25 @@ export class ProjectSettingsService { return getExistingConnectedRepo() .andThen((connectedRepo) => { const installationId = Number(connectedRepo.repository.installation.appInstallationId); + const oldStagingBranch = connectedRepo.branchTracking?.staging?.branch; - return ResultAsync.combine([ - validateProductionBranch({ - installationId, - fullRepoName: connectedRepo.repository.fullName, - oldProductionBranch: connectedRepo.branchTracking?.prod?.branch, - }), - validateStagingBranch({ - installationId, - fullRepoName: connectedRepo.repository.fullName, - oldStagingBranch: connectedRepo.branchTracking?.staging?.branch, - }), - this.isPreviewEnvironmentEnabled(projectId), - ]); + return this.isStagingEnvironmentEnabled(projectId).andThen((stagingEnvironmentEnabled) => + ResultAsync.combine([ + validateProductionBranch({ + installationId, + fullRepoName: connectedRepo.repository.fullName, + oldProductionBranch: connectedRepo.branchTracking?.prod?.branch, + }), + stagingEnvironmentEnabled + ? validateStagingBranch({ + installationId, + fullRepoName: connectedRepo.repository.fullName, + oldStagingBranch, + }) + : okAsync(oldStagingBranch), + this.isPreviewEnvironmentEnabled(projectId), + ]) + ); }) .map(([productionBranch, stagingBranch, previewEnvironmentEnabled]) => ({ productionBranch, @@ -317,7 +322,8 @@ export class ProjectSettingsService { }, where: { projectId: projectId, - slug: "preview", + type: "PREVIEW", + parentEnvironmentId: null, }, }), (error) => ({ @@ -326,4 +332,23 @@ export class ProjectSettingsService { }) ).map((previewEnvironment) => previewEnvironment !== null); } + + private isStagingEnvironmentEnabled(projectId: string) { + return fromPromise( + this.#prismaClient.runtimeEnvironment.findFirst({ + select: { + id: true, + }, + where: { + projectId: projectId, + type: "STAGING", + parentEnvironmentId: null, + }, + }), + (error) => ({ + type: "other" as const, + cause: error, + }) + ).map((stagingEnvironment) => stagingEnvironment !== null); + } } diff --git a/apps/webapp/app/services/realtime/sessionRunManager.server.ts b/apps/webapp/app/services/realtime/sessionRunManager.server.ts index a1989a9ef7a..f11bc960205 100644 --- a/apps/webapp/app/services/realtime/sessionRunManager.server.ts +++ b/apps/webapp/app/services/realtime/sessionRunManager.server.ts @@ -310,6 +310,7 @@ async function triggerSessionRun(params: { ...(config.maxDuration !== undefined ? { maxDuration: config.maxDuration } : {}), ...(config.lockToVersion ? { lockToVersion: config.lockToVersion } : {}), ...(config.region ? { region: config.region } : {}), + ...(config.ttl !== undefined ? { ttl: config.ttl } : {}), }, }; diff --git a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts index a5a808e3195..1f0004dda1a 100644 --- a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts +++ b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts @@ -25,13 +25,14 @@ import { getApiVersion } from "~/api/versions"; import { WORKER_HEADERS } from "@trigger.dev/core/v3/runEngineWorker"; import { ServiceValidationError } from "~/v3/services/common.server"; import { EngineServiceValidationError } from "@internal/run-engine"; +import { unroutableIdResponse } from "./unroutableId.server"; import { tenantContext, tenantContextFromAuthEnvironment } from "~/services/tenantContext.server"; // Client aborts and service-level validation errors aren't bugs — they're // expected at API boundaries. Log them at `warn` so they stay in stdout // without flowing to Sentry via Logger.onError. function logBoundaryError( - message: "Error in loader" | "Error in action", + message: "Error in loader" | "Error in action" | "Unroutable id", error: unknown, url: string ) { @@ -451,6 +452,12 @@ export function createLoaderApiRoute< return await wrapResponse(request, error, corsStrategy !== "none"); } + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } + logBoundaryError("Error in loader", error, request.url); return await wrapResponse( @@ -722,6 +729,12 @@ export function createLoaderPATApiRoute< if (error instanceof Response) { return await wrapResponse(request, error, corsStrategy !== "none"); } + + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } return await wrapResponse( request, json({ error: "Internal Server Error" }, { status: 500 }), @@ -996,6 +1009,12 @@ export function createActionPATApiRoute< return await wrapResponse(request, error, corsStrategy !== "none"); } + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } + logBoundaryError("Error in action", error, request.url); // Typed validation errors map to their own status (default 400); @@ -1346,6 +1365,12 @@ export function createActionApiRoute< return await wrapResponse(request, error, corsStrategy !== "none"); } + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } + logBoundaryError("Error in action", error, request.url); return await wrapResponse( @@ -1612,6 +1637,12 @@ export function createMultiMethodApiRoute< return await wrapResponse(request, error, corsStrategy !== "none"); } + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } + logBoundaryError("Error in action", error, request.url); return await wrapResponse( diff --git a/apps/webapp/app/services/routeBuilders/dashboardBuilder.server.ts b/apps/webapp/app/services/routeBuilders/dashboardBuilder.server.ts index 01bdb6d9b53..8698b56bcd1 100644 --- a/apps/webapp/app/services/routeBuilders/dashboardBuilder.server.ts +++ b/apps/webapp/app/services/routeBuilders/dashboardBuilder.server.ts @@ -5,6 +5,7 @@ import { json, redirect } from "@remix-run/server-runtime"; import type { RbacAbility } from "@trigger.dev/rbac"; +import { env } from "~/env.server"; import { rbac } from "~/services/rbac.server"; import { getUserId } from "~/services/session.server"; import { permissionDeniedResponse } from "~/utils/permissionDenied"; @@ -23,7 +24,7 @@ function loginRedirectFor(request: Request, override?: string): Response { function isAuthorized(ability: RbacAbility, authorization: AuthorizationOption): boolean { if ("requireSuper" in authorization) { - return ability.canSuper(); + return env.ADMIN_DASHBOARD_ENABLED && ability.canSuper(); } return ability.can(authorization.action, authorization.resource); } diff --git a/apps/webapp/app/services/routeBuilders/permissions.server.ts b/apps/webapp/app/services/routeBuilders/permissions.server.ts index 37a70272c17..b94c7a25fac 100644 --- a/apps/webapp/app/services/routeBuilders/permissions.server.ts +++ b/apps/webapp/app/services/routeBuilders/permissions.server.ts @@ -1,4 +1,5 @@ import type { RbacAbility, RbacResource } from "@trigger.dev/rbac"; +import { env } from "~/env.server"; /** * A single permission check, mirroring the `authorization` option the @@ -32,7 +33,9 @@ export function checkPermissions( if (!Object.hasOwn(checks, key)) continue; const check = checks[key]; result[key] = - "requireSuper" in check ? ability.canSuper() : ability.can(check.action, check.resource); + "requireSuper" in check + ? env.ADMIN_DASHBOARD_ENABLED && ability.canSuper() + : ability.can(check.action, check.resource); } return result; } diff --git a/apps/webapp/app/services/routeBuilders/unroutableId.server.ts b/apps/webapp/app/services/routeBuilders/unroutableId.server.ts new file mode 100644 index 00000000000..dcc2d46d7b6 --- /dev/null +++ b/apps/webapp/app/services/routeBuilders/unroutableId.server.ts @@ -0,0 +1,19 @@ +import { json } from "@remix-run/server-runtime"; +import { UnknownShardKey } from "@internal/run-store"; + +/** + * An id naming a shard the topology has no store for cannot be routed, so a read cannot locate + * the row: that is a 404, and matches what an absent gen-1 or cuid id already returns. It must + * not be a 500 — `resolveShard` is pure id-shape, so any base32hex core plus `[a-z0-9]` plus "2" + * parses as gen-2, which lets any caller induce a 5xx, and a 5xx on a read trips canary rollbacks. + * + * The router still throws. Callers log it before returning this, so a genuine misconfiguration — + * a shard key dropped from a config that is meant to be append-only — still alarms. + */ +export function unroutableIdResponse(error: unknown): Response | undefined { + // Explicitly NOT retryable: an id naming an unconfigured shard is not a transient miss, and + // no number of retries makes a topology grow a store. + return error instanceof UnknownShardKey + ? json({ error: "Not Found" }, { status: 404, headers: { "x-should-retry": "false" } }) + : undefined; +} diff --git a/apps/webapp/app/services/runsReplicationInstance.server.ts b/apps/webapp/app/services/runsReplicationInstance.server.ts index 164ce07fb92..7c074b1586b 100644 --- a/apps/webapp/app/services/runsReplicationInstance.server.ts +++ b/apps/webapp/app/services/runsReplicationInstance.server.ts @@ -3,6 +3,7 @@ import { env } from "~/env.server"; import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { singleton } from "~/utils/singleton"; import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server"; +import { nonAliasedShards } from "~/v3/runOpsShards.server"; import { meter, provider } from "~/v3/tracer.server"; import { setRunsReplicationConfiguredSources, @@ -31,6 +32,17 @@ export function buildReplicationSources(args: { newSlotName: string; newPublicationName: string; newOriginGeneration: number; + /** + * Gen-2 shards that own their own database, each with its own slot, publication and origin + * generation. An aliased shard is absent: its target's slot already covers its WAL. + */ + shards?: Array<{ + key: string; + url: string; + /** The DIRECT, non-pooled DSN. Logical replication needs a session-mode connection. */ + directUrl?: string; + replication: { slotName: string; publicationName: string; originGeneration: number }; + }>; }): RunsReplicationSource[] { const legacy: RunsReplicationSource = { id: "legacy", @@ -54,7 +66,29 @@ export function buildReplicationSources(args: { originGeneration: args.newOriginGeneration, }; - return [legacy, next]; + // Shard sources come after the gen-1 pair. Reached only when the new source is on, because + // split is the precondition for a shard to exist at all. The origin generations come from the + // descriptor, which the boot parser already bounds to 2..255 and checks for duplicates; the + // service re-checks uniqueness across every source it is given. + // The DIRECT dsn, not the app writer dsn. A transaction pooler cannot serve the replication + // protocol, and the writer dsn is pooled in a real deployment. Gen-1 keeps the same separation + // through its own RUN_REPLICATION_* variables, and the migration loop prefers directUrl too. + const shardSources: RunsReplicationSource[] = (args.shards ?? []).map((shard) => ({ + id: shardSourceId(shard.key), + pgConnectionUrl: shard.directUrl ?? shard.url, + slotName: shard.replication.slotName, + publicationName: shard.replication.publicationName, + originGeneration: shard.replication.originGeneration, + })); + + return [legacy, next, ...shardSources]; +} + +// The replication source id for a shard. It derives the per-source client name and the key the +// status route probes, so it must be stable and unique across sources. The leader lock is keyed on +// the slot name, not on this id. +function shardSourceId(key: string): string { + return `shard-${key}`; } /** @@ -66,24 +100,106 @@ export function buildReplicationSources(args: { * rather than ship a fleet-wide under-count. */ export class SplitReplicationMisconfiguredError extends Error { - constructor() { + constructor(message?: string) { super( - 'RUN_OPS_SPLIT_ENABLED is on but the runs-replication sources[] has no "new" source: ' + - "run-ops runs on the new DB would not replicate to ClickHouse, under-counting every " + - "ClickHouse-fronted aggregate. Enable the new replication source " + - "(RUN_REPLICATION_NEW_ENABLED / RUN_REPLICATION_RUN_OPS_DATABASE_URL) or turn the split off." + message ?? + 'RUN_OPS_SPLIT_ENABLED is on but the runs-replication sources[] has no "new" source: ' + + "run-ops runs on the new DB would not replicate to ClickHouse, under-counting every " + + "ClickHouse-fronted aggregate. Enable the new replication source " + + "(RUN_REPLICATION_NEW_ENABLED / RUN_REPLICATION_RUN_OPS_DATABASE_URL) or turn the split off." ); this.name = "SplitReplicationMisconfiguredError"; } } +/** + * Two sources that share an identity. The descriptor parser checks uniqueness AMONG shards only, so + * it cannot see the env-configured legacy and new sources: a shard can collide with either. The + * service has its own check, but it throws from the constructor, which the caller reaches only AFTER + * it has shut the bootstrap instance down — leaving the process up with NO replication at all, which + * is the exact silent under-count this family of errors exists to prevent. So the check runs here, + * at the fatal gate, before anything is torn down. + */ +class DuplicateReplicationIdentityError extends SplitReplicationMisconfiguredError { + constructor(field: string, value: unknown) { + super( + `the runs-replication sources[] has two sources with the same ${field} "${String(value)}": ` + + "two consumers on one WAL stream is a data race, and a shared origin generation defeats the " + + "ClickHouse dedup tiebreak. Give every source its own slot, publication and origin generation." + ); + this.name = "DuplicateReplicationIdentityError"; + } +} + +/** + * A configured shard with no replication source of its own. Subclasses the split error on purpose: + * the boot catch site tests `instanceof SplitReplicationMisconfiguredError` to reach + * process.exit(1), and a shard whose runs never reach ClickHouse must take that same exit. + */ +class ShardReplicationMisconfiguredError extends SplitReplicationMisconfiguredError { + constructor(shardKey: string) { + super( + `run-ops shard ${shardKey} is configured but the runs-replication sources[] has no ` + + `"${shardSourceId(shardKey)}" source: runs on that shard would not replicate to ` + + "ClickHouse, under-counting every ClickHouse-fronted aggregate. Give the shard a " + + "replication slot, publication and origin generation, or remove the shard." + ); + this.name = "ShardReplicationMisconfiguredError"; + } +} + +/** + * A shard that replicates but declares no direct dsn. Falling back to its writer dsn is a silent + * trap: if that dsn is pooled, the replication client throws inside start(), which is NOT a + * SplitReplicationMisconfiguredError, so the process stays up with EVERY source down, legacy + * included. Refuse the boot instead. + */ +class ShardDirectUrlMissingError extends SplitReplicationMisconfiguredError { + constructor(shardKey: string) { + super( + `run-ops shard ${shardKey} declares replication but no directUrl: logical replication needs a ` + + "session-mode connection, which a transaction pooler cannot serve. Give the shard a directUrl " + + "pointing at its direct, non-pooled endpoint." + ); + this.name = "ShardDirectUrlMissingError"; + } +} + export function assertReplicationCoversSplit(args: { splitEnabled: boolean; sources: RunsReplicationSource[]; + /** Every configured shard, aliased ones included. An aliased shard needs no source of its own. */ + shards?: Array<{ key: string; aliasOf?: "new"; hasDirectUrl?: boolean }>; }): void { - if (args.splitEnabled && !args.sources.some((s) => s.id === "new")) { + if (!args.splitEnabled) { + return; + } + if (!args.sources.some((s) => s.id === "new")) { throw new SplitReplicationMisconfiguredError(); } + for (const shard of args.shards ?? []) { + // An aliased shard shares its target's database, so the target's slot already carries its WAL. + if (shard.aliasOf !== undefined) continue; + if (!args.sources.some((s) => s.id === shardSourceId(shard.key))) { + throw new ShardReplicationMisconfiguredError(shard.key); + } + if (shard.hasDirectUrl === false) { + throw new ShardDirectUrlMissingError(shard.key); + } + } + + // Cross-source identity, over EVERY source and not only the shards. A correct two-source + // deployment already satisfies this, because two consumers on one WAL slot is a data race that + // cannot work. So this adds a loud failure for a configuration that was already broken silently. + for (const field of ["id", "slotName", "publicationName", "originGeneration"] as const) { + const seen = new Set(); + for (const source of args.sources) { + if (seen.has(source[field])) { + throw new DuplicateReplicationIdentityError(field, source[field]); + } + seen.add(source[field]); + } + } } function initializeRunsReplicationInstance() { @@ -171,6 +287,20 @@ function initializeRunsReplicationInstance() { // The legacy-only instance above is never started in the dual path (no slot/lock // taken). runsReplicationService.server.ts is untouched. The create route also calls // setRunsReplicationGlobal — last-writer-wins is the existing contract. + // An aliased shard replicates through its target's slot, so only the shards that own their own + // database take a source. Coverage is then checked against EVERY descriptor, aliased included. + // The schema requires `replication` on every non-aliased descriptor, so the guard below is a + // type narrowing and not a policy. + const shardReplicationByKey = new Map( + env.RUN_OPS_SHARDS.flatMap((d) => (d.replication ? [[d.key, d.replication] as const] : [])) + ); + const shardsWithReplication = nonAliasedShards(env.RUN_OPS_SHARDS).flatMap((shard) => { + const replication = shardReplicationByKey.get(shard.key); + return replication + ? [{ key: shard.key, url: shard.url, directUrl: shard.directUrl, replication }] + : []; + }); + isSplitEnabled() .then(async (splitEnabled) => { const sources = buildReplicationSources({ @@ -184,10 +314,20 @@ function initializeRunsReplicationInstance() { newSlotName: env.RUN_REPLICATION_NEW_SLOT_NAME, newPublicationName: env.RUN_REPLICATION_NEW_PUBLICATION_NAME, newOriginGeneration: env.RUN_REPLICATION_NEW_ORIGIN_GENERATION, + shards: shardsWithReplication, }); - // Refuse to start replication if split is on but `#new` is not a source. - assertReplicationCoversSplit({ splitEnabled, sources }); + // Refuse to start replication if split is on but `#new` is not a source, or if any shard + // that owns its own database has no source of its own. + assertReplicationCoversSplit({ + splitEnabled, + sources, + shards: env.RUN_OPS_SHARDS.map((d) => ({ + key: d.key, + aliasOf: d.aliasOf, + hasDirectUrl: d.directUrl !== undefined, + })), + }); if (sources.length > 1) { // Release the bootstrap instance's eager replication client (Redis + Redlock) diff --git a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts index 2e911e5e958..02b3e014466 100644 --- a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts @@ -1,4 +1,4 @@ -import { type ClickhouseQueryBuilder } from "@internal/clickhouse"; +import { type ClickhouseQueryBuilder, isClickhouseResourceLimitError } from "@internal/clickhouse"; import { ErrorId, RunId } from "@trigger.dev/core/v3/isomorphic"; import { type FilterRunsOptions, @@ -10,6 +10,7 @@ import { type RunsRepositoryOptions, type TagListOptions, convertRunListInputOptionsToFilterRunsOptions, + RunsListQueryError, } from "./runsRepository.server"; import parseDuration from "parse-duration"; import { decodeRunsCursor, encodeRunsCursor } from "./runsCursor.server"; @@ -19,6 +20,18 @@ import { type PrismaClientOrTransaction } from "~/db.server"; import { boundedIn, type Prisma } from "@trigger.dev/database"; type RunCursorRow = { runId: string; createdAt: number }; +/** + * Re-throws a runs-list query error, converting a ClickHouse resource-limit rejection (execution + * time or memory) into a typed {@link RunsListQueryError} so callers can surface an actionable 4xx + * instead of an opaque 500. Any other error is re-thrown unchanged. + */ +function rethrowRunsListQueryError(queryError: unknown): never { + if (isClickhouseResourceLimitError(queryError)) { + throw new RunsListQueryError(undefined, { cause: queryError }); + } + throw queryError; +} + /** * Default hydrate select for the runs list, used when a caller does not derive * one from the visible columns (bulk actions, the live poll). Kept in sync with @@ -102,7 +115,7 @@ export class ClickHouseRunsRepository implements IRunsRepository { const [queryError, result] = await queryBuilder.execute(); if (queryError) { - throw queryError; + rethrowRunsListQueryError(queryError); } return (result?.length ?? 0) > 0; @@ -166,7 +179,7 @@ export class ClickHouseRunsRepository implements IRunsRepository { const [queryError, result] = await queryBuilder.execute(); if (queryError) { - throw queryError; + rethrowRunsListQueryError(queryError); } return result.map((row) => ({ runId: row.run_id, createdAt: row.created_at_ms })); @@ -349,7 +362,7 @@ export class ClickHouseRunsRepository implements IRunsRepository { const [queryError, result] = await queryBuilder.execute(); if (queryError) { - throw queryError; + rethrowRunsListQueryError(queryError); } if (result.length === 0) { @@ -402,7 +415,7 @@ export class ClickHouseRunsRepository implements IRunsRepository { const [queryError, result] = await queryBuilder.execute(); if (queryError) { - throw queryError; + rethrowRunsListQueryError(queryError); } return { @@ -411,6 +424,22 @@ export class ClickHouseRunsRepository implements IRunsRepository { } } +/** + * Builds the shared filter clauses for the runs list against `task_runs_v2 FINAL`. + * + * A filter may go in PREWHERE only if its truth value can never flip true->false across a run's + * versions, because PREWHERE is evaluated before FINAL reconciles versions and would otherwise keep + * a stale matching version and drop the winning one. That holds for columns that are only ever set + * once and never change: trigger-time identity columns (task_identifier, task_version, schedule_id, + * is_test, root_run_id, batch_id, friendly_id, queue, task_kind), append-only arrays under + * `hasAny`/`hasAll` (tags, bulk_action_group_ids), `region` (set once at dequeue, `''` -> value, + * never changes), and `error_fingerprint` (empty until a terminal error status, then fixed). Those + * go in PREWHERE to filter (and, for tags, use the skip index) before FINAL and before materialising + * the wide columns, which is what bounds memory on these scans. Columns that change as a run runs + * stay in WHERE (post-FINAL): `status` (lifecycle) and `machine_preset` (escalates on OOM retry). + * The `(organization_id, project_id, environment_id)` primary-key prefix and the `created_at` range + * also stay in WHERE so they keep driving primary-key and partition pruning. + */ function applyRunFiltersToQueryBuilder( queryBuilder: ClickhouseQueryBuilder, options: FilterRunsOptions @@ -426,28 +455,14 @@ function applyRunFiltersToQueryBuilder( environmentId: options.environmentId, }); - if (options.tasks && options.tasks.length > 0) { - queryBuilder.where("task_identifier IN {tasks: Array(String)}", { tasks: options.tasks }); - } - - if (options.versions && options.versions.length > 0) { - queryBuilder.where("task_version IN {versions: Array(String)}", { - versions: options.versions, - }); - } - if (options.statuses && options.statuses.length > 0) { queryBuilder.where("status IN {statuses: Array(String)}", { statuses: options.statuses }); } - if (options.tags && options.tags.length > 0) { - // Both hasAny and hasAll are served by the tags bloom_filter skip index. - const tagsFn = options.tagsMatch === "all" ? "hasAll" : "hasAny"; - queryBuilder.where(`${tagsFn}(tags, {tags: Array(String)})`, { tags: options.tags }); - } - - if (options.scheduleId) { - queryBuilder.where("schedule_id = {scheduleId: String}", { scheduleId: options.scheduleId }); + if (options.machines && options.machines.length > 0) { + queryBuilder.where("machine_preset IN {machines: Array(String)}", { + machines: options.machines, + }); } // Period is a number of milliseconds duration @@ -467,49 +482,63 @@ function applyRunFiltersToQueryBuilder( queryBuilder.where("created_at <= fromUnixTimestamp64Milli({to: Int64})", { to: options.to }); } + if (options.tasks && options.tasks.length > 0) { + queryBuilder.prewhere("task_identifier IN {tasks: Array(String)}", { tasks: options.tasks }); + } + + if (options.versions && options.versions.length > 0) { + queryBuilder.prewhere("task_version IN {versions: Array(String)}", { + versions: options.versions, + }); + } + + if (options.tags && options.tags.length > 0) { + // Both hasAny and hasAll are served by the tags bloom_filter skip index. + const tagsFn = options.tagsMatch === "all" ? "hasAll" : "hasAny"; + queryBuilder.prewhere(`${tagsFn}(tags, {tags: Array(String)})`, { tags: options.tags }); + } + + if (options.scheduleId) { + queryBuilder.prewhere("schedule_id = {scheduleId: String}", { + scheduleId: options.scheduleId, + }); + } + if (typeof options.isTest === "boolean") { - queryBuilder.where("is_test = {isTest: Boolean}", { isTest: options.isTest }); + queryBuilder.prewhere("is_test = {isTest: Boolean}", { isTest: options.isTest }); } if (options.rootOnly) { - queryBuilder.where("root_run_id = ''"); + queryBuilder.prewhere("root_run_id = ''"); } if (options.batchId) { - queryBuilder.where("batch_id = {batchId: String}", { batchId: options.batchId }); + queryBuilder.prewhere("batch_id = {batchId: String}", { batchId: options.batchId }); } if (options.bulkId) { - queryBuilder.where("hasAny(bulk_action_group_ids, {bulkActionGroupIds: Array(String)})", { + queryBuilder.prewhere("hasAny(bulk_action_group_ids, {bulkActionGroupIds: Array(String)})", { bulkActionGroupIds: [options.bulkId], }); } if (options.runId && options.runId.length > 0) { // it's important that in the query it's "runIds", otherwise it clashes with the cursor which is called "runId" - queryBuilder.where("friendly_id IN {runIds: Array(String)}", { + queryBuilder.prewhere("friendly_id IN {runIds: Array(String)}", { runIds: options.runId.map((runId) => RunId.toFriendlyId(runId)), }); } if (options.queues && options.queues.length > 0) { - queryBuilder.where("queue IN {queues: Array(String)}", { queues: options.queues }); + queryBuilder.prewhere("queue IN {queues: Array(String)}", { queues: options.queues }); } if (options.regions && options.regions.length > 0) { - queryBuilder.where("if(region != '', region, worker_queue) IN {regions: Array(String)}", { - regions: options.regions, - }); - } - - if (options.machines && options.machines.length > 0) { - queryBuilder.where("machine_preset IN {machines: Array(String)}", { - machines: options.machines, - }); + queryBuilder.prewhere("region IN {regions: Array(String)}", { regions: options.regions }); } if (options.errorId) { - queryBuilder.where("error_fingerprint = {errorFingerprint: String}", { + queryBuilder.prewhere("error_fingerprint = {errorFingerprint: String}", { errorFingerprint: ErrorId.toId(options.errorId), }); } @@ -520,11 +549,11 @@ function applyRunFiltersToQueryBuilder( const effectiveKinds = includesStandard ? [...options.taskKinds, ""] : options.taskKinds; if (effectiveKinds.length === 1) { - queryBuilder.where("task_kind = {taskKind: String}", { + queryBuilder.prewhere("task_kind = {taskKind: String}", { taskKind: effectiveKinds[0]!, }); } else { - queryBuilder.where("task_kind IN {taskKinds: Array(String)}", { + queryBuilder.prewhere("task_kind IN {taskKinds: Array(String)}", { taskKinds: effectiveKinds, }); } diff --git a/apps/webapp/app/services/runsRepository/runsRepository.server.ts b/apps/webapp/app/services/runsRepository/runsRepository.server.ts index 0b1049125dd..1aeeb96dbc1 100644 --- a/apps/webapp/app/services/runsRepository/runsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/runsRepository.server.ts @@ -13,6 +13,33 @@ import { runStore as defaultRunStore } from "~/v3/runStore.server"; import { startActiveSpan } from "~/v3/tracer.server"; import { ClickHouseRunsRepository } from "./clickhouseRunsRepository.server"; +/** + * User-facing message when a runs-list query exceeds a ClickHouse resource limit. It tells the + * caller how to recover (a narrower time range restores partition pruning), and is safe to show + * on the dashboard and return from the public API. + */ +const RUNS_LIST_QUERY_TOO_EXPENSIVE_MESSAGE = + "This query was too expensive to run over the selected time range. Narrow the time window (a shorter period, or a smaller createdAt from/to range) and try again."; + +/** + * Thrown when a runs-list ClickHouse query hits a server-side resource limit (execution time or + * memory). It is the caller's query being too broad, not a service fault, so it carries a 4xx + * status and a recovery message rather than surfacing as a 500. + */ +export class RunsListQueryError extends Error { + public readonly name = "RunsListQueryError"; + public readonly status = 422; + constructor( + message: string = RUNS_LIST_QUERY_TOO_EXPENSIVE_MESSAGE, + options?: { cause?: unknown } + ) { + super(message); + if (options?.cause !== undefined) { + this.cause = options.cause; + } + } +} + export type RunsRepositoryOptions = { clickhouse: ClickHouse; prisma: PrismaClientOrTransaction; diff --git a/apps/webapp/app/services/vercelIntegration.server.ts b/apps/webapp/app/services/vercelIntegration.server.ts index 336519af03b..a95bb12b491 100644 --- a/apps/webapp/app/services/vercelIntegration.server.ts +++ b/apps/webapp/app/services/vercelIntegration.server.ts @@ -20,6 +20,8 @@ import { VercelProjectIntegrationDataSchema, envTypeToSlug, createDefaultVercelIntegrationData, + getAvailableEnvSlugs, + restrictConfigToAvailableEnvSlugs, SKEW_PROTECTION_ENV_VAR_KEY, } from "~/v3/vercel/vercelProjectIntegrationSchema"; @@ -129,6 +131,17 @@ export class VercelIntegrationService { .filter((i): i is VercelProjectIntegrationWithProject => i !== null); } + async #getAvailableEnvSlugs(projectId: string): Promise { + const environments = await this.#prismaClient.runtimeEnvironment.findMany({ + where: { projectId, type: { in: ["STAGING", "PREVIEW"] }, parentEnvironmentId: null }, + select: { type: true }, + }); + + const types = new Set(environments.map((environment) => environment.type)); + + return getAvailableEnvSlugs(types.has("STAGING"), types.has("PREVIEW")); + } + async createVercelProjectIntegration(params: { organizationIntegrationId: string; projectId: string; @@ -142,7 +155,8 @@ export class VercelIntegrationService { params.vercelProjectId, params.vercelProjectName, params.vercelTeamId, - params.vercelTeamSlug + params.vercelTeamSlug, + await this.#getAvailableEnvSlugs(params.projectId) ); return this.#prismaClient.organizationProjectIntegration.create({ @@ -183,6 +197,8 @@ export class VercelIntegrationService { () => undefined ); + const availableEnvSlugs = await this.#getAvailableEnvSlugs(params.projectId); + // Use a serializable transaction to prevent duplicate project integrations // from concurrent selectVercelProject calls (read-then-write race condition). const txResult = await $transaction( @@ -236,7 +252,8 @@ export class VercelIntegrationService { params.vercelProjectId, params.vercelProjectName, teamId, - vercelTeamSlug + vercelTeamSlug, + availableEnvSlugs ); const created = await tx.organizationProjectIntegration.create({ @@ -320,7 +337,10 @@ export class VercelIntegrationService { const updatedConfig = { ...existing.parsedIntegrationData.config, - ...configUpdates, + ...restrictConfigToAvailableEnvSlugs( + configUpdates, + await this.#getAvailableEnvSlugs(projectId) + ), }; const updatedData: VercelProjectIntegrationData = { @@ -578,14 +598,20 @@ export class VercelIntegrationService { prod: {}, preview: {}, }; + const availableEnvSlugs = await this.#getAvailableEnvSlugs(projectId); const updatedData: VercelProjectIntegrationData = { ...existing.parsedIntegrationData, config: { ...existing.parsedIntegrationData.config, - pullEnvVarsBeforeBuild: params.pullEnvVarsBeforeBuild ?? null, - atomicBuilds: params.atomicBuilds ?? null, - discoverEnvVars: params.discoverEnvVars ?? null, - vercelStagingEnvironment: params.vercelStagingEnvironment ?? null, + ...restrictConfigToAvailableEnvSlugs( + { + pullEnvVarsBeforeBuild: params.pullEnvVarsBeforeBuild ?? null, + atomicBuilds: params.atomicBuilds ?? null, + discoverEnvVars: params.discoverEnvVars ?? null, + vercelStagingEnvironment: params.vercelStagingEnvironment ?? null, + }, + availableEnvSlugs + ), }, //This is intentionally not updated here, in case of resetting the onboarding it should not override the existing mapping with an empty one syncEnvVarsMapping: existing.parsedIntegrationData.syncEnvVarsMapping, @@ -610,7 +636,7 @@ export class VercelIntegrationService { projectId, vercelProjectId: updatedData.vercelProjectId, teamId, - vercelStagingEnvironment: params.vercelStagingEnvironment, + vercelStagingEnvironment: updatedData.config.vercelStagingEnvironment, syncEnvVarsMapping, orgIntegration, }); diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts new file mode 100644 index 00000000000..89a0a674a04 --- /dev/null +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -0,0 +1,118 @@ +import { BuildServerMetadata } from "@trigger.dev/core/v3"; + +/** + * Attribute names for the `deployment.finished` and `deployment.initialized` + * telemetry events (emitted by services/recordDeploymentFinished.server.ts). + * This module is the single owner of these names — external queries, + * dashboards, and monitors reference them, so treat renames as breaking. + * + * Query gotchas: dedup with `arg_max(_time, *) by deployment.id` (job retries + * can double-emit); the span's `_time` is the deployment's createdAt, so a + * TIMED_OUT event lands backdated by up to the full deploy timeout — monitor + * windows must exceed it; phase durations are omitted (not zero) when a + * boundary timestamp is missing, and `total_ms` excludes local-bundle's + * pre-init client work (esbuild + upload) until the CLI reports timings. + */ +export const DeploymentTelemetryAttributes = { + ORG_ID: "$trigger.org.id", + ORG_SLUG: "$trigger.org.slug", + PROJECT_ID: "$trigger.project.id", + PROJECT_NAME: "$trigger.project.name", + // Project external ref ("proj_…") + PROJECT_REF: "$trigger.project.ref", + ENV_ID: "$trigger.env.id", + // PRODUCTION / STAGING / PREVIEW / DEVELOPMENT + ENV_TYPE: "$trigger.env.type", + // Deployment friendly id — the dedup key + DEPLOYMENT_ID: "deployment.id", + VERSION: "deployment.version", + // finished: terminal status; initialized: initial status (PENDING/BUILDING) + STATUS: "deployment.status", + // status === DEPLOYED; CANCELED is excluded from failure rates + SUCCESS: "deployment.success", + // depot / native / native_local_bundle (see deriveBuildPath) + BUILD_PATH: "deployment.build_path", + // V1 / MANAGED (run engine) + WORKER_TYPE: "deployment.worker_type", + RUNTIME: "deployment.runtime", + // Set at indexing; null for pre-index failures + RUNTIME_VERSION: "deployment.runtime_version", + // From x-trigger-cli-version at init; null for pre-column history + CLI_VERSION: "deployment.cli_version", + TRIGGERED_VIA: "deployment.triggered_via", + COMMIT_SHA: "deployment.commit_sha", + // error.* only on FAILED/TIMED_OUT; CANCELED uses canceled_reason + ERROR_NAME: "deployment.error.name", + ERROR_MESSAGE: "deployment.error.message", + CANCELED_REASON: "deployment.canceled_reason", + // createdAt → terminal (also the span's own duration) + DURATION_TOTAL_MS: "deployment.duration.total_ms", + // createdAt → startedAt; ≈0 when created directly in BUILDING (depot) + DURATION_QUEUE_MS: "deployment.duration.queue_ms", + // startedAt → installedAt; build-server paths only (depot never sets it) + DURATION_INSTALL_MS: "deployment.duration.install_ms", + // (installedAt ?? startedAt) → builtAt + DURATION_BUILDING_MS: "deployment.duration.building_ms", + // builtAt → terminal; for depot dominated by the server-side registry push + DURATION_DEPLOYING_MS: "deployment.duration.deploying_ms", +} as const; + +export type DeploymentBuildPath = "native_local_bundle" | "native" | "depot"; + +/** + * Everything that is not a native-build-server deployment falls into the depot + * bucket, including rare `--local-build` deploys (their flag is not persisted). + * `externalBuildData` is NOT a usable depot signal: init writes a placeholder + * for every path. + */ +export function deriveBuildPath(buildServerMetadata: unknown): DeploymentBuildPath { + const metadata = BuildServerMetadata.safeParse(buildServerMetadata); + + if (metadata.success && metadata.data.isNativeBuild) { + return metadata.data.fromBundle ? "native_local_bundle" : "native"; + } + + return "depot"; +} + +export type DeploymentTimestamps = { + createdAt: Date; + startedAt?: Date | null; + installedAt?: Date | null; + builtAt?: Date | null; +}; + +export type DeploymentDurations = { + totalMs: number; + queueMs?: number; + installMs?: number; + buildingMs?: number; + deployingMs?: number; +}; + +/** + * Timestamp chains are path-shaped (e.g. depot never sets installedAt), so + * each phase is derived only when both of its boundary timestamps exist and + * are ordered. + */ +export function deriveDeploymentDurations( + timestamps: DeploymentTimestamps, + terminalAt: Date +): DeploymentDurations { + const { createdAt, startedAt, installedAt, builtAt } = timestamps; + const buildingFrom = installedAt ?? startedAt; + + return { + totalMs: Math.max(terminalAt.getTime() - createdAt.getTime(), 0), + queueMs: msBetween(createdAt, startedAt), + installMs: msBetween(startedAt, installedAt), + buildingMs: msBetween(buildingFrom, builtAt), + deployingMs: msBetween(builtAt, terminalAt), + }; +} + +function msBetween(from?: Date | null, to?: Date | null): number | undefined { + if (!from || !to) return undefined; + const ms = to.getTime() - from.getTime(); + return ms >= 0 ? ms : undefined; +} diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index fdd302ede84..152a14b6496 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -1,15 +1,18 @@ import { type z } from "zod"; import type { PrismaClient } from "@trigger.dev/database"; -import { prisma, type PrismaClientOrTransaction } from "~/db.server"; import { FEATURE_FLAG, type FeatureFlagCatalogSchema, type FeatureFlagKey, FeatureFlagCatalog, + GLOBAL_LOCKED_FLAGS, + GRACED_FLAG_GROUPS, validatePartialFeatureFlags, } from "~/v3/featureFlags"; import { env } from "~/env.server"; import { stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace"; +import { stampMintShardSetFlip } from "~/v3/runOpsMigration/mintShardGrace"; +import { $transaction, boundedIn, prisma, type PrismaClientOrTransaction } from "~/db.server"; export type FlagsOptions = { key: T; @@ -181,46 +184,202 @@ export function makeSetMultipleFlags(_prisma: PrismaClientOrTransaction = prisma }; } -// Read -> stamp -> write the global mint-kind grace metadata in one transaction. The three -// FeatureFlag rows may not exist yet, so a row FOR UPDATE can't lock them; an advisory xact lock -// serializes concurrent global flips so one can't clobber another's grace stamp (mirrors per-org). -export async function applyGlobalMintKindFlip( +// The key topology lives in the shared module, because the admin page needs it too. This adds +// the stamping behaviour, which is server-only. +const GRACED_GLOBAL_GROUPS = GRACED_FLAG_GROUPS.map((group) => ({ + ...group, + stamp: group.primary === FEATURE_FLAG.runOpsMintKind ? stampMintKindFlip : stampMintShardSetFlip, +})); + +const GRACED_GLOBAL_KEYS: FeatureFlagKey[] = GRACED_GLOBAL_GROUPS.flatMap((g) => [ + g.primary, + ...g.derived, +]); + +function gracedGroupFor(key: FeatureFlagKey) { + return GRACED_GLOBAL_GROUPS.find((g) => g.primary === key || g.derived.includes(key)); +} + +// True when a save changes any graced group, and therefore needs the stamped path. Derived from +// the group table, so adding a group cannot leave a caller silently writing an unstamped flip. +export function touchesGracedGroup(requestedFlags: Record): boolean { + return GRACED_GLOBAL_GROUPS.some((group) => requestedFlags[group.primary] !== undefined); +} + +// Strips every derived key: a grace stamp is computed here, never accepted from a caller. +// Only the flags whose stored value differs. Each write is a round trip inside an interactive +// transaction, so writing an unchanged flag costs a round trip for nothing. +export function flagsNeedingWrite( + requested: Record, + existing: Record +): Record { + const out: Record = {}; + for (const [key, value] of Object.entries(requested)) { + if (JSON.stringify(existing[key] ?? null) !== JSON.stringify(value ?? null)) { + out[key] = value; + } + } + return out; +} + +export function withoutDerivedKeys( + requestedFlags: Partial> +): Record { + const out: Record = { ...requestedFlags }; + for (const group of GRACED_GLOBAL_GROUPS) { + for (const derived of group.derived) { + delete out[derived]; + } + } + return out; +} + +// The rows may not exist yet, so a row FOR UPDATE cannot lock them; an advisory xact lock +// serializes concurrent global flips instead, so one cannot clobber another's stamp. +// +// Two lock ids are taken, in a fixed order. The FIRST is the operative one: an older release +// takes only that id, and a deploy rolls for hours, so it is the id that serializes across both +// versions. The second is this release's name and adds nothing until every writer takes it. +// Renaming without keeping the old id is what would leave the two versions unserialized. Remove +// the legacy id one release after this one ships, when nothing takes it alone. +async function lockGracedGroups(tx: PrismaClientOrTransaction): Promise { + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-mint-kind-flip'))`; + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-graced-flag-flip'))`; +} + +// Reads each group's current rows and returns the requested flags plus a fresh stamp for every +// group the save actually changes. A group whose primary the save omits is left untouched. +async function stampGracedGroups( + tx: PrismaClientOrTransaction, + requestedFlags: Record, + graceMs: number +): Promise> { + const existingRows = await tx.featureFlag.findMany({ + where: { key: { in: boundedIn(GRACED_GLOBAL_KEYS) } }, + select: { key: true, value: true }, + }); + const existingGlobal: Record = {}; + for (const row of existingRows) { + existingGlobal[row.key] = row.value; + } + + // Anchor the cutover to the control-plane DB clock, not this process's wall clock. A rolling + // deploy spans hours, so every pod must date the window against one shared clock. + const [{ now }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`; + + let stamped: Record = { ...requestedFlags }; + for (const group of GRACED_GLOBAL_GROUPS) { + stamped = group.stamp(existingGlobal, stamped, now.getTime(), graceMs); + } + return stamped; +} + +// Merge-semantics write: sets what the caller asked for, stamps any graced group it changes, and +// touches nothing else. Used by the JSON admin API. +export async function applyGlobalGracedFlips( client: PrismaClient, requestedFlags: Partial>, graceMs: number ): Promise<{ key: string; value: any }[]> { - return client.$transaction(async (tx) => { - await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-mint-kind-flip'))`; + const applied = await $transaction(client, "applyGlobalGracedFlips", async (tx) => { + await lockGracedGroups(tx); + const stamped = await stampGracedGroups(tx, withoutDerivedKeys(requestedFlags), graceMs); + return makeSetMultipleFlags(tx)(stamped as Partial>); + }); - const existingRows = await tx.featureFlag.findMany({ - where: { - key: { - in: [ - FEATURE_FLAG.runOpsMintKind, - FEATURE_FLAG.runOpsMintKindPrev, - FEATURE_FLAG.runOpsMintKindFlippedAt, - ], - }, - }, - select: { key: true, value: true }, - }); - const existingGlobal: Record = {}; - for (const row of existingRows) { - existingGlobal[row.key] = row.value; + // The helper resolves undefined rather than throwing when Prisma swallows an infrastructure + // error. This write stamps a cutover window, so a transaction that did not run must be loud. + if (!applied) { + throw new Error("applyGlobalGracedFlips: transaction did not complete"); + } + return applied; +} + +// Replace-semantics write for the global admin flags page: submitted flags upsert, omitted ones +// delete unless protected. One transaction covers the stamp, the upserts and the deletes, so a +// save cannot half-apply. +// +// A graced group is all-or-nothing. Submitting its primary writes the group with a fresh stamp. +// Omitting its primary deletes the primary AND its stamp together, because a stamp left behind +// without its primary keeps being served: {set: [], prevSet: [a], flippedAt: t} resolves to [a] +// for the rest of the window, which would mint into a shard the operator just removed. The +// delete ignores `isProtected` for the derived keys for the same reason. +export async function replaceGlobalFeatureFlags( + client: PrismaClient, + params: { + requestedFlags: Record; + catalogKeys: FeatureFlagKey[]; + isManagedCloud: boolean; + unlockLockedFlags: boolean; + graceMs: number; + } +): Promise { + const requestedFlags = withoutDerivedKeys(params.requestedFlags); + + // A locked flag absent from the payload means the page never offered it, not that the admin + // unset it, so it survives. Only a self-hosted page that says it unlocked them may delete one. + const canDeleteLocked = params.unlockLockedFlags && !params.isManagedCloud; + const isProtected = (key: FeatureFlagKey) => + !canDeleteLocked && GLOBAL_LOCKED_FLAGS.includes(key); + + const applied = await $transaction(client, "replaceGlobalFeatureFlags", async (tx) => { + await lockGracedGroups(tx); + const stamped = await stampGracedGroups(tx, requestedFlags, params.graceMs); + + const toWrite: Record = {}; + const keysToDelete: string[] = []; + + for (const key of params.catalogKeys) { + const group = gracedGroupFor(key); + + if (group) { + if (requestedFlags[group.primary] !== undefined) { + if (stamped[key] !== undefined) { + toWrite[key] = stamped[key]; + } + } else if (!isProtected(group.primary)) { + keysToDelete.push(key); + } + continue; + } + + if (key in requestedFlags) { + toWrite[key] = requestedFlags[key]; + } else if (!isProtected(key)) { + keysToDelete.push(key); + } } - // Anchor the cutover to the control-plane DB clock, not this process's wall clock. - const [{ now }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`; + // One round trip to learn the stored values, then a write only for what actually differs. + // makeSetMultipleFlags upserts sequentially, so an unchanged flag costs a round trip for + // nothing, and this transaction is interactive and holds a pooled connection. + const writeKeys = Object.keys(toWrite); + if (writeKeys.length > 0) { + const storedRows = await tx.featureFlag.findMany({ + where: { key: { in: boundedIn(writeKeys) } }, + select: { key: true, value: true }, + }); + const stored: Record = {}; + for (const row of storedRows) { + stored[row.key] = row.value; + } + + await makeSetMultipleFlags(tx)( + flagsNeedingWrite(toWrite, stored) as Partial> + ); + } - const stamped = stampMintKindFlip( - existingGlobal, - { ...requestedFlags }, - now.getTime(), - graceMs - ) as Partial>; + if (keysToDelete.length > 0) { + await tx.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } }); + } - return makeSetMultipleFlags(tx)(stamped); + return true; }); + + // This write deletes flags, so a transaction that did not run must reach the caller. + if (!applied) { + throw new Error("replaceGlobalFeatureFlags: transaction did not complete"); + } } /** The global flag set, with the env-var defaults this app applies. */ diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 7c775799178..3a88beb54bc 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -26,6 +26,16 @@ export const FEATURE_FLAG = { // Grace-linger stamp carried alongside runOpsMintKind on flip. See mintFlipGrace.ts. runOpsMintKindPrev: "runOpsMintKindPrev", runOpsMintKindFlippedAt: "runOpsMintKindFlippedAt", + // Gen-2 mint shard pins, read from the org override blob only. See runOpsMintShard.server.ts. + runOpsMintShard: "runOpsMintShard", + runOpsMintShardEnvPins: "runOpsMintShardEnvPins", + // The active mint-shard list, global only. Lives here rather than in the environment because a + // rolling deploy runs two environment values at once for hours. See mintShardGrace.ts. + runOpsMintShardSet: "runOpsMintShardSet", + runOpsMintShardSetPrev: "runOpsMintShardSetPrev", + runOpsMintShardSetFlippedAt: "runOpsMintShardSetFlippedAt", + // Fleet-wide pin for the complete cutover. Beats every per-org and per-env pin. + runOpsMintShardOverride: "runOpsMintShardOverride", queueMetricsUiEnabled: "queueMetricsUiEnabled", // Per-organization rollout for creating additional environment API keys. additionalApiKeysEnabled: "additionalApiKeysEnabled", @@ -89,6 +99,52 @@ export const FeatureFlagCatalog = { // by stampMintKindFlip on a genuine flip. Display-only (see ORG_LOCKED_FLAGS). [FEATURE_FLAG.runOpsMintKindPrev]: z.enum(["cuid", "runOpsId"]), [FEATURE_FLAG.runOpsMintKindFlippedAt]: z.string().datetime(), + // Pins one org to a gen-2 mint shard. "new" holds the org on gen-1 run-ops ids, which is how + // a canary keeps the fleet's default while one org moves. Only honored while the key is in + // the active list; a drained key falls through to the hash. + [FEATURE_FLAG.runOpsMintShard]: z + .string() + .refine((v) => v === "new" || /^[a-z0-9]$/.test(v), 'must be a single [a-z0-9] char, or "new"'), + // Per-environment pins as JSON: {"": ""}. A JSON string because + // this catalog is scalar-only. Rejected at write, so a typo cannot silently un-pin an env. + [FEATURE_FLAG.runOpsMintShardEnvPins]: z.string().superRefine((raw, ctx) => { + const fail = (message: string) => ctx.addIssue({ code: z.ZodIssueCode.custom, message }); + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return fail("must be valid JSON"); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return fail("must be a JSON object mapping environment id to shard key"); + } + for (const [environmentId, value] of Object.entries(parsed)) { + if (typeof value !== "string" || !(value === "new" || /^[a-z0-9]$/.test(value))) { + fail(`"${environmentId}" must map to a single [a-z0-9] char, or "new"`); + } + } + }), + // CSV of the shard keys eligible for root minting right now. Empty means no gen-2 minting. + // Reserved keys are rejected, because "new" already means gen-1. + [FEATURE_FLAG.runOpsMintShardSet]: z.string().refine( + (v) => + v + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + .every((k) => /^[a-z0-9]$/.test(k)), + "must be a CSV of single [a-z0-9] chars" + ), + // Grace stamp: the previously-effective list and the flip time, written by + // stampMintShardSetFlip on a genuine change. Display-only (see ORG_LOCKED_FLAGS). + [FEATURE_FLAG.runOpsMintShardSetPrev]: z.string(), + [FEATURE_FLAG.runOpsMintShardSetFlippedAt]: z.string().datetime(), + // Sends every environment to one shard, outranking every pin, so a cutover needs no per-org + // visit. "new" holds the whole fleet on gen-1. Only honored while the key is in the active set. + [FEATURE_FLAG.runOpsMintShardOverride]: z + .string() + .refine((v) => v === "new" || /^[a-z0-9]$/.test(v), 'must be a single [a-z0-9] char, or "new"'), // Per-org access to the Queue Metrics dashboard UI (view only; emission is global and // separate). Off unless enabled for the org. [FEATURE_FLAG.queueMetricsUiEnabled]: z.coerce.boolean(), @@ -101,11 +157,20 @@ export const FeatureFlagCatalog = { export type FeatureFlagKey = keyof typeof FeatureFlagCatalog; -// Infrastructure flags that are read-only on the global flags page. -// Shown with current/resolved value but no controls. +// Infrastructure flags, plus org-scoped-only flags, that are read-only on the global flags +// page. Shown with current/resolved value but no controls. An org-scoped-only flag belongs +// here because its resolver never reads a global row, so an editable global control would +// offer a setting that does nothing. export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [ FEATURE_FLAG.defaultWorkerInstanceGroupId, FEATURE_FLAG.taskEventRepository, + FEATURE_FLAG.runOpsMintShard, + FEATURE_FLAG.runOpsMintShardEnvPins, + // Grace stamps are computed server-side. An editable control here would discard what it saves. + FEATURE_FLAG.runOpsMintKindPrev, + FEATURE_FLAG.runOpsMintKindFlippedAt, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, ]; // Flags that are read-only on the org-level dialog. @@ -118,8 +183,53 @@ export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [ // System-wide only — orgs must not be able to override these kill switches. FEATURE_FLAG.additionalApiKeyIssuanceEnabled, FEATURE_FLAG.additionalApiKeyLookupEnabled, + // The active mint-shard list is deployment-wide; only the pins are per-org. + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + FEATURE_FLAG.runOpsMintShardOverride, +]; + +/** + * Flag groups where the operator sets a `primary` and the server computes the rest. The topology + * lives here, not in the server module, because the admin page needs it too: unsetting a primary + * clears its stamps, and the page has to disclose that. + */ +export const GRACED_FLAG_GROUPS: ReadonlyArray<{ + primary: FeatureFlagKey; + derived: readonly FeatureFlagKey[]; +}> = [ + { + primary: FEATURE_FLAG.runOpsMintKind, + derived: [FEATURE_FLAG.runOpsMintKindPrev, FEATURE_FLAG.runOpsMintKindFlippedAt], + }, + { + primary: FEATURE_FLAG.runOpsMintShardSet, + derived: [FEATURE_FLAG.runOpsMintShardSetPrev, FEATURE_FLAG.runOpsMintShardSetFlippedAt], + }, ]; +/** The stamps deleted alongside `primary`. Empty unless `primary` is a graced primary. */ +export function derivedFlagsClearedWith(primary: string): FeatureFlagKey[] { + const group = GRACED_FLAG_GROUPS.find((g) => g.primary === primary); + return group ? [...group.derived] : []; +} + +/** + * Locked flags present in a payload the global page must refuse. On managed cloud the page never + * offers them, so their presence means the request did not come from that page. Locally an admin + * may unlock and edit them, so nothing is refused. + */ +export function lockedFlagsInPayload( + payloadKeys: string[], + isManagedCloud: boolean +): FeatureFlagKey[] { + if (!isManagedCloud) return []; + return payloadKeys.filter((key): key is FeatureFlagKey => + GLOBAL_LOCKED_FLAGS.includes(key as FeatureFlagKey) + ); +} + // Create a Zod schema from the existing catalog export const FeatureFlagCatalogSchema = z.object(FeatureFlagCatalog); export type FeatureFlagCatalog = z.infer; diff --git a/apps/webapp/app/v3/runEngineHandlersShared.server.ts b/apps/webapp/app/v3/runEngineHandlersShared.server.ts index 4ce8cc2de8a..d8999e2332a 100644 --- a/apps/webapp/app/v3/runEngineHandlersShared.server.ts +++ b/apps/webapp/app/v3/runEngineHandlersShared.server.ts @@ -35,7 +35,8 @@ export async function readRunForEvent( deps: EventReadDeps ): Promise | null> { const result = await readThroughRun>({ - runId, + id: runId, + idKind: "run", environmentId, readNew: (client) => deps.store.findRun({ id: runId }, { select }, client), readLegacy: (replica) => deps.store.findRun({ id: runId }, { select }, replica), @@ -47,7 +48,7 @@ export async function readRunForEvent( }, }); - return result.source === "not-found" || result.source === "past-retention" ? null : result.value; + return result.found ? result.value : null; } /** diff --git a/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts b/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts index 37301f68743..d08f96529c9 100644 --- a/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts @@ -34,6 +34,7 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { const emit = vi.fn(); await assertControlPlaneCoresidencyAdvisory({ ...urls, + shards: [], expectSplit: false, probe: async () => ({ coresident: "true" }), emit, @@ -46,6 +47,7 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { await expect( assertControlPlaneCoresidencyAdvisory({ ...urls, + shards: [], expectSplit: true, probe: async () => ({ coresident: "true" }), emit: vi.fn(), @@ -58,6 +60,7 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { const emit = vi.fn(); await assertControlPlaneCoresidencyAdvisory({ ...urls, + shards: [], expectSplit: true, probe: async () => ({ coresident: "unknown", reason: "denied" }), emit, @@ -71,6 +74,7 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { const warn = vi.fn(); await assertControlPlaneCoresidencyAdvisory({ ...urls, + shards: [], expectSplit: true, probe: async () => { throw new Error("probe blew up"); @@ -86,9 +90,12 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { const emit = vi.fn(); const probe = vi.fn(); await assertControlPlaneCoresidencyAdvisory({ - legacyUrl: undefined, + // "" and not undefined: ?? only guards nullish, so undefined would read the ambient + // RUN_OPS_LEGACY_DATABASE_URL and this test would depend on the developer's .env. + legacyUrl: "", controlPlaneUrl: "postgres://cp", expectSplit: true, + shards: [], probe, emit, log: noopLog, @@ -97,3 +104,120 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { expect(emit).not.toHaveBeenCalled(); }); }); + +describe("assertControlPlaneCoresidencyAdvisory at N shards", () => { + const urls = { legacyUrl: "postgres://legacy", controlPlaneUrl: "postgres://cp" }; + const shardA = { key: "a", url: "postgres://shard-a" }; + const shardB = { key: "b", url: "postgres://shard-b" }; + + it("emits the legacy verdict with NO shard key, so today's series is unchanged", async () => { + const emit = vi.fn(); + await assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: false, + shards: [], + probe: async () => ({ coresident: "false" }), + emit, + log: noopLog, + }); + expect(emit).toHaveBeenCalledTimes(1); + expect(emit).toHaveBeenCalledWith("false"); + }); + + it("emits one tagged verdict per shard, plus the untagged legacy verdict", async () => { + const emit = vi.fn(); + await assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: false, + shards: [shardA, shardB], + probe: async () => ({ coresident: "false" }), + emit, + log: noopLog, + }); + expect(emit).toHaveBeenCalledTimes(3); + expect(emit).toHaveBeenCalledWith("false"); + expect(emit).toHaveBeenCalledWith("false", "a"); + expect(emit).toHaveBeenCalledWith("false", "b"); + }); + + it("probes each shard against the control plane", async () => { + const probe = vi.fn().mockResolvedValue({ coresident: "false" }); + await assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: false, + shards: [shardA], + probe, + emit: vi.fn(), + log: noopLog, + }); + expect(probe).toHaveBeenCalledWith("postgres://legacy", "postgres://cp", expect.anything()); + expect(probe).toHaveBeenCalledWith("postgres://shard-a", "postgres://cp", expect.anything()); + }); + + it("names the offending shard when enforcement is opted in and a shard is co-resident", async () => { + await expect( + assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: true, + shards: [shardA], + probe: async (url: string) => + url === "postgres://shard-a" + ? ({ coresident: "true", reason: "same db" } as const) + : ({ coresident: "false" } as const), + emit: vi.fn(), + log: noopLog, + }) + ).rejects.toThrow(/shard a/i); + }); + + it("emits every store before it throws, so no store loses its metric", async () => { + const emit = vi.fn(); + await expect( + assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: true, + shards: [shardA, shardB], + probe: async (url: string) => + url === "postgres://shard-a" + ? ({ coresident: "true", reason: "same db" } as const) + : ({ coresident: "false" } as const), + emit, + log: noopLog, + }) + ).rejects.toThrow(); + expect(emit).toHaveBeenCalledTimes(3); + expect(emit).toHaveBeenCalledWith("false", "b"); + }); + + it("degrades one shard's throwing probe to unknown and still reports the others", async () => { + const emit = vi.fn(); + await assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: true, + shards: [shardA, shardB], + probe: async (url: string) => { + if (url === "postgres://shard-a") throw new Error("probe blew up"); + return { coresident: "false" } as const; + }, + emit, + log: { info: () => {}, warn: () => {} }, + }); + expect(emit).toHaveBeenCalledWith("unknown", "a"); + expect(emit).toHaveBeenCalledWith("false", "b"); + }); + + it("still probes the shards when there is no legacy DSN", async () => { + const emit = vi.fn(); + await assertControlPlaneCoresidencyAdvisory({ + legacyUrl: "", + controlPlaneUrl: "postgres://cp", + expectSplit: false, + shards: [shardA], + probe: async () => ({ coresident: "false" }), + emit, + log: noopLog, + }); + expect(emit).toHaveBeenCalledTimes(1); + expect(emit).toHaveBeenCalledWith("false", "a"); + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.ts b/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.ts index 1fb741b0e67..870f51c75bc 100644 --- a/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.ts @@ -12,6 +12,7 @@ import type { Counter } from "@opentelemetry/api"; import { getMeter } from "@internal/tracing"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; +import { nonAliasedShards, type ShardTarget } from "~/v3/runOpsShards.server"; import { probeControlPlaneCoresidency, type CoresidencyProbeResult, @@ -39,12 +40,14 @@ export type CoresidencyEnforcement = { throw: false } | { throw: true; message: export function resolveCoresidencyEnforcement(args: { coresident: CoresidencyVerdict; expectSplit: boolean; + /** Omitted for the legacy store, so its message stays exactly as it was. */ + shardKey?: string; }): CoresidencyEnforcement { if (args.expectSplit && args.coresident === "true") { + const store = args.shardKey === undefined ? "legacy run-ops DB" : `shard ${args.shardKey}`; return { throw: true, - message: - "RUN_OPS_EXPECT_CONTROL_PLANE_SPLIT is on but the legacy run-ops DB is still co-resident with the control-plane DB; refusing to start.", + message: `RUN_OPS_EXPECT_CONTROL_PLANE_SPLIT is on but the ${store} is still co-resident with the control-plane DB; refusing to start.`, }; } return { throw: false }; @@ -57,46 +60,84 @@ type AdvisoryLogger = { export async function assertControlPlaneCoresidencyAdvisory(deps?: { probe?: typeof probeControlPlaneCoresidency; - emit?: (verdict: CoresidencyVerdict) => void; + /** shardKey is omitted for the legacy store, so its metric series is unchanged at N=0. */ + emit?: (verdict: CoresidencyVerdict, shardKey?: string) => void; log?: AdvisoryLogger; expectSplit?: boolean; legacyUrl?: string; controlPlaneUrl?: string; + shards?: ShardTarget[]; }): Promise { const log = deps?.log ?? logger; const legacyUrl = deps?.legacyUrl ?? env.RUN_OPS_LEGACY_DATABASE_URL; const controlPlaneUrl = deps?.controlPlaneUrl ?? env.CONTROL_PLANE_DATABASE_URL ?? env.DATABASE_URL; - // No legacy DSN (single-DB / self-host) or no control-plane DSN -> nothing to compare. - if (!legacyUrl || !controlPlaneUrl) return; + const shards = deps?.shards ?? nonAliasedShards(env.RUN_OPS_SHARDS); + // No control-plane DSN -> nothing to compare against, for any store. + if (!controlPlaneUrl) return; + + // The legacy store carries NO shard key, so its metric series and its message are unchanged. + // An aliased shard is already absent from `shards`: it shares its target's database on purpose, + // so a co-residency verdict for it would duplicate its target's verdict. + const stores: Array<{ url: string; shardKey?: string }> = [ + ...(legacyUrl ? [{ url: legacyUrl }] : []), + ...shards.map((shard) => ({ url: shard.url, shardKey: shard.key })), + ]; + if (stores.length === 0) return; const probe = deps?.probe ?? probeControlPlaneCoresidency; const emit = deps?.emit ?? - ((verdict: CoresidencyVerdict) => getCoresidentCounter().add(1, { result: verdict })); + ((verdict: CoresidencyVerdict, shardKey?: string) => + getCoresidentCounter().add( + 1, + shardKey === undefined ? { result: verdict } : { result: verdict, shard: shardKey } + )); const expectSplit = deps?.expectSplit ?? env.RUN_OPS_EXPECT_CONTROL_PLANE_SPLIT; - let result: CoresidencyProbeResult; - try { - result = await probe(legacyUrl, controlPlaneUrl, { logger: log }); - } catch (error) { - // Any unexpected throw still degrades to "unknown" — the advisory arm must never crash boot. - log.warn("run-ops control-plane co-residency probe threw; reporting unknown", { error }); - result = { coresident: "unknown", reason: String(error) }; - } + const results = await Promise.all( + stores.map(async (store) => { + let result: CoresidencyProbeResult; + try { + result = await probe(store.url, controlPlaneUrl, { logger: log }); + } catch (error) { + // Any unexpected throw degrades THAT store to "unknown" — the advisory arm must never + // crash boot, and one store's denied probe must not hide another store's verdict. + log.warn("run-ops control-plane co-residency probe threw; reporting unknown", { + error, + shard: store.shardKey, + }); + result = { coresident: "unknown", reason: String(error) }; + } + return { store, result }; + }) + ); - emit(result.coresident); - log.info("run_ops_legacy_control_plane_coresident", { - coresident: result.coresident, - reason: "reason" in result ? result.reason : undefined, - expectSplit, - }); + // Emit and log EVERY store before any enforcement throw, so a failing store never costs + // another store its metric. + for (const { store, result } of results) { + // One argument for the legacy store, so its emission is byte-identical to today's. + if (store.shardKey === undefined) { + emit(result.coresident); + } else { + emit(result.coresident, store.shardKey); + } + log.info("run_ops_legacy_control_plane_coresident", { + coresident: result.coresident, + reason: "reason" in result ? result.reason : undefined, + expectSplit, + shard: store.shardKey, + }); + } - const enforcement = resolveCoresidencyEnforcement({ - coresident: result.coresident, - expectSplit, - }); - if (enforcement.throw) { - throw new Error(enforcement.message); + for (const { store, result } of results) { + const enforcement = resolveCoresidencyEnforcement({ + coresident: result.coresident, + expectSplit, + shardKey: store.shardKey, + }); + if (enforcement.throw) { + throw new Error(enforcement.message); + } } } diff --git a/apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts b/apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts index 4b2bfd9d986..ed7fb0cb237 100644 --- a/apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts @@ -62,32 +62,46 @@ export async function probeControlPlaneCoresidency( } } -export async function probeDistinctDatabases( - legacyUrl: string, - newUrl: string, +export type DistinctTarget = { id: string; url: string }; + +/** + * Set uniqueness over every store that owns its own database. Fail-closed: a probe that cannot + * answer returns NOT distinct, because "distinct" is a positive claim a failed probe cannot support. + * + * Same-cluster-different-database policy (unchanged from the pairwise probe): two databases inside + * the SAME cluster (same system identifier, different current_database()) are reported distinct. + * They are genuinely separate Postgres databases with separate WAL-visible state for our purposes. + * + * An ALIASED shard never appears in `targets`. It shares its target's client by reference, so it is + * not its own database and inclusion would guarantee a duplicate. See nonAliasedShards. + */ +export async function probeDistinctStores( + targets: DistinctTarget[], opts?: { logger?: { warn: (msg: string, meta?: Record) => void } } ): Promise<{ distinct: true } | { distinct: false; reason: string }> { + if (targets.length < 2) { + return { distinct: true }; + } + try { - const [legacy, next] = await Promise.all([ - readDatabaseFingerprint(legacyUrl), - readDatabaseFingerprint(newUrl), - ]); - const sameCluster = legacy.systemIdentifier === next.systemIdentifier; - const sameDb = sameCluster && legacy.databaseName === next.databaseName; - // Same-cluster-different-database policy: two databases inside the SAME cluster - // (same system identifier, different current_database()) are reported distinct: true. - // That is acceptable — they are genuinely separate Postgres databases with separate - // WAL-visible state for our purposes, and the Cloud topology always uses separate - // clusters anyway. A stricter "must be a different cluster" policy would gate on - // sameCluster alone; that is flagged as an open question, not decided here. - if (sameDb) { - const reason = - "run-ops legacy and new URLs resolve to the SAME physical database " + - `(systemIdentifier=${legacy.systemIdentifier}, database=${legacy.databaseName}); ` + - "refusing to enable split — pooler/replica likely."; - opts?.logger?.warn(reason); - return { distinct: false, reason }; + const fingerprints = await Promise.all(targets.map((t) => readDatabaseFingerprint(t.url))); + + const seen = new Map(); + for (const [index, target] of targets.entries()) { + const fingerprint = fingerprints[index]; + const key = `${fingerprint.systemIdentifier}/${fingerprint.databaseName}`; + const first = seen.get(key); + if (first !== undefined) { + const reason = + `run-ops stores "${first}" and "${target.id}" resolve to the SAME physical database ` + + `(systemIdentifier=${fingerprint.systemIdentifier}, database=${fingerprint.databaseName}); ` + + "refusing to enable split — pooler/replica likely."; + opts?.logger?.warn(reason); + return { distinct: false, reason }; + } + seen.set(key, target.id); } + return { distinct: true }; } catch (error) { const reason = `distinct-db sentinel probe failed; failing closed (single-DB). ${String(error)}`; @@ -95,3 +109,19 @@ export async function probeDistinctDatabases( return { distinct: false, reason }; } } + +// The gen-1 pairwise entry point, kept as a thin delegate over a 2-element target list. Set +// uniqueness over one pair IS the pairwise compare, and this function's tests are the proof. +export async function probeDistinctDatabases( + legacyUrl: string, + newUrl: string, + opts?: { logger?: { warn: (msg: string, meta?: Record) => void } } +): Promise<{ distinct: true } | { distinct: false; reason: string }> { + return probeDistinctStores( + [ + { id: "legacy", url: legacyUrl }, + { id: "new", url: newUrl }, + ], + opts + ); +} diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts new file mode 100644 index 00000000000..a4e4a64d36d --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts @@ -0,0 +1,512 @@ +import { describe, expect, it } from "vitest"; +import { + computeMintShard, + resolveMintShardWith, + type MintShardCache, + type MintShardDeps, + type ResolveMintShardDeps, +} from "./mintShardAssignment"; +import { type MintShardSetResolution } from "./mintShardGrace"; + +const GRACE_MS = 90_000; +const T = 1_000_000; + +// Cuid-shaped ids, not sequential integers: a sequential space does not model the real +// key distribution the hash has to spread. +function envIds(count: number): string[] { + const ids: string[] = []; + for (let i = 0; i < count; i++) { + ids.push(`cm${(i * 2654435761).toString(36).padStart(10, "0")}${i.toString(36)}zzq`); + } + return ids; +} + +function deps( + resolution: MintShardSetResolution, + overrides: Partial = {} +): MintShardDeps { + return { + resolution, + nowMs: T + GRACE_MS + 1, + graceMs: GRACE_MS, + orgFeatureFlags: undefined, + ...overrides, + }; +} + +function orgFlags(flags: Record) { + return { orgFeatureFlags: flags }; +} + +function place(ids: string[], resolution: MintShardSetResolution): Map { + const out = new Map(); + for (const id of ids) { + out.set(id, computeMintShard({ id }, deps(resolution))); + } + return out; +} + +describe("computeMintShard — the no-shards answer", () => { + it("returns new when the live list is empty", () => { + expect(computeMintShard({ id: "env_1" }, deps({ set: [] }))).toBe("new"); + }); + + it("returns new when a stale stamp is present but both lists are empty", () => { + const resolution: MintShardSetResolution = { set: [], prevSet: [], flippedAtMs: T }; + expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new"); + }); + + it("returns new when the grace serves an empty list", () => { + const resolution: MintShardSetResolution = { set: ["a"], prevSet: [], flippedAtMs: T }; + expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new"); + }); + + it("returns new when the grace serves an empty prevSet", () => { + const resolution: MintShardSetResolution = { set: ["a"], prevSet: [], flippedAtMs: T }; + expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new"); + }); +}); + +describe("computeMintShard — determinism", () => { + it("returns the same value for the same environment on every call", () => { + const resolution: MintShardSetResolution = { set: ["a", "b", "c"] }; + const first = computeMintShard({ id: "env_stable" }, deps(resolution)); + for (let i = 0; i < 1000; i++) { + expect(computeMintShard({ id: "env_stable" }, deps(resolution))).toBe(first); + } + }); + + it("ignores the order the operator listed the keys in", () => { + const ids = envIds(200); + const canonical = place(ids, { set: ["a", "b", "c"] }); + for (const permutation of [ + ["c", "b", "a"], + ["b", "a", "c"], + ["a", "c", "b"], + ]) { + expect(place(ids, { set: permutation })).toEqual(canonical); + } + }); +}); + +describe("computeMintShard — pins", () => { + const resolution: MintShardSetResolution = { set: ["a", "b"] }; + + it("lets a per-env pin override the hash", () => { + const ids = envIds(50); + for (const id of ids) { + const pinned = computeMintShard( + { id }, + deps(resolution, orgFlags({ runOpsMintShardEnvPins: JSON.stringify({ [id]: "b" }) })) + ); + expect(pinned).toBe("b"); + } + }); + + it("lets a per-org pin override the hash when no per-env pin is set", () => { + const ids = envIds(50); + for (const id of ids) { + expect(computeMintShard({ id }, deps(resolution, orgFlags({ runOpsMintShard: "a" })))).toBe( + "a" + ); + } + }); + + it("lets a per-env pin beat a per-org pin", () => { + const result = computeMintShard( + { id: "env_1" }, + deps( + resolution, + orgFlags({ + runOpsMintShard: "a", + runOpsMintShardEnvPins: JSON.stringify({ env_1: "b" }), + }) + ) + ); + expect(result).toBe("b"); + }); + + it("holds an environment on gen-1 when the pin is new", () => { + expect( + computeMintShard({ id: "env_1" }, deps(resolution, orgFlags({ runOpsMintShard: "new" }))) + ).toBe("new"); + expect( + computeMintShard( + { id: "env_1" }, + deps(resolution, orgFlags({ runOpsMintShardEnvPins: JSON.stringify({ env_1: "new" }) })) + ) + ).toBe("new"); + }); + + it("falls through to the hash and reports when the pin is outside the active set", () => { + // Honouring a drained pin would leak the drain; throwing would fail customer triggers. + const rejected: string[] = []; + const result = computeMintShard( + { id: "env_1" }, + deps(resolution, { + ...orgFlags({ runOpsMintShard: "z" }), + onPinRejected: (info) => rejected.push(info.pin), + }) + ); + expect(result).toBe(computeMintShard({ id: "env_1" }, deps(resolution))); + expect(rejected).toEqual(["z"]); + }); + + it("honours a pin to a drained key for the whole grace window, then falls through", () => { + const draining: MintShardSetResolution = { set: ["a"], prevSet: ["a", "b"], flippedAtMs: T }; + const pinnedToB = orgFlags({ runOpsMintShard: "b" }); + expect(computeMintShard({ id: "env_1" }, deps(draining, { ...pinnedToB, nowMs: T + 1 }))).toBe( + "b" + ); + expect( + computeMintShard({ id: "env_1" }, deps(draining, { ...pinnedToB, nowMs: T + GRACE_MS })) + ).not.toBe("b"); + }); + + it("ignores an unparseable pin blob rather than un-pinning silently", () => { + const result = computeMintShard( + { id: "env_1" }, + deps(resolution, orgFlags({ runOpsMintShard: "a", runOpsMintShardEnvPins: "{not json" })) + ); + expect(result).toBe("a"); + }); + + it("falls back to the org pin when the blob holds an invalid value for this env", () => { + const result = computeMintShard( + { id: "env_1" }, + deps( + resolution, + orgFlags({ + runOpsMintShard: "a", + runOpsMintShardEnvPins: JSON.stringify({ env_1: "LEGACY" }), + }) + ) + ); + expect(result).toBe("a"); + }); + + it("ignores an invalid org pin value", () => { + const result = computeMintShard( + { id: "env_1" }, + deps(resolution, orgFlags({ runOpsMintShard: "legacy" })) + ); + expect(result).toBe(computeMintShard({ id: "env_1" }, deps(resolution))); + }); +}); + +describe("computeMintShard — rendezvous properties", () => { + const ids = envIds(10_000); + + it("spreads roughly evenly across the active set", () => { + for (const set of [ + ["a", "b"], + ["a", "b", "c"], + ["a", "b", "c", "d"], + ]) { + const counts = new Map(); + for (const shard of place(ids, { set }).values()) { + counts.set(shard, (counts.get(shard) ?? 0) + 1); + } + expect(counts.size).toBe(set.length); + const expected = ids.length / set.length; + for (const count of counts.values()) { + expect(Math.abs(count - expected) / expected).toBeLessThan(0.1); + } + } + }); + + it("moves about 1/(N+1) of environments when a shard is added", () => { + const cases: Array<{ from: string[]; to: string[]; expected: number }> = [ + { from: ["a"], to: ["a", "b"], expected: 1 / 2 }, + { from: ["a", "b"], to: ["a", "b", "c"], expected: 1 / 3 }, + { from: ["a", "b", "c"], to: ["a", "b", "c", "d"], expected: 1 / 4 }, + ]; + + for (const { from, to, expected } of cases) { + const before = place(ids, { set: from }); + const after = place(ids, { set: to }); + const added = to.filter((k) => !from.includes(k)); + let moved = 0; + for (const id of ids) { + if (before.get(id) === after.get(id)) continue; + moved++; + // HRW's defining property: a mover lands on the ADDED shard, never on a survivor. + expect(added).toContain(after.get(id)); + } + expect(Math.abs(moved / ids.length - expected) / expected).toBeLessThan(0.1); + } + }); + + it("moves only the environments that hashed to a removed shard", () => { + const before = place(ids, { set: ["a", "b", "c"] }); + const after = place(ids, { set: ["a", "b"] }); + for (const id of ids) { + if (before.get(id) === "c") { + expect(after.get(id)).not.toBe("c"); + } else { + expect(after.get(id)).toBe(before.get(id)); + } + } + }); + + it("also moves pinned environments when their shard is removed", () => { + // Criterion 6 is a property of the hash only. A pin to a removed key moves too. + const pinnedToC = orgFlags({ runOpsMintShard: "c" }); + expect(computeMintShard({ id: "env_1" }, deps({ set: ["a", "b", "c"] }, pinnedToC))).toBe("c"); + expect(computeMintShard({ id: "env_1" }, deps({ set: ["a", "b"] }, pinnedToC))).not.toBe("c"); + }); +}); + +describe("resolveMintShardWith — cache, read failure and fail-safe", () => { + function wrapperDeps( + overrides: Partial = {} + ): ResolveMintShardDeps & { reads: number } { + const state = { + readFlags: async () => ({ runOpsMintShardSet: "a,b" }), + cache: { current: undefined as MintShardCache }, + nowMs: T, + ttlMs: 30_000, + graceMs: GRACE_MS, + orgFeatureFlags: undefined as unknown, + reads: 0, + ...overrides, + }; + const wrapped = state.readFlags; + state.readFlags = async () => { + state.reads++; + return wrapped(); + }; + return state; + } + + it("reads once, then serves the cache until the TTL expires", async () => { + const deps = wrapperDeps(); + await resolveMintShardWith({ id: "env_1" }, deps); + await resolveMintShardWith({ id: "env_2" }, deps); + await resolveMintShardWith({ id: "env_3" }, deps); + expect(deps.reads).toBe(1); + }); + + it("reads again once the TTL expires", async () => { + const deps = wrapperDeps(); + await resolveMintShardWith({ id: "env_1" }, deps); + deps.nowMs = T + 30_000; + await resolveMintShardWith({ id: "env_1" }, deps); + expect(deps.reads).toBe(2); + }); + + it("falls back to gen-1 when the read throws, and does not poison the cache", async () => { + // A blip must not move every environment's placement, so it returns gen-1 rather than guess. + let fail = true; + const deps = wrapperDeps({ + readFlags: async () => { + if (fail) throw new Error("db down"); + return { runOpsMintShardSet: "a,b" }; + }, + }); + const failures: unknown[] = []; + deps.onReadFailed = (error) => failures.push(error); + + expect(await resolveMintShardWith({ id: "env_1" }, deps)).toBe("new"); + expect(failures).toHaveLength(1); + + fail = false; + expect(["a", "b"]).toContain(await resolveMintShardWith({ id: "env_1" }, deps)); + }); + + it("returns gen-1 when the stored list is empty", async () => { + const deps = wrapperDeps({ readFlags: async () => ({ runOpsMintShardSet: "" }) }); + expect(await resolveMintShardWith({ id: "env_1" }, deps)).toBe("new"); + }); + + it("coalesces concurrent misses into ONE read", async () => { + // Two misses must share a single read. Otherwise a slower read landing after a faster one + // writes its older snapshot back into the cache for a whole TTL. + let release: (flags: Record) => void = () => {}; + const gate = new Promise>((resolve) => { + release = resolve; + }); + const deps = wrapperDeps({ readFlags: () => gate }); + + const both = Promise.all([ + resolveMintShardWith({ id: "env_1" }, deps), + resolveMintShardWith({ id: "env_2" }, deps), + ]); + release({ runOpsMintShardSet: "a,b" }); + await both; + + expect(deps.reads).toBe(1); + }); + + it("does not let a slower read overwrite a newer one", async () => { + // The slow read starts first and finishes last. Its result must not become the cached + // value, because the fast read already published a newer snapshot. + let releaseSlow: (flags: Record) => void = () => {}; + const slow = new Promise>((resolve) => { + releaseSlow = resolve; + }); + let call = 0; + const deps = wrapperDeps({ + readFlags: () => { + call++; + return call === 1 ? slow : Promise.resolve({ runOpsMintShardSet: "c" }); + }, + }); + + const first = resolveMintShardWith({ id: "env_1" }, deps); + const second = resolveMintShardWith({ id: "env_2" }, deps); + releaseSlow({ runOpsMintShardSet: "a" }); + await Promise.all([first, second]); + + // One read served both, so there is no second snapshot to race with. + expect(deps.reads).toBe(1); + expect(deps.cache.current?.value.resolution.set).toEqual(["a"]); + }); + + it("clears the in-flight refresh after a failure, so the next call retries", async () => { + let fail = true; + const deps = wrapperDeps({ + readFlags: async () => { + if (fail) throw new Error("db down"); + return { runOpsMintShardSet: "a,b" }; + }, + }); + deps.onReadFailed = () => {}; + + expect(await resolveMintShardWith({ id: "env_1" }, deps)).toBe("new"); + fail = false; + expect(["a", "b"]).toContain(await resolveMintShardWith({ id: "env_1" }, deps)); + expect(deps.reads).toBe(2); + }); + + it("agrees with the pure core for the same inputs", async () => { + const deps = wrapperDeps(); + const viaWrapper = await resolveMintShardWith({ id: "env_1" }, deps); + const viaCore = computeMintShard( + { id: "env_1" }, + { + resolution: { set: ["a", "b"] }, + nowMs: T, + graceMs: GRACE_MS, + orgFeatureFlags: undefined, + } + ); + expect(viaWrapper).toBe(viaCore); + }); +}); + +describe("computeMintShard — the global override wins the complete cutover", () => { + const resolution: MintShardSetResolution = { set: ["a", "b"] }; + + it("beats the hash for every environment", () => { + for (const id of envIds(200)) { + expect(computeMintShard({ id }, deps(resolution, { globalOverride: "b" }))).toBe("b"); + } + }); + + it("beats a per-org pin", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps(resolution, { globalOverride: "b", orgFeatureFlags: { runOpsMintShard: "a" } }) + ); + expect(shard).toBe("b"); + }); + + it("beats a per-env pin, which is the whole point of a cutover", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps(resolution, { + globalOverride: "b", + orgFeatureFlags: { runOpsMintShardEnvPins: JSON.stringify({ env_1: "a" }) }, + }) + ); + expect(shard).toBe("b"); + }); + + it("holds the whole fleet on gen-1 when set to new, whatever any org pinned", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps(resolution, { globalOverride: "new", orgFeatureFlags: { runOpsMintShard: "a" } }) + ); + expect(shard).toBe("new"); + }); + + it("is ignored, and reported, when it names a key outside the active set", () => { + // Honouring it would mint into a drained or unroutable shard. Explicit pins still apply. + const rejected: string[] = []; + const shard = computeMintShard( + { id: "env_1" }, + deps(resolution, { + globalOverride: "z", + orgFeatureFlags: { runOpsMintShard: "a" }, + onOverrideRejected: (info) => rejected.push(info.override), + }) + ); + expect(shard).toBe("a"); + expect(rejected).toEqual(["z"]); + }); + + it("reports a bad override WITHOUT the environment id, so one line covers the fleet", () => { + // Keying the report by environment would log once per environment for a fleet-wide setting. + const seen: Array<{ override: string }> = []; + for (const id of envIds(50)) { + computeMintShard( + { id }, + deps(resolution, { globalOverride: "z", onOverrideRejected: (i) => seen.push(i) }) + ); + } + expect(seen).toHaveLength(50); + expect(new Set(seen.map((i) => i.override))).toEqual(new Set(["z"])); + expect(seen.every((i) => !("environmentId" in i))).toBe(true); + }); + + it("is ignored when it is not a legal value", () => { + for (const bad of ["legacy", "AB", "", "a,b"]) { + const shard = computeMintShard({ id: "env_1" }, deps(resolution, { globalOverride: bad })); + expect(shard).toBe(computeMintShard({ id: "env_1" }, deps(resolution))); + } + }); + + it("cannot resurrect minting when the list is empty", () => { + expect(computeMintShard({ id: "env_1" }, deps({ set: [] }, { globalOverride: "b" }))).toBe( + "new" + ); + }); +}); + +describe("routableKeys bound (the shard descriptor keys this deployment can route)", () => { + it("drops an active key that is not routable, so the hash never returns it", () => { + // "z" is in the active list but not configured as a descriptor -> only "a" is selectable. + const ids = envIds(200); + for (const id of ids) { + const shard = computeMintShard({ id }, deps({ set: ["a", "z"] }, { routableKeys: ["a"] })); + expect(shard).toBe("a"); + } + }); + + it("returns new when the active list holds only non-routable keys (fail-safe to gen-1)", () => { + expect(computeMintShard({ id: "env_1" }, deps({ set: ["z"] }, { routableKeys: ["a"] }))).toBe( + "new" + ); + }); + + it("rejects a per-org pin to a non-routable key and falls through to the hash", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps({ set: ["a", "z"] }, { ...orgFlags({ runOpsMintShard: "z" }), routableKeys: ["a"] }) + ); + expect(shard).toBe("a"); + }); + + it("with no routableKeys given, behaviour is unchanged", () => { + const ids = envIds(200); + for (const id of ids) { + const withBound = computeMintShard( + { id }, + deps({ set: ["a", "b"] }, { routableKeys: ["a", "b"] }) + ); + const without = computeMintShard({ id }, deps({ set: ["a", "b"] })); + expect(withBound).toBe(without); + } + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts new file mode 100644 index 00000000000..2855f936249 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts @@ -0,0 +1,222 @@ +// PURE module: no env, no clock, no database. Kept separate from the .server wrapper so a test +// can drive it without evaluating env.server, whose schema parse demands a full environment. +import { createHash } from "node:crypto"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { + effectiveMintShardSet, + GEN_1_PIN_VALUE, + isValidPinValue, + readMintShardSetResolution, + type MintShardSetResolution, +} from "./mintShardGrace"; + +export type MintShardDeps = { + // The live list, from the control-plane database. + resolution: MintShardSetResolution; + // Fleet-wide pin that beats every per-org and per-env pin. The complete-cutover lever. + globalOverride?: unknown; + nowMs: number; + graceMs: number; + orgFeatureFlags: unknown; + // The shard keys this deployment can actually route (the RUN_OPS_SHARDS descriptor keys). The + // active set is bounded to these, so a stored key with no descriptor is never minted into. + // Undefined means "no bound" (today's behaviour). + routableKeys?: readonly string[]; + onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; + onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; +}; + +function asRecord(value: unknown): Record | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + return value as Record; +} + +// Map keys are environment INTERNAL ids (cuids), not friendly ids. An unparseable blob, or a +// blob whose value for this environment is invalid, yields no per-env pin and lets the +// per-org scalar decide — never a silent un-pin straight to the hash. +function readEnvPin(raw: unknown, environmentId: string): ShardKey | undefined { + if (typeof raw !== "string") return undefined; + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return undefined; + } + + const pins = asRecord(parsed); + const pin = pins?.[environmentId]; + return isValidPinValue(pin) ? pin : undefined; +} + +// Both pins live in the org override blob the trigger path already holds, so resolving a mint +// shard costs no query. +function readPin(orgFeatureFlags: unknown, environmentId: string): ShardKey | undefined { + const blob = asRecord(orgFeatureFlags); + if (!blob) return undefined; + + const envPin = readEnvPin(blob[FEATURE_FLAG.runOpsMintShardEnvPins], environmentId); + if (envPin !== undefined) return envPin; + + const scalar = blob[FEATURE_FLAG.runOpsMintShard]; + return isValidPinValue(scalar) ? scalar : undefined; +} + +// 64 bits: a 32-bit score collides at this system's environment count, and an undetected tie +// would resolve by iteration order. The NUL separates the fields so no two input pairs can +// concatenate alike. This hash input is FROZEN once gen-2 minting is live: changing it +// re-places every environment, silently. +function shardScore(environmentId: string, key: string): bigint { + return createHash("sha256").update(`${environmentId}\0${key}`).digest().readBigUInt64BE(0); +} + +function hrwSelect(environmentId: string, activeSet: string[]): string { + let bestKey = activeSet[0]; + let bestScore = shardScore(environmentId, bestKey); + + for (let i = 1; i < activeSet.length; i++) { + const key = activeSet[i]; + const score = shardScore(environmentId, key); + if (score > bestScore || (score === bestScore && key > bestKey)) { + bestKey = key; + bestScore = score; + } + } + + return bestKey; +} + +// PURE CORE — no env, no clock, no I/O; tests drive this directly. Deterministic for fixed +// deps, which is what lets run minting and token minting agree on one answer. +// +// An empty list is the off state, and it is the state of every deployment that has not set the +// flag. Bounding the list against the shard keys this deployment can actually route belongs with +// the shard descriptors, which own that information; nothing here mints, so nothing can misroute. +// +// A pin outside the active set falls through to the hash rather than throwing: honouring it +// would leak the drain the active list performs, and throwing would fail customer triggers +// whenever a pinned shard drains. +export function computeMintShard(environment: { id: string }, deps: MintShardDeps): ShardKey { + const rawActiveSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs); + // Empty check BEFORE the bound, so an unconfigured deployment returns "new" exactly as today. + if (rawActiveSet.length === 0) { + return "new"; + } + + // Bound the active set to the keys this deployment can route. A stored key with no descriptor is + // dropped, never minted into. If nothing survives, fall back to gen-1 (fail-safe, never a throw). + const activeSet = deps.routableKeys + ? rawActiveSet.filter((key) => deps.routableKeys!.includes(key)) + : rawActiveSet; + if (activeSet.length === 0) { + return "new"; + } + + // The global override outranks every pin, so one flag completes a cutover without visiting + // each org. An override outside the active set is ignored, so explicit pins still apply. + if (isValidPinValue(deps.globalOverride)) { + const override = deps.globalOverride; + if (override === GEN_1_PIN_VALUE) { + return "new"; + } + if (activeSet.includes(override)) { + return override; + } + // Fleet-wide, so it is reported once for the value, not once per environment. + deps.onOverrideRejected?.({ override, activeSet }); + } + + const pin = readPin(deps.orgFeatureFlags, environment.id); + if (pin !== undefined) { + if (pin === GEN_1_PIN_VALUE) { + return "new"; + } + if (activeSet.includes(pin)) { + return pin; + } + deps.onPinRejected?.({ environmentId: environment.id, pin, activeSet }); + } + + return hrwSelect(environment.id, activeSet); +} + +// Read together so the override costs no extra query beyond the list it is bounded by. + +type GlobalShardConfig = { resolution: MintShardSetResolution; override: unknown }; + +export type MintShardCache = { value: GlobalShardConfig; expiresAt: number } | undefined; + +type MintShardCacheHandle = { + current: MintShardCache; + // The refresh currently in flight, if any. Concurrent misses share it. + inFlight?: Promise; +}; + +export type ResolveMintShardDeps = { + // Reads the list rows. Injected so the cache and the fail-safe are testable without a + // database, the same way computeRunIdMintKind takes its flag reader. + readFlags: () => Promise>; + cache: MintShardCacheHandle; + nowMs: number; + ttlMs: number; + graceMs: number; + orgFeatureFlags: unknown; + routableKeys?: readonly string[]; + onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; + onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; + onReadFailed?: (error: unknown) => void; +}; + +// The live list is org-independent, so one process-wide entry serves every mint: one query per +// process per TTL, over one round-trip. Two processes can therefore disagree for the TTL PLUS the +// replica lag behind the read, which can exceed graceMs. That is tolerable here and only here, +// because a gen-2 id carries its own shard key, so disagreement cannot misroute an existing run; +// it only decides where the next root lands, and every failure direction is toward gen-1. +// +// A failed read falls back to gen-1 rather than guessing a list. Guessing would move every +// environment's placement for the length of one blip. +async function refreshConfig(deps: ResolveMintShardDeps): Promise { + try { + const flags = await deps.readFlags(); + const config: GlobalShardConfig = { + resolution: readMintShardSetResolution(flags), + override: flags[FEATURE_FLAG.runOpsMintShardOverride], + }; + deps.cache.current = { value: config, expiresAt: deps.nowMs + deps.ttlMs }; + return config; + } finally { + deps.cache.inFlight = undefined; + } +} + +export async function resolveMintShardWith( + environment: { id: string; orgFeatureFlags?: unknown }, + deps: ResolveMintShardDeps +): Promise { + let config: GlobalShardConfig; + const cached = deps.cache.current; + if (cached && cached.expiresAt > deps.nowMs) { + config = cached.value; + } else { + try { + // Single-flight. Without it, two misses both read, and a slower read landing after a + // faster one puts its older snapshot back into the cache for a whole TTL. + config = await (deps.cache.inFlight ??= refreshConfig(deps)); + } catch (error) { + deps.onReadFailed?.(error); + return "new"; + } + } + + return computeMintShard(environment, { + resolution: config.resolution, + globalOverride: config.override, + nowMs: deps.nowMs, + graceMs: deps.graceMs, + orgFeatureFlags: deps.orgFeatureFlags, + routableKeys: deps.routableKeys, + onPinRejected: deps.onPinRejected, + onOverrideRejected: deps.onOverrideRejected, + }); +} diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardGrace.test.ts b/apps/webapp/app/v3/runOpsMigration/mintShardGrace.test.ts new file mode 100644 index 00000000000..3d26cb9fd1a --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/mintShardGrace.test.ts @@ -0,0 +1,234 @@ +import { describe, expect, it } from "vitest"; +import { generateRunOpsIdV2 } from "@trigger.dev/core/v3/isomorphic"; +import { + effectiveMintShardSet, + isValidPinValue, + parseShardCsv, + readMintShardSetResolution, + SHARD_KEY_PATTERN, + stampMintShardSetFlip, + type MintShardSetResolution, +} from "./mintShardGrace"; + +const GRACE_MS = 90_000; +const T = 1_000_000; + +describe("parseShardCsv", () => { + it("returns an empty list for unset, empty and whitespace input", () => { + expect(parseShardCsv(undefined)).toEqual([]); + expect(parseShardCsv("")).toEqual([]); + expect(parseShardCsv(" ")).toEqual([]); + expect(parseShardCsv(",,")).toEqual([]); + }); + + it("trims, dedupes and SORTS, so operator typing order cannot change HRW", () => { + expect(parseShardCsv("b, a ,b")).toEqual(["a", "b"]); + expect(parseShardCsv("a,b,c")).toEqual(parseShardCsv("c,b,a")); + expect(parseShardCsv("b,c,a")).toEqual(parseShardCsv("a,c,b")); + }); + + it("accepts every one of the 36 legal shard keys", () => { + const all = "abcdefghijklmnopqrstuvwxyz0123456789".split(""); + expect(parseShardCsv(all.join(","))).toEqual([...all].sort()); + }); + + it("throws on a key outside [a-z0-9]", () => { + // generateRunOpsIdV2 throws on these; an unvalidated key MUST fail at boot, not at mint. + expect(() => parseShardCsv("A")).toThrow(/shard key/i); + expect(() => parseShardCsv("ab")).toThrow(/shard key/i); + expect(() => parseShardCsv("a,-")).toThrow(/shard key/i); + expect(() => parseShardCsv("a,_")).toThrow(/shard key/i); + }); + + it("rejects the reserved keys by name", () => { + expect(() => parseShardCsv("new")).toThrow(/reserved/i); + expect(() => parseShardCsv("a,legacy")).toThrow(/reserved/i); + }); +}); + +// Core does not export its shard-char pattern, so pin the local one to the real minter. +describe("shard alphabet agrees with the core minter", () => { + it("accepts exactly the characters generateRunOpsIdV2 accepts", () => { + const candidates = [ + ..."abcdefghijklmnopqrstuvwxyz0123456789".split(""), + ..."ABZ-_. +/é!".split(""), + "", + "ab", + ]; + + for (const candidate of candidates) { + let minterAccepts = true; + try { + generateRunOpsIdV2(candidate); + } catch { + minterAccepts = false; + } + + expect(SHARD_KEY_PATTERN.test(candidate)).toBe(minterAccepts); + } + }); +}); + +describe("isValidPinValue", () => { + it('accepts a shard key, and accepts "new" as the gen-1 hold value', () => { + expect(isValidPinValue("a")).toBe(true); + expect(isValidPinValue("7")).toBe(true); + expect(isValidPinValue("new")).toBe(true); + }); + + it("rejects legacy, and rejects anything outside the alphabet", () => { + expect(isValidPinValue("legacy")).toBe(false); + expect(isValidPinValue("A")).toBe(false); + expect(isValidPinValue("ab")).toBe(false); + expect(isValidPinValue("")).toBe(false); + }); +}); + +describe("effectiveMintShardSet", () => { + it("returns set when there is no stamp", () => { + const r: MintShardSetResolution = { set: ["a", "b"] }; + expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual(["a", "b"]); + }); + + it("returns set when flippedAtMs is absent even though prevSet is present", () => { + const r: MintShardSetResolution = { set: ["a", "b"], prevSet: ["a"] }; + expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual(["a", "b"]); + }); + + it("serves prevSet inside the window and set at/after the boundary", () => { + const r: MintShardSetResolution = { set: ["a", "b"], prevSet: ["a"], flippedAtMs: T }; + expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual(["a"]); + expect(effectiveMintShardSet(r, T + GRACE_MS - 1, GRACE_MS)).toEqual(["a"]); + // Boundary is exclusive on the prev side, so every process crosses it together. + expect(effectiveMintShardSet(r, T + GRACE_MS, GRACE_MS)).toEqual(["a", "b"]); + expect(effectiveMintShardSet(r, T + GRACE_MS + 1, GRACE_MS)).toEqual(["a", "b"]); + }); + + it("represents a graced first activation as an empty prevSet", () => { + const r: MintShardSetResolution = { set: ["a"], prevSet: [], flippedAtMs: T }; + expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual([]); + expect(effectiveMintShardSet(r, T + GRACE_MS, GRACE_MS)).toEqual(["a"]); + }); + + it("serves a drain through the window", () => { + const r: MintShardSetResolution = { set: ["a"], prevSet: ["a", "b"], flippedAtMs: T }; + expect(effectiveMintShardSet(r, T + 1, GRACE_MS)).toEqual(["a", "b"]); + expect(effectiveMintShardSet(r, T + GRACE_MS, GRACE_MS)).toEqual(["a"]); + }); +}); + +describe("readMintShardSetResolution", () => { + it("returns an empty set for an absent record", () => { + expect(readMintShardSetResolution(undefined)).toEqual({ set: [] }); + expect(readMintShardSetResolution({})).toEqual({ set: [] }); + }); + + it("reads and sorts the trio", () => { + const r = readMintShardSetResolution({ + runOpsMintShardSet: "b,a", + runOpsMintShardSetPrev: "c,a", + runOpsMintShardSetFlippedAt: new Date(T).toISOString(), + }); + expect(r).toEqual({ set: ["a", "b"], prevSet: ["a", "c"], flippedAtMs: T }); + }); + + it("omits prevSet when no flip timestamp is stored", () => { + // A prevSet with no timestamp can never apply, so it MUST NOT linger. + const r = readMintShardSetResolution({ + runOpsMintShardSet: "a,b", + runOpsMintShardSetPrev: "a", + }); + expect(r).toEqual({ set: ["a", "b"], prevSet: undefined, flippedAtMs: undefined }); + }); + + it("keeps an empty prevSet when a timestamp IS stored, which graces a first activation", () => { + const r = readMintShardSetResolution({ + runOpsMintShardSet: "a", + runOpsMintShardSetPrev: "", + runOpsMintShardSetFlippedAt: new Date(T).toISOString(), + }); + expect(r).toEqual({ set: ["a"], prevSet: [], flippedAtMs: T }); + }); + + it("degrades a stored value it cannot parse to an empty list instead of throwing", () => { + // Boot may throw on a bad env var. The mint path must never throw on a bad stored value. + expect(() => readMintShardSetResolution({ runOpsMintShardSet: "NOPE" })).not.toThrow(); + expect(readMintShardSetResolution({ runOpsMintShardSet: "NOPE" }).set).toEqual([]); + expect(readMintShardSetResolution({ runOpsMintShardSet: 42 }).set).toEqual([]); + expect( + readMintShardSetResolution({ + runOpsMintShardSet: "a", + runOpsMintShardSetFlippedAt: "not-a-date", + }) + ).toEqual({ set: ["a"], prevSet: undefined, flippedAtMs: undefined }); + }); +}); + +describe("stampMintShardSetFlip", () => { + it("does nothing when the save omits the set", () => { + // Omitting the set is an unrelated flag change; it must not inject a default or reset the clock. + const outgoing = { someOtherFlag: true } as Record; + expect(stampMintShardSetFlip({ runOpsMintShardSet: "a" }, outgoing, T, GRACE_MS)).toEqual({ + someOtherFlag: true, + }); + }); + + it("stamps prev and flippedAt on a genuine change", () => { + const stamped = stampMintShardSetFlip( + { runOpsMintShardSet: "a" }, + { runOpsMintShardSet: "a,b" }, + T, + GRACE_MS + ); + expect(stamped.runOpsMintShardSetPrev).toBe("a"); + expect(stamped.runOpsMintShardSetFlippedAt).toBe(new Date(T).toISOString()); + }); + + it("stamps an empty prev on a first activation", () => { + const stamped = stampMintShardSetFlip({}, { runOpsMintShardSet: "a" }, T, GRACE_MS); + expect(stamped.runOpsMintShardSetPrev).toBe(""); + expect(stamped.runOpsMintShardSetFlippedAt).toBe(new Date(T).toISOString()); + }); + + it("treats a reordered list as no change", () => { + const stamped = stampMintShardSetFlip( + { runOpsMintShardSet: "a,b" }, + { runOpsMintShardSet: "b,a" }, + T, + GRACE_MS + ); + expect(stamped.runOpsMintShardSetFlippedAt).toBeUndefined(); + }); + + it("carries an in-flight stamp forward rather than resetting the cutover clock", () => { + const existing = { + runOpsMintShardSet: "a,b", + runOpsMintShardSetPrev: "a", + runOpsMintShardSetFlippedAt: new Date(T).toISOString(), + }; + const stamped = stampMintShardSetFlip( + existing, + { runOpsMintShardSet: "a,b" }, + T + 1000, + GRACE_MS + ); + expect(stamped.runOpsMintShardSetPrev).toBe("a"); + expect(stamped.runOpsMintShardSetFlippedAt).toBe(new Date(T).toISOString()); + }); + + it("stamps prev as the CURRENTLY-EFFECTIVE set when a second flip lands mid-window", () => { + // Two flips inside one window must not strand the original prev; prev is what readers serve now. + const existing = { + runOpsMintShardSet: "a,b", + runOpsMintShardSetPrev: "a", + runOpsMintShardSetFlippedAt: new Date(T).toISOString(), + }; + const stamped = stampMintShardSetFlip( + existing, + { runOpsMintShardSet: "a,b,c" }, + T + 1000, + GRACE_MS + ); + expect(stamped.runOpsMintShardSetPrev).toBe("a"); + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardGrace.ts b/apps/webapp/app/v3/runOpsMigration/mintShardGrace.ts new file mode 100644 index 00000000000..65d8af6f417 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/mintShardGrace.ts @@ -0,0 +1,133 @@ +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; + +// Index 24 of a gen-2 id sits inside the pod name `runner-`, and a DNS-1123 label accepts +// lowercase only, so the alphabet is 36 keys and no wider. Core keeps its copy private; +// mintShardGrace.test.ts pins this pattern to generateRunOpsIdV2 instead. +export const SHARD_KEY_PATTERN = /^[a-z0-9]$/; + +// Neither may enter the active set: "new" already means "mint a gen-1 run-ops id" and +// "legacy" means the cuid store, which minting never selects. +const RESERVED_SHARD_KEYS: readonly string[] = ["new", "legacy"]; + +// "new" IS legal as a PIN, holding one org or environment on gen-1 while the rest of the fleet +// mints gen-2. Without it a non-empty active set moves every environment at once. +export const GEN_1_PIN_VALUE = "new"; + +export type MintShardSetResolution = { + set: string[]; + prevSet?: string[]; + flippedAtMs?: number; +}; + +// Flag keys holding the active set and its grace stamp. Named here so the pure module can read +// a flag record without importing the catalog. +const SET_KEY = "runOpsMintShardSet"; +const SET_PREV_KEY = "runOpsMintShardSetPrev"; +const SET_FLIPPED_AT_KEY = "runOpsMintShardSetFlippedAt"; + +export function isValidPinValue(value: unknown): value is ShardKey { + if (typeof value !== "string") return false; + return value === GEN_1_PIN_VALUE || SHARD_KEY_PATTERN.test(value); +} + +// Throws rather than dropping a bad key: generateRunOpsIdV2 throws on an out-of-alphabet char, +// so an unvalidated key must fail at boot and never at mint. +export function parseShardCsv(raw: string | undefined | null): string[] { + const keys = (raw ?? "") + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + + const unique = new Set(); + for (const key of keys) { + if (RESERVED_SHARD_KEYS.includes(key)) { + throw new Error(`"${key}" is a reserved key and cannot be an active mint shard`); + } + if (!SHARD_KEY_PATTERN.test(key)) { + throw new Error(`invalid shard key "${key}": must be a single char in [a-z0-9]`); + } + unique.add(key); + } + + // Sorted so no placement can depend on the order an operator typed the CSV in. + return [...unique].sort(); +} + +// Cutover boundary, mirroring effectiveMintKind. `nowMs` is the reader's wall clock while +// `flippedAtMs` is operator-supplied, so this assumes NTP-synced hosts with skew << graceMs, +// letting every process cross [flippedAtMs, flippedAtMs + graceMs) together (OLD then NEW). +export function effectiveMintShardSet( + r: MintShardSetResolution, + nowMs: number, + graceMs: number +): string[] { + if (r.prevSet === undefined || r.flippedAtMs === undefined) { + return r.set; + } + return nowMs < r.flippedAtMs + graceMs ? r.prevSet : r.set; +} + +// The active set lives in the control-plane database, not in the environment. A deploy rolls +// for hours, so two pods can hold different environment values at the same time; only a shared +// row lets every pod agree on one set. Boot may reject a bad environment value, but the mint +// path must never throw on a bad stored value, so an unreadable list degrades to empty. +function readStoredCsv(value: unknown): string[] { + if (typeof value !== "string") return []; + try { + return parseShardCsv(value); + } catch { + return []; + } +} + +// Reads the { set, prevSet, flippedAtMs } trio out of one flag record. Pure. A prevSet with no +// timestamp can never apply, so it is dropped. A timestamp with an EMPTY prevSet is meaningful: +// it graces a first activation, serving no shards for the window. +export function readMintShardSetResolution( + flags: Record | null | undefined +): MintShardSetResolution { + const source = flags ?? {}; + const flippedAtRaw = source[SET_FLIPPED_AT_KEY]; + const parsed = typeof flippedAtRaw === "string" ? Date.parse(flippedAtRaw) : NaN; + const flippedAtMs = Number.isNaN(parsed) ? undefined : parsed; + + return { + set: readStoredCsv(source[SET_KEY]), + prevSet: flippedAtMs === undefined ? undefined : readStoredCsv(source[SET_PREV_KEY]), + flippedAtMs, + }; +} + +// Stamps a grace window only when the outgoing set differs from the stored one. prev becomes the +// set readers serve right now, so a second flip inside one window cannot strand the first. A save +// that leaves the set unchanged carries any in-flight stamp forward, so it cannot reset the clock. +export function stampMintShardSetFlip( + existingFlags: Record | null | undefined, + outgoingFlags: Record, + nowMs: number, + graceMs: number +): Record { + // Only act when the save actually SETS the list. Omitting it must not inject a default. + if (typeof outgoingFlags[SET_KEY] !== "string") { + return outgoingFlags; + } + + const existing = existingFlags ?? {}; + const outgoingSet = readStoredCsv(outgoingFlags[SET_KEY]); + const storedSet = readStoredCsv(existing[SET_KEY]); + + if (outgoingSet.join(",") !== storedSet.join(",")) { + const effective = effectiveMintShardSet(readMintShardSetResolution(existing), nowMs, graceMs); + outgoingFlags[SET_PREV_KEY] = effective.join(","); + outgoingFlags[SET_FLIPPED_AT_KEY] = new Date(nowMs).toISOString(); + return outgoingFlags; + } + + if (existing[SET_PREV_KEY] !== undefined) { + outgoingFlags[SET_PREV_KEY] = existing[SET_PREV_KEY]; + } + if (existing[SET_FLIPPED_AT_KEY] !== undefined) { + outgoingFlags[SET_FLIPPED_AT_KEY] = existing[SET_FLIPPED_AT_KEY]; + } + return outgoingFlags; +} diff --git a/apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts b/apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts index f7f7c43a530..8a657060ef9 100644 --- a/apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts @@ -13,6 +13,21 @@ vi.setConfig({ testTimeout: 60_000 }); // 25-char cuid body → LEGACY residency. 26-char v1 body (version "1" at index 25) → NEW residency. const LEGACY_RUN_ID = "run_" + "a".repeat(25); const NEW_RUN_ID = "run_" + "b".repeat(24) + "01"; +// 26-char gen-2 body: shard char at index 24, version "2" at index 25. +const SHARD_A_RUN_ID = "run_" + "c".repeat(24) + "a2"; +const SHARD_Z_RUN_ID = "run_" + "c".repeat(24) + "z2"; +const LEGACY_WAITPOINT_ID = "waitpoint_" + "d".repeat(25); + +function throwingClient(label: string) { + return vi.fn(async (): Promise<{ marker: number } | null> => { + throw new Error(`${label} must never be read`); + }); +} + +function collectingLogger() { + const errors: { message: string; meta?: unknown }[] = []; + return { errors, error: (message: string, meta?: unknown) => errors.push({ message, meta }) }; +} // Lightweight real read: a trivial `$queryRaw` that genuinely hits the given container. // `hit` controls whether the read "finds" the run, so we exercise routing without @@ -28,14 +43,7 @@ async function realRead( // A presenter-shaped mapping: both "not-found" and "past-retention" collapse to the // same 404-ish surface, so an old run after termination yields the normal response. function toHttpish(result: ReadThroughResult): { status: number; value?: T } { - switch (result.source) { - case "new": - case "legacy-replica": - return { status: 200, value: result.value }; - case "not-found": - case "past-retention": - return { status: 404 }; - } + return result.found ? { status: 200, value: result.value } : { status: 404 }; } describe("readThroughRun (legacy replica + new DB)", () => { @@ -46,7 +54,8 @@ describe("readThroughRun (legacy replica + new DB)", () => { // read resolving through `legacyReplica` (prisma14) IS the structural guarantee // that the primary is never touched. const result = await readThroughRun({ - runId: LEGACY_RUN_ID, + id: LEGACY_RUN_ID, + idKind: "run", environmentId: "env_1", readNew: (c) => realRead(c, false), readLegacy: (c) => realRead(c, true), @@ -57,7 +66,7 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(result.source).toBe("legacy-replica"); + expect(result.found && result.source).toBe("legacy-replica"); expect(toHttpish(result).status).toBe(200); } ); @@ -66,7 +75,8 @@ describe("readThroughRun (legacy replica + new DB)", () => { "post-termination past-retention returns the normal not-found surface", async ({ prisma14, prisma17 }) => { const pastRetentionResult = await readThroughRun({ - runId: LEGACY_RUN_ID, + id: LEGACY_RUN_ID, + idKind: "run", environmentId: "env_1", readNew: (c) => realRead(c, false), readLegacy: (c) => realRead(c, false), // legacy gone / retention elapsed @@ -78,11 +88,14 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(pastRetentionResult.source).toBe("past-retention"); + expect(pastRetentionResult.found === false && pastRetentionResult.reason).toBe( + "past-retention" + ); // A run that is simply absent (not past retention) yields not-found. const notFoundResult = await readThroughRun({ - runId: LEGACY_RUN_ID, + id: LEGACY_RUN_ID, + idKind: "run", environmentId: "env_1", readNew: (c) => realRead(c, false), readLegacy: (c) => realRead(c, false), @@ -94,7 +107,7 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(notFoundResult.source).toBe("not-found"); + expect(notFoundResult.found === false && notFoundResult.reason).toBe("not-found"); // Both collapse to the same 404-ish surface. expect(toHttpish(pastRetentionResult).status).toBe(toHttpish(notFoundResult).status); expect(toHttpish(pastRetentionResult).status).toBe(404); @@ -110,7 +123,8 @@ describe("readThroughRun (legacy replica + new DB)", () => { const newRead = vi.fn((c: PrismaReplicaClient) => realRead(c, true)); const result = await readThroughRun({ - runId: LEGACY_RUN_ID, + id: LEGACY_RUN_ID, + idKind: "run", environmentId: "env_1", readNew: newRead, readLegacy: throwingLegacy, @@ -121,7 +135,7 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(result.source).toBe("new"); + expect(result.found && result.source).toBe("new"); expect(newRead).toHaveBeenCalledTimes(1); expect(throwingLegacy).not.toHaveBeenCalled(); } @@ -135,7 +149,152 @@ describe("readThroughRun (legacy replica + new DB)", () => { }); const result = await readThroughRun({ - runId: NEW_RUN_ID, + id: NEW_RUN_ID, + idKind: "run", + environmentId: "env_1", + readNew: (c) => realRead(c, true), + readLegacy: throwingLegacy, + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + }, + }); + + expect(result.found && result.source).toBe("new"); + expect(throwingLegacy).not.toHaveBeenCalled(); + } + ); + + heteroPostgresTest( + "gen-2 id reads its OWN shard replica once and probes no other store", + async ({ prisma14, prisma17 }) => { + const throwingNew = throwingClient("the gen-1 new store"); + const throwingLegacy = throwingClient("the legacy replica"); + const shardRead = vi.fn((c: PrismaReplicaClient) => realRead(c, true)); + + const result = await readThroughRun({ + id: SHARD_A_RUN_ID, + idKind: "run", + environmentId: "env_1", + // One closure serves both the gen-1 new store and a shard: a shard is the same + // dedicated schema. The throwing clients prove WHICH client it was handed. + readNew: (c) => shardRead(c), + readLegacy: throwingLegacy, + deps: { + splitEnabled: true, + newClient: throwingNew as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", prisma17 as unknown as PrismaReplicaClient]]), + }, + }); + + expect(result.found && result.source).toBe("shard:a"); + expect(shardRead).toHaveBeenCalledTimes(1); + // Identity, not deep equality: a Prisma client is too large to deep-compare. + expect(shardRead.mock.calls[0][0]).toBe(prisma17); + expect(throwingLegacy).not.toHaveBeenCalled(); + } + ); + + heteroPostgresTest( + "gen-2 id on an UNCONFIGURED shard key logs an error and returns not-found, never throws", + async ({ prisma14, prisma17 }) => { + const logger = collectingLogger(); + const throwingLegacy = throwingClient("the legacy replica"); + const newRead = vi.fn((c: PrismaReplicaClient) => realRead(c, true)); + + // Shard "z" is not configured. A 500 here would be inducible by any caller that + // guesses a shard char, so the layer must degrade rather than throw. + const result = await readThroughRun({ + id: SHARD_Z_RUN_ID, + idKind: "run", + environmentId: "env_1", + readNew: newRead, + readLegacy: throwingLegacy, + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", prisma17 as unknown as PrismaReplicaClient]]), + logger, + }, + }); + + expect(result.found).toBe(false); + expect(result.found === false && result.reason).toBe("not-found"); + expect(logger.errors).toHaveLength(1); + expect(logger.errors[0].meta).toMatchObject({ shardKey: "z", configured: ["a"] }); + // It must not silently fall back onto a gen-1 store. + expect(newRead).not.toHaveBeenCalled(); + expect(throwingLegacy).not.toHaveBeenCalled(); + } + ); + + heteroPostgresTest( + "gen-1 RUN id reads the legacy replica only and never probes the new store", + async ({ prisma14 }) => { + const throwingNew = throwingClient("the new store"); + const legacyRead = vi.fn((c: PrismaReplicaClient) => realRead(c, true)); + + const result = await readThroughRun({ + id: LEGACY_RUN_ID, + idKind: "run", + environmentId: "env_1", + readNew: throwingNew, + readLegacy: legacyRead, + deps: { + splitEnabled: true, + newClient: prisma14 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + }, + }); + + expect(result.found && result.source).toBe("legacy-replica"); + expect(throwingNew).not.toHaveBeenCalled(); + expect(legacyRead).toHaveBeenCalledTimes(1); + } + ); + + heteroPostgresTest( + "cuid WAITPOINT id keeps the new-FIRST pair probe (frozen: cuid waitpoints co-locate on new)", + async ({ prisma14, prisma17 }) => { + const calls: string[] = []; + const newRead = vi.fn(async (c: PrismaReplicaClient) => { + calls.push("new"); + return realRead(c, false); + }); + const legacyRead = vi.fn(async (c: PrismaReplicaClient) => { + calls.push("legacy"); + return realRead(c, true); + }); + + const result = await readThroughRun({ + id: LEGACY_WAITPOINT_ID, + idKind: "waitpoint", + environmentId: "env_1", + readNew: newRead, + readLegacy: legacyRead, + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + }, + }); + + expect(result.found && result.source).toBe("legacy-replica"); + expect(calls).toEqual(["new", "legacy"]); + } + ); + + heteroPostgresTest( + "a cuid waitpoint found on the new store returns it without touching legacy", + async ({ prisma14, prisma17 }) => { + const throwingLegacy = throwingClient("the legacy replica"); + + const result = await readThroughRun({ + id: LEGACY_WAITPOINT_ID, + idKind: "waitpoint", environmentId: "env_1", readNew: (c) => realRead(c, true), readLegacy: throwingLegacy, @@ -146,7 +305,7 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(result.source).toBe("new"); + expect(result.found && result.source).toBe("new"); expect(throwingLegacy).not.toHaveBeenCalled(); } ); diff --git a/apps/webapp/app/v3/runOpsMigration/readThrough.server.ts b/apps/webapp/app/v3/runOpsMigration/readThrough.server.ts index f15230ec442..6e1beaae62c 100644 --- a/apps/webapp/app/v3/runOpsMigration/readThrough.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/readThrough.server.ts @@ -3,12 +3,18 @@ * (which carries the read load we are shedding). Disabled entirely when isSplitEnabled() * is false (single-DB passthrough). * - * During the retention window, old run-ops rows are served off the legacy read replica. - * Residency is decided purely by id-shape: a run-ops id (NEW) id reads new only, a cuid - * (LEGACY) id reads legacy only. An unclassifiable id falls back to a new-then-legacy - * probe. After termination, past-retention runs return the normal not-found response. - * Patterned on `mollifier/resolveRunForMutation.server.ts` (`?? default` DI), but with - * the legacy-primary/writer fallback deliberately removed: this layer has NO legacy-writer + * Residency is decided purely by id-shape, via `resolveShard`: a gen-2 body names its own + * shard (ONE read there), a gen-1 v1 body reads new only, everything else is legacy and + * routes on `idKind`. + * + * `idKind` is required because a cuid gives no way to tell a run id from a waitpoint id, + * and the two must route differently: a legacy-classified RUN id is legacy-resident (there + * is no cuid run migration), while a cuid WAITPOINT can be co-located with its run on the + * new store, which is what makes the new-first probe load-bearing for it. No default — + * a default would pick one of those arms silently. + * + * Patterned on `mollifier/resolveRunForMutation.server.ts` (`?? default` DI), but with the + * legacy-primary/writer fallback deliberately removed: this layer has NO legacy-writer * handle at all (structural guarantee). */ import type { PrismaReplicaClient } from "~/db.server"; @@ -17,90 +23,118 @@ import { runOpsNewReplica as defaultNewClient, } from "~/db.server"; import { logger as defaultLogger } from "~/services/logger.server"; -import { ownerEngine, UnclassifiableRunId } from "@trigger.dev/core/v3/isomorphic"; +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import { isSplitEnabled } from "./splitMode.server"; +import { runOpsShardReplicas } from "./shardHandles.server"; + +type ShardSource = `shard:${string}`; -type ReadThroughSource = "new" | "legacy-replica"; +type ReadThroughSource = "new" | "legacy-replica" | ShardSource; +/** + * `found` carries hit/miss STRUCTURALLY. `source` is open-ended once shards exist, so a + * consumer testing found-ness by listing hit sources reads a gen-2 hit as a miss; + * discriminating on `found` makes that a compile error instead. + */ export type ReadThroughResult = - | { source: ReadThroughSource; value: T } - | { source: "not-found" } - | { source: "past-retention" }; + | { found: true; source: ReadThroughSource; value: T } + | { found: false; reason: "not-found" | "past-retention" }; type ReadThroughDeps = { newClient?: PrismaReplicaClient; legacyReplica?: PrismaReplicaClient; + /** + * Gen-2 shard replicas by shard char; empty (RUN_OPS_SHARDS unset) makes the gen-2 arm + * unreachable. Load-bearing only for callers whose closures read a client DIRECTLY: + * `RoutingRunStore` never forwards a caller's client, so for store-backed closures the + * client picked here is only a read-your-writes signal. Not dead weight. + */ + shardReplicas?: ReadonlyMap; /** Resolved boot constant; never `await`ed per-request when supplied. */ splitEnabled?: boolean; - isPastRetention?: (runId: string) => boolean; - logger?: { warn: (m: string, meta?: unknown) => void }; + isPastRetention?: (id: string) => boolean; + logger?: { error: (m: string, meta?: Record) => void }; /** Saturation-signal emit hook: called on each legacy-replica hit. */ - onLegacyReplicaRead?: (runId: string) => void; + onLegacyReplicaRead?: (id: string) => void; }; type ReadThroughRunInput = { - runId: string; + id: string; + idKind: "run" | "waitpoint"; environmentId: string; readNew: (client: PrismaReplicaClient) => Promise; readLegacy: (replica: PrismaReplicaClient) => Promise; deps?: ReadThroughDeps; }; +function hit(source: ReadThroughSource, value: T): ReadThroughResult { + return { found: true, source, value }; +} + +function miss(reason: "not-found" | "past-retention"): ReadThroughResult { + return { found: false, reason }; +} + export async function readThroughRun( input: ReadThroughRunInput ): Promise> { - const { runId, deps } = input; + const { id, idKind, deps } = input; const newClient = deps?.newClient ?? defaultNewClient; const legacyReplica = deps?.legacyReplica ?? defaultLegacyReplica; + const shardReplicas = deps?.shardReplicas ?? runOpsShardReplicas; const logger = deps?.logger ?? defaultLogger; const splitEnabled = deps?.splitEnabled ?? (await isSplitEnabled()); - // Passthrough: single plain read against the one collapsed store. No legacy read, - // no second connection. + // Passthrough: single plain read against the one collapsed store. if (!splitEnabled) { const v = await input.readNew(newClient); - return v != null ? { source: "new", value: v } : { source: "not-found" }; + return v != null ? hit("new", v) : miss("not-found"); } - // Split is on. Classify residency; an unclassifiable id is treated as LEGACY - // (conservative — probe rather than drop a real run). - let residency: "LEGACY" | "NEW"; - try { - residency = ownerEngine(runId); - } catch (e) { - if (e instanceof UnclassifiableRunId) { - logger.warn("readThroughRun: UnclassifiableRunId, treating as LEGACY", { - runId, - valueLength: e.valueLength, + // Total: an unclassifiable id resolves to "legacy" (probe rather than drop a real run). + const shardKey = resolveShard(id); + + if (shardKey !== "new" && shardKey !== "legacy") { + const shardReplica = shardReplicas.get(shardKey); + if (shardReplica === undefined) { + // Deliberately not a throw: this id arrives from the caller (a URL param on the + // waitpoint route) and any base32hex core + [a-z0-9] + "2" parses as gen-2, so a + // throw is a 500 any client can induce. An error-logged not-found is neither silent + // nor a misroute. Throwing stays correct on the router path, where ids are minted. + logger.error("readThroughRun: gen-2 id resolved to an unconfigured shard key", { + id, + shardKey, + configured: [...shardReplicas.keys()], }); - residency = "LEGACY"; - } else { - throw e; + return miss("not-found"); } + // A gen-2 shard is a dedicated-schema store, exactly like `new`, so `readNew` fits. + const v = await input.readNew(shardReplica); + return v != null ? hit(`shard:${shardKey}`, v) : miss("not-found"); } - // A run-ops id can only live on the new DB — skip the legacy replica entirely. - if (residency === "NEW") { + if (shardKey === "new") { const v = await input.readNew(newClient); - return v != null ? { source: "new", value: v } : { source: "not-found" }; + return v != null ? hit("new", v) : miss("not-found"); } - // LEGACY (or unclassifiable→LEGACY) fan-out: new first. - const v = await input.readNew(newClient); - if (v != null) { - return { source: "new", value: v }; + if (idKind === "waitpoint") { + const v = await input.readNew(newClient); + if (v != null) { + return hit("new", v); + } } // Legacy READ REPLICA only — never a legacy writer/primary (no such handle exists). const lv = await input.readLegacy(legacyReplica); if (lv != null) { - deps?.onLegacyReplicaRead?.(runId); - return { source: "legacy-replica", value: lv }; + deps?.onLegacyReplicaRead?.(id); + return hit("legacy-replica", lv); } - if (deps?.isPastRetention?.(runId)) { - return { source: "past-retention" }; + if (deps?.isPastRetention?.(id)) { + return miss("past-retention"); } - return { source: "not-found" }; + return miss("not-found"); } diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts new file mode 100644 index 00000000000..c1c2b9ddd48 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts @@ -0,0 +1,94 @@ +import { $replica, boundedIn } from "~/db.server"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { BoundedTtlCache } from "~/services/realtime/boundedTtlCache"; +import { singleton } from "~/utils/singleton"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { resolveMintShardWith, type MintShardCache } from "./mintShardAssignment"; + +// A misconfiguration is reported again after this long, so a still-broken pin stays visible +// without logging on every trigger. +const REPORT_TTL_MS = 3_600_000; +const REPORT_MAX_ENTRIES = 10_000; + +const GLOBAL_SHARD_KEYS = [ + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + FEATURE_FLAG.runOpsMintShardOverride, +]; + +const liveCache = singleton("runOpsMintShardCache", (): { current: MintShardCache } => ({ + current: undefined, +})); + +async function readSetFlags(): Promise> { + const rows = await $replica.featureFlag.findMany({ + where: { key: { in: boundedIn(GLOBAL_SHARD_KEYS) } }, + select: { key: true, value: true }, + }); + const flags: Record = {}; + for (const row of rows) { + flags[row.key] = row.value; + } + return flags; +} + +// A stale pin sits on the root-trigger path, so it would otherwise log on every trigger for that +// environment forever. Bounded, because the set of pinned environments is operator-controlled but +// not operator-bounded, and an unbounded Set on this path is a leak. +const reportedPins = singleton( + "runOpsMintShardReportedPins", + () => new BoundedTtlCache(REPORT_TTL_MS, REPORT_MAX_ENTRIES) +); + +function reportPinRejected(info: { + environmentId: string; + pin: string; + activeSet: string[]; +}): void { + if (reportedPins.get(info.environmentId) !== undefined) return; + reportedPins.set(info.environmentId, true); + logger.error("[runOpsMintShard] pinned shard is not in the active set; using the hash", info); +} + +// Keyed by the override value, not by environment: one bad override applies to the whole fleet, +// so one line is the correct volume. Keying by environment would log once per environment. +const reportedOverrides = singleton( + "runOpsMintShardReportedOverrides", + () => new BoundedTtlCache(REPORT_TTL_MS, REPORT_MAX_ENTRIES) +); + +function reportOverrideRejected(info: { override: string; activeSet: string[] }): void { + if (reportedOverrides.get(info.override) !== undefined) return; + reportedOverrides.set(info.override, true); + logger.error("[runOpsMintShard] override shard is not in the active set; ignoring it", info); +} + +/** + * Which shard an environment mints new roots into. Call only after resolveRunIdMintKind has + * returned "runOpsId". Returns "new" to mean a gen-1 run-ops id, which is today's behaviour. + * + * @knipignore the gen-2 write-path change is the first production caller; drop this tag there. + */ +export async function resolveMintShard(environment: { + id: string; + // Pass environment.organization.featureFlags from the trigger call site. + orgFeatureFlags?: unknown; +}): Promise { + return resolveMintShardWith(environment, { + readFlags: readSetFlags, + cache: liveCache, + nowMs: Date.now(), + ttlMs: env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS, + graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS, + orgFeatureFlags: environment.orgFeatureFlags, + // Bound the active list to the shards this deployment can actually route. + routableKeys: env.RUN_OPS_SHARDS.map((shard) => shard.key), + onPinRejected: reportPinRejected, + onOverrideRejected: reportOverrideRejected, + onReadFailed: (error) => + logger.error("[runOpsMintShard] shard-set read failed; minting gen-1 (fail-safe)", { error }), + }); +} diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts b/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts index 0872a256508..e70dc29f449 100644 --- a/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts +++ b/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts @@ -8,6 +8,12 @@ export function computeRunOpsSplitReadEnabled(args: { controlPlaneReplica: unknown; hasNewUrl: boolean; hasLegacyUrl: boolean; + /** + * Gen-2 shard handles. Observability only: a non-distinct shard handle WARNS and never changes the + * returned verdict. A gen-2 fault must not disable the proven gen-1 read fan-out, and the + * distinctness sentinel already fail-closes the boot when two stores share a database. + */ + shardHandles?: Array<{ key: string; writer?: unknown; replica: unknown; aliasOf?: "new" }>; logger?: { warn: (msg: string, meta?: Record) => void }; }): boolean { const newIsDistinctDedicatedClient = @@ -24,5 +30,36 @@ export function computeRunOpsSplitReadEnabled(args: { ); } + // An aliased shard shares its target's client by reference, so identity equality is its correct + // state and never a fault. Keyed on the declared field, not on object identity. + for (const shard of args.shardHandles ?? []) { + if (shard.aliasOf !== undefined) continue; + + // A shard with no replica URL takes its own writer as its replica handle, so its reads go to + // its primary. This is the per-shard analogue of the existing legacy-primary warning. + if (shard.writer !== undefined && shard.replica === shard.writer) { + args.logger?.warn( + `run-ops shard ${shard.key} has no read replica handle; reads for that shard will hit the ` + + "shard primary. Set the shard's replicaUrl to keep replica reads off its primary." + ); + continue; + } + + // Unreachable by construction today: a non-aliased shard always gets a freshly built client. + // Kept as a regression guard, so a future control-plane fallback for shards cannot silently + // route a shard's reads to another database. + if ( + shard.replica === args.controlPlaneWriter || + shard.replica === args.controlPlaneReplica || + shard.replica === args.newReplica + ) { + args.logger?.warn( + `run-ops shard ${shard.key} declares its own database but its replica client is not a ` + + "distinct instance from the control-plane or gen-1 new client; reads for that shard " + + "would not reach its database." + ); + } + } + return enabled; } diff --git a/apps/webapp/app/v3/runOpsMigration/shardHandles.server.test.ts b/apps/webapp/app/v3/runOpsMigration/shardHandles.server.test.ts new file mode 100644 index 00000000000..b90c1dfe7c4 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/shardHandles.server.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { buildShardHandleMaps } from "./shardHandles.server"; + +// Two distinct sentinels per shard: the maps must not cross writer and replica. +function handle(key: string) { + return { + key, + writer: { tag: `${key}-writer` } as never, + replica: { tag: `${key}-replica` } as never, + }; +} + +describe("buildShardHandleMaps", () => { + it("yields empty maps when no shard is configured", () => { + const { replicas, writers } = buildShardHandleMaps([]); + + expect(replicas.size).toBe(0); + expect(writers.size).toBe(0); + }); + + it("keys each shard's replica and writer under its shard char", () => { + const { replicas, writers } = buildShardHandleMaps([handle("a"), handle("b")]); + + expect([...replicas.keys()].sort()).toEqual(["a", "b"]); + expect([...writers.keys()].sort()).toEqual(["a", "b"]); + expect(replicas.get("a")).toEqual({ tag: "a-replica" }); + expect(writers.get("a")).toEqual({ tag: "a-writer" }); + expect(replicas.get("b")).toEqual({ tag: "b-replica" }); + expect(writers.get("b")).toEqual({ tag: "b-writer" }); + }); + + it("never places a writer in the replica map", () => { + const { replicas } = buildShardHandleMaps([handle("a")]); + + expect(replicas.get("a")).not.toEqual({ tag: "a-writer" }); + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/shardHandles.server.ts b/apps/webapp/app/v3/runOpsMigration/shardHandles.server.ts new file mode 100644 index 00000000000..cbe827be4dc --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/shardHandles.server.ts @@ -0,0 +1,46 @@ +/** + * Gen-2 shard client handles, keyed by shard char, for the consumers that route by + * `resolveShard` outside the run-store boundary: read-through and the two cross-seam + * batch hydration sites. Both maps are empty unless RUN_OPS_SHARDS is configured, which + * is what keeps every gen-2 arm unreachable today. + */ +import type { PrismaClient } from "@trigger.dev/database"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import type { PrismaReplicaClient } from "~/db.server"; +import { runOpsShardHandles } from "~/db.server"; + +type ShardHandle = { + key: string; + writer: unknown; + replica: unknown; +}; + +export function buildShardHandleMaps(handles: ShardHandle[]): { + replicas: ReadonlyMap; + writers: ReadonlyMap; +} { + const replicas = new Map(); + const writers = new Map(); + for (const handle of handles) { + replicas.set(handle.key, handle.replica as PrismaReplicaClient); + writers.set(handle.key, handle.writer as PrismaClient); + } + return { replicas, writers }; +} + +// A gen-2 shard is the same dedicated subset schema as the gen-1 new store, so these casts +// carry exactly the precedent (and the same residual risk) as `runOpsNewPrisma`'s. +// The try/catch mirrors `runStore.server.ts`'s handle resolution: a minimal `db.server` mock +// does not define this export at all, and accessing an undefined mock export throws. +function resolveShardHandles(): ShardHandle[] { + try { + return runOpsShardHandles ?? []; + } catch { + return []; + } +} + +const maps = buildShardHandleMaps(resolveShardHandles()); + +export const runOpsShardReplicas = maps.replicas; +export const runOpsShardWriters = maps.writers; diff --git a/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts b/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts index 688f95bac03..b9a4e3dfdf2 100644 --- a/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts @@ -6,12 +6,15 @@ */ import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; -import { probeDistinctDatabases as defaultProbe } from "./distinctDbSentinel.server"; +import { probeDistinctStores as defaultProbe } from "./distinctDbSentinel.server"; +import { nonAliasedShards, type ShardTarget } from "~/v3/runOpsShards.server"; export type SplitModeConfig = { flagEnabled: boolean; legacyUrl?: string; newUrl?: string; + /** Gen-2 shards that own their own database. Empty (the default) is today's gen-1 pair. */ + shards?: ShardTarget[]; }; export type SplitModeDeps = { @@ -34,9 +37,16 @@ export async function computeSplitEnabled( ); return false; } - // Hard gate #2: runtime sentinel must confirm physically-distinct DBs. + // Hard gate #2: runtime sentinel must confirm physically-distinct DBs. At N stores this is set + // uniqueness over every store that owns its own database, not a compare of the gen-1 pair. An + // aliased shard is already absent from `shards` — it shares its target's client by reference. const probe = deps.probe ?? defaultProbe; - const result = await probe(config.legacyUrl, config.newUrl, { logger: deps.logger }); + const targets = [ + { id: "legacy", url: config.legacyUrl }, + { id: "new", url: config.newUrl }, + ...(config.shards ?? []).map((shard) => ({ id: `shard-${shard.key}`, url: shard.url })), + ]; + const result = await probe(targets, { logger: deps.logger }); return result.distinct === true; } @@ -72,6 +82,7 @@ export function isSplitEnabled(): Promise { flagEnabled: env.RUN_OPS_SPLIT_ENABLED, legacyUrl: env.RUN_OPS_LEGACY_DATABASE_URL, newUrl: env.RUN_OPS_DATABASE_URL, + shards: nonAliasedShards(env.RUN_OPS_SHARDS), }, { logger } ); diff --git a/apps/webapp/app/v3/runOpsMigration/track1-baseline.json b/apps/webapp/app/v3/runOpsMigration/track1-baseline.json index 63d27dbadca..26d038678ea 100644 --- a/apps/webapp/app/v3/runOpsMigration/track1-baseline.json +++ b/apps/webapp/app/v3/runOpsMigration/track1-baseline.json @@ -68,19 +68,19 @@ "WaitpointTag.project" ], "totals": { - "violations": 4, - "detectorI": 4, + "violations": 5, + "detectorI": 5, "detectorII": 0, "detectorIII": 0, "write": 0, - "read": 4, + "read": 5, "files": 1, "legacyAnnotations": 0 }, "violations": [ { "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", - "line": 89, + "line": 93, "model": "BatchTaskRun", "delegate": "batchTaskRun", "callKind": "read", @@ -89,7 +89,7 @@ }, { "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", - "line": 150, + "line": 154, "model": "BatchTaskRun", "delegate": "batchTaskRun", "callKind": "read", @@ -98,16 +98,25 @@ }, { "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", - "line": 184, + "line": 214, "model": "TaskRun", "delegate": "taskRun", "callKind": "read", "detector": "i", - "snippet": "const newRows = (await newClient.taskRun.findMany({" + "snippet": "? ((await newClient.taskRun.findMany({" }, { "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", - "line": 196, + "line": 224, + "model": "TaskRun", + "delegate": "taskRun", + "callKind": "read", + "detector": "i", + "snippet": "(await shardReplicas.get(shardKey)!.taskRun.findMany({" + }, + { + "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", + "line": 241, "model": "TaskRun", "delegate": "taskRun", "callKind": "read", diff --git a/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts b/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts index 9ea849c8058..42dca92e1ce 100644 --- a/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts @@ -138,7 +138,8 @@ describe("public wait-token resolution across the split boundary", () => { expect(gated?.id).toBe(waitpointId); const passthrough = await readThroughRun({ - runId: waitpointId, + id: waitpointId, + idKind: "waitpoint", environmentId: environment.id, readNew: (c) => read(c), readLegacy: (r) => read(r), @@ -150,7 +151,7 @@ describe("public wait-token resolution across the split boundary", () => { }); expect(gated).not.toBeNull(); - expect(passthrough.source).toBe("not-found"); + expect(passthrough.found === false && passthrough.reason).toBe("not-found"); } ); }); diff --git a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts new file mode 100644 index 00000000000..0396af0f6d2 --- /dev/null +++ b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts @@ -0,0 +1,80 @@ +import { env } from "~/env.server"; +import type { RunOpsShardKnobs } from "~/v3/runOpsShards.server"; + +// Pool configuration for one run-ops store (writer + replica). Kept separate from db.server (which +// ~156 tests mock wholesale) so a new export breaks no mock. +export type ResolvedPoolKnobs = { + writerPoolTimeout: number; + writerConnectionTimeout: number; + writerDriverAdapter: boolean; + connectionLimit: number; + replicaConnectionLimit: number; + replicaPoolTimeout: number; + replicaConnectionTimeout: number; + replicaDriverAdapter: boolean; +}; + +type Role = "new" | "legacy"; + +// PURE: overlay a gen-2 shard's descriptor knobs on a role's resolved defaults. This holds the only +// logic (per-field override), so a test drives it with literal defaults and literal overrides — +// no env import, no circular assertion against the same env expression the impl reads. +export function applyPoolKnobOverrides( + defaults: ResolvedPoolKnobs, + k?: RunOpsShardKnobs +): ResolvedPoolKnobs { + return { + writerPoolTimeout: k?.writerPoolTimeout ?? defaults.writerPoolTimeout, + writerConnectionTimeout: k?.writerConnectionTimeout ?? defaults.writerConnectionTimeout, + writerDriverAdapter: k?.writerDriverAdapter ?? defaults.writerDriverAdapter, + connectionLimit: k?.connectionLimit ?? defaults.connectionLimit, + replicaConnectionLimit: k?.replicaConnectionLimit ?? defaults.replicaConnectionLimit, + replicaPoolTimeout: k?.replicaPoolTimeout ?? defaults.replicaPoolTimeout, + replicaConnectionTimeout: k?.replicaConnectionTimeout ?? defaults.replicaConnectionTimeout, + replicaDriverAdapter: k?.replicaDriverAdapter ?? defaults.replicaDriverAdapter, + }; +} + +// The env-derived defaults for a role, reproducing today's run-ops builder expressions exactly. A +// flat mapping (no logic), verified by inspection against the former builders. Transaction +// resilience is a SEPARATE mechanism (resolveTransactionResilience) and is not here. +function poolKnobDefaults(role: Role): ResolvedPoolKnobs { + if (role === "legacy") { + return { + writerPoolTimeout: + env.RUN_OPS_LEGACY_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + writerConnectionTimeout: + env.RUN_OPS_LEGACY_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaConnectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaPoolTimeout: + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + replicaConnectionTimeout: + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? + env.DATABASE_CONNECTION_TIMEOUT, + replicaDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + }; + } + + return { + writerPoolTimeout: env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + writerConnectionTimeout: + env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaConnectionLimit: + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT, + replicaPoolTimeout: env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + replicaConnectionTimeout: + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + replicaDriverAdapter: env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + }; +} + +export function resolveRunOpsPoolKnobs( + role: Role, + descriptorKnobs?: RunOpsShardKnobs +): ResolvedPoolKnobs { + return applyPoolKnobOverrides(poolKnobDefaults(role), descriptorKnobs); +} diff --git a/apps/webapp/app/v3/runOpsShardTable.ts b/apps/webapp/app/v3/runOpsShardTable.ts new file mode 100644 index 00000000000..6741b3cb5b0 --- /dev/null +++ b/apps/webapp/app/v3/runOpsShardTable.ts @@ -0,0 +1,28 @@ +// Pure boot-table helpers. Dependency-free (no db.server, no env) so a test of these two string +// functions never constructs a Prisma client. db.server imports them for the boot log. + +// A host:port/db address, with NO username and NO query params — never a secret, and deliberately +// NOT an identity claim (two DSNs can share an address yet be different databases; that proof is the +// distinctness sentinel's, not this line's). Same tuple sameDatabaseTarget compares, kept in step. +export function runOpsAddressFingerprint(url: string): string { + try { + const u = new URL(url); + return `${u.hostname}:${u.port || "5432"}${u.pathname}`; + } catch { + return "unparseable"; + } +} + +export type RunOpsShardTableRow = { key: string; fingerprint: string; role: string }; + +// The resolved shard table for the boot log: one row per descriptor. An alias reports its role and +// carries no address (it shares the new store's pool). +export function buildRunOpsShardTable( + descriptors: Array<{ key: string; url?: string; aliasOf?: "new" }> +): RunOpsShardTableRow[] { + return descriptors.map((d) => + d.aliasOf + ? { key: d.key, fingerprint: "alias(new)", role: "alias(new)" } + : { key: d.key, fingerprint: runOpsAddressFingerprint(d.url ?? ""), role: "shard" } + ); +} diff --git a/apps/webapp/app/v3/runOpsShards.server.ts b/apps/webapp/app/v3/runOpsShards.server.ts new file mode 100644 index 00000000000..12cc460aa06 --- /dev/null +++ b/apps/webapp/app/v3/runOpsShards.server.ts @@ -0,0 +1,154 @@ +import { z } from "zod"; +import { isValidShardChar } from "@trigger.dev/core/v3/isomorphic"; +import { isValidDatabaseUrl } from "~/utils/db"; + +const KnobsSchema = z + .object({ + writerPoolTimeout: z.number().int().optional(), + writerConnectionTimeout: z.number().int().optional(), + writerDriverAdapter: z.boolean().optional(), + connectionLimit: z.number().int().optional(), + replicaConnectionLimit: z.number().int().optional(), + replicaPoolTimeout: z.number().int().optional(), + replicaConnectionTimeout: z.number().int().optional(), + replicaDriverAdapter: z.boolean().optional(), + transactionMaxWaitMs: z.number().int().optional(), + transactionStartRetryEnabled: z.boolean().optional(), + transactionStartRetryMaxAttempts: z.number().int().optional(), + transactionStartRetryBackoffMinMs: z.number().int().optional(), + transactionStartRetryBackoffMaxMs: z.number().int().optional(), + transactionStartRetryBudgetPerSec: z.number().int().optional(), + transactionStartRetryBudgetBurst: z.number().int().optional(), + }) + .strict(); +export type RunOpsShardKnobs = z.infer; + +const ReplicationSchema = z.object({ + slotName: z.string().min(1), + publicationName: z.string().min(1), + originGeneration: z.number().int().min(2).max(255), +}); + +const DescriptorSchema = z + .object({ + key: z.string().refine(isValidShardChar, "shard key must be a single [a-z0-9] char"), + region: z.string().min(1), + url: z.string().refine(isValidDatabaseUrl, "url is invalid").optional(), + replicaUrl: z.string().refine(isValidDatabaseUrl, "replicaUrl is invalid").optional(), + directUrl: z.string().refine(isValidDatabaseUrl, "directUrl is invalid").optional(), + replication: ReplicationSchema.optional(), + knobs: KnobsSchema.optional(), + aliasOf: z.literal("new").optional(), + }) + .strict() + .superRefine((d, ctx) => { + const hasUrl = d.url !== undefined; + const hasAlias = d.aliasOf !== undefined; + if (hasUrl === hasAlias) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "exactly one of url or aliasOf is required", + }); + } + if (!hasAlias && d.replication === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "replication is required unless aliasOf is set", + }); + } + }); + +export type RunOpsShardDescriptor = z.infer; + +// Boot-validated transform, in the style of parseMachinePresetCsv. Undefined and "" both mean the +// off state and resolve to []. The undefined guard is load-bearing: an unguarded JSON.parse would +// kill every single-DB boot, which never sets this variable. +export function parseRunOpsShards( + raw: string | undefined, + ctx: z.RefinementCtx +): RunOpsShardDescriptor[] { + if (raw === undefined || raw.trim() === "") return []; + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "RUN_OPS_SHARDS is not valid JSON" }); + return z.NEVER; + } + + const result = z.array(DescriptorSchema).safeParse(parsed); + if (!result.success) { + for (const issue of result.error.issues) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS[${issue.path.join(".")}]: ${issue.message}`, + }); + } + return z.NEVER; + } + + const keys = new Set(); + const gens = new Set(); + for (const d of result.data) { + if (keys.has(d.key)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS: duplicate key ${d.key}`, + }); + return z.NEVER; + } + keys.add(d.key); + if (d.replication) { + if (gens.has(d.replication.originGeneration)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS: duplicate originGeneration ${d.replication.originGeneration}`, + }); + return z.NEVER; + } + gens.add(d.replication.originGeneration); + } + } + + return result.data; +} + +// A non-empty shard list requires the gen-1 new store, because gen-1 v1 ids resolve to "new" +// forever (append-only). Pure so the boot refinement and its test share one rule. +export function validateShardListAgainstNewUrl( + shards: RunOpsShardDescriptor[], + newUrl: string | undefined +): boolean { + return shards.length === 0 || !!newUrl; +} + +// A shard that owns its own physical database. Every boot check that must not treat two handles +// over one database as two databases derives its target list from here: the distinctness sentinel, +// the coresidency loop, the read gate, the replication sources and the migration loop. +export type ShardTarget = { + key: string; + url: string; + replicaUrl?: string; + directUrl?: string; +}; + +// An aliased shard shares its target's client BY REFERENCE, so it is never its own database. The +// exemption keys on the declared `aliasOf` field, never on client object identity: two store objects +// can sit over one database, which identity comparison cannot see. +export function nonAliasedShards(shards: RunOpsShardDescriptor[]): ShardTarget[] { + const targets: ShardTarget[] = []; + for (const shard of shards) { + if (shard.aliasOf !== undefined) continue; + // Unreachable for a valid descriptor (the schema requires exactly one of url/aliasOf); this is + // the type narrowing, not a second policy. + if (shard.url === undefined) continue; + targets.push({ + key: shard.key, + url: shard.url, + ...(shard.replicaUrl !== undefined ? { replicaUrl: shard.replicaUrl } : {}), + ...(shard.directUrl !== undefined ? { directUrl: shard.directUrl } : {}), + }); + } + return targets; +} diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index 9ccf84b5117..7f51a940cec 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -1,5 +1,12 @@ -import { PostgresRunStore, RoutingRunStore, type RunStore } from "@internal/run-store"; -import { ownerEngine, type Residency } from "@trigger.dev/core/v3/isomorphic"; +import { + PostgresRunStore, + RoutingRunStore, + type RoutingStoreMetrics, + type RunStore, +} from "@internal/run-store"; +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { Counter } from "prom-client"; +import { metricsRegister } from "~/metrics.server"; import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import { @@ -9,6 +16,7 @@ import { runOpsLegacyReplica, runOpsNewPrismaClient, runOpsNewReplicaClient, + runOpsShardHandles, } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; @@ -29,8 +37,18 @@ type BuildRunStoreDeps = { /** Single-DB store handles (control-plane pair). Used verbatim when split is OFF. */ singleWriter: PrismaClient; singleReplica: PrismaReplicaClient; - /** Residency classifier; defaults to ownerEngine inside RoutingRunStore. */ - classify?: (id: string) => Residency; + /** Id-to-shard-key resolver; defaults to the core resolveShard inside RoutingRunStore. */ + resolveShard?: (id: string) => ShardKey; + /** Gen-2 shard handles. When non-empty, buildRunStore builds one dedicated store per descriptor and + * hands them to the N-way router; an aliased shard shares its target's database. Empty/absent keeps + * the two-store compat router. */ + shards?: Array<{ + key: ShardKey; + writer: RunOpsPrismaClient; + replica: RunOpsPrismaClient; + aliasOf?: ShardKey; + resilience?: TransactionResilienceConfig; + }>; /** Per-pool transaction-resilience configs threaded into the store(s) this builds (IoC). */ singleResilience?: TransactionResilienceConfig; newResilience?: TransactionResilienceConfig; @@ -79,13 +97,50 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { transactionStartRetry: deps.legacyResilience?.startRetry, }); + // Gen-2 shards: one dedicated store per descriptor, handed to the N-way router. An aliased shard + // builds a store over its target's (shared) client and carries aliasOf, so the router dedups it out + // of fan-out sums by declaration. No shards -> the two-store compat router (byte-identical). + const shardStores = (deps.shards ?? []).map((shard) => ({ + key: shard.key, + store: new PostgresRunStore({ + prisma: shard.writer, + readOnlyPrisma: shard.replica, + schemaVariant: "dedicated" as const, + maxWait: shard.resilience?.maxWait, + transactionStartRetry: shard.resilience?.startRetry, + }), + aliasOf: shard.aliasOf, + })); + return new RoutingRunStore({ new: newStore, legacy: legacyStore, - classify: deps.classify ?? ownerEngine, + shards: shardStores.length > 0 ? shardStores : undefined, + resolveShard: deps.resolveShard ?? resolveShard, + metrics: routingStoreMetrics, }); } +// singleton: module-scope Counter registration double-registers under dev HMR. +const routingStoreMetrics: RoutingStoreMetrics = singleton("routingStoreMetrics", () => { + const duplicateId = new Counter({ + name: "runops_shard_duplicate_id_total", + help: "One id was returned by two run-ops shards that must be disjoint (a routing-invariant violation).", + labelNames: ["shard_keys"], + registers: [metricsRegister], + }); + const probeFallback = new Counter({ + name: "runops_waitpoint_probe_fallback_total", + help: "A waitpoint was not on the run-ops store its id named and was found by a fallback probe.", + labelNames: ["from", "to"], + registers: [metricsRegister], + }); + return { + recordDuplicateId: (shardKeys) => duplicateId.inc({ shard_keys: shardKeys.join(",") }), + recordWaitpointProbeFallback: (from, to) => probeFallback.inc({ from, to }), + }; +}); + // Build the routing store whenever BOTH run-ops DBs are configured, independent of // RUN_OPS_SPLIT_ENABLED. Reads must fan out across both DBs so a run that lives on the new // DB stays visible even with the flag off (matches the db.server topology factory). The flag @@ -110,6 +165,8 @@ function tryResolveRunOpsHandles() { newReplica: runOpsNewReplicaClient, legacyWriter: runOpsLegacyPrisma, legacyReplica: runOpsLegacyReplica, + // Absent under a minimal db.server mock; default to no shards so the compat router is built. + shardHandles: runOpsShardHandles ?? [], }; } catch { return null; @@ -127,9 +184,17 @@ export const runStore: RunStore = singleton("RunStore", () => { singleResilience: resilienceForClient(prisma), }); } + const { shardHandles, ...storeHandles } = handles; return buildRunStore({ splitEnabled: true, - ...handles, + ...storeHandles, + shards: shardHandles.map((shard) => ({ + key: shard.key, + writer: shard.writer, + replica: shard.replica, + aliasOf: shard.aliasOf, + resilience: resilienceForClient(shard.writer), + })), singleWriter: prisma, singleReplica: $replica, singleResilience: resilienceForClient(prisma), diff --git a/apps/webapp/app/v3/services/artifacts.server.ts b/apps/webapp/app/v3/services/artifacts.server.ts index 9e82af51234..2d1ef190978 100644 --- a/apps/webapp/app/v3/services/artifacts.server.ts +++ b/apps/webapp/app/v3/services/artifacts.server.ts @@ -24,16 +24,19 @@ const objectStoreClient = const artifactKeyPrefixByType = { deployment_context: "deployments", + // The key prefix is the one bundle signal that survives schema skew + deployment_bundle: "bundles", } as const; const artifactBytesSizeLimitByType = { - deployment_context: 100 * 1024 * 1024, // 100MB + deployment_context: env.DEPLOYMENT_CONTEXT_ARTIFACT_SIZE_LIMIT_BYTES, + deployment_bundle: env.DEPLOYMENT_BUNDLE_ARTIFACT_SIZE_LIMIT_BYTES, } as const; export class ArtifactsService extends BaseService { private readonly bucket = env.ARTIFACTS_OBJECT_STORE_BUCKET; public createArtifact( - type: "deployment_context", + type: "deployment_context" | "deployment_bundle", authenticatedEnv: AuthenticatedEnvironment, contentLength?: number ) { diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts index 066af4be101..6eff2f9336f 100644 --- a/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts @@ -39,7 +39,7 @@ export async function getBillableEnvironmentsForBillingLimit( export async function createBillingLimitRunsRepository(organizationId: string) { const clickhouse = await clickhouseFactory.getClickhouseForOrganization( organizationId, - "standard" + "runsList" ); return new RunsRepository({ @@ -95,7 +95,7 @@ export async function countBillableQueuedRunsForOrganization( ): Promise { const client = clickhouse ?? - (await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard")); + (await clickhouseFactory.getClickhouseForOrganization(organizationId, "runsList")); const queryBuilder = client.taskRuns.countQueryBuilder({ settings: { max_execution_time: BILLING_LIMIT_QUEUED_COUNT_MAX_EXECUTION_S }, diff --git a/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts b/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts index 99d4cfd2dd7..779a7234748 100644 --- a/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts +++ b/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts @@ -13,6 +13,8 @@ vi.setConfig({ testTimeout: 60_000 }); // 25-char cuid body → LEGACY residency. 26-char v1 body (version "1" at index 25) → NEW residency. const LEGACY_RUN_ID = "run_" + "a".repeat(25); const NEW_RUN_ID = "run_" + "b".repeat(24) + "01"; +// 26-char gen-2 body: shard char at index 24, version "2" at index 25. +const SHARD_A_RUN_ID = "run_" + "c".repeat(24) + "a2"; type Row = { id: string }; @@ -90,4 +92,109 @@ describe("hydrateRunsAcrossSeam (PG14 legacy replica + PG17 new)", () => { expect(throwingLegacy).not.toHaveBeenCalled(); } ); + + heteroPostgresTest( + "(c) a gen-2 id hydrates from its OWN shard and is never read from the gen-1 stores", + async ({ prisma14, prisma17 }) => { + // Before the shard arm existed a gen-2 id joined the `new` group, missed, and was + // never legacy-probed either — so it vanished from the page with no error. + const onShardA = new Set([SHARD_A_RUN_ID]); + + const readNew = vi.fn(async (client: PrismaReplicaClient, ids: string[]): Promise => { + if ( + ids.includes(SHARD_A_RUN_ID) && + client !== (prisma17 as unknown as PrismaReplicaClient) + ) { + throw new Error("a gen-2 id must only be read on its own shard"); + } + return realReadFiltered(client, ids, onShardA); + }); + const readLegacyReplica = vi.fn( + async (_replica: PrismaReplicaClient, ids: string[]): Promise => { + if (ids.includes(SHARD_A_RUN_ID)) { + throw new Error("a gen-2 id must never reach the legacy probe"); + } + return []; + } + ); + + const rows = await hydrateRunsAcrossSeam({ + runIds: [SHARD_A_RUN_ID], + readNew, + readLegacyReplica, + deps: { + splitEnabled: true, + newClient: prisma14 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", prisma17 as unknown as PrismaReplicaClient]]), + }, + }); + + expect(rows.map((r) => r.id)).toEqual([SHARD_A_RUN_ID]); + expect(readLegacyReplica).not.toHaveBeenCalled(); + } + ); + + heteroPostgresTest( + "(d) a mixed gen-1 and gen-2 page hydrates every member", + async ({ prisma14, prisma17 }) => { + const onGenOneNew = new Set([NEW_RUN_ID]); + const onLegacy = new Set([LEGACY_RUN_ID]); + const onShardA = new Set([SHARD_A_RUN_ID]); + + const readNew = vi.fn(async (client: PrismaReplicaClient, ids: string[]): Promise => { + const present = ids.includes(SHARD_A_RUN_ID) ? onShardA : onGenOneNew; + return realReadFiltered(client, ids, present); + }); + const readLegacyReplica = vi.fn( + async (replica: PrismaReplicaClient, ids: string[]): Promise => + realReadFiltered(replica, ids, onLegacy) + ); + + const rows = await hydrateRunsAcrossSeam({ + runIds: [NEW_RUN_ID, LEGACY_RUN_ID, SHARD_A_RUN_ID], + readNew, + readLegacyReplica, + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", prisma17 as unknown as PrismaReplicaClient]]), + }, + }); + + expect(rows.map((r) => r.id).sort()).toEqual( + [NEW_RUN_ID, LEGACY_RUN_ID, SHARD_A_RUN_ID].sort() + ); + } + ); + + heteroPostgresTest( + "(e) a gen-2 id on an unconfigured shard is dropped with a logged error, not read elsewhere", + async ({ prisma14, prisma17 }) => { + const errors: unknown[] = []; + const readNew = vi.fn(async (client: PrismaReplicaClient, ids: string[]): Promise => { + if (ids.includes(SHARD_A_RUN_ID)) { + throw new Error("an unconfigured gen-2 id must not fall back to a gen-1 store"); + } + return realReadFiltered(client, ids, new Set([NEW_RUN_ID])); + }); + + const rows = await hydrateRunsAcrossSeam({ + runIds: [NEW_RUN_ID, SHARD_A_RUN_ID], + readNew, + readLegacyReplica: async () => [], + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map(), + logger: { error: (_m, meta) => errors.push(meta) }, + }, + }); + + expect(rows.map((r) => r.id)).toEqual([NEW_RUN_ID]); + expect(errors).toHaveLength(1); + } + ); }); diff --git a/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.ts b/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.ts index c7a0dc735e8..bc476cebf31 100644 --- a/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.ts +++ b/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.ts @@ -20,7 +20,8 @@ import { runOpsLegacyReplica as defaultLegacyReplica, runOpsNewReplica as defaultNewClient, } from "~/db.server"; -import { ownerEngine, UnclassifiableRunId } from "@trigger.dev/core/v3/isomorphic"; +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { runOpsShardReplicas as defaultShardReplicas } from "~/v3/runOpsMigration/shardHandles.server"; type SeamReadDeps = { /** @@ -30,7 +31,9 @@ type SeamReadDeps = { splitEnabled: boolean; newClient?: PrismaReplicaClient; legacyReplica?: PrismaReplicaClient; - logger?: { warn: (m: string, meta?: unknown) => void }; + /** Gen-2 shard replicas by shard char; empty (RUN_OPS_SHARDS unset) keeps today's behaviour. */ + shardReplicas?: ReadonlyMap; + logger?: { error: (m: string, meta?: Record) => void }; }; type HydrateRunsAcrossSeamInput = { @@ -61,28 +64,30 @@ export async function hydrateRunsAcrossSeam(input: HydrateRunsAcrossSeamInput return input.readNew(newClient, runIds); } - // Split is on. Classify residency; unclassifiable → LEGACY (probe rather than drop). + // Split is on. Partition by shard key; `resolveShard` is total, so an unclassifiable id + // resolves to "legacy" (probe rather than drop). A gen-2 id goes to its OWN shard and to + // no other store: it is directly routable, so it joins neither gen-1 group. + const shardReplicas = deps.shardReplicas ?? defaultShardReplicas; const newIds: string[] = []; const legacyCandidateIds: string[] = []; + const idsByShard = new Map(); for (const runId of runIds) { - let residency: "LEGACY" | "NEW"; - try { - residency = ownerEngine(runId); - } catch (e) { - if (e instanceof UnclassifiableRunId) { - deps.logger?.warn("hydrateRunsAcrossSeam: UnclassifiableRunId, treating as LEGACY", { - runId, - valueLength: e.valueLength, - }); - residency = "LEGACY"; - } else { - throw e; - } - } - if (residency === "NEW") { + const shardKey = resolveShard(runId); + if (shardKey === "new") { newIds.push(runId); - } else { + } else if (shardKey === "legacy") { legacyCandidateIds.push(runId); + } else if (shardReplicas.has(shardKey)) { + const group = idsByShard.get(shardKey); + group ? group.push(runId) : idsByShard.set(shardKey, [runId]); + } else { + // Not routable and not a gen-1 shape. Reading a gen-1 store would query the wrong + // database, so the id is dropped from the page — loudly, never silently. + deps.logger?.error("hydrateRunsAcrossSeam: gen-2 id on an unconfigured shard key", { + runId, + shardKey, + configured: [...shardReplicas.keys()], + }); } } @@ -103,6 +108,16 @@ export async function hydrateRunsAcrossSeam(input: HydrateRunsAcrossSeamInput legacyRows = await input.readLegacyReplica(legacyReplica, legacyToProbe); } + // Each configured shard is read once, in parallel: the groups are disjoint by id, so the + // results need no dedupe. + const shardRows = ( + await Promise.all( + [...idsByShard.entries()].map(([shardKey, ids]) => + input.readNew(shardReplicas.get(shardKey)!, ids) + ) + ) + ).flat(); + // Order within the page is irrelevant (downstream pMap does not depend on it). - return [...newRows, ...legacyRows]; + return [...newRows, ...legacyRows, ...shardRows]; } diff --git a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts index d3cd77b143c..b19f6e4bc69 100644 --- a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts +++ b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts @@ -115,7 +115,7 @@ export class BulkActionService extends BaseService { // Count the runs that will be affected by the bulk action const clickhouse = await clickhouseFactory.getClickhouseForOrganization( organizationId, - "standard" + "runsList" ); const runsRepository = new RunsRepository({ clickhouse, @@ -275,7 +275,7 @@ export class BulkActionService extends BaseService { const clickhouse = await clickhouseFactory.getClickhouseForOrganization( group.project.organizationId, - "standard" + "runsList" ); const runsRepository = new RunsRepository({ clickhouse, diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index 305ee45ce25..bf2c95745f3 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -1,9 +1,10 @@ import type { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3"; import { logger, tryCatch } from "@trigger.dev/core/v3"; -import type { - BackgroundWorker, - PrismaClientOrTransaction, - WorkerDeployment, +import { + Prisma, + type BackgroundWorker, + type PrismaClientOrTransaction, + type WorkerDeployment, } from "@trigger.dev/database"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { type TaskMetadataCache } from "~/services/taskMetadataCache.server"; @@ -17,7 +18,7 @@ import { } from "./createBackgroundWorker.server"; import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { webhookPrisma } from "~/db.server"; @@ -297,6 +298,12 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { error: Error, environment: AuthenticatedEnvironment ) { + const failedAt = new Date(); + const errorData = { + name: error.name, + message: error.message, + }; + // Guarded BUILDING → FAILED transition, symmetric with the BUILDING → DEPLOYING // transition in `call()`. With idempotent retries, two attempts can run side-by-side; // without the predicate, one attempt's failure could downgrade the deployment after @@ -308,11 +315,9 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { }, data: { status: "FAILED", - failedAt: new Date(), - errorData: { - name: error.name, - message: error.message, - }, + failedAt, + errorData, + buildEnvVars: Prisma.DbNull, }, }); @@ -330,13 +335,18 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { // BUILDING → DEPLOYING transition. await TimeoutDeploymentService.dequeue(deployment.id, this._prisma); - recordDeploymentOutcome({ + recordDeploymentFinished({ status: "FAILED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: environment.organizationId, - projectId: environment.projectId, - environmentId: environment.id, - environmentType: environment.type, + deployment: { ...deployment, status: "FAILED", failedAt, errorData }, + environment: { + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + projectId: environment.projectId, + projectName: environment.project.name, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, reason: error.message, }); } diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index c67d7778568..0eeaa82b3d6 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -1,7 +1,7 @@ import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { BaseService } from "./baseService.server"; import { errAsync, fromPromise, okAsync, type ResultAsync } from "neverthrow"; -import { type WorkerDeployment, type Project, boundedIn } from "@trigger.dev/database"; +import { Prisma, type WorkerDeployment, type Project, boundedIn } from "@trigger.dev/database"; import { BuildServerMetadata, logger, @@ -9,6 +9,7 @@ import { type DeploymentEvent, } from "@trigger.dev/core/v3"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { createRemoteImageBuild } from "../remoteImageBuilder.server"; import { FINAL_DEPLOYMENT_STATUSES } from "./failDeployment.server"; @@ -195,10 +196,8 @@ export class DeploymentService extends BaseService { friendlyId: string, data?: Partial> ) { - const validateDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } + const validateDeployment = >( + deployment: T ) => { if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) { logger.warn("Attempted cancelling deployment in a final state", { @@ -210,11 +209,7 @@ export class DeploymentService extends BaseService { return okAsync(deployment); }; - const cancelDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } - ) => + const cancelDeployment = >(deployment: T) => fromPromise( this._prisma.workerDeployment.updateMany({ where: { @@ -227,6 +222,7 @@ export class DeploymentService extends BaseService { status: "CANCELED", canceledAt: new Date(), canceledReason: data?.canceledReason, + buildEnvVars: Prisma.DbNull, }, }), (error) => ({ @@ -249,7 +245,27 @@ export class DeploymentService extends BaseService { return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(cancelDeployment) - .andThen(({ deployment }) => + .andTee(({ deployment }) => + recordDeploymentFinished({ + status: "CANCELED", + deployment: { + ...deployment, + status: "CANCELED", + canceledAt: new Date(), + canceledReason: data?.canceledReason ?? null, + }, + environment: { + organizationId: deployment.environment.project.organizationId, + organizationSlug: deployment.environment.organization.slug, + projectId: deployment.environment.project.id, + projectName: deployment.environment.project.name, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environment.id, + environmentType: deployment.environment.type, + }, + }) + ) + .andTee(({ deployment }) => this.appendToEventLog(deployment.environment.project, deployment, [ { type: "finalized", @@ -258,14 +274,11 @@ export class DeploymentService extends BaseService { message: data?.canceledReason ?? undefined, }, }, - ]) - .orElse((error) => { - logger.error("Failed to append event to deployment event log", { error }); - return okAsync(deployment); - }) - .map(() => deployment) + ]).orTee((error) => { + logger.error("Failed to append event to deployment event log", { error }); + }) ) - .andThen(deleteTimeout) + .andThen(({ deployment }) => deleteTimeout(deployment)) .map(() => undefined); } @@ -339,6 +352,7 @@ export class DeploymentService extends BaseService { options: { skipPromotion?: boolean; configFilePath?: string; + fromBundle?: boolean; } ) { return fromPromise( @@ -482,6 +496,23 @@ export class DeploymentService extends BaseService { select: { status: true, id: true, + friendlyId: true, + version: true, + type: true, + createdAt: true, + startedAt: true, + installedAt: true, + builtAt: true, + deployedAt: true, + failedAt: true, + canceledAt: true, + canceledReason: true, + errorData: true, + runtime: true, + runtimeVersion: true, + cliVersion: true, + triggeredVia: true, + commitSHA: true, buildServerMetadata: true, imageReference: true, shortCode: true, @@ -489,9 +520,15 @@ export class DeploymentService extends BaseService { include: { project: { select: { + id: true, + name: true, + organizationId: true, externalRef: true, }, }, + organization: { + select: { slug: true }, + }, }, }, }, diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index 87b7618d76d..b5a6bbeb24c 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -1,11 +1,11 @@ import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { BaseService } from "./baseService.server"; import { logger } from "~/services/logger.server"; -import { type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { boundedIn, Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [ "CANCELED", @@ -41,24 +41,55 @@ export class FailDeploymentService extends BaseService { return; } - const failedDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + + // Guarded: a concurrent terminal transition can win after the check above + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: { notIn: boundedIn(FINAL_DEPLOYMENT_STATUSES) }, }, data: { status: "FAILED", - failedAt: new Date(), + failedAt, errorData: params.error, + buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment reached a final state concurrently, skipping fail", { + id: deployment.id, + friendlyId, + }); + return; + } + + // Re-read: the row can gain phase timestamps between the read and the guarded write + const failedDeployment = await this._prisma.workerDeployment.findFirst({ + where: { id: deployment.id }, + }); + + if (!failedDeployment) { + logger.error("Worker deployment disappeared after fail transition", { + id: deployment.id, + friendlyId, + }); + return; + } + + recordDeploymentFinished({ status: "FAILED", - deploymentFriendlyId: friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: failedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + organizationSlug: authenticatedEnv.organization.slug, + projectId: authenticatedEnv.projectId, + projectName: authenticatedEnv.project.name, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, reason: params.error.message, }); diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index 0595cee1e2b..c9d6e7f3932 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -1,4 +1,5 @@ import type { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; +import { Prisma } from "@trigger.dev/database"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { updateEnvConcurrencyLimits } from "../runQueue.server"; @@ -9,7 +10,7 @@ import { projectPubSub } from "./projectPubSub.server"; import { FailDeploymentService } from "./failDeployment.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { engine } from "../runEngine.server"; import { tryCatch } from "@trigger.dev/core"; import { externalDeploymentCacheInstance } from "~/services/externalDeploymentCacheInstance.server"; @@ -65,27 +66,53 @@ export class FinalizeDeploymentService extends BaseService { } const imageDigest = validatedImageDigest(body.imageDigest); + const deployedAt = new Date(); + const imageReference = imageDigest + ? `${deployment.imageReference}@${imageDigest}` + : deployment.imageReference; - // Link the deployment with the background worker - const finalizedDeployment = await this._prisma.workerDeployment.update({ + // Guarded: stops a concurrent transition (e.g. a late timeout) from double-committing + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: "DEPLOYING", }, data: { status: "DEPLOYED", - deployedAt: new Date(), + deployedAt, // Only add the digest, if any - imageReference: imageDigest ? `${deployment.imageReference}@${imageDigest}` : undefined, + imageReference: imageDigest ? imageReference : undefined, + buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment left DEPLOYING concurrently, skipping finalize", { + id: deployment.id, + }); + throw new ServiceValidationError("Worker deployment is not in DEPLOYING status"); + } + + const finalizedDeployment = { + ...deployment, + status: "DEPLOYED" as const, + deployedAt, + imageReference, + buildEnvVars: null, + }; + + recordDeploymentFinished({ status: "DEPLOYED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: finalizedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + organizationSlug: authenticatedEnv.organization.slug, + projectId: authenticatedEnv.projectId, + projectName: authenticatedEnv.project.name, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, }); const deploymentService = new DeploymentService(); diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index c5b01c6084b..ca7ce966006 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -6,6 +6,7 @@ import { import { customAlphabet } from "nanoid"; import { env } from "~/env.server"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { encryptSecret } from "~/services/secrets/secretStore.server"; import { logger } from "~/services/logger.server"; import { generateFriendlyId } from "../friendlyIdentifiers"; import { createRemoteImageBuild, remoteBuildsEnabled } from "../remoteImageBuilder.server"; @@ -15,6 +16,7 @@ import { getDeploymentImageRef } from "../getDeploymentImageRef.server"; import { tryCatch } from "@trigger.dev/core"; import { getRegistryConfig } from "../registryConfig.server"; import { DeploymentService } from "./deployment.server"; +import { recordDeploymentInitialized } from "./recordDeploymentFinished.server"; import { createDeploymentWithNextVersion } from "./initializeDeployment/createDeploymentWithNextVersion.server"; import { cancelSupersededDeployments, @@ -55,7 +57,8 @@ export type InitializeDeploymentResult = export class InitializeDeploymentService extends BaseService { public async call( environment: AuthenticatedEnvironment, - payload: InitializeDeploymentRequestBody + payload: InitializeDeploymentRequestBody, + options?: { cliVersion?: string } ): Promise { return this.traceWithEnv("call", environment, async (span) => { if (payload.externalId) { @@ -268,6 +271,38 @@ export class InitializeDeploymentService extends BaseService { } : undefined; + let encryptedBuildEnvVars: Awaited> | undefined; + + if ( + payload.isNativeBuild && + payload.fromBundle && + payload.buildEnvVars && + Object.keys(payload.buildEnvVars).length > 0 + ) { + const buildEnvVars = payload.buildEnvVars; + + const keyCount = Object.keys(buildEnvVars).length; + if (keyCount > env.DEPLOYMENT_BUILD_ENV_VARS_MAX_KEYS) { + throw new ServiceValidationError( + `Build environment variable count (${keyCount}) exceeds the allowed limit of ${env.DEPLOYMENT_BUILD_ENV_VARS_MAX_KEYS}. Reach out to us if you are seeing this error consistently.` + ); + } + + const serialized = JSON.stringify(buildEnvVars); + const serializedBytes = Buffer.byteLength(serialized, "utf8"); + if (serializedBytes > env.DEPLOYMENT_BUILD_ENV_VARS_SIZE_LIMIT_BYTES) { + const sizeKB = parseFloat((serializedBytes / 1024).toFixed(1)); + const limitKB = parseFloat( + (env.DEPLOYMENT_BUILD_ENV_VARS_SIZE_LIMIT_BYTES / 1024).toFixed(1) + ); + throw new ServiceValidationError( + `Build environment variables size (${sizeKB} KB) exceeds the allowed limit of ${limitKB} KB. Reach out to us if you are seeing this error consistently.` + ); + } + + encryptedBuildEnvVars = await encryptSecret(env.ENCRYPTION_KEY, serialized); + } + const buildServerMetadata: BuildServerMetadata | undefined = payload.isNativeBuild || payload.buildId ? { @@ -279,6 +314,7 @@ export class InitializeDeploymentService extends BaseService { skipPromotion: payload.skipPromotion, configFilePath: payload.configFilePath, skipEnqueue: payload.skipEnqueue, + fromBundle: payload.fromBundle, } : {}), } @@ -343,6 +379,7 @@ export class InitializeDeploymentService extends BaseService { projectId: environment.projectId, externalBuildData, buildServerMetadata, + buildEnvVars: encryptedBuildEnvVars, triggeredById: triggeredBy?.id, type: payload.type, imageReference: imageRef, @@ -351,12 +388,26 @@ export class InitializeDeploymentService extends BaseService { commitSHA: payload.gitMeta?.commitSha ?? undefined, externalId: payload.externalId, runtime: payload.runtime ?? environment.project.defaultRuntime ?? undefined, + cliVersion: options?.cliVersion, triggeredVia: payload.triggeredVia ?? undefined, startedAt: initialStatus === "BUILDING" ? new Date() : undefined, }; } ); + recordDeploymentInitialized({ + deployment, + environment: { + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + projectId: environment.projectId, + projectName: environment.project.name, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, + }); + const timeoutMs = deployment.status === "PENDING" ? env.DEPLOY_QUEUE_TIMEOUT_MS : env.DEPLOY_TIMEOUT_MS; @@ -373,6 +424,7 @@ export class InitializeDeploymentService extends BaseService { .enqueueBuild(environment, deployment, payload.artifactKey, { skipPromotion: payload.skipPromotion, configFilePath: payload.configFilePath, + fromBundle: payload.fromBundle, }) .orElse((error) => { logger.error("Failed to enqueue build", { diff --git a/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts new file mode 100644 index 00000000000..4bea471269c --- /dev/null +++ b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts @@ -0,0 +1,189 @@ +import { ROOT_CONTEXT, SpanStatusCode } from "@opentelemetry/api"; +import { type WorkerDeployment, type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { logger } from "~/services/logger.server"; +import { SEMINTATTRS_FORCE_RECORDING, tracer } from "~/v3/tracer.server"; +import { + DeploymentTelemetryAttributes as ATTRS, + deriveBuildPath, + deriveDeploymentDurations, +} from "~/v3/deploymentTelemetry"; + +type TerminalDeploymentStatus = Extract< + WorkerDeploymentStatus, + "DEPLOYED" | "FAILED" | "TIMED_OUT" | "CANCELED" +>; + +type FinishedDeployment = Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "startedAt" + | "installedAt" + | "builtAt" + | "deployedAt" + | "failedAt" + | "canceledAt" + | "canceledReason" + | "errorData" + | "runtime" + | "runtimeVersion" + | "cliVersion" + | "triggeredVia" + | "commitSHA" +> & { buildServerMetadata: unknown }; + +type EnvironmentInfo = { + organizationId?: string; + organizationSlug?: string; + projectId?: string; + projectName?: string; + projectRef?: string; + environmentId?: string; + environmentType?: string; +}; + +/** + * Records a deployment's terminal transition as a single wide + * `deployment.finished` span, backdated createdAt → terminal (attribute + * contract in ../deploymentTelemetry.ts). Call exactly once, only after a + * guarded status write confirmed this caller won the transition. Emitted on + * ROOT_CONTEXT with forceRecording so the sampler can never drop it; never + * throws. + */ +export function recordDeploymentFinished(params: { + status: TerminalDeploymentStatus; + deployment: FinishedDeployment; + environment: EnvironmentInfo; + reason?: string; +}): void { + try { + const { status, deployment, environment, reason } = params; + + const isFailure = status === "FAILED" || status === "TIMED_OUT"; + const terminalAt = + deployment.deployedAt ?? deployment.failedAt ?? deployment.canceledAt ?? new Date(); + const durations = deriveDeploymentDurations(deployment, terminalAt); + const errorData = parseErrorData(deployment.errorData); + + const span = tracer.startSpan( + "deployment.finished", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.ORG_SLUG]: environment.organizationSlug, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_NAME]: environment.projectName, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: status, + [ATTRS.SUCCESS]: status === "DEPLOYED", + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.RUNTIME_VERSION]: deployment.runtimeVersion ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + [ATTRS.COMMIT_SHA]: deployment.commitSHA ?? undefined, + [ATTRS.ERROR_NAME]: isFailure ? errorData?.name : undefined, + [ATTRS.ERROR_MESSAGE]: isFailure ? (reason ?? errorData?.message) : undefined, + [ATTRS.CANCELED_REASON]: deployment.canceledReason ?? undefined, + [ATTRS.DURATION_TOTAL_MS]: durations.totalMs, + [ATTRS.DURATION_QUEUE_MS]: durations.queueMs, + [ATTRS.DURATION_INSTALL_MS]: durations.installMs, + [ATTRS.DURATION_BUILDING_MS]: durations.buildingMs, + [ATTRS.DURATION_DEPLOYING_MS]: durations.deployingMs, + }, + }, + ROOT_CONTEXT + ); + + // CANCELED is deliberately not an error: it stays out of failure rates + if (isFailure) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: reason ?? errorData?.message, + }); + } + + span.end(terminalAt); + } catch (error) { + logger.debug("recordDeploymentFinished failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +/** + * Records a deployment's creation as a zero-duration `deployment.initialized` + * event — the funnel counterpart to `deployment.finished` for detecting + * stuck deployments. Never throws. + */ +export function recordDeploymentInitialized(params: { + deployment: Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "runtime" + | "cliVersion" + | "triggeredVia" + > & { buildServerMetadata: unknown }; + environment: EnvironmentInfo; +}): void { + try { + const { deployment, environment } = params; + + const span = tracer.startSpan( + "deployment.initialized", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.ORG_SLUG]: environment.organizationSlug, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_NAME]: environment.projectName, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: deployment.status, + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + }, + }, + ROOT_CONTEXT + ); + + span.end(deployment.createdAt); + } catch (error) { + logger.debug("recordDeploymentInitialized failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +function parseErrorData(errorData: unknown): { name?: string; message?: string } | undefined { + if (!errorData || typeof errorData !== "object") return undefined; + const record = errorData as Record; + return { + name: typeof record.name === "string" ? record.name : undefined, + message: typeof record.message === "string" ? record.message : undefined, + }; +} diff --git a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts b/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts deleted file mode 100644 index e66a7a6a9f9..00000000000 --- a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { SpanStatusCode } from "@opentelemetry/api"; -import { type WorkerDeploymentStatus } from "@trigger.dev/database"; -import { logger } from "~/services/logger.server"; -import { tracer } from "~/v3/tracer.server"; - -type TerminalDeploymentStatus = Extract< - WorkerDeploymentStatus, - "DEPLOYED" | "FAILED" | "TIMED_OUT" ->; - -/** - * Records a deployment's terminal status as a `deployment.outcome` span so - * deploy success/failure is queryable from traces (no DB read). Call after each - * terminal-status write. Org/project/env are best-effort; never throws. - */ -export function recordDeploymentOutcome(params: { - status: TerminalDeploymentStatus; - deploymentFriendlyId: string; - organizationId?: string; - projectId?: string; - environmentId?: string; - environmentType?: string; - reason?: string; -}): void { - try { - const span = tracer.startSpan("deployment.outcome", { - attributes: { - "$trigger.org.id": params.organizationId, - "$trigger.project.id": params.projectId, - "$trigger.env.id": params.environmentId, - "$trigger.env.type": params.environmentType, - "deployment.outcome.status": params.status, - "deployment.outcome.success": params.status === "DEPLOYED", - "deployment.outcome.deployment_id": params.deploymentFriendlyId, - "deployment.outcome.reason": params.reason, - }, - }); - - if (params.status !== "DEPLOYED") { - span.setStatus({ code: SpanStatusCode.ERROR, message: params.reason }); - } - - span.end(); - } catch (error) { - logger.debug("recordDeploymentOutcome failed", { - deploymentFriendlyId: params.deploymentFriendlyId, - error: error instanceof Error ? error.message : String(error), - }); - } -} diff --git a/apps/webapp/app/v3/services/timeoutDeployment.server.ts b/apps/webapp/app/v3/services/timeoutDeployment.server.ts index fa3de698e36..cc935cc4712 100644 --- a/apps/webapp/app/v3/services/timeoutDeployment.server.ts +++ b/apps/webapp/app/v3/services/timeoutDeployment.server.ts @@ -1,10 +1,11 @@ +import { Prisma } from "@trigger.dev/database"; import { logger } from "~/services/logger.server"; import { BaseService } from "./baseService.server"; import { commonWorker } from "../commonWorker.server"; import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { type PrismaClientOrTransaction } from "~/db.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export class TimeoutDeploymentService extends BaseService { public async call(id: string, fromStatus: string, errorMessage: string) { @@ -16,6 +17,9 @@ export class TimeoutDeploymentService extends BaseService { environment: { include: { project: true, + organization: { + select: { slug: true }, + }, }, }, }, @@ -37,24 +41,51 @@ export class TimeoutDeploymentService extends BaseService { return; } - const timedOutDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + const errorData = { message: errorMessage, name: "TimeoutError" }; + + // Guarded: keeps the fromStatus check atomic with the write + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: deployment.status, }, data: { status: "TIMED_OUT", - failedAt: new Date(), - errorData: { message: errorMessage, name: "TimeoutError" }, + failedAt, + errorData, + buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Deployment moved out of the expected state concurrently, skipping timeout", { + id: deployment.id, + fromStatus, + }); + return; + } + + const timedOutDeployment = { + ...deployment, + status: "TIMED_OUT" as const, + failedAt, + errorData, + buildEnvVars: null, + }; + + recordDeploymentFinished({ status: "TIMED_OUT", - deploymentFriendlyId: deployment.friendlyId, - organizationId: deployment.environment.project.organizationId, - projectId: deployment.environment.projectId, - environmentId: deployment.environmentId, - environmentType: deployment.environment.type, + deployment: timedOutDeployment, + environment: { + organizationId: deployment.environment.project.organizationId, + organizationSlug: deployment.environment.organization.slug, + projectId: deployment.environment.projectId, + projectName: deployment.environment.project.name, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environmentId, + environmentType: deployment.environment.type, + }, reason: errorMessage, }); diff --git a/apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts b/apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts new file mode 100644 index 00000000000..cdbb17f0087 --- /dev/null +++ b/apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts @@ -0,0 +1,273 @@ +import type { PrismaClientOrTransaction } from "@trigger.dev/database"; +import { normalizeExternalDeploymentId, tryCatch } from "@trigger.dev/core/v3"; +import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/isomorphic"; +import pMap from "p-map"; +import { logger } from "~/services/logger.server"; + +type BackfillEnvironmentResult = { + /** An environment id, or a project id when `scope` is "project". */ + id: string; + /** Only set when the failure happened before any environment was resolved. */ + scope?: "project"; + action: "updated" | "would_update" | "skipped_nothing_eligible" | "error"; + eligible?: number; + written?: number; + error?: string; +}; + +export type BackfillResult = { + projects: number; + environments: BackfillEnvironmentResult[]; + summary: Record; + deployments: { eligible: number; written: number }; + next?: string; + done?: boolean; +}; + +export type BackfillOptions = { + prisma: PrismaClientOrTransaction; + replica: PrismaClientOrTransaction; + cursor?: string; + limit: number; + recentPerEnvironment: number; + parallelism: number; + dryRun: boolean; +}; + +type Candidate = { id: string; externalId: string }; + +/** + * Copy `commitSHA` into `externalId` for Vercel deployments that predate skew + * protection, one keyset page of connected projects at a time. + * + * Resolution reads (environmentId, externalId, status=DEPLOYED) and a miss parks + * the run rather than falling back, so a deployment that stores a commit SHA but + * no external id is unreachable to an app that sends one. + * + * `cursor` and `limit` are in OrganizationProjectIntegration ids, so a page is N + * connected projects and yields however many environments those hold. + */ +export async function backfillVercelExternalIds(options: BackfillOptions): Promise { + const { replica, cursor, limit, parallelism } = options; + + // Paginate over the connected projects rather than over environments. Driving + // from RuntimeEnvironment means "is this Vercel-connected" sits two joins away + // from the ordered column, so no index can serve filter and order together and + // every page has to build the whole matching set and sort it. Here the keyset + // runs on this table's primary key and the page is bounded by `take`. + const integrations = await replica.organizationProjectIntegration.findMany({ + where: { + deletedAt: null, + organizationIntegration: { service: "VERCEL", deletedAt: null }, + id: cursor ? { gt: cursor } : undefined, + }, + select: { id: true, projectId: true }, + orderBy: { id: "asc" }, + take: limit, + }); + + if (integrations.length === 0) { + return { + projects: 0, + environments: [], + summary: {}, + deployments: { eligible: 0, written: 0 }, + done: true, + }; + } + + const next = integrations[integrations.length - 1]?.id; + + // A project can hold more than one connection row, and reconnecting leaves the + // old one behind. Deduping keeps a page from walking the same environments twice. + const projectIds = [...new Set(integrations.map((integration) => integration.projectId))]; + + // One equality lookup per project rather than a single `projectId IN (...)`. A + // wide IN list tips the planner into seq-scanning RuntimeEnvironment, whereas an + // equality always rides projectId's index. These run concurrently anyway. + // Nothing in this mapper may throw. `stopOnError: false` does not isolate a + // rejected mapper: pMap still rejects the whole call with an AggregateError, + // which would cost the page its results and its `next` cursor. + const perProject = await pMap( + projectIds, + async (projectId): Promise => { + const [lookupError, environments] = await tryCatch( + replica.runtimeEnvironment.findMany({ + where: { projectId, type: { not: "DEVELOPMENT" } }, + select: { id: true }, + orderBy: { id: "asc" }, + }) + ); + + if (lookupError) { + logger.error("Vercel external id backfill could not list environments", { + projectId, + error: lookupError, + }); + return [{ id: projectId, scope: "project", action: "error", error: lookupError.message }]; + } + + const results: BackfillEnvironmentResult[] = []; + for (const environment of environments) { + results.push(await backfillEnvironment(environment.id, options)); + } + return results; + }, + { concurrency: parallelism, stopOnError: false } + ); + + const results = perProject.flat(); + + if (results.length === 0) { + return { + projects: projectIds.length, + environments: [], + summary: {}, + deployments: { eligible: 0, written: 0 }, + next, + }; + } + + const summary = results.reduce>((acc, result) => { + acc[result.action] = (acc[result.action] ?? 0) + 1; + return acc; + }, {}); + + const deployments = results.reduce( + (acc, result) => ({ + eligible: acc.eligible + (result.eligible ?? 0), + written: acc.written + (result.written ?? 0), + }), + { eligible: 0, written: 0 } + ); + + return { + projects: projectIds.length, + environments: results, + summary, + deployments, + next, + }; +} + +async function backfillEnvironment( + environmentId: string, + options: BackfillOptions +): Promise { + const [readError, candidates] = await tryCatch(findCandidates(environmentId, options)); + + if (readError) { + logger.error("Vercel external id backfill could not read deployments", { + environmentId, + error: readError, + }); + return { id: environmentId, action: "error", error: readError.message }; + } + + if (candidates.length === 0) { + return { id: environmentId, action: "skipped_nothing_eligible", eligible: 0 }; + } + + if (options.dryRun) { + return { id: environmentId, action: "would_update", eligible: candidates.length }; + } + + let written = 0; + + for (const candidate of candidates) { + const [writeError, result] = await tryCatch( + options.prisma.workerDeployment.updateMany({ + // Re-checking externalId lets a deploy landing mid-backfill keep the id it set. + where: { id: candidate.id, externalId: null }, + data: { externalId: candidate.externalId }, + }) + ); + + if (writeError) { + logger.error("Vercel external id backfill could not write a deployment", { + environmentId, + deploymentId: candidate.id, + error: writeError, + }); + return { + id: environmentId, + action: "error", + eligible: candidates.length, + written, + error: writeError.message, + }; + } + + written += result.count; + } + + return { id: environmentId, action: "updated", eligible: candidates.length, written }; +} + +/** + * The deployment holding the `current` promotion, plus the most recent DEPLOYED + * ones. Only DEPLOYED deployments are ever resolved, and `current` plus a recent + * window is what can still receive traffic. The window is there for Vercel + * instant-rollback, where the live app is an older commit than `current`. + */ +async function findCandidates( + environmentId: string, + { replica, recentPerEnvironment }: BackfillOptions +): Promise { + const select = { + id: true, + externalId: true, + commitSHA: true, + workerId: true, + status: true, + } as const; + + const [promotion, recent] = await Promise.all([ + replica.workerDeploymentPromotion.findFirst({ + where: { environmentId, label: CURRENT_DEPLOYMENT_LABEL }, + select: { deployment: { select } }, + }), + recentPerEnvironment > 0 + ? replica.workerDeployment.findMany({ + where: { environmentId, status: "DEPLOYED" }, + select, + // id DESC, not createdAt: it matches [environmentId, status, id] exactly, so + // status stays in the index condition and the LIMIT bounds the scan. cuids sort + // by creation, and resolveExternalDeployment orders its candidates the same way. + orderBy: { id: "desc" }, + take: recentPerEnvironment, + }) + : Promise.resolve([]), + ]); + + const byId = new Map(); + for (const deployment of recent) { + byId.set(deployment.id, deployment); + } + if (promotion?.deployment) { + byId.set(promotion.deployment.id, promotion.deployment); + } + + const candidates: Candidate[] = []; + + for (const deployment of byId.values()) { + if ( + deployment.externalId !== null || + deployment.workerId === null || + deployment.status !== "DEPLOYED" + ) { + continue; + } + + // Reusing the live normalizer keeps a backfilled id byte-identical to what a + // build would have written. + const externalId = normalizeExternalDeploymentId(deployment.commitSHA ?? undefined); + if (!externalId) { + continue; + } + + candidates.push({ id: deployment.id, externalId }); + } + + return candidates; +} diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index cbf9a937c03..7047d65d2b9 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -36,7 +36,9 @@ import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto"; import { BatchSpanProcessor, ParentBasedSampler, + type ReadableSpan, type Sampler, + type Span as SdkTraceSpan, SamplingDecision, type SamplingResult, SimpleSpanProcessor, @@ -69,7 +71,7 @@ import { metricsRegister } from "~/metrics.server"; import { collectDatabaseClientMetrics } from "~/utils/databaseMetrics.server"; import { performance } from "node:perf_hooks"; -const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; +export const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; export const DATASOURCE_CONTEXT_KEY = createContextKey("trigger.db.datasource"); @@ -89,6 +91,29 @@ class DatasourceAttributeSpanProcessor implements SpanProcessor { } } +// Mirrors name-prefixed spans into a second exporter; they still flow to the main one +class SpanNamePrefixMirrorProcessor implements SpanProcessor { + constructor( + private readonly _inner: SpanProcessor, + private readonly _prefix: string + ) {} + + onStart(span: SdkTraceSpan, parentContext: Context): void { + this._inner.onStart(span, parentContext); + } + onEnd(span: ReadableSpan): void { + if (span.name.startsWith(this._prefix)) { + this._inner.onEnd(span); + } + } + shutdown(): Promise { + return this._inner.shutdown(); + } + forceFlush(): Promise { + return this._inner.forceFlush(); + } +} + class CustomWebappSampler implements Sampler { constructor(private readonly _baseSampler: Sampler) {} @@ -270,6 +295,30 @@ function setupTelemetry() { } } + if (env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL) { + const deploymentEventExporter = new OTLPTraceExporter({ + url: env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL, + timeoutMillis: 15_000, + headers: parseInternalDeploymentEventHeaders() ?? {}, + }); + + spanProcessors.push( + new SpanNamePrefixMirrorProcessor( + new BatchSpanProcessor(deploymentEventExporter, { + maxExportBatchSize: 64, + scheduledDelayMillis: 1000, + exportTimeoutMillis: 30000, + maxQueueSize: 2048, + }), + "deployment." + ) + ); + + console.log( + `🔦 Tracer: deployment-event exporter enabled to ${env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL}` + ); + } + const ratioSampler = new TraceIdRatioBasedSampler(samplingRate); const provider = new NodeTracerProvider({ @@ -341,6 +390,13 @@ function setupTelemetry() { instrumentations, }); + // Without this flush every shutdown drops the last batch of spans + const flushOnShutdown = () => { + provider.forceFlush().catch(() => {}); + }; + process.once("SIGTERM", flushOnShutdown); + process.once("SIGINT", flushOnShutdown); + return { tracer: provider.getTracer("trigger.dev", "3.3.12"), logger: logs.getLogger("trigger.dev", "3.3.12"), @@ -874,6 +930,19 @@ function parseInternalTraceHeaders(): Record | undefined { } } +function parseInternalDeploymentEventHeaders(): Record | undefined { + try { + return env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS + ? (JSON.parse(env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS) as Record< + string, + string + >) + : undefined; + } catch { + return; + } +} + function parseInternalMetricsHeaders(): Record | undefined { try { return env.INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS diff --git a/apps/webapp/app/v3/transactionResilience.server.ts b/apps/webapp/app/v3/transactionResilience.server.ts index ae5678c987c..eabde6691d8 100644 --- a/apps/webapp/app/v3/transactionResilience.server.ts +++ b/apps/webapp/app/v3/transactionResilience.server.ts @@ -17,8 +17,11 @@ export type TransactionResilienceConfig = { startRetry: TransactionStartRetryConfig; }; -function resolveTransactionResilience( - pool: "control-plane" | "run-ops" | "run-ops-legacy", +// Exported so the topology singleton can build a per-shard config (each call creates its OWN +// TokenBucketRetryBudget, so one shard's retry storm cannot drain another's). `pool` is a free +// string — it only labels a log line, never keys any behaviour. +export function resolveTransactionResilience( + pool: string, overrides: { maxWaitMs?: number; enabled?: boolean; @@ -64,6 +67,44 @@ export const runOpsTransactionResilience = resolveTransactionResilience("run-ops budgetBurst: env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_BURST, }); +// A gen-2 shard's resilience. Defaults to the RUN_OPS_DATABASE_TRANSACTION_* values (so a shard with +// no overrides matches the gen-1 new store), then applies the descriptor's per-shard overrides. Each +// call builds its OWN budget, so a storm on one shard cannot drain another's. +export function resolveShardResilience( + key: string, + overrides?: { + transactionMaxWaitMs?: number; + transactionStartRetryEnabled?: boolean; + transactionStartRetryMaxAttempts?: number; + transactionStartRetryBackoffMinMs?: number; + transactionStartRetryBackoffMaxMs?: number; + transactionStartRetryBudgetPerSec?: number; + transactionStartRetryBudgetBurst?: number; + } +): TransactionResilienceConfig { + return resolveTransactionResilience(`run-ops-shard-${key}`, { + maxWaitMs: overrides?.transactionMaxWaitMs ?? env.RUN_OPS_DATABASE_TRANSACTION_MAX_WAIT_MS, + enabled: + overrides?.transactionStartRetryEnabled ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_ENABLED, + maxAttempts: + overrides?.transactionStartRetryMaxAttempts ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_MAX_ATTEMPTS, + backoffMinMs: + overrides?.transactionStartRetryBackoffMinMs ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BACKOFF_MIN_MS, + backoffMaxMs: + overrides?.transactionStartRetryBackoffMaxMs ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BACKOFF_MAX_MS, + budgetPerSec: + overrides?.transactionStartRetryBudgetPerSec ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_PER_SEC, + budgetBurst: + overrides?.transactionStartRetryBudgetBurst ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_BURST, + }); +} + export const runOpsLegacyTransactionResilience = resolveTransactionResilience("run-ops-legacy", { maxWaitMs: env.RUN_OPS_LEGACY_DATABASE_TRANSACTION_MAX_WAIT_MS, enabled: env.RUN_OPS_LEGACY_DATABASE_TRANSACTION_START_RETRY_ENABLED, diff --git a/apps/webapp/app/v3/vercel/vercelProjectIntegrationSchema.ts b/apps/webapp/app/v3/vercel/vercelProjectIntegrationSchema.ts index cde9f708163..c428c7956a7 100644 --- a/apps/webapp/app/v3/vercel/vercelProjectIntegrationSchema.ts +++ b/apps/webapp/app/v3/vercel/vercelProjectIntegrationSchema.ts @@ -85,13 +85,16 @@ export function createDefaultVercelIntegrationData( vercelProjectId: string, vercelProjectName: string, vercelTeamId: string | null, - vercelTeamSlug?: string + vercelTeamSlug?: string, + availableEnvSlugs: EnvSlug[] = ALL_ENV_SLUGS ): VercelProjectIntegrationData { + const defaultOn = (["prod", "preview"] as EnvSlug[]).filter((s) => availableEnvSlugs.includes(s)); + return { config: { atomicBuilds: [], - pullEnvVarsBeforeBuild: ["prod", "preview"], - discoverEnvVars: ["prod", "preview"], + pullEnvVarsBeforeBuild: defaultOn, + discoverEnvVars: defaultOn, vercelStagingEnvironment: null, autoPromote: true, }, @@ -142,6 +145,26 @@ export function getAvailableEnvSlugs( }); } +export function restrictConfigToAvailableEnvSlugs( + config: Partial, + availableEnvSlugs: EnvSlug[] +): Partial { + const restricted = { ...config }; + + for (const key of ["atomicBuilds", "pullEnvVarsBeforeBuild", "discoverEnvVars"] as const) { + const slugs = restricted[key]; + if (slugs) { + restricted[key] = slugs.filter((slug) => availableEnvSlugs.includes(slug)); + } + } + + if ("vercelStagingEnvironment" in restricted && !availableEnvSlugs.includes("stg")) { + restricted.vercelStagingEnvironment = null; + } + + return restricted; +} + export function getAvailableEnvSlugsForBuildSettings( hasStagingEnvironment: boolean, hasPreviewEnvironment: boolean diff --git a/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts b/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts new file mode 100644 index 00000000000..b5bfd8ea052 --- /dev/null +++ b/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts @@ -0,0 +1,155 @@ +// The page posts only the flags its UI manages, so how the action reads an absent key is the whole +// bug surface. These drive the real exported action against a real Postgres and assert on the rows +// it leaves behind. The only module substituted is the auth wrapper, so the handler can be called +// without a super-admin session; the database is the genuine article, injected into db.server. +import { boundedIn, $transaction as realTransaction } from "@trigger.dev/database"; +import type { PrismaClient } from "@trigger.dev/database"; +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; + +vi.setConfig({ testTimeout: 60_000 }); + +const db = vi.hoisted(() => ({ client: null as unknown as PrismaClient })); + +vi.mock("~/services/routeBuilders/dashboardBuilder", () => ({ + dashboardAction: (_options: unknown, handler: unknown) => handler, + dashboardLoader: (_options: unknown, handler: unknown) => handler, +})); + +vi.mock("~/db.server", () => ({ + get prisma() { + return db.client; + }, + boundedIn, + // Delegates to the SAME shared implementation the production helper wraps, so the + // transactional semantics, the nesting case and the retry behaviour are the real ones rather + // than a reimplementation. Only the webapp wrapper's tracing span and its infrastructure-error + // logging are absent, and neither is asserted here. + $transaction: ( + client: PrismaClient, + nameOrFn: unknown, + fnOrOptions?: unknown, + options?: unknown + ) => { + const fn = (typeof nameOrFn === "function" ? nameOrFn : fnOrOptions) as Parameters< + typeof realTransaction + >[1]; + const opts = (typeof nameOrFn === "function" ? fnOrOptions : options) as Parameters< + typeof realTransaction + >[3]; + return realTransaction(client, fn, () => {}, opts); + }, +})); + +import { action } from "~/routes/admin.feature-flags"; + +const WORKER_GROUP_ID = "clwg000000000000000000000"; + +async function post(host: string, body: unknown) { + const request = new Request(`https://${host}/admin/feature-flags`, { + method: "POST", + body: JSON.stringify(body), + headers: { "content-type": "application/json" }, + }); + return (await (action as any)({ request, params: {}, context: {} })) as Response; +} + +async function readFlag(prisma: PrismaClient, key: string) { + const row = await prisma.featureFlag.findFirst({ where: { key }, select: { value: true } }); + return row?.value; +} + +async function seed(prisma: PrismaClient) { + db.client = prisma; + await prisma.featureFlag.createMany({ + data: [ + { id: "ff_locked", key: FEATURE_FLAG.defaultWorkerInstanceGroupId, value: WORKER_GROUP_ID }, + { id: "ff_plain", key: FEATURE_FLAG.mollifierEnabled, value: true }, + ], + }); +} + +describe("admin feature flags action", () => { + postgresTest("keeps the locked flag when the page did not unlock it", async ({ prisma }) => { + await seed(prisma); + + const response = await post("localhost:3030", { flags: {} }); + + expect(response.status).toBe(200); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe(WORKER_GROUP_ID); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBeUndefined(); + }); + + postgresTest("keeps the locked flag when the body omits the unlock field", async ({ prisma }) => { + await seed(prisma); + + // A tab opened before the field existed posts the old shape. + const response = await post("localhost:3030", { flags: {}, unlockLockedFlags: undefined }); + + expect(response.status).toBe(200); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe(WORKER_GROUP_ID); + }); + + postgresTest("deletes the locked flag when the page unlocked it", async ({ prisma }) => { + await seed(prisma); + + await post("localhost:3030", { flags: {}, unlockLockedFlags: true }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBeUndefined(); + }); + + postgresTest( + "keeps the locked flag on managed cloud despite the unlock claim", + async ({ prisma }) => { + await seed(prisma); + + await post("cloud.trigger.dev", { flags: {}, unlockLockedFlags: true }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe( + WORKER_GROUP_ID + ); + } + ); + + postgresTest( + "rejects a locked flag submitted to managed cloud, writing nothing", + async ({ prisma }) => { + await seed(prisma); + + const response = await post("cloud.trigger.dev", { + flags: { [FEATURE_FLAG.defaultWorkerInstanceGroupId]: "clwg999" }, + }); + + expect(response.status).toBe(400); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe( + WORKER_GROUP_ID + ); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(true); + } + ); + + postgresTest("rejects a value the catalog refuses, writing nothing", async ({ prisma }) => { + await seed(prisma); + + const response = await post("localhost:3030", { + flags: { [FEATURE_FLAG.realtimeBackend]: "not-a-backend" }, + }); + + expect(response.status).toBe(400); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(true); + }); + + postgresTest("upserts what was submitted and sweeps what was not", async ({ prisma }) => { + await seed(prisma); + + await post("localhost:3030", { + flags: { [FEATURE_FLAG.hasAiAccess]: true }, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.hasAiAccess)).toBe(true); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBeUndefined(); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe(WORKER_GROUP_ID); + }); +}); diff --git a/apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts b/apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts index eb322c48a1c..ade925f239c 100644 --- a/apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts +++ b/apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts @@ -3,10 +3,10 @@ // RESULTS READ assembles correctly when one batch's members are genuinely split across the real // dedicated run-ops subset schema (prisma17 / RunOpsPrismaClient) and the full control-plane // schema (prisma14) — not a mirrored full schema on both sides. No mocks. -import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import { heteroRunOpsPostgresTest, makeNShardRunOpsPostgresTest } from "@internal/testcontainers"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import type { PrismaClient } from "@trigger.dev/database"; -import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { generateRunOpsId, generateRunOpsIdV2 } from "@trigger.dev/core/v3/isomorphic"; import { describe, expect, vi } from "vitest"; import type { PrismaReplicaClient } from "~/db.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; @@ -209,6 +209,10 @@ async function seedBatchOnNew( return batch; } +// One real gen-2 shard on its OWN database, so a member seeded there is genuinely absent +// from the gen-1 `new` store rather than merely routed away from it. +const oneShardTest = makeNShardRunOpsPostgresTest(1); + const env = (ctx: SeedCtx) => ({ id: ctx.environment.id, @@ -334,4 +338,141 @@ describe("ApiBatchResultsPresenter split mode — real run-ops dedicated schema expect(result!.items[0]).toMatchObject({ ok: true, id: "run_present" }); } ); + + // A gen-2 member is directly routable to its own shard. Before the shard arm existed it + // joined the gen-1 `new` read, missed there, and — classifying dedicated-family — never + // reached the legacy probe either, so it vanished from the batch results with no error. + oneShardTest( + "a gen-2 member is hydrated from its own shard database alongside a legacy-resident member", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const shardPrisma = shardPrismas[0]!; + const ctx = await seedLegacyEnv(legacyPrisma, "gen2-shard"); + await relaxLegacyAttemptFk(legacyPrisma); + await relaxNewBatchItemFk(newPrisma); + + const shardMemberId = generateRunOpsIdV2("a"); + const legacyMemberId = generateLegacyCuid(); + + // The gen-2 member exists ONLY on the shard database. The gen-1 `new` store below is a + // different database, so routing this id there would genuinely miss. + await seedNewMember( + shardPrisma, + { envId: ctx.environment.id, orgId: ctx.organization.id, projectId: ctx.project.id }, + { + id: shardMemberId, + friendlyId: "run_shard_member", + status: "COMPLETED_SUCCESSFULLY", + output: JSON.stringify({ from: "shard-a" }), + } + ); + await seedLegacyMember(legacyPrisma, ctx, { + id: legacyMemberId, + friendlyId: "run_legacy_member", + status: "COMPLETED_WITH_ERRORS", + error: { type: "BUILT_IN_ERROR", name: "Err", message: "boom", stackTrace: "" }, + }); + + const batchFriendlyId = "batch_gen2_shard"; + await seedBatchOnNew(newPrisma, ctx.environment.id, batchFriendlyId, [ + shardMemberId, + legacyMemberId, + ]); + + const presenter = new ApiBatchResultsPresenter(throwingPrisma, throwingPrisma, { + splitEnabled: true, + newClient: newPrisma as unknown as PrismaReplicaClient, + legacyReplica: legacyPrisma as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", shardPrisma as unknown as PrismaReplicaClient]]), + }); + + const result = await presenter.call(batchFriendlyId, env(ctx)); + + expect(result).toBeDefined(); + expect(result!.items).toHaveLength(2); + const [first, second] = result!.items; + expect(first).toEqual({ + ok: true, + id: "run_shard_member", + taskIdentifier: "my-task", + output: JSON.stringify({ from: "shard-a" }), + outputType: "application/json", + }); + expect(second).toMatchObject({ ok: false, id: "run_legacy_member" }); + }, + 180_000 + ); + + // A gen-2 id naming a shard that is NOT configured must not fall back onto a gen-1 store: + // that reads the wrong database, misses, and (being dedicated-family) never reaches the + // legacy probe, so the member disappears with no error. Drop it, but loudly. + oneShardTest( + "a gen-2 member on an unconfigured shard is dropped without being read from a gen-1 store", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const shardPrisma = shardPrismas[0]!; + const ctx = await seedLegacyEnv(legacyPrisma, "gen2-unconfigured"); + await relaxLegacyAttemptFk(legacyPrisma); + await relaxNewBatchItemFk(newPrisma); + + // Shard "z" is not in the configured map; shard "a" is. + const unconfiguredId = generateRunOpsIdV2("z"); + const legacyMemberId = generateLegacyCuid(); + + await seedNewMember( + shardPrisma, + { envId: ctx.environment.id, orgId: ctx.organization.id, projectId: ctx.project.id }, + { id: unconfiguredId, friendlyId: "run_unconfigured", status: "COMPLETED_SUCCESSFULLY" } + ); + await seedLegacyMember(legacyPrisma, ctx, { + id: legacyMemberId, + friendlyId: "run_legacy_member", + status: "COMPLETED_SUCCESSFULLY", + }); + + const batchFriendlyId = "batch_gen2_unconfigured"; + await seedBatchOnNew(newPrisma, ctx.environment.id, batchFriendlyId, [ + unconfiguredId, + legacyMemberId, + ]); + + // A closure-based recorder, not a mock: it records the id sets each store is asked for, + // so the assertion is about real reads rather than about a test double's behaviour. + const askedOf = (label: string, target: RunOpsPrismaClient | PrismaClient) => { + const asked: string[][] = []; + const handle = { + ...target, + taskRun: { + findMany: (args: { where?: { id?: { in?: string[] } } }) => { + asked.push(args.where?.id?.in ?? []); + return (target as unknown as PrismaReplicaClient).taskRun.findMany(args as never); + }, + }, + } as unknown as PrismaReplicaClient; + return { label, asked, handle }; + }; + const genOneNew = askedOf("new", newPrisma); + const legacy = askedOf("legacy", legacyPrisma); + + const presenter = new ApiBatchResultsPresenter(throwingPrisma, throwingPrisma, { + splitEnabled: true, + newClient: genOneNew.handle, + legacyReplica: legacy.handle, + shardReplicas: new Map([["a", shardPrisma as unknown as PrismaReplicaClient]]), + }); + + const result = await presenter.call(batchFriendlyId, env(ctx)); + + // The legacy member still resolves; the unconfigured gen-2 member is dropped. + expect(result).toBeDefined(); + expect(result!.items).toHaveLength(1); + expect(result!.items[0]).toMatchObject({ ok: true, id: "run_legacy_member" }); + + // The unconfigured id was never asked of a gen-1 store. + for (const store of [genOneNew, legacy]) { + for (const ids of store.asked) { + expect(ids).not.toContain(unconfiguredId); + } + } + }, + 180_000 + ); }); diff --git a/apps/webapp/test/clickhouseQueryMetrics.test.ts b/apps/webapp/test/clickhouseQueryMetrics.test.ts new file mode 100644 index 00000000000..29e4bc85835 --- /dev/null +++ b/apps/webapp/test/clickhouseQueryMetrics.test.ts @@ -0,0 +1,93 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; +import { + createRun, + insertTaskRunV2Rows, + seedParents, +} from "./helpers/apiRunListPresenterTestHelpers"; +import { createInMemoryMetrics } from "./utils/tracing"; +import { histogramCount, latestMetrics, metricSum } from "./otlpMetrics.helpers"; + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); + +vi.setConfig({ testTimeout: 90_000 }); + +describe("clickhouse query metrics", () => { + containerTest( + "records duration + read_rows on success and an error metric with the ClickHouse error type", + async ({ clickhouseContainer, prisma }) => { + const ctx = await seedParents(prisma, "chm"); + const run = await createRun(prisma, ctx, { friendlyId: "run_chm" }); + + const seedClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "clickhouse-metrics-seed", + }); + await insertTaskRunV2Rows(seedClient, [{ ...run, createdAt: new Date() }]); + + const listArgs = { + page: { size: 10 } as const, + organizationId: ctx.organizationId, + projectId: ctx.projectId, + environmentId: ctx.environmentId, + }; + + const okMetrics = createInMemoryMetrics(); + const okClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "clickhouse-metrics-ok", + meter: okMetrics.meter, + }); + const okRepo = new RunsRepository({ prisma, clickhouse: okClient }); + const result = await okRepo.listRuns(listArgs); + expect(result.runs.map((r) => r.friendlyId)).toEqual(["run_chm"]); + + await vi.waitFor( + async () => { + const rm = await latestMetrics(okMetrics); + expect( + histogramCount(rm, "clickhouse.query.duration", { + client: "clickhouse-metrics-ok", + status: "ok", + }) + ).toBeGreaterThanOrEqual(1); + }, + { timeout: 5000, interval: 50 } + ); + const okRm = await latestMetrics(okMetrics); + expect( + histogramCount(okRm, "clickhouse.query.read_rows", { client: "clickhouse-metrics-ok" }) + ).toBeGreaterThanOrEqual(1); + expect( + histogramCount(okRm, "clickhouse.query.memory_usage", { client: "clickhouse-metrics-ok" }) + ).toBeGreaterThanOrEqual(1); + await okMetrics.shutdown(); + + const errMetrics = createInMemoryMetrics(); + const cappedClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "clickhouse-metrics-capped", + clickhouseSettings: { max_memory_usage: "1" }, + meter: errMetrics.meter, + }); + const errRepo = new RunsRepository({ prisma, clickhouse: cappedClient }); + await expect(errRepo.listRuns(listArgs)).rejects.toThrow(); + + await vi.waitFor( + async () => { + const rm = await latestMetrics(errMetrics); + expect( + metricSum(rm, "clickhouse.query.errors", { + client: "clickhouse-metrics-capped", + error_type: "MEMORY_LIMIT_EXCEEDED", + }) + ).toBeGreaterThanOrEqual(1); + }, + { timeout: 5000, interval: 50 } + ); + await errMetrics.shutdown(); + } + ); +}); diff --git a/apps/webapp/test/deploymentTelemetry.test.ts b/apps/webapp/test/deploymentTelemetry.test.ts new file mode 100644 index 00000000000..9020f0c7102 --- /dev/null +++ b/apps/webapp/test/deploymentTelemetry.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import { deriveBuildPath, deriveDeploymentDurations } from "~/v3/deploymentTelemetry"; + +describe("deriveBuildPath", () => { + it("classifies fromBundle native builds as native_local_bundle", () => { + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: true })).toBe("native_local_bundle"); + }); + + it("classifies native builds without fromBundle as native", () => { + expect(deriveBuildPath({ isNativeBuild: true })).toBe("native"); + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: false })).toBe("native"); + }); + + it("classifies everything else as depot", () => { + expect(deriveBuildPath(null)).toBe("depot"); + expect(deriveBuildPath(undefined)).toBe("depot"); + expect(deriveBuildPath({})).toBe("depot"); + expect(deriveBuildPath({ buildId: "depot-build-id" })).toBe("depot"); + expect(deriveBuildPath({ isNativeBuild: false })).toBe("depot"); + // fromBundle alone (skewed writer) must not count as native_local_bundle + expect(deriveBuildPath({ fromBundle: true })).toBe("depot"); + expect(deriveBuildPath("garbage")).toBe("depot"); + }); +}); + +describe("deriveDeploymentDurations", () => { + const t = (seconds: number) => new Date(1_700_000_000_000 + seconds * 1000); + + it("derives all phases for the full build-server chain", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(10), installedAt: t(40), builtAt: t(100) }, + t(130) + ); + + expect(durations).toEqual({ + totalMs: 130_000, + queueMs: 10_000, + installMs: 30_000, + buildingMs: 60_000, + deployingMs: 30_000, + }); + }); + + it("omits install and measures building from startedAt when installedAt is missing (depot)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(0), installedAt: null, builtAt: t(90) }, + t(120) + ); + + expect(durations).toEqual({ + totalMs: 120_000, + queueMs: 0, + installMs: undefined, + buildingMs: 90_000, + deployingMs: 30_000, + }); + }); + + it("omits phases whose boundaries are missing (failed before building)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(5), installedAt: null, builtAt: null }, + t(20) + ); + + expect(durations).toEqual({ + totalMs: 20_000, + queueMs: 5_000, + installMs: undefined, + buildingMs: undefined, + deployingMs: undefined, + }); + }); + + it("never returns negative durations on clock skew", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(10), startedAt: t(5), installedAt: null, builtAt: null }, + t(3) + ); + + expect(durations.totalMs).toBe(0); + expect(durations.queueMs).toBeUndefined(); + }); +}); diff --git a/apps/webapp/test/globalFeatureFlagsLockedFlags.test.ts b/apps/webapp/test/globalFeatureFlagsLockedFlags.test.ts new file mode 100644 index 00000000000..ce18ef800e3 --- /dev/null +++ b/apps/webapp/test/globalFeatureFlagsLockedFlags.test.ts @@ -0,0 +1,150 @@ +// With "Unlock read-only flags" off, the page strips GLOBAL_LOCKED_FLAGS from its payload, so an +// omitted locked key means "the UI never offered it", not "the admin unset it". +import type { PrismaClient } from "@trigger.dev/database"; +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { FEATURE_FLAG, FeatureFlagCatalog, type FeatureFlagKey } from "~/v3/featureFlags"; +import { makeSetMultipleFlags, replaceGlobalFeatureFlags } from "~/v3/featureFlags.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +const CATALOG_KEYS = Object.keys(FeatureFlagCatalog) as FeatureFlagKey[]; +const WORKER_GROUP_ID = "clwg000000000000000000000"; + +async function readFlag(prisma: PrismaClient, key: FeatureFlagKey): Promise { + const row = await prisma.featureFlag.findFirst({ where: { key }, select: { value: true } }); + return row?.value; +} + +describe("replaceGlobalFeatureFlags — locked flags the UI never submitted", () => { + postgresTest( + "keeps defaultWorkerInstanceGroupId when a locked flag is absent from the payload", + async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + [FEATURE_FLAG.mollifierEnabled]: true, + }); + + // What the page posts when an admin unsets mollifierEnabled on a self-hosted instance. + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe( + WORKER_GROUP_ID + ); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBeUndefined(); + } + ); + + postgresTest("an unlocked self-hosted page can still unset a locked flag", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: true, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBeUndefined(); + }); + + postgresTest( + "managed cloud keeps locked flags even when unlocking is claimed", + async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + isManagedCloud: true, + unlockLockedFlags: true, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe( + WORKER_GROUP_ID + ); + } + ); + + postgresTest("managed cloud still sweeps ordinary flags it was not sent", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + [FEATURE_FLAG.hasAiAccess]: true, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.mollifierEnabled]: true }, + catalogKeys: CATALOG_KEYS, + isManagedCloud: true, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe(WORKER_GROUP_ID); + expect(await readFlag(prisma, FEATURE_FLAG.hasAiAccess)).toBeUndefined(); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(true); + }); + + // The upsert and the sweep share one statement, which is only safe while no key is in both. + postgresTest("a submitted key is never also swept", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.mollifierEnabled]: true, + [FEATURE_FLAG.hasAiAccess]: true, + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { + [FEATURE_FLAG.mollifierEnabled]: false, + [FEATURE_FLAG.hasAiAccess]: true, + }, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: true, + }); + + // Both submitted keys survive with their new values rather than being swept by the same + // statement that wrote them. + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(false); + expect(await readFlag(prisma, FEATURE_FLAG.hasAiAccess)).toBe(true); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBeUndefined(); + }); + + postgresTest("writes nothing when there is nothing to write", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.mollifierEnabled]: true }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: [], + isManagedCloud: false, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(true); + }); + + postgresTest("submitted flags are upserted and omitted ones swept", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.mollifierEnabled]: true, + [FEATURE_FLAG.hasAiAccess]: true, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.mollifierEnabled]: false }, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(false); + expect(await readFlag(prisma, FEATURE_FLAG.hasAiAccess)).toBeUndefined(); + }); +}); diff --git a/apps/webapp/test/globalFlagChangeList.test.ts b/apps/webapp/test/globalFlagChangeList.test.ts new file mode 100644 index 00000000000..269774e28d3 --- /dev/null +++ b/apps/webapp/test/globalFlagChangeList.test.ts @@ -0,0 +1,142 @@ +// Two properties of a global flag save that the admin page had no way to state. +// +// 1. Unsetting a graced primary clears its server-computed stamps too. Those keys are locked, so +// they are absent from the page's editable set, and the confirm dialog listed one removal +// while three rows were deleted. +// 2. A save should write only the flags whose value actually changed. Writing every submitted +// flag costs one round trip each inside an interactive transaction. +import { describe, expect, it } from "vitest"; +import { FEATURE_FLAG, derivedFlagsClearedWith } from "~/v3/featureFlags"; +import { flagsNeedingWrite } from "~/v3/featureFlags.server"; +import { buildFlagChangeList } from "~/components/admin/flagChangeList"; + +const LOCKED = [ + FEATURE_FLAG.runOpsMintKindPrev, + FEATURE_FLAG.runOpsMintKindFlippedAt, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, +] as string[]; + +// Sorted, as the dialog sorts before calling: the builder preserves the order it is given. +const EDITABLE = [ + FEATURE_FLAG.runOpsMintKind, + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.mollifierEnabled, +].sort() as string[]; + +describe("derivedFlagsClearedWith", () => { + it("names the stamps that go with a graced primary", () => { + expect(derivedFlagsClearedWith(FEATURE_FLAG.runOpsMintKind)).toEqual([ + FEATURE_FLAG.runOpsMintKindPrev, + FEATURE_FLAG.runOpsMintKindFlippedAt, + ]); + expect(derivedFlagsClearedWith(FEATURE_FLAG.runOpsMintShardSet)).toEqual([ + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + ]); + }); + + it("names nothing for an ordinary flag, or for a stamp itself", () => { + expect(derivedFlagsClearedWith(FEATURE_FLAG.mollifierEnabled)).toEqual([]); + expect(derivedFlagsClearedWith(FEATURE_FLAG.runOpsMintKindPrev)).toEqual([]); + }); +}); + +describe("buildFlagChangeList — what the confirm dialog must show", () => { + it("lists an added, a changed and a removed flag", () => { + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: { mollifierEnabled: true, runOpsMintShardSet: "a" }, + storedValues: { mollifierEnabled: true, runOpsMintShardSet: "a" }, + newValues: { runOpsMintShardSet: "a,b", runOpsMintKind: "runOpsId" }, + }); + + expect(changes).toEqual([ + { key: FEATURE_FLAG.mollifierEnabled, type: "removed", oldVal: "true" }, + { key: FEATURE_FLAG.runOpsMintKind, type: "added", newVal: "runOpsId" }, + { key: FEATURE_FLAG.runOpsMintShardSet, type: "changed", oldVal: "a", newVal: "a,b" }, + ]); + }); + + it("discloses the stamps cleared alongside an unset graced primary", () => { + // Three rows are deleted, so three removals must be shown, not one. The caller filters + // locked keys OUT of initialValues, so the stamps are only visible in storedValues. + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: { runOpsMintShardSet: "a,b" }, + storedValues: { + runOpsMintShardSet: "a,b", + runOpsMintShardSetPrev: "a", + runOpsMintShardSetFlippedAt: "2026-08-24T00:00:00.000Z", + }, + newValues: {}, + }); + + expect(changes.map((c) => c.key)).toEqual([ + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + ]); + expect(changes.every((c) => c.type === "removed")).toBe(true); + }); + + it("does not disclose a stamp that is not stored", () => { + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: { runOpsMintShardSet: "a,b" }, + storedValues: { runOpsMintShardSet: "a,b" }, + newValues: {}, + }); + expect(changes.map((c) => c.key)).toEqual([FEATURE_FLAG.runOpsMintShardSet]); + }); + + it("does not disclose stamps when the primary is only CHANGED", () => { + // A change re-stamps rather than clearing, so nothing is removed. + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: { runOpsMintShardSet: "a" }, + storedValues: { runOpsMintShardSet: "a", runOpsMintShardSetPrev: "" }, + newValues: { runOpsMintShardSet: "a,b" }, + }); + expect(changes.map((c) => c.key)).toEqual([FEATURE_FLAG.runOpsMintShardSet]); + }); + + it("never lists a locked key on its own", () => { + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: {}, + storedValues: { runOpsMintShardSetPrev: "a" }, + newValues: {}, + }); + expect(changes).toEqual([]); + }); +}); + +describe("flagsNeedingWrite — one round trip per CHANGED flag, not per submitted flag", () => { + it("drops a submitted flag whose stored value already matches", () => { + const out = flagsNeedingWrite( + { mollifierEnabled: true, hasAiAccess: true }, + { mollifierEnabled: true, hasAiAccess: false } + ); + expect(out).toEqual({ hasAiAccess: true }); + }); + + it("keeps a flag that is absent from storage", () => { + expect(flagsNeedingWrite({ mollifierEnabled: true }, {})).toEqual({ mollifierEnabled: true }); + }); + + it("returns nothing when a save changes nothing", () => { + expect(flagsNeedingWrite({ mollifierEnabled: true }, { mollifierEnabled: true })).toEqual({}); + }); + + it("compares by value, not by reference, so a CSV rewritten the same way is not a write", () => { + expect(flagsNeedingWrite({ runOpsMintShardSet: "a,b" }, { runOpsMintShardSet: "a,b" })).toEqual( + {} + ); + }); +}); diff --git a/apps/webapp/test/globalFlagWriteRouting.test.ts b/apps/webapp/test/globalFlagWriteRouting.test.ts new file mode 100644 index 00000000000..3193919b5ca --- /dev/null +++ b/apps/webapp/test/globalFlagWriteRouting.test.ts @@ -0,0 +1,110 @@ +// Both global write routes used to carry their own copy of "which keys are graced" and "which +// keys are derived", so a new group needed an edit in three places and missing one meant an +// unstamped flip or a stamp taken from a request body. These tests cover the two helpers the +// routes now call. They do NOT reach a route: both actions sit behind admin auth, so that the +// routes call these helpers rather than their own copies is held by review, not by a test. +import { describe, expect, it } from "vitest"; +import { FEATURE_FLAG, lockedFlagsInPayload } from "~/v3/featureFlags"; +import { touchesGracedGroup, withoutDerivedKeys } from "~/v3/featureFlags.server"; + +describe("touchesGracedGroup — decides whether a save needs the stamped path", () => { + it("is true for a mint-kind change", () => { + expect(touchesGracedGroup({ [FEATURE_FLAG.runOpsMintKind]: "runOpsId" })).toBe(true); + }); + + it("is true for a shard-list change", () => { + expect(touchesGracedGroup({ [FEATURE_FLAG.runOpsMintShardSet]: "a,b" })).toBe(true); + }); + + it("is false for an ordinary flag, which writes directly", () => { + expect(touchesGracedGroup({ [FEATURE_FLAG.mollifierEnabled]: true })).toBe(false); + expect(touchesGracedGroup({})).toBe(false); + }); + + it("is false when only a DERIVED key is present", () => { + // A body carrying only a stamp changes no group. Treating it as a flip would let a caller + // reset a cutover clock without touching the value the clock dates. + expect(touchesGracedGroup({ [FEATURE_FLAG.runOpsMintKindPrev]: "cuid" })).toBe(false); + expect(touchesGracedGroup({ [FEATURE_FLAG.runOpsMintShardSetPrev]: "a" })).toBe(false); + }); + + it("recognises every graced primary the group table declares", () => { + const gracedPrimaries = [FEATURE_FLAG.runOpsMintKind, FEATURE_FLAG.runOpsMintShardSet]; + for (const key of gracedPrimaries) { + expect(touchesGracedGroup({ [key]: "x" })).toBe(true); + } + // Every key the strip removes belongs to a group whose primary is one of the above. + const derived = Object.keys( + withoutDerivedKeys({ + [FEATURE_FLAG.runOpsMintKindPrev]: "cuid", + [FEATURE_FLAG.runOpsMintKindFlippedAt]: "t", + [FEATURE_FLAG.runOpsMintShardSetPrev]: "a", + [FEATURE_FLAG.runOpsMintShardSetFlippedAt]: "t", + } as Record) + ); + expect(derived).toEqual([]); + }); +}); + +describe("withoutDerivedKeys — a stamp is never taken from a request body", () => { + it("strips both stamps and keeps everything else", () => { + const out = withoutDerivedKeys({ + [FEATURE_FLAG.runOpsMintKind]: "runOpsId", + [FEATURE_FLAG.runOpsMintKindPrev]: "spoofed", + [FEATURE_FLAG.runOpsMintKindFlippedAt]: "1999-01-01T00:00:00.000Z", + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + [FEATURE_FLAG.runOpsMintShardSetPrev]: "spoofed", + [FEATURE_FLAG.runOpsMintShardSetFlippedAt]: "1999-01-01T00:00:00.000Z", + [FEATURE_FLAG.mollifierEnabled]: true, + } as Record); + + expect(out).toEqual({ + [FEATURE_FLAG.runOpsMintKind]: "runOpsId", + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + [FEATURE_FLAG.mollifierEnabled]: true, + }); + }); + + it("does not mutate its input", () => { + const input = { [FEATURE_FLAG.runOpsMintKindPrev]: "cuid" } as Record; + withoutDerivedKeys(input); + expect(input[FEATURE_FLAG.runOpsMintKindPrev]).toBe("cuid"); + }); +}); + +describe("lockedFlagsInPayload — what the global page refuses", () => { + it("refuses a locked flag on managed cloud, where the page never offers one", () => { + const refused = lockedFlagsInPayload( + [FEATURE_FLAG.taskEventRepository, FEATURE_FLAG.mollifierEnabled], + true + ); + expect(refused).toEqual([FEATURE_FLAG.taskEventRepository]); + }); + + it("refuses the mint-shard pins, which are per-org only", () => { + expect(lockedFlagsInPayload([FEATURE_FLAG.runOpsMintShard], true)).toEqual([ + FEATURE_FLAG.runOpsMintShard, + ]); + expect(lockedFlagsInPayload([FEATURE_FLAG.runOpsMintShardEnvPins], true)).toEqual([ + FEATURE_FLAG.runOpsMintShardEnvPins, + ]); + }); + + it("refuses a grace stamp, which the server owns", () => { + expect(lockedFlagsInPayload([FEATURE_FLAG.runOpsMintShardSetFlippedAt], true)).toEqual([ + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + ]); + }); + + it("allows the shard list, because that is the page's ramp lever", () => { + expect(lockedFlagsInPayload([FEATURE_FLAG.runOpsMintShardSet], true)).toEqual([]); + }); + + it("refuses nothing when not managed cloud, where an admin may unlock and edit", () => { + expect(lockedFlagsInPayload([FEATURE_FLAG.taskEventRepository], false)).toEqual([]); + }); + + it("refuses nothing for an empty payload", () => { + expect(lockedFlagsInPayload([], true)).toEqual([]); + }); +}); diff --git a/apps/webapp/test/impersonationDisabled.test.ts b/apps/webapp/test/impersonationDisabled.test.ts new file mode 100644 index 00000000000..233ef533f47 --- /dev/null +++ b/apps/webapp/test/impersonationDisabled.test.ts @@ -0,0 +1,92 @@ +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { env } from "~/env.server"; +import { clearImpersonation, redirectWithImpersonation } from "~/models/admin.server"; +import { + commitImpersonationSession, + getImpersonationId, + getImpersonationState, + getRawImpersonationId, + setImpersonationId, +} from "~/services/impersonation.server"; + +vi.setConfig({ testTimeout: 30_000 }); + +function suffix() { + return Math.random().toString(36).slice(2, 10); +} + +// ADMIN_DASHBOARD_ENABLED=false: starting 404s, cookies resolve to nothing, +// stopping still works so lingering sessions can be terminated. +describe("impersonation disabled", () => { + postgresTest("the flag defaults to enabled", async () => { + // Flipping the default would kill the admin dashboard on every existing deployment. + expect(env.ADMIN_DASHBOARD_ENABLED).toBe(true); + }); + + postgresTest("starting impersonation 404s and cookies are inert", async ({ prisma }) => { + const admin = await prisma.user.create({ + data: { + email: `admin-${suffix()}@test.local`, + authenticationMethod: "MAGIC_LINK", + admin: true, + }, + }); + const target = await prisma.user.create({ + data: { + email: `target-${suffix()}@test.local`, + authenticationMethod: "MAGIC_LINK", + confirmedBasicDetails: true, + }, + }); + + // A cookie minted while the flag was on, e.g. carried over or replayed. + const session = await setImpersonationId(target.id, new Request("http://localhost:3030/admin")); + const cookie = await commitImpersonationSession(session); + const requestWithCookie = () => + new Request("http://localhost:3030/", { headers: { Cookie: cookie } }); + + expect(await getImpersonationId(requestWithCookie())).toBe(target.id); + // resolvedUserId must be the impersonated id or the state is false even + // with the flag on, making the disabled assertion below vacuous. + const enabledState = await getImpersonationState(requestWithCookie(), target.id); + expect(enabledState.isImpersonating).toBe(true); + + const original = env.ADMIN_DASHBOARD_ENABLED; + // @ts-expect-error deliberately flipping the parsed env for the test + env.ADMIN_DASHBOARD_ENABLED = false; + try { + await expect( + redirectWithImpersonation( + new Request("http://localhost:3030/admin/impersonate", { method: "POST" }), + target.id, + "/", + { id: admin.id, admin: true }, + prisma + ) + ).rejects.toMatchObject({ status: 404 }); + + // No audit log: the gate fires before anything is recorded. + expect(await prisma.impersonationAuditLog.count()).toBe(0); + + expect(await getImpersonationId(requestWithCookie())).toBeUndefined(); + const disabledState = await getImpersonationState(requestWithCookie(), target.id); + expect(disabledState.isImpersonating).toBe(false); + + // The ungated reader still sees the cookie (stop/scrub paths need it). + expect(await getRawImpersonationId(requestWithCookie())).toBe(target.id); + + // Stopping works with the flag off and clears the cookie. + const response = await clearImpersonation(requestWithCookie(), "/"); + const setCookie = response.headers.get("set-cookie"); + expect(setCookie).toContain("__impersonate="); + const clearedRequest = new Request("http://localhost:3030/", { + headers: { Cookie: setCookie!.split(";")[0] }, + }); + expect(await getRawImpersonationId(clearedRequest)).toBeUndefined(); + } finally { + // @ts-expect-error restore the parsed env + env.ADMIN_DASHBOARD_ENABLED = original; + } + }); +}); diff --git a/apps/webapp/test/readRunForEvent.replicaLag.test.ts b/apps/webapp/test/readRunForEvent.replicaLag.test.ts index 877f920e40f..9817a1b6e55 100644 --- a/apps/webapp/test/readRunForEvent.replicaLag.test.ts +++ b/apps/webapp/test/readRunForEvent.replicaLag.test.ts @@ -195,4 +195,76 @@ describe("readRunForEvent tolerates replica lag on its event-enrichment read", ( expect(onPrimary.friendlyId).toBe("run_rrfe_missing"); } ); + + // (c) SPLIT MODE, the gen-1 run fast path. A cuid run id classifies legacy, and there is no cuid + // run migration, so the new-store probe cannot find it. readRunForEvent declares idKind "run", + // which reads the legacy replica ONLY. The observable difference is the number of reads: one on + // the fast path, two on the old new-then-legacy pair probe. Counted by delegating through the + // real store rather than replacing it. + containerTest( + "readRunForEvent takes ONE read for a cuid run id under split, not a new-then-legacy pair", + async ({ prisma }) => { + const { organization, project, environment } = await seedEnvironment(prisma, "rrfe_split"); + + const runId = "d".repeat(25); // cuid-shaped -> classifies legacy + const friendlyId = "run_rrfe_split"; + + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: "trace_split", + spanId: "span_split", + queue: "task/my-task", + runtimeEnvironmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + environmentType: "DEVELOPMENT", + isTest: false, + taskEventStore: "taskEvent", + }, + }); + + const realStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma as never }); + let findRunCalls = 0; + const countingStore = new Proxy(realStore, { + get(target, prop, receiver) { + if (prop === "findRun") { + return (...args: unknown[]) => { + findRunCalls += 1; + return (target.findRun as (...a: unknown[]) => unknown)(...args); + }; + } + return Reflect.get(target, prop, receiver); + }, + }); + + // The new side MUST miss for the two arms to be distinguishable: a pair probe that finds the + // row on its first read short-circuits and looks identical to the fast path. `missing` makes + // the new-store read return nothing, exactly as it would for a legacy-resident run. + const missingOnNew = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + + const deps: EventReadDeps = { + store: countingStore as never, + newReplica: missingOnNew.client as never, + legacyReplica: prisma as never, + splitEnabled: true, + }; + + const run = await readRunForEvent(runId, environment.id, EVENT_SELECT, deps); + + // The run still resolves — the fast path must not cost the read. + expect(run).not.toBeNull(); + expect(run!.id).toBe(runId); + expect(run!.friendlyId).toBe(friendlyId); + + // ONE read. Two would mean the new store was probed first, which is the arm this removes. + expect(findRunCalls).toBe(1); + } + ); }); diff --git a/apps/webapp/test/realtimeServices.replicaLag.test.ts b/apps/webapp/test/realtimeServices.replicaLag.test.ts index 6a302dcfd9c..0de58c0403d 100644 --- a/apps/webapp/test/realtimeServices.replicaLag.test.ts +++ b/apps/webapp/test/realtimeServices.replicaLag.test.ts @@ -395,7 +395,7 @@ describe("realtime-svc — replica-lag guards", () => { environmentType: "DEVELOPMENT", organizationId: seed.organization.id, taskIdentifier: "my-task", - triggerConfig: { basePayload: {} }, + triggerConfig: { basePayload: {}, ttl: "2m" }, currentRunId: callingRunId, currentRunVersion: 0, streamBasinName: "session-pinned-basin", @@ -429,6 +429,8 @@ describe("realtime-svc — replica-lag guards", () => { // previousRunId forwarded to the triggered run is the calling run's cuid (documented fallback). expect(triggerState.calls).toHaveLength(1); expect(triggerState.calls[0]!.body.payload.previousRunId).toBe(callingRunId); + // The session's ttl reaches the trigger options, so an undequeued run expires. + expect(triggerState.calls[0]!.body.options.ttl).toBe("2m"); expect(versionCalls.at(-1)).toEqual({ requested: "v2", basin: null }); expect(replica.wasHit("taskRun")).toBe(true); diff --git a/apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts b/apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts index c0b627262f7..09c3327ab57 100644 --- a/apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts +++ b/apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts @@ -1,7 +1,7 @@ import { heteroRunOpsPostgresTest, postgresTest } from "@internal/testcontainers"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import type { PrismaClient } from "@trigger.dev/database"; -import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { generateRunOpsId, generateRunOpsIdV2 } from "@trigger.dev/core/v3/isomorphic"; import { describe, expect, vi } from "vitest"; import type { PrismaReplicaClient } from "~/db.server"; import { resolveWaitpointThroughReadThrough } from "~/runEngine/concerns/resolveWaitpointThroughReadThrough.server"; @@ -286,4 +286,105 @@ describe("resolveWaitpointThroughReadThrough (hetero PG14 legacy + dedicated run expect(legacy.calls.length).toBe(0); } ); + + heteroRunOpsPostgresTest( + "gen-2 waitpoint resolves on its OWN shard replica; the gen-1 new store is never read", + async ({ prisma17, prisma14 }) => { + const id = generateRunOpsIdV2("a"); + const environmentId = generateRunOpsId(); + const projectId = generateRunOpsId(); + const seeded = await seedWaitpoint(prisma17, id, { id: environmentId, projectId }); + + // The gen-1 new store and the legacy replica are both forbidden: a gen-2 id must + // take one read on its shard and probe nothing else. + const newClient = recording(prisma14, { forbidden: true }); + const legacyReplica = recording(prisma14, { forbidden: true }); + const shardReplica = recording(prisma17); + + const result = await resolveWaitpointThroughReadThrough({ + waitpointId: id, + environmentId, + read: read(id, environmentId), + deps: { + splitEnabled: true, + newClient: newClient.handle, + legacyReplica: legacyReplica.handle, + newPrimary: newClient.handle, + shardReplicas: new Map([["a", shardReplica.handle]]), + }, + }); + + expect(result).not.toBeNull(); + expect(result!.id).toBe(seeded.id); + expect(shardReplica.calls.length).toBe(1); + expect(newClient.calls.length).toBe(0); + expect(legacyReplica.calls.length).toBe(0); + } + ); + + heteroRunOpsPostgresTest( + "a gen-2 waitpoint missing its shard REPLICA falls back to that shard's WRITER, not the gen-1 new writer", + async ({ prisma17, prisma14 }) => { + // Read-your-writes: a token completed immediately after mint may not have replicated. + // The fallback must read the shard's own primary. Reading the gen-1 new writer would + // query the wrong database and return null. + const id = generateRunOpsIdV2("a"); + const environmentId = generateRunOpsId(); + const projectId = generateRunOpsId(); + const seeded = await seedWaitpoint(prisma17, id, { id: environmentId, projectId }); + + const shardReplica = recording(prisma14); // lags: does not have the row + const shardWriter = recording(prisma17); // has the row + const forbiddenNewPrimary = recording(prisma17, { forbidden: true }); + + const result = await resolveWaitpointThroughReadThrough({ + waitpointId: id, + environmentId, + read: read(id, environmentId), + deps: { + splitEnabled: true, + newClient: recording(prisma14, { forbidden: true }).handle, + legacyReplica: recording(prisma14, { forbidden: true }).handle, + newPrimary: forbiddenNewPrimary.handle, + shardReplicas: new Map([["a", shardReplica.handle]]), + shardWriters: new Map([["a", shardWriter.handle]]), + }, + }); + + expect(result).not.toBeNull(); + expect(result!.id).toBe(seeded.id); + expect(shardReplica.calls.length).toBe(1); + expect(shardWriter.calls.length).toBe(1); + expect(forbiddenNewPrimary.calls.length).toBe(0); + } + ); + + heteroRunOpsPostgresTest( + "a gen-2 waitpoint with no configured shard writer returns null instead of reading a wrong database", + async ({ prisma17, prisma14 }) => { + const id = generateRunOpsIdV2("a"); + const environmentId = generateRunOpsId(); + const projectId = generateRunOpsId(); + await seedWaitpoint(prisma17, id, { id: environmentId, projectId }); + + const forbiddenNewPrimary = recording(prisma17, { forbidden: true }); + + const result = await resolveWaitpointThroughReadThrough({ + waitpointId: id, + environmentId, + read: read(id, environmentId), + deps: { + splitEnabled: true, + newClient: recording(prisma14, { forbidden: true }).handle, + legacyReplica: recording(prisma14, { forbidden: true }).handle, + newPrimary: forbiddenNewPrimary.handle, + shardReplicas: new Map([["a", recording(prisma14).handle]]), + shardWriters: new Map(), + }, + }); + + expect(result).toBeNull(); + expect(forbiddenNewPrimary.calls.length).toBe(0); + } + ); }); diff --git a/apps/webapp/test/runOpsDbTopology.test.ts b/apps/webapp/test/runOpsDbTopology.test.ts index 8890fdbb662..f2bcc0bf5ea 100644 --- a/apps/webapp/test/runOpsDbTopology.test.ts +++ b/apps/webapp/test/runOpsDbTopology.test.ts @@ -142,6 +142,75 @@ describe("selectRunOpsTopology (pure)", () => { expect(topo.legacyRunOps.replica).toBe(legacyWriter); expect(buildLegacyReplica).not.toHaveBeenCalled(); }); + + const baseSplit = { + splitEnabled: true, + legacyUrl: "postgres://legacy", + newUrl: "postgres://new", + }; + const baseBuilders = () => ({ + controlPlane: cp, + buildNewWriter: vi.fn().mockReturnValue({ tag: "nw" } as any), + buildNewReplica: vi.fn().mockReturnValue({ tag: "nr" } as any), + buildLegacyWriter: vi.fn().mockReturnValue({ tag: "lw" } as any), + buildLegacyReplica: vi.fn().mockReturnValue({ tag: "lr" } as any), + }); + + it("no descriptors: the shards map is empty", () => { + const topo = selectRunOpsTopology(baseSplit, baseBuilders()); + expect(topo.shards.size).toBe(0); + }); + + it("two descriptors: two shard client pairs, each built once", () => { + const buildShardWriter = vi.fn((s: any) => ({ tag: `w:${s.key}` }) as any); + const buildShardReplica = vi.fn((s: any) => ({ tag: `r:${s.key}` }) as any); + const topo = selectRunOpsTopology( + { + ...baseSplit, + shards: [ + { key: "a", url: "postgres://a", replicaUrl: "postgres://a-r" }, + { key: "b", url: "postgres://b" }, + ], + }, + { ...baseBuilders(), buildShardWriter, buildShardReplica } + ); + expect(topo.shards.size).toBe(2); + expect(topo.shards.get("a")!.writer).toEqual({ tag: "w:a" }); + // b has no replicaUrl, so its replica falls back to its writer (buildShardReplica not called for b). + expect(topo.shards.get("b")!.replica).toEqual({ tag: "w:b" }); + expect(buildShardWriter).toHaveBeenCalledTimes(2); + expect(buildShardReplica).toHaveBeenCalledTimes(1); + }); + + it("an alias descriptor reuses newRunOps by reference and calls no shard builder", () => { + const buildShardWriter = vi.fn(); + const buildShardReplica = vi.fn(); + const topo = selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a", aliasOf: "new" }] }, + { ...baseBuilders(), buildShardWriter, buildShardReplica } + ); + expect(topo.shards.get("a")).toBe(topo.newRunOps); + expect(buildShardWriter).not.toHaveBeenCalled(); + expect(buildShardReplica).not.toHaveBeenCalled(); + }); + + it("throws when a non-aliased shard has no url (guards the shard.url non-null assertion)", () => { + expect(() => + selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a" }] }, + { ...baseBuilders(), buildShardWriter: vi.fn(), buildShardReplica: vi.fn() } + ) + ).toThrow(/shard "a" needs a url/); + }); + + it("throws when a non-aliased shard is configured but the shard builders are absent", () => { + expect(() => + selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a", url: "postgres://a" }] }, + baseBuilders() + ) + ).toThrow(/shard "a" needs a url and shard builders/); + }); }); describe("sameDatabaseTarget", () => { diff --git a/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts b/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts index 1492fd02c43..b3acd7e45eb 100644 --- a/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts +++ b/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts @@ -5,7 +5,7 @@ import type { PrismaClient } from "@trigger.dev/database"; import { postgresTest } from "@internal/testcontainers"; import { describe, expect, vi } from "vitest"; import { FEATURE_FLAG } from "~/v3/featureFlags"; -import { applyGlobalMintKindFlip, makeSetMultipleFlags } from "~/v3/featureFlags.server"; +import { applyGlobalGracedFlips, makeSetMultipleFlags } from "~/v3/featureFlags.server"; vi.setConfig({ testTimeout: 60_000 }); @@ -25,11 +25,11 @@ async function readGlobalMint(prisma: PrismaClient): Promise { +describe("applyGlobalGracedFlips — transactional stamp + serialized flips", () => { postgresTest("a genuine global flip stamps prev + flippedAt", async ({ prisma }) => { await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintKind]: "cuid" }); - await applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000); + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000); const m = await readGlobalMint(prisma); expect(m[FEATURE_FLAG.runOpsMintKind]).toBe("runOpsId"); @@ -44,7 +44,7 @@ describe("applyGlobalMintKindFlip — transactional stamp + serialized flips", ( await Promise.all( Array.from({ length: 8 }, () => - applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000) + applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000) ) ); diff --git a/apps/webapp/test/runOpsMintShardFlags.test.ts b/apps/webapp/test/runOpsMintShardFlags.test.ts new file mode 100644 index 00000000000..98b76eb19c8 --- /dev/null +++ b/apps/webapp/test/runOpsMintShardFlags.test.ts @@ -0,0 +1,118 @@ +// The mint-shard flags carry two safety claims that only the catalog can enforce: a bad value is +// rejected at WRITE (so no unroutable key and no silently-unpinned environment can ever be +// stored), and each key is locked at the scope its resolver does not read. Pure, no containers. +import { describe, expect, it } from "vitest"; +import { + FEATURE_FLAG, + FeatureFlagCatalog, + GLOBAL_LOCKED_FLAGS, + ORG_LOCKED_FLAGS, + validateFeatureFlagValue, +} from "~/v3/featureFlags"; + +describe("runOpsMintShard — the per-org pin", () => { + const key = FEATURE_FLAG.runOpsMintShard; + + it("accepts every legal shard key", () => { + for (const c of "abcdefghijklmnopqrstuvwxyz0123456789") { + expect(validateFeatureFlagValue(key, c).success).toBe(true); + } + }); + + it('accepts "new", which holds an org on gen-1', () => { + expect(validateFeatureFlagValue(key, "new").success).toBe(true); + }); + + it("rejects a value that could never be stamped into an id", () => { + for (const bad of ["A", "ab", "", "-", "legacy", " a", "a,b"]) { + expect(validateFeatureFlagValue(key, bad).success).toBe(false); + } + }); +}); + +describe("runOpsMintShardEnvPins — the per-environment pins", () => { + const key = FEATURE_FLAG.runOpsMintShardEnvPins; + + it("accepts a map of environment id to shard key", () => { + expect( + validateFeatureFlagValue(key, JSON.stringify({ env_1: "a", env_2: "new" })).success + ).toBe(true); + expect(validateFeatureFlagValue(key, "{}").success).toBe(true); + }); + + it("rejects a blob that is not JSON, so a typo cannot silently un-pin every environment", () => { + for (const bad of ["{not json", "", "null", "[]", '"a"', "42"]) { + expect(validateFeatureFlagValue(key, bad).success).toBe(false); + } + }); + + it("rejects a map whose value is not a legal pin", () => { + for (const bad of [{ env_1: "AB" }, { env_1: "legacy" }, { env_1: 1 }, { env_1: "" }]) { + expect(validateFeatureFlagValue(key, JSON.stringify(bad)).success).toBe(false); + } + }); +}); + +describe("runOpsMintShardSet — the active list", () => { + const key = FEATURE_FLAG.runOpsMintShardSet; + + it("accepts an empty list and a CSV of legal keys", () => { + expect(validateFeatureFlagValue(key, "").success).toBe(true); + expect(validateFeatureFlagValue(key, "a").success).toBe(true); + expect(validateFeatureFlagValue(key, "a,b, c").success).toBe(true); + }); + + it("rejects a CSV holding a key that cannot be routed", () => { + for (const bad of ["A", "ab", "a,B", "a,legacy", "a,new", "a;b"]) { + expect(validateFeatureFlagValue(key, bad).success).toBe(false); + } + }); +}); + +describe("runOpsMintShardOverride — the complete-cutover lever", () => { + const key = FEATURE_FLAG.runOpsMintShardOverride; + + it("accepts a shard key and accepts new", () => { + expect(validateFeatureFlagValue(key, "a").success).toBe(true); + expect(validateFeatureFlagValue(key, "new").success).toBe(true); + }); + + it("rejects anything that is not a single legal key", () => { + for (const bad of ["A", "ab", "", "legacy", "a,b"]) { + expect(validateFeatureFlagValue(key, bad).success).toBe(false); + } + }); +}); + +describe("scope locks match what each resolver actually reads", () => { + it("locks the pins globally, because the resolver reads them from the org blob only", () => { + expect(GLOBAL_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShard); + expect(GLOBAL_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardEnvPins); + }); + + it("locks the list and the override per-org, because both are deployment-wide", () => { + expect(ORG_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardSet); + expect(ORG_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardSetPrev); + expect(ORG_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardSetFlippedAt); + // An org that could override the cutover lever would defeat its purpose. + expect(ORG_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardOverride); + }); + + it("keeps the pins settable per-org, which is the canary lever", () => { + expect(ORG_LOCKED_FLAGS).not.toContain(FEATURE_FLAG.runOpsMintShard); + expect(ORG_LOCKED_FLAGS).not.toContain(FEATURE_FLAG.runOpsMintShardEnvPins); + }); + + it("registers every new key in the catalog, so the admin pages render it", () => { + for (const key of [ + FEATURE_FLAG.runOpsMintShard, + FEATURE_FLAG.runOpsMintShardEnvPins, + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + FEATURE_FLAG.runOpsMintShardOverride, + ]) { + expect(FeatureFlagCatalog).toHaveProperty(key); + } + }); +}); diff --git a/apps/webapp/test/runOpsMintShardSetFlip.test.ts b/apps/webapp/test/runOpsMintShardSetFlip.test.ts new file mode 100644 index 00000000000..6dc126fc0bd --- /dev/null +++ b/apps/webapp/test/runOpsMintShardSetFlip.test.ts @@ -0,0 +1,279 @@ +// The active mint-shard list lives in the control-plane database, not in the environment: a +// rolling deploy runs two environment values at once for hours, so only a shared row lets every +// pod agree on one list. A change must therefore read -> stamp -> write under an advisory lock, +// and must never be writable as a bare upsert from a request body. Real testcontainers Postgres. +import type { PrismaClient } from "@trigger.dev/database"; +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { FEATURE_FLAG, type FeatureFlagKey } from "~/v3/featureFlags"; +import { + applyGlobalGracedFlips, + makeSetMultipleFlags, + replaceGlobalFeatureFlags, +} from "~/v3/featureFlags.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +const SET_KEYS: FeatureFlagKey[] = [ + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, +]; + +const MINT_KIND_KEYS: FeatureFlagKey[] = [ + FEATURE_FLAG.runOpsMintKind, + FEATURE_FLAG.runOpsMintKindPrev, + FEATURE_FLAG.runOpsMintKindFlippedAt, +]; + +const CATALOG_KEYS: FeatureFlagKey[] = [ + ...SET_KEYS, + ...MINT_KIND_KEYS, + FEATURE_FLAG.mollifierEnabled, +]; + +// Self-hosted with the lock left on: locked flags survive omission, everything else sweeps. +const SELF_HOSTED = { isManagedCloud: false, unlockLockedFlags: false } as const; + +async function readFlags( + prisma: PrismaClient, + keys: FeatureFlagKey[] +): Promise> { + const rows = await prisma.featureFlag.findMany({ + where: { key: { in: keys } }, + select: { key: true, value: true }, + }); + const m: Record = {}; + for (const row of rows) m[row.key] = row.value; + return m; +} + +describe("applyGlobalGracedFlips — the shard-set list is stamped, not bare-written", () => { + postgresTest("a genuine list change stamps prev + flippedAt", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintShardSet]: "a" }); + + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, 60_000); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b"); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe("a"); + expect(typeof m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe("string"); + }); + + postgresTest("a first activation stamps an empty prev, which graces it", async ({ prisma }) => { + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a" }, 60_000); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe(""); + expect(typeof m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe("string"); + }); + + postgresTest( + "a reordered list is not a change, so the clock is not reset", + async ({ prisma }) => { + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, 60_000); + const first = await readFlags(prisma, SET_KEYS); + + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "b,a" }, 60_000); + const second = await readFlags(prisma, SET_KEYS); + + expect(second[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe( + first[FEATURE_FLAG.runOpsMintShardSetFlippedAt] + ); + } + ); + + postgresTest("both graced groups stamp in ONE save", async ({ prisma }) => { + // A save that flips the kind and the list must not stamp one and lose the other. + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.runOpsMintKind]: "cuid", + [FEATURE_FLAG.runOpsMintShardSet]: "a", + }); + + await applyGlobalGracedFlips( + prisma, + { + [FEATURE_FLAG.runOpsMintKind]: "runOpsId", + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + }, + 60_000 + ); + + const m = await readFlags(prisma, [...SET_KEYS, ...MINT_KIND_KEYS]); + expect(m[FEATURE_FLAG.runOpsMintKindPrev]).toBe("cuid"); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe("a"); + expect(typeof m[FEATURE_FLAG.runOpsMintKindFlippedAt]).toBe("string"); + expect(typeof m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe("string"); + }); + + postgresTest("concurrent list changes do not interleave", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintShardSet]: "a" }); + + await Promise.all([ + applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, 60_000), + applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a,c" }, 60_000), + ]); + + const m = await readFlags(prisma, SET_KEYS); + const set = m[FEATURE_FLAG.runOpsMintShardSet]; + const prev = m[FEATURE_FLAG.runOpsMintShardSetPrev]; + + // The pair must be a consistent history, not a mix of the two writers. The winner's set is + // one of the two, and prev is what the OTHER writer left behind: either the original "a", or + // the loser's set when the loser committed first. "a,b" beside prev "a,b" would mean one + // writer read its own uncommitted state, and prev naming the winner's own set is incoherent. + expect(["a,b", "a,c"]).toContain(set); + expect(["a", "a,b", "a,c"]).toContain(prev); + expect(prev).not.toBe(set); + expect(typeof m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe("string"); + }); +}); + +describe("replaceGlobalFeatureFlags — the admin page cannot bypass the stamp", () => { + postgresTest("a list change through the page is stamped", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintShardSet]: "a" }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b"); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe("a"); + }); + + postgresTest("a body-supplied stamp is ignored and recomputed", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintShardSet]: "a" }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + [FEATURE_FLAG.runOpsMintShardSetPrev]: "zzz", + [FEATURE_FLAG.runOpsMintShardSetFlippedAt]: "1999-01-01T00:00:00.000Z", + }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe("a"); + expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).not.toBe("1999-01-01T00:00:00.000Z"); + }); + + postgresTest("a co-submitted flag does not disturb a resubmitted list", async ({ prisma }) => { + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + const first = await readFlags(prisma, SET_KEYS); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + [FEATURE_FLAG.mollifierEnabled]: true, + }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b"); + // Resubmitting the same list is not a flip, so the cutover clock is not reset. + expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe( + first[FEATURE_FLAG.runOpsMintShardSetFlippedAt] + ); + }); + + postgresTest( + "omitting the list DELETES it, so unset still turns minting off", + async ({ prisma }) => { + // The admin page's unset button omits the key. If the save skipped it, unset would be a + // silent no-op and gen-2 minting would stay armed. + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + // The stamp goes with it: a stamp without its list keeps being served for the whole window. + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBeUndefined(); + } + ); + + postgresTest("omitting the mint kind still deletes its trio", async ({ prisma }) => { + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, MINT_KIND_KEYS); + expect(m[FEATURE_FLAG.runOpsMintKind]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintKindPrev]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintKindFlippedAt]).toBeUndefined(); + }); + + postgresTest("unlocking does not orphan a stamp from its list", async ({ prisma }) => { + // With the lock off, a locked key may be swept. The stamps are locked, so this is the case + // where they could be deleted while the list survives, which would keep serving prevSet. + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: true, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b"); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBeDefined(); + expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBeDefined(); + }); + + postgresTest("an ordinary flag keeps replace semantics", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.mollifierEnabled]: true }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, [FEATURE_FLAG.mollifierEnabled]); + expect(m[FEATURE_FLAG.mollifierEnabled]).toBeUndefined(); + }); +}); diff --git a/apps/webapp/test/runOpsPoolKnobs.test.ts b/apps/webapp/test/runOpsPoolKnobs.test.ts new file mode 100644 index 00000000000..38353682cb2 --- /dev/null +++ b/apps/webapp/test/runOpsPoolKnobs.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { applyPoolKnobOverrides, type ResolvedPoolKnobs } from "~/v3/runOpsPoolKnobs.server"; + +// Literal defaults, so the assertions lock the override logic against fixed values rather than +// against the same env expression the implementation reads. No env import (webapp test rule). +const DEFAULTS: ResolvedPoolKnobs = { + writerPoolTimeout: 10, + writerConnectionTimeout: 20, + writerDriverAdapter: false, + connectionLimit: 30, + replicaConnectionLimit: 40, + replicaPoolTimeout: 50, + replicaConnectionTimeout: 60, + replicaDriverAdapter: false, +}; + +describe("applyPoolKnobOverrides", () => { + it("returns the defaults verbatim when no descriptor knobs are given", () => { + expect(applyPoolKnobOverrides(DEFAULTS)).toEqual(DEFAULTS); + expect(applyPoolKnobOverrides(DEFAULTS, {})).toEqual(DEFAULTS); + }); + + it("overrides only the fields the descriptor sets", () => { + const result = applyPoolKnobOverrides(DEFAULTS, { + connectionLimit: 999, + writerDriverAdapter: true, + replicaPoolTimeout: 555, + }); + expect(result.connectionLimit).toBe(999); + expect(result.writerDriverAdapter).toBe(true); + expect(result.replicaPoolTimeout).toBe(555); + // Untouched fields keep the defaults. + expect(result.writerPoolTimeout).toBe(10); + expect(result.replicaConnectionLimit).toBe(40); + expect(result.replicaDriverAdapter).toBe(false); + }); + + it("does not read the transaction knobs off the descriptor", () => { + const result = applyPoolKnobOverrides(DEFAULTS, { transactionMaxWaitMs: 1234 }); + expect(result).toEqual(DEFAULTS); + }); +}); diff --git a/apps/webapp/test/runOpsShardBootTable.test.ts b/apps/webapp/test/runOpsShardBootTable.test.ts new file mode 100644 index 00000000000..3031608d701 --- /dev/null +++ b/apps/webapp/test/runOpsShardBootTable.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { runOpsAddressFingerprint, buildRunOpsShardTable } from "~/v3/runOpsShardTable"; + +describe("runOpsAddressFingerprint", () => { + it("returns host:port/db with no username or query params", () => { + const fp = runOpsAddressFingerprint( + "postgres://user:pw@host.example:5433/mydb?schema=public&pool_timeout=20" + ); + expect(fp).toBe("host.example:5433/mydb"); + expect(fp).not.toContain("user"); + expect(fp).not.toContain("pool_timeout"); + }); + it("defaults the port to 5432", () => { + expect(runOpsAddressFingerprint("postgres://h/db")).toBe("h:5432/db"); + }); + it("returns a marker on unparseable input rather than throwing", () => { + expect(runOpsAddressFingerprint("not a url")).toBe("unparseable"); + }); +}); + +describe("buildRunOpsShardTable", () => { + it("one row per descriptor, with key, fingerprint, and role", () => { + const rows = buildRunOpsShardTable([ + { key: "a", url: "postgres://user:pw@h/adb?schema=public" }, + { key: "b", aliasOf: "new" }, + ]); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual({ key: "a", fingerprint: "h:5432/adb", role: "shard" }); + expect(rows[1]).toEqual({ key: "b", fingerprint: "alias(new)", role: "alias(new)" }); + }); + it("is empty for an empty descriptor list", () => { + expect(buildRunOpsShardTable([])).toEqual([]); + }); +}); diff --git a/apps/webapp/test/runOpsShardDsns.test.ts b/apps/webapp/test/runOpsShardDsns.test.ts new file mode 100644 index 00000000000..a2cef335b1d --- /dev/null +++ b/apps/webapp/test/runOpsShardDsns.test.ts @@ -0,0 +1,191 @@ +import { describe, expect, it } from "vitest"; +// The entrypoint calls this file with plain `node`, so it takes no path alias and no bundler. +import { shardMigrationDsns } from "../../../docker/scripts/runOpsShardDsns.mjs"; + +const shardA = { + key: "a", + region: "us-east-1", + url: "postgres://h/a", + replication: { slotName: "sa", publicationName: "pa", originGeneration: 2 }, +}; + +describe("shardMigrationDsns", () => { + it("returns nothing when the variable is unset", () => { + expect(shardMigrationDsns(undefined)).toEqual([]); + }); + + it("returns nothing when the variable is blank", () => { + expect(shardMigrationDsns(" ")).toEqual([]); + }); + + it("returns nothing for an empty array", () => { + expect(shardMigrationDsns("[]")).toEqual([]); + }); + + it("throws on invalid JSON, so the entrypoint stops before it migrates", () => { + expect(() => shardMigrationDsns("{not json")).toThrow(/not valid JSON/i); + }); + + it("throws when the value is JSON but not an array", () => { + expect(() => shardMigrationDsns('{"key":"a"}')).toThrow(/not a JSON array/i); + }); + + it("returns the url of a shard that owns its own database", () => { + expect(shardMigrationDsns(JSON.stringify([shardA]))).toEqual(["postgres://h/a"]); + }); + + it("prefers directUrl over url, because migrations must not go through a pooler", () => { + const withDirect = { ...shardA, directUrl: "postgres://h/a-direct" }; + expect(shardMigrationDsns(JSON.stringify([withDirect]))).toEqual(["postgres://h/a-direct"]); + }); + + it("skips an aliased shard, because its target's invocation already migrates it", () => { + const aliased = { key: "z", region: "us-east-1", aliasOf: "new" }; + expect(shardMigrationDsns(JSON.stringify([shardA, aliased]))).toEqual(["postgres://h/a"]); + }); + + it("rejects a descriptor with neither url nor aliasOf", () => { + const noUrl = { key: "b", region: "us-east-1" }; + expect(() => shardMigrationDsns(JSON.stringify([shardA, noUrl]))).toThrow(/exactly one/i); + }); + + it("keeps declaration order across several shards", () => { + const shardB = { ...shardA, key: "b", url: "postgres://h/b" }; + expect(shardMigrationDsns(JSON.stringify([shardA, shardB]))).toEqual([ + "postgres://h/a", + "postgres://h/b", + ]); + }); + + it("throws when an entry is not an object", () => { + expect(() => shardMigrationDsns('["postgres://h/a"]')).toThrow(/not an object/i); + }); +}); + +describe("shardMigrationDsns line protocol", () => { + // One DSN per line is the protocol with entrypoint.sh, so a line break would split one DSN into + // two bogus ones. The URL parser strips ASCII line breaks, so nothing upstream rejects this. + it("throws when a DSN holds a line break", () => { + const bad = { ...shardA, url: "postgres://h/a\npostgres://evil/db" }; + expect(() => shardMigrationDsns(JSON.stringify([bad]))).toThrow(/line break/i); + }); + + it("throws when a directUrl holds a carriage return", () => { + const bad = { ...shardA, directUrl: "postgres://h/a\rx" }; + expect(() => shardMigrationDsns(JSON.stringify([bad]))).toThrow(/line break/i); + }); +}); + +// The script and the boot schema validate the same variable, so they must agree. If the script is +// laxer, the entrypoint migrates a database and the application then refuses to start, which breaks +// the fail-before-migration contract the entrypoint exists to hold. +describe("shardMigrationDsns matches the descriptor contract", () => { + it("rejects an aliasOf value the schema does not allow", () => { + const bad = [{ key: "a", region: "r", url: "postgres://h/a", aliasOf: "other" }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/aliasOf/i); + }); + + it("rejects a shard that owns its database but declares no replication", () => { + const bad = [{ key: "b", region: "r", url: "postgres://h/b" }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/replication/i); + }); + + it("rejects a descriptor that sets both url and aliasOf", () => { + const bad = [{ key: "c", region: "r", url: "postgres://h/c", aliasOf: "new" }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/exactly one/i); + }); + + it("rejects a descriptor that sets neither url nor aliasOf", () => { + const bad = [{ key: "d", region: "r" }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/exactly one/i); + }); + + it("still accepts a valid aliased descriptor and skips it", () => { + const ok = [{ key: "z", region: "r", aliasOf: "new" }]; + expect(shardMigrationDsns(JSON.stringify(ok))).toEqual([]); + }); + + it("still accepts a valid owning descriptor", () => { + expect(shardMigrationDsns(JSON.stringify([shardA]))).toEqual(["postgres://h/a"]); + }); +}); + +// The boot schema URL-validates url, replicaUrl and directUrl with isValidDatabaseUrl. The script +// must not be laxer, or a valid descriptor ahead of an invalid one gets its database migrated before +// the configuration is rejected. +describe("shardMigrationDsns validates descriptor values", () => { + const rep = { slotName: "s", publicationName: "p", originGeneration: 2 }; + + it("rejects a url that is not a parseable URL", () => { + const bad = [{ key: "a", region: "r", url: "not a URL", replication: rep }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/url/i); + }); + + it("rejects a directUrl that is not a parseable URL", () => { + const bad = [ + { key: "a", region: "r", url: "postgres://h/a", directUrl: "nope", replication: rep }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/directUrl/i); + }); + + it("rejects a replicaUrl that is not a parseable URL", () => { + const bad = [ + { key: "a", region: "r", url: "postgres://h/a", replicaUrl: "nope", replication: rep }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/replicaUrl/i); + }); + + it("rejects an empty schema search param, matching the boot schema", () => { + const bad = [{ key: "a", region: "r", url: "postgres://h/a?schema=", replication: rep }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/url/i); + }); + + it("rejects a multi-char shard key", () => { + const bad = [{ key: "ab", region: "r", url: "postgres://h/a", replication: rep }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/key/i); + }); + + it("rejects an origin generation outside 2..255", () => { + for (const gen of [1, 256]) { + const bad = [ + { + key: "a", + region: "r", + url: "postgres://h/a", + replication: { ...rep, originGeneration: gen }, + }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/originGeneration/i); + } + }); + + it("rejects a replication block with a blank slot name", () => { + const bad = [ + { key: "a", region: "r", url: "postgres://h/a", replication: { ...rep, slotName: "" } }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/slotName/i); + }); + + // The failure must come BEFORE any DSN is handed back, so no database is migrated first. + it("emits nothing when a later descriptor is invalid", () => { + const mixed = [ + { key: "a", region: "r", url: "postgres://h/a", replication: rep }, + { key: "b", region: "r", url: "not a URL", replication: { ...rep, originGeneration: 3 } }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(mixed))).toThrow(/url/i); + }); + + it("still accepts a fully valid descriptor with all three URLs", () => { + const ok = [ + { + key: "a", + region: "r", + url: "postgres://h/a?schema=public", + replicaUrl: "postgres://h/a-replica?schema=public", + directUrl: "postgres://h/a-direct?schema=public", + replication: rep, + }, + ]; + expect(shardMigrationDsns(JSON.stringify(ok))).toEqual(["postgres://h/a-direct?schema=public"]); + }); +}); diff --git a/apps/webapp/test/runOpsShards.test.ts b/apps/webapp/test/runOpsShards.test.ts new file mode 100644 index 00000000000..c7fd0e0defa --- /dev/null +++ b/apps/webapp/test/runOpsShards.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { + nonAliasedShards, + parseRunOpsShards, + validateShardListAgainstNewUrl, + type RunOpsShardDescriptor, +} from "~/v3/runOpsShards.server"; + +function run(raw: string | undefined) { + const schema = z.string().optional().transform(parseRunOpsShards); + return schema.safeParse(raw); +} + +const valid = { + key: "a", + region: "us-east-1", + url: "postgres://h/db", + replication: { slotName: "s", publicationName: "p", originGeneration: 2 }, +}; + +describe("parseRunOpsShards", () => { + it("returns [] for undefined", () => { + const r = run(undefined); + expect(r.success && r.data).toEqual([]); + }); + it("returns [] for an empty array literal", () => { + const r = run("[]"); + expect(r.success && r.data).toEqual([]); + }); + it("parses a valid single descriptor", () => { + const r = run(JSON.stringify([valid])); + expect(r.success).toBe(true); + if (r.success) expect(r.data[0].key).toBe("a"); + }); + it("fails on malformed JSON", () => { + expect(run("{not json").success).toBe(false); + }); + it("fails on a multi-char key", () => { + expect(run(JSON.stringify([{ ...valid, key: "ab" }])).success).toBe(false); + }); + it("fails on duplicate keys", () => { + const b = { + ...valid, + replication: { slotName: "s2", publicationName: "p2", originGeneration: 3 }, + }; + expect(run(JSON.stringify([valid, b])).success).toBe(false); + }); + it("fails on duplicate origin generations", () => { + const b = { ...valid, key: "b", url: "postgres://h/b" }; + expect(run(JSON.stringify([valid, b])).success).toBe(false); + }); + it("fails when both url and aliasOf are set", () => { + expect( + run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db", aliasOf: "new" }])) + .success + ).toBe(false); + }); + it("accepts aliasOf without url or replication", () => { + expect(run(JSON.stringify([{ key: "a", region: "x", aliasOf: "new" }])).success).toBe(true); + }); + it("fails on an origin generation below 2 or above 255", () => { + const mk = (g: number) => + run( + JSON.stringify([ + { ...valid, replication: { slotName: "s", publicationName: "p", originGeneration: g } }, + ]) + ); + expect(mk(1).success).toBe(false); + expect(mk(256).success).toBe(false); + }); + it("fails when a non-aliased descriptor omits replication", () => { + expect(run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db" }])).success).toBe( + false + ); + }); +}); + +describe("validateShardListAgainstNewUrl", () => { + it("passes when the list is empty and no new url", () => { + expect(validateShardListAgainstNewUrl([], undefined)).toBe(true); + }); + it("passes when the list is non-empty and new url is set", () => { + expect(validateShardListAgainstNewUrl([valid as never], "postgres://h/new")).toBe(true); + }); + it("fails when the list is non-empty and new url is unset", () => { + expect(validateShardListAgainstNewUrl([valid as never], undefined)).toBe(false); + }); +}); + +describe("nonAliasedShards", () => { + const shardA: RunOpsShardDescriptor = { + key: "a", + region: "us-east-1", + url: "postgres://h/a", + replication: { slotName: "sa", publicationName: "pa", originGeneration: 2 }, + }; + const shardB: RunOpsShardDescriptor = { + key: "b", + region: "us-west-2", + url: "postgres://h/b", + replicaUrl: "postgres://h/b-replica", + directUrl: "postgres://h/b-direct", + replication: { slotName: "sb", publicationName: "pb", originGeneration: 3 }, + }; + const aliased: RunOpsShardDescriptor = { + key: "z", + region: "us-east-1", + aliasOf: "new", + }; + + it("returns [] for no descriptors", () => { + expect(nonAliasedShards([])).toEqual([]); + }); + + it("keeps a shard that owns its own database", () => { + expect(nonAliasedShards([shardA])).toEqual([{ key: "a", url: "postgres://h/a" }]); + }); + + it("carries the replica and direct URLs when the descriptor sets them", () => { + expect(nonAliasedShards([shardB])).toEqual([ + { + key: "b", + url: "postgres://h/b", + replicaUrl: "postgres://h/b-replica", + directUrl: "postgres://h/b-direct", + }, + ]); + }); + + it("drops an aliased shard, because it shares its target's database", () => { + expect(nonAliasedShards([aliased])).toEqual([]); + }); + + it("keeps declaration order across a mixed list", () => { + expect(nonAliasedShards([shardA, aliased, shardB]).map((s) => s.key)).toEqual(["a", "b"]); + }); +}); diff --git a/apps/webapp/test/runOpsSplitMode.test.ts b/apps/webapp/test/runOpsSplitMode.test.ts index 7ce2bec3a5d..fd7da6f356c 100644 --- a/apps/webapp/test/runOpsSplitMode.test.ts +++ b/apps/webapp/test/runOpsSplitMode.test.ts @@ -61,6 +61,77 @@ describe("computeSplitEnabled (pure)", () => { }); }); +describe("computeSplitEnabled shard targets", () => { + const shardA = { key: "a", url: "postgres://shard-a" }; + const shardB = { key: "b", url: "postgres://shard-b" }; + + it("probes the gen-1 pair only when no shard is configured", async () => { + const probe = vi.fn().mockResolvedValue({ distinct: true }); + await computeSplitEnabled( + { flagEnabled: true, legacyUrl: "postgres://a", newUrl: "postgres://b" }, + { probe } + ); + expect(probe).toHaveBeenCalledWith( + [ + { id: "legacy", url: "postgres://a" }, + { id: "new", url: "postgres://b" }, + ], + expect.anything() + ); + }); + + it("appends one target per shard, keyed by shard id", async () => { + const probe = vi.fn().mockResolvedValue({ distinct: true }); + await computeSplitEnabled( + { + flagEnabled: true, + legacyUrl: "postgres://a", + newUrl: "postgres://b", + shards: [shardA, shardB], + }, + { probe } + ); + expect(probe).toHaveBeenCalledWith( + [ + { id: "legacy", url: "postgres://a" }, + { id: "new", url: "postgres://b" }, + { id: "shard-a", url: "postgres://shard-a" }, + { id: "shard-b", url: "postgres://shard-b" }, + ], + expect.anything() + ); + }); + + it("stays single-DB when a shard duplicates another store", async () => { + const probe = vi.fn().mockResolvedValue({ distinct: false, reason: "same DB" }); + expect( + await computeSplitEnabled( + { + flagEnabled: true, + legacyUrl: "postgres://a", + newUrl: "postgres://b", + shards: [shardA], + }, + { probe } + ) + ).toBe(false); + }); + + it("never probes a shard when the flag is off", async () => { + const probe = vi.fn(); + await computeSplitEnabled( + { + flagEnabled: false, + legacyUrl: "postgres://a", + newUrl: "postgres://b", + shards: [shardA], + }, + { probe } + ); + expect(probe).not.toHaveBeenCalled(); + }); +}); + describe("assertSplitRealtimeInterlock (pure)", () => { it("throws when split is on but the native realtime backend is off", () => { expect(() => diff --git a/apps/webapp/test/runOpsSplitReadGate.test.ts b/apps/webapp/test/runOpsSplitReadGate.test.ts index 4deb0bb5329..430abbff859 100644 --- a/apps/webapp/test/runOpsSplitReadGate.test.ts +++ b/apps/webapp/test/runOpsSplitReadGate.test.ts @@ -165,3 +165,136 @@ describe("computeRunOpsSplitReadEnabled", () => { }); }); }); + +describe("computeRunOpsSplitReadEnabled shard handles", () => { + const shardA = { __tag: "shard-a" }; + const shardB = { __tag: "shard-b" }; + const base = { + newReplica: dedicatedNew, + controlPlaneWriter: cpWriter, + controlPlaneReplica: cpReplica, + hasNewUrl: true, + hasLegacyUrl: true, + }; + + it("does not warn when every shard handle is a distinct instance", () => { + const warn = vi.fn(); + const enabled = computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [ + { key: "a", replica: shardA }, + { key: "b", replica: shardB }, + ], + logger: { warn }, + }); + expect(enabled).toBe(true); + expect(warn).not.toHaveBeenCalled(); + }); + + it("warns, naming the shard, when a shard replica aliases a control-plane handle", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", replica: cpReplica }], + logger: { warn }, + }); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toMatch(/shard a/i); + }); + + it("warns when a shard replica aliases the gen-1 new replica", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", replica: dedicatedNew }], + logger: { warn }, + }); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it("does NOT warn for an aliased shard, because sharing is its purpose", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "z", replica: dedicatedNew, aliasOf: "new" as const }], + logger: { warn }, + }); + expect(warn).not.toHaveBeenCalled(); + }); + + // The distinctness sentinel already fail-closes the boot on this condition. A gen-2 fault must + // not disable the proven gen-1 read fan-out on top of that. + it("keeps the gen-1 verdict when a shard handle is not distinct", () => { + expect( + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", replica: cpReplica }], + }) + ).toBe(true); + }); + + // The reachable case. A shard with no replicaUrl gets its own WRITER as its replica handle, so its + // reads go to its primary. selectRunOpsTopology does exactly that (db.server.ts), which makes this + // the per-shard analogue of the existing legacy "reads will hit the legacy primary" warning. + it("warns when a shard has no distinct replica handle, so its reads hit its primary", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", writer: shardA, replica: shardA }], + logger: { warn }, + }); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toMatch(/shard a/i); + expect(warn.mock.calls[0][0]).toMatch(/primary/i); + }); + + it("does not warn when a shard has its own distinct replica handle", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", writer: shardA, replica: shardB }], + logger: { warn }, + }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("does NOT warn about primary reads for an aliased shard", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [ + { key: "z", writer: dedicatedNew, replica: dedicatedNew, aliasOf: "new" as const }, + ], + logger: { warn }, + }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("keeps the gen-1 verdict when a shard reads from its primary", () => { + expect( + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", writer: shardA, replica: shardA }], + }) + ).toBe(true); + }); + + it("warns once per offending shard", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [ + { key: "a", replica: cpReplica }, + { key: "b", replica: cpWriter }, + ], + logger: { warn }, + }); + expect(warn).toHaveBeenCalledTimes(2); + }); + + it("is unchanged when no shard handle is supplied", () => { + const warn = vi.fn(); + expect(computeRunOpsSplitReadEnabled({ ...base, logger: { warn } })).toBe(true); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/webapp/test/runStoreShardWiring.test.ts b/apps/webapp/test/runStoreShardWiring.test.ts new file mode 100644 index 00000000000..728f8ee5623 --- /dev/null +++ b/apps/webapp/test/runStoreShardWiring.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { RoutingRunStore } from "@internal/run-store"; +import { buildRunStore } from "~/v3/runStore.server"; + +// Construction-only: buildRunStore wraps clients but never connects, so stub handles suffice. This +// asserts the wiring shape (compat router vs N-way router), not query behaviour. +const stub = () => ({}) as any; + +const baseSplit = { + splitEnabled: true as const, + newWriter: stub(), + newReplica: stub(), + legacyWriter: stub(), + legacyReplica: stub(), + singleWriter: stub(), + singleReplica: stub(), +}; + +describe("buildRunStore shard wiring", () => { + it("split ON with no shards builds the two-store compat router", () => { + const store = buildRunStore(baseSplit); + expect(store).toBeInstanceOf(RoutingRunStore); + }); + + it("split ON with two shard descriptors builds the N-way router", () => { + const store = buildRunStore({ + ...baseSplit, + shards: [ + { key: "a", writer: stub(), replica: stub() }, + { key: "b", writer: stub(), replica: stub() }, + ], + }); + expect(store).toBeInstanceOf(RoutingRunStore); + }); + + it("split OFF builds the single-store passthrough (not a router)", () => { + const store = buildRunStore({ + splitEnabled: false, + singleWriter: stub(), + singleReplica: stub(), + }); + expect(store).not.toBeInstanceOf(RoutingRunStore); + }); +}); diff --git a/apps/webapp/test/runsListClickhouseSettings.test.ts b/apps/webapp/test/runsListClickhouseSettings.test.ts new file mode 100644 index 00000000000..fc066b60369 --- /dev/null +++ b/apps/webapp/test/runsListClickhouseSettings.test.ts @@ -0,0 +1,66 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { clickhouseTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { z } from "zod"; + +vi.setConfig({ testTimeout: 60_000 }); + +describe("runs-list ClickHouse protection settings", () => { + clickhouseTest( + "server-side max_execution_time kills a slow read, and readonly=2 does not block the caps", + async ({ clickhouseContainer }) => { + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-list-settings-test", + requestTimeoutMs: 30_000, + clickhouseSettings: { + max_execution_time: 1, + timeout_before_checking_execution_speed: 0, + max_threads: 2, + readonly: "2", + }, + }); + + const slow = clickhouse.reader.query({ + name: "slow-read", + query: "SELECT sum(number) AS total FROM numbers(1000000000000)", + schema: z.object({ total: z.number() }), + }); + const [slowError] = await slow({}); + + expect(slowError).not.toBeNull(); + expect(slowError?.message.toLowerCase()).toMatch(/timeout|exceeded/); + + const fast = clickhouse.reader.query({ + name: "fast-read", + query: "SELECT 1 AS one", + schema: z.object({ one: z.number() }), + }); + const [fastError, rows] = await fast({}); + + expect(fastError).toBeNull(); + expect(rows).toEqual([{ one: 1 }]); + } + ); + + clickhouseTest( + "readonly=2 rejects writes while permitting reads", + async ({ clickhouseContainer }) => { + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-list-readonly-test", + clickhouseSettings: { readonly: "2" }, + }); + + const write = clickhouse.reader.query({ + name: "write-under-readonly", + query: "CREATE TABLE trigger_dev.runs_list_readonly_probe (id UInt8) ENGINE = Memory", + schema: z.object({}), + }); + const [writeError] = await write({}); + + expect(writeError).not.toBeNull(); + expect(writeError?.message.toLowerCase()).toMatch(/readonly|read-only|read only/); + } + ); +}); diff --git a/apps/webapp/test/runsListQueryError.test.ts b/apps/webapp/test/runsListQueryError.test.ts new file mode 100644 index 00000000000..89e0486bf72 --- /dev/null +++ b/apps/webapp/test/runsListQueryError.test.ts @@ -0,0 +1,52 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { + RunsListQueryError, + RunsRepository, +} from "~/services/runsRepository/runsRepository.server"; +import { + createRun, + insertTaskRunV2Rows, + seedParents, +} from "./helpers/apiRunListPresenterTestHelpers"; + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); + +vi.setConfig({ testTimeout: 90_000 }); + +describe("runs list query error handling", () => { + containerTest( + "a ClickHouse resource-limit error surfaces as RunsListQueryError", + async ({ clickhouseContainer, prisma }) => { + const ctx = await seedParents(prisma, "qerr"); + const run = await createRun(prisma, ctx, { friendlyId: "run_qerr" }); + + const seedClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-list-query-error-seed", + }); + await insertTaskRunV2Rows(seedClient, [{ ...run, createdAt: new Date() }]); + + const listArgs = { + page: { size: 10 } as const, + organizationId: ctx.organizationId, + projectId: ctx.projectId, + environmentId: ctx.environmentId, + }; + + const cappedClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-list-query-error-capped", + clickhouseSettings: { max_memory_usage: "1" }, + }); + const capped = new RunsRepository({ prisma, clickhouse: cappedClient }); + await expect(capped.listRuns(listArgs)).rejects.toBeInstanceOf(RunsListQueryError); + await expect(capped.countRuns(listArgs)).rejects.toBeInstanceOf(RunsListQueryError); + + const ok = new RunsRepository({ prisma, clickhouse: seedClient }); + const result = await ok.listRuns(listArgs); + expect(result.runs.map((r) => r.friendlyId)).toEqual(["run_qerr"]); + } + ); +}); diff --git a/apps/webapp/test/runsListQueryShape.test.ts b/apps/webapp/test/runsListQueryShape.test.ts new file mode 100644 index 00000000000..c0411b6fc7a --- /dev/null +++ b/apps/webapp/test/runsListQueryShape.test.ts @@ -0,0 +1,124 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; +import { + createRun, + insertTaskRunV2Rows, + seedParents, +} from "./helpers/apiRunListPresenterTestHelpers"; + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); + +vi.setConfig({ testTimeout: 90_000 }); + +const DAY_MS = 24 * 60 * 60 * 1000; + +describe("runs list query shape (PREWHERE routing under FINAL)", () => { + containerTest( + "keeps status post-FINAL and returns old pending runs (no date clamp)", + async ({ clickhouseContainer, prisma }) => { + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "query-shape-test", + }); + + const ctx = await seedParents(prisma, "shape"); + + const completed = await createRun(prisma, ctx, { friendlyId: "run_completed" }); + const pendingRecent = await createRun(prisma, ctx, { friendlyId: "run_pending_recent" }); + const pendingOld = await createRun(prisma, ctx, { friendlyId: "run_pending_old" }); + + const base = { + taskIdentifier: "webhook.deliver", + runTags: ["booking:T"], + createdAt: new Date(Date.now() - 1 * DAY_MS), + }; + + await insertTaskRunV2Rows(clickhouse, [ + { ...completed, ...base, status: "PENDING", updatedAt: new Date(Date.now() - 2 * DAY_MS) }, + { + ...completed, + ...base, + status: "COMPLETED", + updatedAt: new Date(Date.now() - 1 * DAY_MS), + }, + { + ...pendingRecent, + ...base, + status: "PENDING", + updatedAt: new Date(Date.now() - 1 * DAY_MS), + }, + { + ...pendingOld, + ...base, + status: "PENDING", + createdAt: new Date(Date.now() - 60 * DAY_MS), + updatedAt: new Date(Date.now() - 60 * DAY_MS), + }, + ]); + + const repository = new RunsRepository({ prisma, clickhouse }); + + const { runIds } = await repository.listRunIds({ + page: { size: 10 }, + period: "365d", + organizationId: ctx.organizationId, + projectId: ctx.projectId, + environmentId: ctx.environmentId, + tasks: ["webhook.deliver"], + tags: ["booking:T"], + statuses: ["PENDING", "DELAYED"], + }); + + expect(runIds.sort()).toEqual([pendingOld.id, pendingRecent.id].sort()); + } + ); + + containerTest( + "region filter matches a dequeued run by its region; a still-queued run is not matched", + async ({ clickhouseContainer, prisma }) => { + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "query-shape-region-test", + }); + + const ctx = await seedParents(prisma, "region"); + const dequeued = await createRun(prisma, ctx, { friendlyId: "run_dequeued" }); + const queued = await createRun(prisma, ctx, { friendlyId: "run_queued" }); + + const shared = { + taskIdentifier: "webhook.deliver", + createdAt: new Date(Date.now() - 1 * DAY_MS), + }; + + await insertTaskRunV2Rows(clickhouse, [ + { ...dequeued, ...shared, region: "", updatedAt: new Date(Date.now() - 2 * DAY_MS) }, + { + ...dequeued, + ...shared, + region: "us-east-1", + updatedAt: new Date(Date.now() - 1 * DAY_MS), + }, + ]); + await insertTaskRunV2Rows(clickhouse, [ + { ...queued, ...shared, region: "", updatedAt: new Date(Date.now() - 1 * DAY_MS) }, + ]); + + const repository = new RunsRepository({ prisma, clickhouse }); + const listArgs = { + page: { size: 10 } as const, + period: "365d", + organizationId: ctx.organizationId, + projectId: ctx.projectId, + environmentId: ctx.environmentId, + }; + + const byRegion = await repository.listRunIds({ ...listArgs, regions: ["us-east-1"] }); + expect(byRegion.runIds).toEqual([dequeued.id]); + + const otherRegion = await repository.listRunIds({ ...listArgs, regions: ["us-west-2"] }); + expect(otherRegion.runIds).toEqual([]); + } + ); +}); diff --git a/apps/webapp/test/runsReplicationInstance.test.ts b/apps/webapp/test/runsReplicationInstance.test.ts index 67f595c597c..86aeb86116e 100644 --- a/apps/webapp/test/runsReplicationInstance.test.ts +++ b/apps/webapp/test/runsReplicationInstance.test.ts @@ -202,6 +202,225 @@ describe("assertReplicationCoversSplit (boot gate-coupling)", () => { }); }); +describe("replication sources at N shards", () => { + const baseArgs = { + legacyUrl: "postgres://legacy", + legacySlotName: "task_runs_to_clickhouse_v1", + legacyPublicationName: "task_runs_to_clickhouse_v1_publication", + legacyOriginGeneration: 0, + newSlotName: "task_runs_to_clickhouse_v2", + newPublicationName: "task_runs_to_clickhouse_v2_publication", + newOriginGeneration: 1, + splitEnabled: true, + newUrl: "postgres://new", + }; + + const shardA = { + key: "a", + url: "postgres://shard-a", + replication: { + slotName: "task_runs_to_clickhouse_shard_a", + publicationName: "task_runs_to_clickhouse_shard_a_publication", + originGeneration: 2, + }, + }; + const shardB = { + key: "b", + url: "postgres://shard-b", + replication: { + slotName: "task_runs_to_clickhouse_shard_b", + publicationName: "task_runs_to_clickhouse_shard_b_publication", + originGeneration: 3, + }, + }; + + it("appends nothing when no shard is configured", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [] }); + expect(sources.map((s) => s.id)).toEqual(["legacy", "new"]); + }); + + it("appends one source per shard, after legacy and new", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [shardA, shardB] }); + expect(sources.map((s) => s.id)).toEqual(["legacy", "new", "shard-a", "shard-b"]); + expect(sources[2]).toEqual({ + id: "shard-a", + pgConnectionUrl: "postgres://shard-a", + slotName: "task_runs_to_clickhouse_shard_a", + publicationName: "task_runs_to_clickhouse_shard_a_publication", + originGeneration: 2, + }); + }); + + it("appends no shard source when the new source is off, because split is the precondition", () => { + const sources = buildReplicationSources({ + ...baseArgs, + splitEnabled: false, + shards: [shardA], + }); + expect(sources.map((s) => s.id)).toEqual(["legacy"]); + }); + + it("throws when a shard that owns its database has no source", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "a" }], + }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("names the uncovered shard in the message", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [shardA] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "a" }, { key: "b" }], + }) + ).toThrow(/shard b/i); + }); + + it("does NOT throw when every shard has its own source", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [shardA, shardB] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "a" }, { key: "b" }], + }) + ).not.toThrow(); + }); + + it("does NOT require a source for an aliased shard, because its target's slot covers it", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "z", aliasOf: "new" }], + }) + ).not.toThrow(); + }); + + it("does NOT check shard coverage when split is off", () => { + const sources = buildReplicationSources({ ...baseArgs, splitEnabled: false, shards: [] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: false, + sources, + shards: [{ key: "a" }], + }) + ).not.toThrow(); + }); + + // The catch site keys on `instanceof SplitReplicationMisconfiguredError` to reach + // process.exit(1). A shard with no replication must reach the same exit. + it("raises an error the existing exit path recognizes", () => { + try { + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ ...baseArgs, shards: [] }), + shards: [{ key: "a" }], + }); + expect.unreachable("expected a throw"); + } catch (error) { + expect(error).toBeInstanceOf(SplitReplicationMisconfiguredError); + } + }); + + // F1 class: the descriptor parser checks uniqueness AMONG shards only. It cannot see the + // env-configured legacy and new sources, so a shard can collide with them. The service's own + // check throws too late: the caller has already shut the bootstrap instance down, so the throw + // leaves the process up with NO replication at all. These must fail at the fatal boot gate. + it("throws when a shard's slot name collides with the gen-1 new slot", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [ + { ...shardA, replication: { ...shardA.replication, slotName: baseArgs.newSlotName } }, + ], + }); + expect(() => + assertReplicationCoversSplit({ splitEnabled: true, sources, shards: [{ key: "a" }] }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("throws when a shard's origin generation collides with the gen-1 new generation", () => { + const sources = buildReplicationSources({ + ...baseArgs, + newOriginGeneration: 2, + shards: [shardA], + }); + expect(() => + assertReplicationCoversSplit({ splitEnabled: true, sources, shards: [{ key: "a" }] }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("throws when a shard's publication name collides with the legacy publication", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [ + { + ...shardA, + replication: { ...shardA.replication, publicationName: baseArgs.legacyPublicationName }, + }, + ], + }); + expect(() => + assertReplicationCoversSplit({ splitEnabled: true, sources, shards: [{ key: "a" }] }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("names the colliding field in the message", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [ + { ...shardA, replication: { ...shardA.replication, slotName: baseArgs.newSlotName } }, + ], + }); + expect(() => + assertReplicationCoversSplit({ splitEnabled: true, sources, shards: [{ key: "a" }] }) + ).toThrow(/slotName/); + }); + + it("does NOT throw when every shard's slot, publication and generation are its own", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [shardA, shardB] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "a" }, { key: "b" }], + }) + ).not.toThrow(); + }); + + // The service validates sources before it builds a single replication client, so this needs no + // container. RunsReplicationService itself is untouched by this change: the check already exists. + it("rejects two shards that share an origin generation, via the service's own check", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [shardA, { ...shardB, replication: { ...shardB.replication, originGeneration: 2 } }], + }); + + expect( + () => + new RunsReplicationService({ + clickhouseFactory: new TestReplicationClickhouseFactory( + new ClickHouse({ url: "http://127.0.0.1:1", name: "unused", logLevel: "warn" }) + ), + serviceName: "runs-replication", + pgConnectionUrl: "postgres://legacy", + slotName: "unused", + publicationName: "unused", + redisOptions: { host: "127.0.0.1", port: 1 }, + sources, + logLevel: "warn", + }) + ).toThrow(/duplicate originGeneration/i); + }); +}); + describe("RunsReplication new-source backfill origin generation (integration)", () => { replicationContainerTest( "backfill via the new source tags the ClickHouse row with the new origin generation (gen=1), not gen=0", @@ -434,3 +653,96 @@ describe("RunsReplication multi-source wiring (integration)", () => { } ); }); + +// Logical replication needs a session-mode connection, which a transaction pooler cannot serve. +// The app writer DSN is pooled in a real deployment, so a shard replication source must take the +// shard's DIRECT url. Getting this wrong throws inside service.start(), which is not a +// SplitReplicationMisconfiguredError, so the process stays up with every source down. +describe("shard replication uses the direct connection", () => { + const baseArgs = { + legacyUrl: "postgres://legacy", + legacySlotName: "v1", + legacyPublicationName: "v1_pub", + legacyOriginGeneration: 0, + newSlotName: "v2", + newPublicationName: "v2_pub", + newOriginGeneration: 1, + splitEnabled: true, + newUrl: "postgres://new", + }; + const rep = { slotName: "sa", publicationName: "pa", originGeneration: 2 }; + + it("prefers the shard's directUrl over its pooled url", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [ + { + key: "a", + url: "postgres://pooled:6432/shard_a", + directUrl: "postgres://direct:5432/shard_a", + replication: rep, + }, + ], + }); + const shard = sources.find((s) => s.id === "shard-a"); + expect(shard?.pgConnectionUrl).toBe("postgres://direct:5432/shard_a"); + }); + + it("falls back to url when no directUrl is given", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [{ key: "a", url: "postgres://only-url/shard_a", replication: rep }], + }); + const shard = sources.find((s) => s.id === "shard-a"); + expect(shard?.pgConnectionUrl).toBe("postgres://only-url/shard_a"); + }); + + it("refuses the boot when a replicating shard declares no directUrl", () => { + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ + ...baseArgs, + shards: [{ key: "a", url: "postgres://u", replication: rep }], + }), + shards: [{ key: "a", hasDirectUrl: false }], + }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("names the shard and the reason in that failure", () => { + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ + ...baseArgs, + shards: [{ key: "a", url: "postgres://u", replication: rep }], + }), + shards: [{ key: "a", hasDirectUrl: false }], + }) + ).toThrow(/shard a.*directUrl|directUrl.*shard a/is); + }); + + it("does NOT refuse when the replicating shard declares a directUrl", () => { + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ + ...baseArgs, + shards: [{ key: "a", url: "postgres://u", directUrl: "postgres://d", replication: rep }], + }), + shards: [{ key: "a", hasDirectUrl: true }], + }) + ).not.toThrow(); + }); + + it("does NOT require a directUrl for an aliased shard", () => { + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ ...baseArgs, shards: [] }), + shards: [{ key: "z", aliasOf: "new", hasDirectUrl: false }], + }) + ).not.toThrow(); + }); +}); diff --git a/apps/webapp/test/transactionResilience.test.ts b/apps/webapp/test/transactionResilience.test.ts new file mode 100644 index 00000000000..a033f3be985 --- /dev/null +++ b/apps/webapp/test/transactionResilience.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { resolveTransactionResilience } from "~/v3/transactionResilience.server"; + +describe("resolveTransactionResilience per-shard", () => { + it("builds a distinct budget per call, so one shard's storm cannot drain another's", () => { + const a = resolveTransactionResilience("run-ops-shard-a", {}); + const b = resolveTransactionResilience("run-ops-shard-b", {}); + expect(a.startRetry.budget).not.toBe(b.startRetry.budget); + }); + + it("accepts an arbitrary pool label and honours a maxWait override", () => { + expect(() => + resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 }) + ).not.toThrow(); + expect(resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 }).maxWait).toBe(1234); + }); +}); diff --git a/apps/webapp/test/unroutableIdStatus.test.ts b/apps/webapp/test/unroutableIdStatus.test.ts new file mode 100644 index 00000000000..89135b44e15 --- /dev/null +++ b/apps/webapp/test/unroutableIdStatus.test.ts @@ -0,0 +1,40 @@ +// `resolveShard` is pure id-shape, so any base32hex core plus `[a-z0-9]` plus "2" parses as gen-2 +// and names a shard — including one a caller invents. The routing store throws for a key it has no +// store for, which is correct and deliberately loud, but a read route that lets it reach the +// boundary answered 500 for caller-supplied input. These tests pin the boundary status. +import { describe, expect, it } from "vitest"; +import { json } from "@remix-run/server-runtime"; +import { UnknownShardKey } from "@internal/run-store"; +import { unroutableIdResponse } from "~/services/routeBuilders/unroutableId.server"; + +describe("unroutableIdResponse", () => { + it("answers 404 for an id naming a shard with no configured store", async () => { + const response = unroutableIdResponse(new UnknownShardKey("z", ["legacy", "new"])); + + expect(response).toBeDefined(); + expect(response!.status).toBe(404); + // Not retryable: no number of retries makes a topology grow a store. Contrast the + // waitpoint wait route, whose 404 IS retryable because a miss there can be replica lag. + expect(response!.headers.get("x-should-retry")).toBe("false"); + await expect(response!.json()).resolves.toEqual({ error: "Not Found" }); + }); + + it("declines an unrelated error so it still reaches the 500 path", () => { + expect(unroutableIdResponse(new Error("db down"))).toBeUndefined(); + expect(unroutableIdResponse(undefined)).toBeUndefined(); + expect(unroutableIdResponse("a string")).toBeUndefined(); + }); + + it("declines a deliberately thrown Response, which carries its own status", () => { + expect(unroutableIdResponse(json({ error: "nope" }, { status: 422 }))).toBeUndefined(); + }); + + it("keeps the key and the configured set on the error for the operator", () => { + // A 404 to the caller must not cost the operator what separates a forged id from a shard + // key dropped out of a config that is meant to be append-only. + const error = new UnknownShardKey("z", ["legacy", "new", "a"]); + + expect(error.shardKey).toBe("z"); + expect(error.configured).toContain("a"); + }); +}); diff --git a/apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts b/apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts index d2baaa6404a..562d50b63d5 100644 --- a/apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts +++ b/apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts @@ -1,7 +1,10 @@ import { heteroPostgresTest } from "@internal/testcontainers"; import { PrismaClient } from "@trigger.dev/database"; import { describe, expect, vi } from "vitest"; -import { probeDistinctDatabases } from "~/v3/runOpsMigration/distinctDbSentinel.server"; +import { + probeDistinctDatabases, + probeDistinctStores, +} from "~/v3/runOpsMigration/distinctDbSentinel.server"; // Spinning up two separate postgres clusters and probing each can exceed the 5s default. vi.setConfig({ testTimeout: 60_000 }); @@ -62,3 +65,113 @@ describe("probeDistinctDatabases", () => { } ); }); + +describe("probeDistinctStores (set uniqueness at N)", () => { + heteroPostgresTest( + "reports distinct for two separate physical clusters", + async ({ uri14, uri17 }) => { + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + ]); + expect(result).toEqual({ distinct: true }); + } + ); + + heteroPostgresTest("reports distinct for a single target", async ({ uri14 }) => { + const result = await probeDistinctStores([{ id: "legacy", url: uri14 }]); + expect(result).toEqual({ distinct: true }); + }); + + heteroPostgresTest("reports distinct for an empty target list", async () => { + const result = await probeDistinctStores([]); + expect(result).toEqual({ distinct: true }); + }); + + heteroPostgresTest( + "reports NOT distinct, naming both ids, when two targets resolve to one database", + async ({ uri14, uri17 }) => { + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + { id: "shard-a", url: uri14 }, + ]); + expect(result.distinct).toBe(false); + if (result.distinct === false) { + expect(result.reason).toMatch(/same physical database/i); + expect(result.reason).toMatch(/legacy/); + expect(result.reason).toMatch(/shard-a/); + } + } + ); + + // A pairwise implementation that only ever compares the first two targets passes every other + // case in this file and fails this one: legacy vs new is clean, and the duplicate pair is + // shard against shard on a third database. + heteroPostgresTest( + "catches a duplicate between two SHARDS while the gen-1 pair is clean", + async ({ postgresContainer14, uri14, uri17 }) => { + const shardDb = `sentinel_shard_dupe_${Date.now()}`; + const admin = new PrismaClient({ + datasources: { + db: { url: urlWithDatabase(postgresContainer14.getConnectionUri(), "postgres") }, + }, + }); + try { + await admin.$executeRawUnsafe(`CREATE DATABASE "${shardDb}"`); + } finally { + await admin.$disconnect(); + } + const shardUrl = urlWithDatabase(uri14, shardDb); + + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + { id: "shard-a", url: shardUrl }, + { id: "shard-b", url: shardUrl }, + ]); + expect(result.distinct).toBe(false); + if (result.distinct === false) { + expect(result.reason).toMatch(/shard-a/); + expect(result.reason).toMatch(/shard-b/); + } + } + ); + + heteroPostgresTest( + "reports distinct for two databases in the SAME cluster", + async ({ postgresContainer14, uri14, uri17 }) => { + const otherDb = `sentinel_set_other_${Date.now()}`; + const admin = new PrismaClient({ + datasources: { + db: { url: urlWithDatabase(postgresContainer14.getConnectionUri(), "postgres") }, + }, + }); + try { + await admin.$executeRawUnsafe(`CREATE DATABASE "${otherDb}"`); + } finally { + await admin.$disconnect(); + } + + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + { id: "shard-a", url: urlWithDatabase(uri14, otherDb) }, + ]); + expect(result).toEqual({ distinct: true }); + } + ); + + heteroPostgresTest( + "fails closed to NOT distinct when one target cannot be reached", + async ({ uri14, uri17 }) => { + const unreachable = "postgresql://nobody:nobody@127.0.0.1:1/does_not_exist"; + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + { id: "shard-a", url: unreachable }, + ]); + expect(result.distinct).toBe(false); + } + ); +}); diff --git a/apps/webapp/test/vercelExternalIdBackfill.test.ts b/apps/webapp/test/vercelExternalIdBackfill.test.ts new file mode 100644 index 00000000000..ec087d6cbd4 --- /dev/null +++ b/apps/webapp/test/vercelExternalIdBackfill.test.ts @@ -0,0 +1,419 @@ +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { backfillVercelExternalIds } from "~/v3/services/vercelExternalIdBackfill.server"; + +let seedCounter = 0; + +const SHA_A = "a".repeat(40); +const SHA_B = "b".repeat(40); + +type SeedOptions = { + vercelConnected?: boolean; + environmentType?: "PRODUCTION" | "STAGING" | "PREVIEW" | "DEVELOPMENT"; +}; + +async function seedEnv(prisma: PrismaClient, slug: string, options: SeedOptions = {}) { + const { vercelConnected = true, environmentType = "PRODUCTION" } = options; + const n = seedCounter++; + + const organization = await prisma.organization.create({ + data: { title: `Org ${slug}`, slug: `org-${slug}-${n}` }, + }); + + const project = await prisma.project.create({ + data: { + name: `Proj ${slug}`, + slug: `proj-${slug}-${n}`, + organizationId: organization.id, + externalRef: `ext-${slug}-${n}`, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: `env-${slug}-${n}`, + type: environmentType, + projectId: project.id, + organizationId: organization.id, + apiKey: `api-${slug}-${n}`, + pkApiKey: `pk-${slug}-${n}`, + shortcode: `sc-${slug}-${n}`, + }, + }); + + if (vercelConnected) { + const tokenReference = await prisma.secretReference.create({ + data: { key: `secret-${slug}-${n}` }, + }); + + const organizationIntegration = await prisma.organizationIntegration.create({ + data: { + friendlyId: `oi-${slug}-${n}`, + service: "VERCEL", + integrationData: {}, + tokenReferenceId: tokenReference.id, + organizationId: organization.id, + }, + }); + + await prisma.organizationProjectIntegration.create({ + data: { + organizationIntegrationId: organizationIntegration.id, + projectId: project.id, + externalEntityId: `vercel-project-${n}`, + integrationData: {}, + }, + }); + } + + return { organization, project, environment }; +} + +type SeedCtx = Awaited>; + +type DeploymentOptions = { + version: string; + commitSHA?: string | null; + externalId?: string | null; + status?: "DEPLOYED" | "FAILED" | "BUILDING"; + withWorker?: boolean; + createdAt?: Date; +}; + +async function seedDeployment(prisma: PrismaClient, ctx: SeedCtx, options: DeploymentOptions) { + const { + version, + commitSHA = SHA_A, + externalId = null, + status = "DEPLOYED", + withWorker = true, + createdAt, + } = options; + const n = seedCounter++; + + let workerId: string | undefined; + if (withWorker) { + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker-${n}`, + contentHash: `hash-${n}`, + projectId: ctx.project.id, + runtimeEnvironmentId: ctx.environment.id, + version, + metadata: {}, + }, + }); + workerId = worker.id; + } + + return prisma.workerDeployment.create({ + data: { + contentHash: `hash-${n}`, + friendlyId: `deployment-${n}`, + shortCode: `short-${n}`, + version, + status, + projectId: ctx.project.id, + environmentId: ctx.environment.id, + commitSHA, + externalId, + workerId, + ...(createdAt ? { createdAt } : {}), + }, + }); +} + +function run( + prisma: PrismaClient, + overrides: Partial[0]> = {} +) { + return backfillVercelExternalIds({ + prisma, + replica: prisma, + limit: 100, + recentPerEnvironment: 10, + parallelism: 5, + dryRun: false, + ...overrides, + }); +} + +describe("backfillVercelExternalIds", () => { + postgresTest("copies commitSHA into externalId for a Vercel deployment", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "copy"); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const result = await run(prisma); + + expect(result.deployments.written).toBe(1); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBe(SHA_A); + }); + + postgresTest("a dry run reports the work and writes nothing", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "dry"); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const result = await run(prisma, { dryRun: true }); + + expect(result.summary.would_update).toBe(1); + expect(result.deployments.eligible).toBe(1); + expect(result.deployments.written).toBe(0); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("never overwrites an existing externalId", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "keep"); + const deployment = await seedDeployment(prisma, ctx, { + version: "20260101.1", + commitSHA: SHA_A, + externalId: SHA_B, + }); + + const result = await run(prisma); + + expect(result.deployments.written).toBe(0); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBe(SHA_B); + }); + + postgresTest("skips projects with no Vercel integration", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "novercel", { vercelConnected: false }); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const result = await run(prisma); + + expect(result.environments.find((e) => e.id === ctx.environment.id)).toBeUndefined(); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("skips development environments", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "dev", { environmentType: "DEVELOPMENT" }); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + await run(prisma); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("skips deployments that are not DEPLOYED", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "failed"); + const deployment = await seedDeployment(prisma, ctx, { + version: "20260101.1", + status: "FAILED", + }); + + await run(prisma); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("skips deployments with no usable commitSHA", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "nosha"); + const missing = await seedDeployment(prisma, ctx, { version: "20260101.1", commitSHA: null }); + const blank = await seedDeployment(prisma, ctx, { version: "20260101.2", commitSHA: " " }); + const tooLong = await seedDeployment(prisma, ctx, { + version: "20260101.3", + commitSHA: "c".repeat(129), + }); + + const result = await run(prisma); + + expect(result.deployments.written).toBe(0); + for (const deployment of [missing, blank, tooLong]) { + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + } + }); + + postgresTest("skips deployments with no worker", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "noworker"); + const deployment = await seedDeployment(prisma, ctx, { + version: "20260101.1", + withWorker: false, + }); + + await run(prisma); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest( + "recentPerEnvironment bounds the window, and current is always included", + async ({ prisma }) => { + const ctx = await seedEnv(prisma, "window"); + + const oldest = await seedDeployment(prisma, ctx, { + version: "20260101.1", + createdAt: new Date("2026-01-01T00:00:00Z"), + }); + const newest = await seedDeployment(prisma, ctx, { + version: "20260101.2", + createdAt: new Date("2026-06-01T00:00:00Z"), + }); + + // Promote the oldest, so it can only be reached via the promotion arm. + await prisma.workerDeploymentPromotion.create({ + data: { + label: "current", + deploymentId: oldest.id, + environmentId: ctx.environment.id, + }, + }); + + const result = await run(prisma, { recentPerEnvironment: 1 }); + + expect(result.deployments.written).toBe(2); + + const afterOldest = await prisma.workerDeployment.findFirst({ where: { id: oldest.id } }); + const afterNewest = await prisma.workerDeployment.findFirst({ where: { id: newest.id } }); + expect(afterOldest?.externalId).toBe(SHA_A); + expect(afterNewest?.externalId).toBe(SHA_A); + } + ); + + postgresTest( + "recentPerEnvironment 0 backfills only the current promotion", + async ({ prisma }) => { + const ctx = await seedEnv(prisma, "currentonly"); + + const promoted = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + const other = await seedDeployment(prisma, ctx, { version: "20260101.2" }); + + await prisma.workerDeploymentPromotion.create({ + data: { + label: "current", + deploymentId: promoted.id, + environmentId: ctx.environment.id, + }, + }); + + const result = await run(prisma, { recentPerEnvironment: 0 }); + + expect(result.deployments.written).toBe(1); + + const afterPromoted = await prisma.workerDeployment.findFirst({ where: { id: promoted.id } }); + const afterOther = await prisma.workerDeployment.findFirst({ where: { id: other.id } }); + expect(afterPromoted?.externalId).toBe(SHA_A); + expect(afterOther?.externalId).toBeNull(); + } + ); + + postgresTest("is idempotent across a second run", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "idem"); + await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const first = await run(prisma); + const second = await run(prisma); + + expect(first.deployments.written).toBe(1); + expect(second.deployments.written).toBe(0); + }); + + postgresTest("paginates by connected project and reports done at the end", async ({ prisma }) => { + const first = await seedEnv(prisma, "page-a"); + const second = await seedEnv(prisma, "page-b"); + await seedDeployment(prisma, first, { version: "20260101.1" }); + await seedDeployment(prisma, second, { version: "20260101.1" }); + + const integrations = await prisma.organizationProjectIntegration.findMany({ + where: { projectId: { in: [first.project.id, second.project.id] } }, + select: { id: true, projectId: true }, + orderBy: { id: "asc" }, + }); + expect(integrations).toHaveLength(2); + + const page = await run(prisma, { limit: 1, dryRun: true }); + expect(page.projects).toBe(1); + expect(page.next).toBe(integrations[0]?.id); + + const firstPageProject = integrations[0]?.projectId; + const expectedEnvironment = + firstPageProject === first.project.id ? first.environment.id : second.environment.id; + expect(page.environments.map((e) => e.id)).toEqual([expectedEnvironment]); + + const secondPage = await run(prisma, { limit: 1, cursor: page.next, dryRun: true }); + expect(secondPage.projects).toBe(1); + expect(secondPage.environments.map((e) => e.id)).not.toEqual([expectedEnvironment]); + + const exhausted = await run(prisma, { cursor: secondPage.next, dryRun: true }); + expect(exhausted.done).toBe(true); + expect(exhausted.environments).toHaveLength(0); + }); + + postgresTest("a project with two connection rows is not walked twice", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "dupe"); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + // Reconnecting leaves the earlier row in place. + const existing = await prisma.organizationProjectIntegration.findFirst({ + where: { projectId: ctx.project.id }, + select: { organizationIntegrationId: true }, + }); + await prisma.organizationProjectIntegration.create({ + data: { + organizationIntegrationId: existing!.organizationIntegrationId, + projectId: ctx.project.id, + externalEntityId: "vercel-project-dupe", + integrationData: {}, + }, + }); + + const result = await run(prisma, { dryRun: true }); + + expect(result.projects).toBe(1); + expect(result.environments.filter((e) => e.id === ctx.environment.id)).toHaveLength(1); + expect(result.deployments.eligible).toBe(1); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("a failed environment lookup does not cost the page", async ({ prisma }) => { + const first = await seedEnv(prisma, "fail-a"); + const second = await seedEnv(prisma, "fail-b"); + await seedDeployment(prisma, first, { version: "20260101.1" }); + await seedDeployment(prisma, second, { version: "20260101.1" }); + + // pMap rejects the whole call when a mapper throws, even with + // stopOnError: false, so the lookup has to convert its own failures. + const replica = new Proxy(prisma, { + get(target, prop, receiver) { + if (prop === "runtimeEnvironment") { + return { + findMany: async () => { + throw new Error("simulated replica failure"); + }, + }; + } + return Reflect.get(target, prop, receiver); + }, + }) as unknown as PrismaClient; + + const result = await backfillVercelExternalIds({ + prisma, + replica, + limit: 100, + recentPerEnvironment: 10, + parallelism: 5, + dryRun: false, + }); + + expect(result.projects).toBeGreaterThanOrEqual(2); + expect(result.next).toBeDefined(); + expect(result.summary.error).toBe(result.projects); + expect(result.environments.every((e) => e.scope === "project")).toBe(true); + expect(result.environments.every((e) => e.error === "simulated replica failure")).toBe(true); + expect(result.deployments.written).toBe(0); + }); +}); diff --git a/apps/webapp/test/vercelIntegrationConfig.test.ts b/apps/webapp/test/vercelIntegrationConfig.test.ts new file mode 100644 index 00000000000..696627d70b2 --- /dev/null +++ b/apps/webapp/test/vercelIntegrationConfig.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect } from "vitest"; +import { + createDefaultVercelIntegrationData, + restrictConfigToAvailableEnvSlugs, +} from "../app/v3/vercel/vercelProjectIntegrationSchema"; + +const STAGING_ENV = { environmentId: "env_123", displayName: "Staging" }; + +describe("restrictConfigToAvailableEnvSlugs", () => { + it("drops slugs the project has no environment for", () => { + const restricted = restrictConfigToAvailableEnvSlugs( + { + atomicBuilds: ["prod", "stg"], + pullEnvVarsBeforeBuild: ["prod", "preview"], + discoverEnvVars: ["dev", "stg", "preview"], + }, + ["dev", "prod"] + ); + + expect(restricted.atomicBuilds).toEqual(["prod"]); + expect(restricted.pullEnvVarsBeforeBuild).toEqual(["prod"]); + expect(restricted.discoverEnvVars).toEqual(["dev"]); + }); + + it("keeps slugs the project does have", () => { + const restricted = restrictConfigToAvailableEnvSlugs( + { atomicBuilds: ["prod", "stg", "preview"] }, + ["dev", "stg", "prod", "preview"] + ); + + expect(restricted.atomicBuilds).toEqual(["prod", "stg", "preview"]); + }); + + it("only touches keys present on the input", () => { + const restricted = restrictConfigToAvailableEnvSlugs({ atomicBuilds: ["stg"] }, ["prod"]); + + expect(restricted).not.toHaveProperty("pullEnvVarsBeforeBuild"); + expect(restricted).not.toHaveProperty("discoverEnvVars"); + expect(restricted).not.toHaveProperty("vercelStagingEnvironment"); + }); + + it("clears the staging environment mapping when staging is unavailable", () => { + const restricted = restrictConfigToAvailableEnvSlugs( + { vercelStagingEnvironment: STAGING_ENV }, + ["dev", "prod", "preview"] + ); + + expect(restricted.vercelStagingEnvironment).toBeNull(); + }); + + it("keeps the staging environment mapping when staging is available", () => { + const restricted = restrictConfigToAvailableEnvSlugs( + { vercelStagingEnvironment: STAGING_ENV }, + ["dev", "stg", "prod"] + ); + + expect(restricted.vercelStagingEnvironment).toEqual(STAGING_ENV); + }); + + it("does not mutate the input", () => { + const config = { atomicBuilds: ["prod", "stg"] as const }; + restrictConfigToAvailableEnvSlugs({ atomicBuilds: [...config.atomicBuilds] }, ["prod"]); + + expect(config.atomicBuilds).toEqual(["prod", "stg"]); + }); +}); + +describe("createDefaultVercelIntegrationData", () => { + it("does not enable preview for a project without a preview environment", () => { + const data = createDefaultVercelIntegrationData("prj_1", "My project", null, undefined, [ + "dev", + "prod", + ]); + + expect(data.config.pullEnvVarsBeforeBuild).toEqual(["prod"]); + expect(data.config.discoverEnvVars).toEqual(["prod"]); + }); + + it("enables preview when the project has a preview environment", () => { + const data = createDefaultVercelIntegrationData("prj_1", "My project", null, undefined, [ + "dev", + "stg", + "prod", + "preview", + ]); + + expect(data.config.pullEnvVarsBeforeBuild).toEqual(["prod", "preview"]); + expect(data.config.discoverEnvVars).toEqual(["prod", "preview"]); + }); + + it("never turns atomic builds on by default", () => { + const data = createDefaultVercelIntegrationData("prj_1", "My project", null, undefined, [ + "dev", + "stg", + "prod", + "preview", + ]); + + expect(data.config.atomicBuilds).toEqual([]); + expect(data.config.vercelStagingEnvironment).toBeNull(); + }); +}); diff --git a/apps/webapp/vite.config.ts b/apps/webapp/vite.config.ts index 56ddae17c02..967fd8fded3 100644 --- a/apps/webapp/vite.config.ts +++ b/apps/webapp/vite.config.ts @@ -75,6 +75,8 @@ export default defineConfig({ clientFiles: ["./app/entry.client.tsx", "./app/root.tsx", "./app/components/**/*.tsx"], ssrFiles: ["./app/entry.server.tsx", "./app/root.tsx"], }, + // In-build calls from local docker (e.g. the indexer) reach the dev webapp via this host + allowedHosts: ["host.docker.internal"], }, build: { sourcemap: true, diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index 1e5c7c7cab0..58d17bc6ce9 100755 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -49,6 +49,44 @@ else echo "RUN_OPS_LEGACY_DIRECT_URL not set, skipping legacy run-ops migrations." fi +# Run-ops shards: migrate every gen-2 shard that owns its own database. Each shard runs the +# identical schema, so this is the existing run-ops migrations against a new DSN. An aliased shard is +# skipped by the DSN script: it IS its target's database. Installs that never set RUN_OPS_SHARDS +# skip this entirely. +{ set +x; } 2>/dev/null +if [ -n "$RUN_OPS_SHARDS" ]; then + set -x + if [ "$SKIP_RUN_OPS_SHARD_MIGRATIONS" != "1" ]; then + echo "Running run-ops shard migrations" + # Tracing stays OFF from here to the end of the loop: `set -x` prints an assignment, so + # capturing a DSN under tracing would put the credentials in the logs. + { set +x; } 2>/dev/null + # A malformed descriptor exits 1 here, so the container stops before it migrates anything. + shard_dsns=$(node scripts/runOpsShardDsns.mjs) + # A `for` loop and NOT `... | while read`: a pipeline subshell would swallow a failed migration + # on any iteration but the last. Here `set -e` stops the boot on the first shard that fails. + # The whole loop runs in a subshell, so the IFS and `set -f` changes need no restore and cannot + # leak into the rest of the entrypoint. IFS is newline-only so a DSN is never split on other + # whitespace, and `set -f` stops a DSN query string (it holds `?`) from acting as a glob. + ( + IFS=' +' + set -f + for shard_dsn in $shard_dsns; do + # Tracing stays off so `set -x` never prints the DSN (with credentials) to the logs. + RUN_OPS_DATABASE_URL="$shard_dsn" DIRECT_URL="$shard_dsn" pnpm --filter @internal/run-ops-database db:migrate:deploy + done + ) + set -x + echo "Run-ops shard migrations done" + else + echo "SKIP_RUN_OPS_SHARD_MIGRATIONS=1, skipping run-ops shard migrations." + fi +else + set -x + echo "RUN_OPS_SHARDS not set, skipping run-ops shard migrations." +fi + if [ "$SKIP_DASHBOARD_AGENT_MIGRATIONS" != "1" ]; then echo "Running dashboard agent migrations" pnpm --filter @internal/dashboard-agent-db db:migrate:deploy diff --git a/docker/scripts/runOpsShardDsns.mjs b/docker/scripts/runOpsShardDsns.mjs new file mode 100644 index 00000000000..dc47d1500f7 --- /dev/null +++ b/docker/scripts/runOpsShardDsns.mjs @@ -0,0 +1,132 @@ +// Print the migration DSN of every run-ops shard that owns its own database, one per line, so +// entrypoint.sh can loop over them. The runner image has no `jq`, and this script is unit-tested, +// which an inline `node -e` string could not be. +// +// Contract: +// RUN_OPS_SHARDS unset or blank -> print nothing, exit 0 (single-DB and gen-1-only installs) +// invalid JSON, or not an array -> message on stderr, exit 1 (the app rejects the same value) +// a descriptor with `aliasOf` -> skipped; it shares its target's database +// the DSN -> `directUrl` if set, else `url`; skipped if neither is set +// +// Never print a DSN to stderr or to a log: stdout is consumed by the caller, nothing else. + +// Mirrors isValidDatabaseUrl in the webapp: parseable by URL(), and no empty `schema` param. +function assertDatabaseUrl(value, field, key) { + try { + const parsed = new URL(value); + if (parsed.searchParams.get("schema") === "") { + throw new Error("empty schema param"); + } + } catch { + throw new Error(`RUN_OPS_SHARDS[${key}]: ${field} is not a valid database URL`); + } +} + +/** + * Mirrors the boot schema's rules for one descriptor. Kept deliberately narrow: it checks the rules + * that make the application REJECT the value at boot, so an invalid descriptor stops the entrypoint + * before any migration runs. It does NOT reject unknown fields, because doing so would fail the + * entrypoint on a descriptor a newer application accepts, which is drift in the other direction. + */ +function assertDescriptor(d) { + const key = typeof d.key === "string" ? d.key : "?"; + if (typeof d.key !== "string" || !/^[a-z0-9]$/.test(d.key)) { + throw new Error(`RUN_OPS_SHARDS[${key}]: key must be a single [a-z0-9] char`); + } + if (typeof d.region !== "string" || d.region === "") { + throw new Error(`RUN_OPS_SHARDS[${key}]: region is required`); + } + + const hasAlias = d.aliasOf !== undefined && d.aliasOf !== null; + if (hasAlias && d.aliasOf !== "new") { + throw new Error(`RUN_OPS_SHARDS[${key}]: aliasOf must be "new", got "${d.aliasOf}"`); + } + + const hasUrl = d.url !== undefined && d.url !== null; + if (hasUrl === hasAlias) { + throw new Error(`RUN_OPS_SHARDS[${key}]: exactly one of url or aliasOf is required`); + } + + for (const field of ["url", "replicaUrl", "directUrl"]) { + const value = d[field]; + if (value === undefined || value === null) continue; + if (typeof value !== "string" || value === "") { + throw new Error(`RUN_OPS_SHARDS[${key}]: ${field} is not a valid database URL`); + } + assertDatabaseUrl(value, field, key); + } + + if (hasAlias) return; + + const rep = d.replication; + if (rep === undefined || rep === null || typeof rep !== "object") { + throw new Error(`RUN_OPS_SHARDS[${key}]: replication is required unless aliasOf is set`); + } + for (const field of ["slotName", "publicationName"]) { + if (typeof rep[field] !== "string" || rep[field] === "") { + throw new Error(`RUN_OPS_SHARDS[${key}]: replication.${field} must be a non-empty string`); + } + } + const gen = rep.originGeneration; + if (!Number.isInteger(gen) || gen < 2 || gen > 255) { + throw new Error( + `RUN_OPS_SHARDS[${key}]: replication.originGeneration must be an integer 2..255` + ); + } +} + +export function shardMigrationDsns(raw) { + if (raw === undefined || raw === null || String(raw).trim() === "") { + return []; + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error("RUN_OPS_SHARDS is not valid JSON"); + } + + if (!Array.isArray(parsed)) { + throw new Error("RUN_OPS_SHARDS is not a JSON array"); + } + + // Validate EVERY descriptor before collecting any DSN, so a valid descriptor ahead of an invalid + // one never gets its database migrated before the configuration is rejected. + for (const descriptor of parsed) { + if (descriptor === null || typeof descriptor !== "object" || Array.isArray(descriptor)) { + throw new Error("RUN_OPS_SHARDS holds an entry that is not an object"); + } + assertDescriptor(descriptor); + } + + const dsns = []; + for (const descriptor of parsed) { + // An aliased shard is the same database as its target, which its own invocation migrates. + if (descriptor.aliasOf !== undefined && descriptor.aliasOf !== null) continue; + + const dsn = descriptor.directUrl ?? descriptor.url; + // One DSN per line IS the protocol with the caller, so a DSN holding a line break would split + // into two bogus DSNs. The URL parser strips ASCII line breaks, so nothing else rejects it. + if (/[\r\n]/.test(dsn)) { + throw new Error("RUN_OPS_SHARDS holds a DSN containing a line break"); + } + dsns.push(dsn); + } + return dsns; +} + +// `import.meta.main` is not available on every supported node, so compare argv instead. +const invokedDirectly = + process.argv[1] !== undefined && process.argv[1].endsWith("runOpsShardDsns.mjs"); + +if (invokedDirectly) { + try { + for (const dsn of shardMigrationDsns(process.env.RUN_OPS_SHARDS)) { + process.stdout.write(`${dsn}\n`); + } + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + } +} diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 6e89dd52aba..8d31694686e 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -184,6 +184,7 @@ mode: "wide" | `MACHINE_PRESETS_OVERRIDE_PATH` | No | — | Path to machine presets override file. See [machine overrides](/self-hosting/overview#machine-overrides). | | `APP_ENV` | No | `NODE_ENV` | App environment. Used for things like the title tag. | | `ADMIN_EMAILS` | No | — | Regex of user emails to automatically promote to admin on signup. Does not apply to existing users. | +| `ADMIN_DASHBOARD_ENABLED` | No | 1 | Set to anything other than `1` or `true` to disable the admin dashboard and user impersonation on this instance. | | `EVENT_LOOP_MONITOR_ENABLED` | No | 1 | Node.js event loop lag monitor. | ## Multi-Provider Object Storage diff --git a/hosting/docker/clickhouse/override.xml b/hosting/docker/clickhouse/override.xml index 41897c29849..253669ab09d 100644 --- a/hosting/docker/clickhouse/override.xml +++ b/hosting/docker/clickhouse/override.xml @@ -9,12 +9,33 @@ 524288000 1 - - - 8192 - 1 - 0 - 0 - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + event_date + INTERVAL 7 DAY DELETE + + + event_date + INTERVAL 7 DAY DELETE + + + event_date + INTERVAL 30 DAY DELETE + + + + diff --git a/hosting/docker/clickhouse/users-override.xml b/hosting/docker/clickhouse/users-override.xml new file mode 100644 index 00000000000..46699505e34 --- /dev/null +++ b/hosting/docker/clickhouse/users-override.xml @@ -0,0 +1,16 @@ + + + + 1 + 8192 + 1 + 0 + 0 + + 0 + 0 + 0 + 0 + + + diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index a8bc1167b77..b893c17eca9 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -177,6 +177,7 @@ services: - clickhouse:/var/lib/clickhouse - ../clickhouse/data-paths.xml:/etc/clickhouse-server/config.d/data-paths.xml:ro - ../clickhouse/override.xml:/etc/clickhouse-server/config.d/override.xml:ro + - ../clickhouse/users-override.xml:/etc/clickhouse-server/users.d/override.xml:ro networks: - webapp healthcheck: diff --git a/internal-packages/clickhouse/src/client/client.ts b/internal-packages/clickhouse/src/client/client.ts index 9949081d504..96db1f4a529 100644 --- a/internal-packages/clickhouse/src/client/client.ts +++ b/internal-packages/clickhouse/src/client/client.ts @@ -7,8 +7,8 @@ import { type BaseQueryParams, type InsertResult, } from "@clickhouse/client"; -import type { Span, Tracer } from "@internal/tracing"; -import { recordSpanError, startSpan, trace } from "@internal/tracing"; +import type { Counter, Histogram, Meter, Span, Tracer, UpDownCounter } from "@internal/tracing"; +import { getMeter, recordSpanError, startSpan, trace } from "@internal/tracing"; import { flattenAttributes, tryCatch, type Result } from "@trigger.dev/core/v3"; import { z } from "zod"; import { InsertError, QueryError } from "./errors.js"; @@ -43,6 +43,7 @@ export type ClickhouseConfig = { httpAgent?: HttpAgent | HttpsAgent; clickhouseSettings?: ClickHouseSettings; logger?: Logger; + meter?: Meter; maxOpenConnections?: number; requestTimeoutMs?: number; logLevel?: LogLevel; @@ -57,11 +58,49 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { private readonly tracer: Tracer; private readonly name: string; private readonly logger: Logger; + private readonly meter: Meter; + private readonly queryInFlight: UpDownCounter; + private readonly queryDuration: Histogram; + private readonly queryServerDuration: Histogram; + private readonly queryReadRows: Histogram; + private readonly queryReadBytes: Histogram; + private readonly queryMemoryUsage: Histogram; + private readonly queryErrors: Counter; constructor(config: ClickhouseConfig) { this.name = config.name; this.logger = config.logger ?? new Logger("ClickhouseClient", config.logLevel ?? "info"); + this.meter = config.meter ?? getMeter("clickhouse"); + this.queryInFlight = this.meter.createUpDownCounter("clickhouse.query.in_flight", { + description: "Concurrent in-flight ClickHouse queries per client, a pool-saturation signal", + }); + this.queryDuration = this.meter.createHistogram("clickhouse.query.duration", { + description: + "Wall-clock ClickHouse query duration, includes client-side connection-pool wait", + unit: "ms", + }); + this.queryServerDuration = this.meter.createHistogram("clickhouse.query.server_duration", { + description: "Server-side ClickHouse query duration from the x-clickhouse-summary elapsed_ns", + unit: "ms", + }); + this.queryReadRows = this.meter.createHistogram("clickhouse.query.read_rows", { + description: "Rows read by a ClickHouse query, from the x-clickhouse-summary header", + unit: "{row}", + }); + this.queryReadBytes = this.meter.createHistogram("clickhouse.query.read_bytes", { + description: "Bytes read by a ClickHouse query, from the x-clickhouse-summary header", + unit: "By", + }); + this.queryMemoryUsage = this.meter.createHistogram("clickhouse.query.memory_usage", { + description: "Peak memory used by a ClickHouse query, from the x-clickhouse-summary header", + unit: "By", + }); + this.queryErrors = this.meter.createCounter("clickhouse.query.errors", { + description: + "ClickHouse query errors by type, e.g. MEMORY_LIMIT_EXCEEDED or TIMEOUT_EXCEEDED", + }); + this.client = createClient({ url: config.url, keep_alive: config.keepAlive, @@ -87,6 +126,40 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { await this.client.close(); } + private recordQueryMetrics( + operation: string, + startedAt: number, + result: { errorType?: string; summary?: Record } + ) { + const attributes = { client: this.name, operation }; + this.queryDuration.record(performance.now() - startedAt, { + ...attributes, + status: result.errorType ? "error" : "ok", + }); + if (result.errorType) { + this.queryErrors.add(1, { ...attributes, error_type: result.errorType }); + } + const summary = result.summary; + if (summary) { + const elapsedNs = Number(summary.elapsed_ns); + if (Number.isFinite(elapsedNs) && elapsedNs > 0) { + this.queryServerDuration.record(elapsedNs / 1_000_000, attributes); + } + const readRows = Number(summary.read_rows); + if (Number.isFinite(readRows)) { + this.queryReadRows.record(readRows, attributes); + } + const readBytes = Number(summary.read_bytes); + if (Number.isFinite(readBytes)) { + this.queryReadBytes.record(readBytes, attributes); + } + const memoryUsage = Number(summary.memory_usage); + if (Number.isFinite(memoryUsage) && memoryUsage > 0) { + this.queryMemoryUsage.record(memoryUsage, attributes); + } + } + } + public query, TOut extends z.ZodSchema>(req: { /** * The name of the operation. @@ -117,124 +190,147 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { }): ClickhouseQueryFunction, z.output> { return async (params, options) => { const queryId = randomUUID(); - - return await startSpan(this.tracer, "query", async (span) => { - this.logger.debug("Querying clickhouse", { - name: req.name, - query: req.query.replace(/\s+/g, " "), - params, - settings: req.settings, - attributes: options?.attributes, - queryId, - }); - - span.setAttributes({ - "clickhouse.clientName": this.name, - "clickhouse.operationName": req.name, - "clickhouse.queryId": queryId, - ...flattenAttributes(req.settings, "clickhouse.settings"), - ...flattenAttributes(options?.attributes), - }); - - const validParams = req.params?.safeParse(params); - - if (validParams?.error) { - recordSpanError(span, validParams.error); - - this.logger.error("Error parsing query params", { + const startedAt = performance.now(); + this.queryInFlight.add(1, { client: this.name }); + let summary: Record | undefined; + + const result = await startSpan( + this.tracer, + "query", + async (span): Promise[], QueryError>> => { + this.logger.debug("Querying clickhouse", { name: req.name, - error: validParams.error, - query: req.query, + query: req.query.replace(/\s+/g, " "), params, + settings: req.settings, + attributes: options?.attributes, queryId, }); - return [ - new QueryError(`Bad params: ${generateErrorMessage(validParams.error.issues)}`, { - query: req.query, - }), - null, - ]; - } - - let unparsedRows: Array = []; - - const [clickhouseError, res] = await tryCatch( - this.client.query({ - query: req.query, - query_params: validParams?.data, - format: "JSONEachRow", - query_id: queryId, - ...options?.params, - clickhouse_settings: { - ...req.settings, - ...options?.params?.clickhouse_settings, - }, - }) - ); - - if (clickhouseError) { - const errorLogFields = { - name: req.name, - error: clickhouseError, - query: req.query, - params, - queryId, - }; + span.setAttributes({ + "clickhouse.clientName": this.name, + "clickhouse.operationName": req.name, + "clickhouse.queryId": queryId, + ...flattenAttributes(req.settings, "clickhouse.settings"), + ...flattenAttributes(options?.attributes), + }); - this.logger.error("Error querying clickhouse", errorLogFields); + const validParams = req.params?.safeParse(params); - recordClickhouseError(span, clickhouseError); + if (validParams?.error) { + recordSpanError(span, validParams.error); - return [ - new QueryError(`Unable to query clickhouse: ${clickhouseError.message}`, { + this.logger.error("Error parsing query params", { + name: req.name, + error: validParams.error, query: req.query, - }), - null, - ]; - } + params, + queryId, + }); + + return [ + new QueryError(`Bad params: ${generateErrorMessage(validParams.error.issues)}`, { + query: req.query, + }), + null, + ]; + } - unparsedRows = await res.json(); + let unparsedRows: Array = []; - span.setAttributes({ - "clickhouse.query_id": res.query_id, - ...flattenAttributes(res.response_headers, "clickhouse.response_headers"), - }); + const [clickhouseError, res] = await tryCatch( + this.client.query({ + query: req.query, + query_params: validParams?.data, + format: "JSONEachRow", + query_id: queryId, + ...options?.params, + clickhouse_settings: { + ...req.settings, + ...options?.params?.clickhouse_settings, + }, + }) + ); + + if (clickhouseError) { + const errorLogFields = { + name: req.name, + error: clickhouseError, + query: req.query, + params, + queryId, + }; + + this.logger.error("Error querying clickhouse", errorLogFields); + + recordClickhouseError(span, clickhouseError); + + return [ + new QueryError( + `Unable to query clickhouse: ${clickhouseError.message}`, + { query: req.query }, + clickhouseError instanceof ClickHouseError ? clickhouseError.type : undefined + ), + null, + ]; + } - const summaryHeader = res.response_headers["x-clickhouse-summary"]; + unparsedRows = await res.json(); - if (typeof summaryHeader === "string") { span.setAttributes({ - ...flattenAttributes(JSON.parse(summaryHeader), "clickhouse.summary"), + "clickhouse.query_id": res.query_id, + ...flattenAttributes(res.response_headers, "clickhouse.response_headers"), }); - } - const parsed = z.array(req.schema).safeParse(unparsedRows); + const summaryHeader = res.response_headers["x-clickhouse-summary"]; - if (parsed.error) { - this.logger.error("Error parsing clickhouse query result", { - name: req.name, - error: parsed.error, - query: req.query, - params, - queryId, - }); + if (typeof summaryHeader === "string") { + summary = JSON.parse(summaryHeader); + span.setAttributes({ + ...flattenAttributes(summary, "clickhouse.summary"), + }); + } - const queryError = new QueryError(generateErrorMessage(parsed.error.issues), { - query: req.query, - }); + const parsed = z.array(req.schema).safeParse(unparsedRows); - recordSpanError(span, queryError); + if (parsed.error) { + this.logger.error("Error parsing clickhouse query result", { + name: req.name, + error: parsed.error, + query: req.query, + params, + queryId, + }); - return [queryError, null]; - } + const queryError = new QueryError(generateErrorMessage(parsed.error.issues), { + query: req.query, + }); - span.setAttributes({ - "clickhouse.rows": unparsedRows.length, - }); + recordSpanError(span, queryError); + + return [queryError, null]; + } - return [null, parsed.data]; + span.setAttributes({ + "clickhouse.rows": unparsedRows.length, + }); + + return [null, parsed.data]; + } + ) + .catch((error) => { + this.recordQueryMetrics(req.name, startedAt, { errorType: "exception" }); + throw error; + }) + .finally(() => this.queryInFlight.add(-1, { client: this.name })); + + this.recordQueryMetrics(req.name, startedAt, { + errorType: + result[0] instanceof QueryError ? (result[0].clickhouseErrorType ?? "other") : undefined, + summary, }); + + return result; }; } @@ -278,163 +374,188 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { }): ClickhouseQueryWithStatsFunction, z.output> { return async (params, options) => { const queryId = randomUUID(); - - return await startSpan(this.tracer, "queryWithStats", async (span) => { - this.logger.debug("Querying clickhouse with stats", { - name: req.name, - query: req.query.replace(/\s+/g, " "), - params, - settings: req.settings, - attributes: options?.attributes, - queryId, - }); - - span.setAttributes({ - "clickhouse.clientName": this.name, - "clickhouse.operationName": req.name, - "clickhouse.queryId": queryId, - ...flattenAttributes(req.settings, "clickhouse.settings"), - ...flattenAttributes(options?.attributes), - }); - - const validParams = req.params?.safeParse(params); - - if (validParams?.error) { - recordSpanError(span, validParams.error); - - this.logger.error("Error parsing query params", { + const startedAt = performance.now(); + this.queryInFlight.add(1, { client: this.name }); + let summary: Record | undefined; + + const result = await startSpan( + this.tracer, + "queryWithStats", + async ( + span + ): Promise[]; stats: QueryStats }, QueryError>> => { + this.logger.debug("Querying clickhouse with stats", { name: req.name, - error: validParams.error, - query: req.query, + query: req.query.replace(/\s+/g, " "), params, + settings: req.settings, + attributes: options?.attributes, queryId, }); - return [ - new QueryError(`Bad params: ${generateErrorMessage(validParams.error.issues)}`, { - query: req.query, - }), - null, - ]; - } - - let unparsedRows: Array = []; + span.setAttributes({ + "clickhouse.clientName": this.name, + "clickhouse.operationName": req.name, + "clickhouse.queryId": queryId, + ...flattenAttributes(req.settings, "clickhouse.settings"), + ...flattenAttributes(options?.attributes), + }); - const [clickhouseError, res] = await tryCatch( - this.client.query({ - query: req.query, - query_params: validParams?.data, - format: "JSONEachRow", - query_id: queryId, - ...options?.params, - clickhouse_settings: { - ...req.settings, - ...options?.params?.clickhouse_settings, - }, - }) - ); + const validParams = req.params?.safeParse(params); - if (clickhouseError) { - const errorLogFields = { - ...req.logFields, - name: req.name, - error: clickhouseError, - query: req.query, - params, - queryId, - }; + if (validParams?.error) { + recordSpanError(span, validParams.error); - switch (classifyClickhouseError(clickhouseError, req.userAuthoredQuery)) { - case "quota": - this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); - break; - case "invalid-sql": - this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); - break; - default: - this.logger.error("Error querying clickhouse", errorLogFields); + this.logger.error("Error parsing query params", { + name: req.name, + error: validParams.error, + query: req.query, + params, + queryId, + }); + + return [ + new QueryError(`Bad params: ${generateErrorMessage(validParams.error.issues)}`, { + query: req.query, + }), + null, + ]; } - recordClickhouseError(span, clickhouseError); + let unparsedRows: Array = []; - return [ - new QueryError(`Unable to query clickhouse: ${clickhouseError.message}`, { + const [clickhouseError, res] = await tryCatch( + this.client.query({ query: req.query, - }), - null, - ]; - } + query_params: validParams?.data, + format: "JSONEachRow", + query_id: queryId, + ...options?.params, + clickhouse_settings: { + ...req.settings, + ...options?.params?.clickhouse_settings, + }, + }) + ); + + if (clickhouseError) { + const errorLogFields = { + ...req.logFields, + name: req.name, + error: clickhouseError, + query: req.query, + params, + queryId, + }; + + switch (classifyClickhouseError(clickhouseError, req.userAuthoredQuery)) { + case "quota": + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + break; + case "invalid-sql": + this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); + break; + default: + this.logger.error("Error querying clickhouse", errorLogFields); + } - unparsedRows = await res.json(); + recordClickhouseError(span, clickhouseError); - span.setAttributes({ - "clickhouse.query_id": res.query_id, - ...flattenAttributes(res.response_headers, "clickhouse.response_headers"), - }); + return [ + new QueryError( + `Unable to query clickhouse: ${clickhouseError.message}`, + { query: req.query }, + clickhouseError instanceof ClickHouseError ? clickhouseError.type : undefined + ), + null, + ]; + } - // Parse the summary header to get stats - const summaryHeader = res.response_headers["x-clickhouse-summary"]; - let stats: QueryStats = { - read_rows: "0", - read_bytes: "0", - written_rows: "0", - written_bytes: "0", - total_rows_to_read: "0", - result_rows: "0", - result_bytes: "0", - elapsed_ns: "0", - byte_seconds: "0", - }; + unparsedRows = await res.json(); - if (typeof summaryHeader === "string") { - const parsedSummary = JSON.parse(summaryHeader); - this.logger.debug("parsedSummary", parsedSummary); - const readBytes = parsedSummary.read_bytes ? parseInt(parsedSummary.read_bytes, 10) : 0; - const elapsedNs = parsedSummary.elapsed_ns ? parseInt(parsedSummary.elapsed_ns, 10) : 0; - const elapsedSeconds = elapsedNs / 1_000_000_000; - const byteSeconds = elapsedSeconds > 0 ? readBytes / elapsedSeconds : 0; - stats = { - read_rows: parsedSummary.read_rows ?? "0", - read_bytes: parsedSummary.read_bytes ?? "0", - written_rows: parsedSummary.written_rows ?? "0", - written_bytes: parsedSummary.written_bytes ?? "0", - total_rows_to_read: parsedSummary.total_rows_to_read ?? "0", - result_rows: parsedSummary.result_rows ?? "0", - result_bytes: parsedSummary.result_bytes ?? "0", - elapsed_ns: parsedSummary.elapsed_ns ?? "0", - byte_seconds: byteSeconds.toString(), - }; span.setAttributes({ - ...flattenAttributes(parsedSummary, "clickhouse.summary"), + "clickhouse.query_id": res.query_id, + ...flattenAttributes(res.response_headers, "clickhouse.response_headers"), }); - } - const parsed = z.array(req.schema).safeParse(unparsedRows); + // Parse the summary header to get stats + const summaryHeader = res.response_headers["x-clickhouse-summary"]; + let stats: QueryStats = { + read_rows: "0", + read_bytes: "0", + written_rows: "0", + written_bytes: "0", + total_rows_to_read: "0", + result_rows: "0", + result_bytes: "0", + elapsed_ns: "0", + byte_seconds: "0", + }; - if (parsed.error) { - this.logger.error("Error parsing clickhouse query result", { - name: req.name, - error: parsed.error, - query: req.query, - params, - queryId, - }); + if (typeof summaryHeader === "string") { + const parsedSummary = JSON.parse(summaryHeader); + summary = parsedSummary; + this.logger.debug("parsedSummary", parsedSummary); + const readBytes = parsedSummary.read_bytes ? parseInt(parsedSummary.read_bytes, 10) : 0; + const elapsedNs = parsedSummary.elapsed_ns ? parseInt(parsedSummary.elapsed_ns, 10) : 0; + const elapsedSeconds = elapsedNs / 1_000_000_000; + const byteSeconds = elapsedSeconds > 0 ? readBytes / elapsedSeconds : 0; + stats = { + read_rows: parsedSummary.read_rows ?? "0", + read_bytes: parsedSummary.read_bytes ?? "0", + written_rows: parsedSummary.written_rows ?? "0", + written_bytes: parsedSummary.written_bytes ?? "0", + total_rows_to_read: parsedSummary.total_rows_to_read ?? "0", + result_rows: parsedSummary.result_rows ?? "0", + result_bytes: parsedSummary.result_bytes ?? "0", + elapsed_ns: parsedSummary.elapsed_ns ?? "0", + byte_seconds: byteSeconds.toString(), + }; + span.setAttributes({ + ...flattenAttributes(parsedSummary, "clickhouse.summary"), + }); + } - const queryError = new QueryError(generateErrorMessage(parsed.error.issues), { - query: req.query, - }); + const parsed = z.array(req.schema).safeParse(unparsedRows); + + if (parsed.error) { + this.logger.error("Error parsing clickhouse query result", { + name: req.name, + error: parsed.error, + query: req.query, + params, + queryId, + }); - recordSpanError(span, queryError); + const queryError = new QueryError(generateErrorMessage(parsed.error.issues), { + query: req.query, + }); - return [queryError, null]; - } + recordSpanError(span, queryError); - span.setAttributes({ - "clickhouse.rows": unparsedRows.length, - }); + return [queryError, null]; + } + + span.setAttributes({ + "clickhouse.rows": unparsedRows.length, + }); - return [null, { rows: parsed.data, stats }]; + return [null, { rows: parsed.data, stats }]; + } + ) + .catch((error) => { + this.recordQueryMetrics(req.name, startedAt, { errorType: "exception" }); + throw error; + }) + .finally(() => this.queryInFlight.add(-1, { client: this.name })); + + this.recordQueryMetrics(req.name, startedAt, { + errorType: + result[0] instanceof QueryError ? (result[0].clickhouseErrorType ?? "other") : undefined, + summary, }); + + return result; }; } @@ -446,103 +567,126 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { }): ClickhouseQueryFunction { return async (params, options) => { const queryId = randomUUID(); - - return await startSpan(this.tracer, "queryFast", async (span) => { - this.logger.debug("Querying clickhouse fast", { - name: req.name, - query: req.query.replace(/\s+/g, " "), - params, - settings: req.settings, - attributes: options?.attributes, - queryId, - }); - - span.setAttributes({ - "clickhouse.clientName": this.name, - "clickhouse.operationName": req.name, - "clickhouse.queryId": queryId, - ...flattenAttributes(req.settings, "clickhouse.settings"), - ...flattenAttributes(options?.attributes), - }); - - const [clickhouseError, resultSet] = await tryCatch( - this.client.query({ - query: req.query, - query_params: params, - format: "JSONCompactEachRow", - query_id: queryId, - ...options?.params, - clickhouse_settings: { - ...req.settings, - ...options?.params?.clickhouse_settings, - }, - }) - ); - - if (clickhouseError) { - const errorLogFields = { + const startedAt = performance.now(); + this.queryInFlight.add(1, { client: this.name }); + let summary: Record | undefined; + + const result = await startSpan( + this.tracer, + "queryFast", + async (span): Promise> => { + this.logger.debug("Querying clickhouse fast", { name: req.name, - error: clickhouseError, - query: req.query, + query: req.query.replace(/\s+/g, " "), params, + settings: req.settings, + attributes: options?.attributes, queryId, - }; - - this.logger.error("Error querying clickhouse", errorLogFields); + }); - recordClickhouseError(span, clickhouseError); + span.setAttributes({ + "clickhouse.clientName": this.name, + "clickhouse.operationName": req.name, + "clickhouse.queryId": queryId, + ...flattenAttributes(req.settings, "clickhouse.settings"), + ...flattenAttributes(options?.attributes), + }); - return [ - new QueryError(`Unable to query clickhouse: ${clickhouseError.message}`, { + const [clickhouseError, resultSet] = await tryCatch( + this.client.query({ query: req.query, - }), - null, - ]; - } - - span.setAttributes({ - "clickhouse.query_id": resultSet.query_id, - ...flattenAttributes(resultSet.response_headers, "clickhouse.response_headers"), - }); - - const summaryHeader = resultSet.response_headers["x-clickhouse-summary"]; + query_params: params, + format: "JSONCompactEachRow", + query_id: queryId, + ...options?.params, + clickhouse_settings: { + ...req.settings, + ...options?.params?.clickhouse_settings, + }, + }) + ); + + if (clickhouseError) { + const errorLogFields = { + name: req.name, + error: clickhouseError, + query: req.query, + params, + queryId, + }; + + this.logger.error("Error querying clickhouse", errorLogFields); + + recordClickhouseError(span, clickhouseError); + + return [ + new QueryError( + `Unable to query clickhouse: ${clickhouseError.message}`, + { query: req.query }, + clickhouseError instanceof ClickHouseError ? clickhouseError.type : undefined + ), + null, + ]; + } - if (typeof summaryHeader === "string") { span.setAttributes({ - ...flattenAttributes(JSON.parse(summaryHeader), "clickhouse.summary"), + "clickhouse.query_id": resultSet.query_id, + ...flattenAttributes(resultSet.response_headers, "clickhouse.response_headers"), }); - } - const resultRows: Array = []; + const summaryHeader = resultSet.response_headers["x-clickhouse-summary"]; - for await (const rows of resultSet.stream()) { - if (rows.length === 0) { - continue; + if (typeof summaryHeader === "string") { + summary = JSON.parse(summaryHeader); + span.setAttributes({ + ...flattenAttributes(summary, "clickhouse.summary"), + }); } - for (const row of rows) { - const rowData = row.json() as any[]; + const resultRows: Array = []; - const hydratedRow: Record = {}; - for (let i = 0; i < req.columns.length; i++) { - const column = req.columns[i]; + for await (const rows of resultSet.stream()) { + if (rows.length === 0) { + continue; + } - if (typeof column === "string") { - hydratedRow[column] = rowData[i]; - } else { - hydratedRow[column.name] = rowData[i]; + for (const row of rows) { + const rowData = row.json() as any[]; + + const hydratedRow: Record = {}; + for (let i = 0; i < req.columns.length; i++) { + const column = req.columns[i]; + + if (typeof column === "string") { + hydratedRow[column] = rowData[i]; + } else { + hydratedRow[column.name] = rowData[i]; + } } + resultRows.push(hydratedRow as TOut); } - resultRows.push(hydratedRow as TOut); } - } - span.setAttributes({ - "clickhouse.rows": resultRows.length, - }); + span.setAttributes({ + "clickhouse.rows": resultRows.length, + }); - return [null, resultRows]; + return [null, resultRows]; + } + ) + .catch((error) => { + this.recordQueryMetrics(req.name, startedAt, { errorType: "exception" }); + throw error; + }) + .finally(() => this.queryInFlight.add(-1, { client: this.name })); + + this.recordQueryMetrics(req.name, startedAt, { + errorType: + result[0] instanceof QueryError ? (result[0].clickhouseErrorType ?? "other") : undefined, + summary, }); + + return result; }; } diff --git a/internal-packages/clickhouse/src/client/errors.ts b/internal-packages/clickhouse/src/client/errors.ts index ff0be4d0d54..dd4de178055 100644 --- a/internal-packages/clickhouse/src/client/errors.ts +++ b/internal-packages/clickhouse/src/client/errors.ts @@ -45,10 +45,39 @@ export class InsertError extends BaseError { export class QueryError extends BaseError<{ query: string }> { public readonly retry = true; public readonly name = QueryError.name; - constructor(message: string, context: { query: string }) { + /** + * The underlying ClickHouse error type (e.g. `TIMEOUT_EXCEEDED`) when the failure came from + * ClickHouse rejecting the query, else undefined. Lets callers distinguish a query that hit a + * server-side resource limit from an unexpected failure. + */ + public readonly clickhouseErrorType?: string; + constructor(message: string, context: { query: string }, clickhouseErrorType?: string) { super({ message, context, }); + this.clickhouseErrorType = clickhouseErrorType; } } + +/** + * ClickHouse error types raised when a query exceeds a server-side resource limit + * (`max_execution_time`, `max_memory_usage`, etc.). These mean the caller's query was too + * expensive, not a service fault, so callers can turn them into an actionable 4xx. + */ +const CLICKHOUSE_RESOURCE_LIMIT_ERROR_TYPES = new Set([ + "MEMORY_LIMIT_EXCEEDED", + "TIMEOUT_EXCEEDED", + "TOO_SLOW", + "TOO_MANY_ROWS", + "TOO_MANY_BYTES", + "TOO_MANY_ROWS_OR_BYTES", +]); + +export function isClickhouseResourceLimitError(error: unknown): boolean { + return ( + error instanceof QueryError && + error.clickhouseErrorType !== undefined && + CLICKHOUSE_RESOURCE_LIMIT_ERROR_TYPES.has(error.clickhouseErrorType) + ); +} diff --git a/internal-packages/clickhouse/src/client/queryBuilder.ts b/internal-packages/clickhouse/src/client/queryBuilder.ts index bcdc68089c9..7b4f172e931 100644 --- a/internal-packages/clickhouse/src/client/queryBuilder.ts +++ b/internal-packages/clickhouse/src/client/queryBuilder.ts @@ -12,6 +12,7 @@ export type WhereCondition = { export class ClickhouseQueryBuilder { private name: string; private baseQuery: string; + private prewhereClauses: string[] = []; private whereClauses: string[] = []; private havingClauses: string[] = []; private params: QueryParams = {}; @@ -42,6 +43,28 @@ export class ClickhouseQueryBuilder { return this; } + /** + * Adds a PREWHERE clause. On a `... FINAL` base query, only use this for columns that are + * immutable or additive across a run's versions (e.g. task_identifier, tags): PREWHERE filters + * rows before FINAL reconciles versions, so a mutable column (e.g. status) would keep a stale + * version and drop the winning one. It filters before materialising the wide columns, which is + * what bounds memory on `task_runs_v2 FINAL` scans. + */ + prewhere(clause: string, params?: QueryParams): this { + this.prewhereClauses.push(clause); + if (params) { + Object.assign(this.params, params); + } + return this; + } + + prewhereIf(condition: any, clause: string, params?: QueryParams): this { + if (condition) { + this.prewhere(clause, params); + } + return this; + } + where(clause: string, params?: QueryParams): this { this.whereClauses.push(clause); if (params) { @@ -117,6 +140,9 @@ export class ClickhouseQueryBuilder { build(): { query: string; params: QueryParams } { let query = this.baseQuery; + if (this.prewhereClauses.length > 0) { + query += " PREWHERE " + this.prewhereClauses.join(" AND "); + } if (this.whereClauses.length > 0) { query += " WHERE " + this.whereClauses.join(" AND "); } diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 407c33135cc..100101a5635 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -75,6 +75,7 @@ import { } from "./errors.js"; export { msToClickHouseInterval } from "./intervals.js"; import { Logger, type LogLevel } from "@trigger.dev/core/logger"; +import type { Meter } from "@internal/tracing"; import type { Agent as HttpAgent } from "http"; import type { Agent as HttpsAgent } from "https"; @@ -123,7 +124,7 @@ export { export type { ColumnFormatType, OutputColumnMetadata } from "@internal/tsql"; // Errors -export { QueryError } from "./client/errors.js"; +export { QueryError, isClickhouseResourceLimitError } from "./client/errors.js"; export type ClickhouseCommonConfig = { keepAlive?: { @@ -133,6 +134,7 @@ export type ClickhouseCommonConfig = { httpAgent?: HttpAgent | HttpsAgent; clickhouseSettings?: ClickHouseSettings; logger?: Logger; + meter?: Meter; logLevel?: LogLevel; compression?: { request?: boolean; @@ -178,6 +180,7 @@ export class ClickHouse { url: config.url, clickhouseSettings: config.clickhouseSettings, logger: this.logger, + meter: config.meter, logLevel: config.logLevel, keepAlive: config.keepAlive, httpAgent: config.httpAgent, @@ -195,6 +198,7 @@ export class ClickHouse { url: config.readerUrl, clickhouseSettings: config.clickhouseSettings, logger: this.logger, + meter: config.meter, logLevel: config.logLevel, keepAlive: config.keepAlive, httpAgent: config.httpAgent, @@ -207,6 +211,7 @@ export class ClickHouse { url: config.writerUrl, clickhouseSettings: config.clickhouseSettings, logger: this.logger, + meter: config.meter, logLevel: config.logLevel, keepAlive: config.keepAlive, httpAgent: config.httpAgent, diff --git a/internal-packages/database/prisma/migrations/20260722155558_add_worker_deployment_build_env_vars/migration.sql b/internal-packages/database/prisma/migrations/20260722155558_add_worker_deployment_build_env_vars/migration.sql new file mode 100644 index 00000000000..49e62e6e20d --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260722155558_add_worker_deployment_build_env_vars/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "public"."WorkerDeployment" ADD COLUMN IF NOT EXISTS "buildEnvVars" JSONB; diff --git a/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql new file mode 100644 index 00000000000..931ea947926 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "public"."WorkerDeployment" ADD COLUMN IF NOT EXISTS "cliVersion" TEXT; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 470f2c251c4..a1423f340c0 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2265,12 +2265,17 @@ model WorkerDeployment { runtime String? runtimeVersion String? + /// CLI version that initiated the deploy, stamped at initialization + cliVersion String? imageReference String? imagePlatform String @default("linux/amd64") externalBuildData Json? buildServerMetadata Json? + /// Encrypted build-time env vars for pre-bundled (fromBundle) deploys, as an + /// EncryptedSecretValue envelope. Cleared when the deployment reaches a terminal status. + buildEnvVars Json? status WorkerDeploymentStatus @default(PENDING) type WorkerDeploymentType @default(V1) diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts index 6fb30b9a4bf..622efe613fc 100644 --- a/internal-packages/redis/src/index.ts +++ b/internal-packages/redis/src/index.ts @@ -1,7 +1,24 @@ -import { Redis, type RedisOptions } from "ioredis"; +import { type Cluster, Redis, type RedisOptions } from "ioredis"; import { Logger } from "@trigger.dev/core/logger"; -export { Redis, type Callback, type RedisOptions, type Result, type RedisCommander } from "ioredis"; +export { + Redis, + Cluster, + type Callback, + type RedisOptions, + type ClusterNode, + type ClusterOptions, + type Result, + type RedisCommander, +} from "ioredis"; + +/** + * Either endpoint shape. A component that only issues key-addressed commands works against both, so + * it should accept this rather than pin itself to a standalone connection. Commands with no key — + * SCAN above all — do NOT fan out across a cluster, so anything that issues one must iterate + * `cluster.nodes("master")` itself. + */ +export type RedisClient = Redis | Cluster; /** * Reply-error -> reconnect mapping. Without this hook, an ElastiCache diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json index 96ace3a0e43..f4cfcd629d1 100644 --- a/internal-packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -43,7 +43,7 @@ }, "scripts": { "clean": "rimraf dist", - "typecheck": "tsc --noEmit -p tsconfig.build.json", + "typecheck": "tsc --noEmit -p tsconfig.build.json && tsc --noEmit -p tsconfig.freeze-test.json", "test": "vitest --sequence.concurrent=false --no-file-parallelism", "test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled", "build": "pnpm run clean && tsc -p tsconfig.build.json", diff --git a/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts b/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts new file mode 100644 index 00000000000..01393ba4f96 --- /dev/null +++ b/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts @@ -0,0 +1,266 @@ +/** + * Waitpoint coordination benchmark. Reports numbers; asserts nothing — on a shared runner + * the timings swing far more than any threshold worth gating on. + * + * Four groups, and only the first two are pairs: + * + * 1. Pending count — the store's SCARD gate against the previous path's + * `COUNT(*) ... WHERE status='PENDING'`, over the same population. Like for like. + * 2. Read amplification — the store's `readBlockState` against a full-payload `SELECT` + * of the same waitpoints. Like for like. + * 3. Store-only write paths — block+complete+deliver and K-watcher fan-out. Absolute + * numbers with NO Postgres counterpart: no single statement on the previous path + * corresponds to a Redis round trip that both blocks a run and delivers to watchers. + * 4. Register cost versus edge count — `registerBlocks` registers each edge with its own + * round trip before the single absorb. This measures whether that serial loop is a + * real cost at a wide fan-in, or a non-issue, at several fan-in widths. + * + * Every Postgres measurement here runs against rows this file inserts. A baseline over an + * empty table measures nothing. + * + * Knobs: BENCH_WP_ITERATIONS, BENCH_WP_FANIN, BENCH_WP_WATCHERS, BENCH_WP_REGISTER_WIDTHS, + * BENCH_WP_REGISTER_SAMPLES. + */ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { + WaitpointStoreCoordinator, + type BlockEdge, + type WaitpointRecordInput, +} from "../waitpointCoordinator/storeCoordinator.js"; +import { setupAuthenticatedEnvironment } from "../tests/setup.js"; + +vi.setConfig({ testTimeout: 900_000 }); + +const ITERATIONS = Number(process.env.BENCH_WP_ITERATIONS ?? 100); +const FANIN = Number(process.env.BENCH_WP_FANIN ?? 1001); +const WATCHERS = Number(process.env.BENCH_WP_WATCHERS ?? 100); +const REGISTER_WIDTHS = (process.env.BENCH_WP_REGISTER_WIDTHS ?? "1,10,100,1001") + .split(",") + .map((raw) => Number(raw.trim())) + .filter((width) => Number.isFinite(width) && width > 0); +const REGISTER_SAMPLES = Number(process.env.BENCH_WP_REGISTER_SAMPLES ?? 20); +const NOW = new Date().toISOString(); + +type Sample = { label: string; count: number; p50: number; p99: number; totalMs: number }; + +function percentile(sorted: number[], p: number): number { + if (sorted.length === 0) return 0; + return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))]!; +} + +async function measure(label: string, count: number, run: (i: number) => Promise) { + const durations: number[] = []; + const started = Date.now(); + for (let i = 0; i < count; i++) { + const t0 = performance.now(); + await run(i); + durations.push(performance.now() - t0); + } + durations.sort((a, b) => a - b); + const sample: Sample = { + label, + count, + p50: percentile(durations, 50), + p99: percentile(durations, 99), + totalMs: Date.now() - started, + }; + console.log( + `[bench] ${sample.label} n=${sample.count} p50=${sample.p50.toFixed(2)}ms ` + + `p99=${sample.p99.toFixed(2)}ms total=${sample.totalMs}ms` + ); + return sample; +} + +function record(id: string, environmentId: string, projectId: string): WaitpointRecordInput { + return { + id, + friendlyId: `waitpoint_${id}`, + type: "MANUAL", + environmentId, + projectId, + createdAt: NOW, + updatedAt: NOW, + userProvidedIdempotencyKey: false, + tags: [], + }; +} + +const completion = { + completedAt: NOW, + outputType: "application/json", + outputIsError: false, + output: { inline: '{"ok":true}' }, +}; + +function edge(waitpointId: string, batchIndex?: number): BlockEdge { + return { waitpointId, batchIndex, createdAt: NOW, type: "MANUAL" }; +} + +async function insertWaitpoints( + prisma: PrismaClient, + ids: string[], + environmentId: string, + projectId: string +) { + await prisma.waitpoint.createMany({ + data: ids.map((id) => ({ + id, + friendlyId: `waitpoint_${id}`, + type: "MANUAL" as const, + idempotencyKey: id, + userProvidedIdempotencyKey: false, + projectId, + environmentId, + })), + }); +} + +containerTest( + "waitpoint coordination: pending count, read amplification, store write paths, register cost", + async ({ prisma, redisOptions }) => { + const env = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const store = new WaitpointStoreCoordinator({ redisOptions }); + const samples: Sample[] = []; + const registerCost: Array<{ + width: number; + p50Ms: number; + p99Ms: number; + perEdgeMsP50: number; + }> = []; + + try { + const ids = Array.from({ length: FANIN }, (_, i) => `bench_w_${i}`); + + // Both stores get the SAME population. A Postgres baseline over an empty table + // measures an index probe against nothing. + await insertWaitpoints(prisma, ids, env.id, env.project.id); + for (const id of ids) { + await store.createIfAbsent({ + record: record(id, env.id, env.project.id), + status: "PENDING", + }); + } + await store.registerBlocks({ + runId: "bench_run_fanin", + edges: ids.map((id, index) => edge(id, index)), + }); + + // --- group 1: the pending-count gate, like for like --- + samples.push( + await measure("store.pendingCount", ITERATIONS, async () => { + await store.absorbBlockers({ runId: "bench_run_fanin", edges: [] }); + }) + ); + samples.push( + await measure("postgres.pendingCount", ITERATIONS, async () => { + await prisma.$queryRaw`SELECT COUNT(*) FROM "Waitpoint" WHERE id = ANY(${ids}::text[]) AND status = 'PENDING'`; + }) + ); + + // --- group 2: read amplification, like for like --- + samples.push( + await measure("store.readBlockState", ITERATIONS, async () => { + await store.readBlockState("bench_run_fanin"); + }) + ); + samples.push( + await measure("postgres.hydrateFullPayload", ITERATIONS, async () => { + // Every column of every waitpoint — the amplification the store removes. + await prisma.waitpoint.findMany({ where: { id: { in: ids } } }); + }) + ); + + // --- group 3: store-only write paths, no Postgres counterpart --- + samples.push( + await measure("store.block+complete+deliver", ITERATIONS, async (i) => { + const id = `bench_cycle_${i}`; + await store.createIfAbsent({ + record: record(id, env.id, env.project.id), + status: "PENDING", + }); + await store.registerBlocks({ runId: `bench_run_${i}`, edges: [edge(id)] }); + const done = await store.complete({ waitpointId: id, completion }); + for (const watcher of done.watchers) { + await store.deliverCompletion({ + runId: watcher.runId, + waitpointId: id, + completion: done.completion!, + }); + } + }) + ); + + const fanOutId = "bench_fanout_w"; + await store.createIfAbsent({ + record: record(fanOutId, env.id, env.project.id), + status: "PENDING", + }); + for (let i = 0; i < WATCHERS; i++) { + await store.registerBlocks({ runId: `bench_watcher_${i}`, edges: [edge(fanOutId)] }); + } + samples.push( + await measure(`store.complete+deliver(watchers=${WATCHERS})`, 1, async () => { + const done = await store.complete({ waitpointId: fanOutId, completion }); + // Serial on purpose: this is the worst case, and it is the number that says + // whether delivery needs to pipeline. + for (const watcher of done.watchers) { + await store.deliverCompletion({ + runId: watcher.runId, + waitpointId: fanOutId, + completion: done.completion!, + }); + } + }) + ); + + // --- group 4: register cost versus edge count --- + // registerBlocks registers each edge with its own round trip, serially, before the + // single absorb. A review flagged that a wide fan-in therefore serializes one round + // trip per edge. This measures the real cost at several widths rather than predicting + // it, so the decision about bounded concurrency is made against a number. + const registerPoolWidth = Math.max(0, ...REGISTER_WIDTHS); + const registerIds = Array.from({ length: registerPoolWidth }, (_, i) => `bench_reg_w_${i}`); + await insertWaitpoints(prisma, registerIds, env.id, env.project.id); + for (const id of registerIds) { + await store.createIfAbsent({ + record: record(id, env.id, env.project.id), + status: "PENDING", + }); + } + + for (const width of REGISTER_WIDTHS) { + const edges = registerIds.slice(0, width).map((id, index) => edge(id, index)); + let call = 0; + const sample = await measure( + `store.registerBlocks(edges=${width})`, + REGISTER_SAMPLES, + async () => { + await store.registerBlocks({ runId: `bench_register_${width}_${call++}`, edges }); + } + ); + samples.push(sample); + registerCost.push({ + width, + p50Ms: sample.p50, + p99Ms: sample.p99, + perEdgeMsP50: sample.p50 / width, + }); + console.log( + `[bench] store.registerBlocks(edges=${width}) implied per-edge cost ` + + `p50=${(sample.p50 / width).toFixed(3)}ms p99=${(sample.p99 / width).toFixed(3)}ms` + ); + } + + console.log( + `[bench] groups 1 and 2 are like-for-like pairs. Group 3 and the register-cost ` + + `group (4) have no Postgres counterpart: no single statement on the previous ` + + `path corresponds to a Redis round trip that blocks, completes and delivers, ` + + `or to a serial per-edge register loop.` + ); + console.log(`[bench] summary\n${JSON.stringify({ samples, registerCost }, null, 2)}`); + } finally { + await store.quit(); + } + } +); diff --git a/internal-packages/run-engine/src/engine/systems/completedWaitpointFreeze.test.ts b/internal-packages/run-engine/src/engine/systems/completedWaitpointFreeze.test.ts new file mode 100644 index 00000000000..a1af90582b6 --- /dev/null +++ b/internal-packages/run-engine/src/engine/systems/completedWaitpointFreeze.test.ts @@ -0,0 +1,664 @@ +// The freeze's executable definition. It runs the real oracle, +// enhanceExecutionSnapshotWithWaitpoints, against a reference resolver over equivalent +// records, and asserts the two agree field for field. The waitpoint lane owns the +// production resolver; this reference exists so the frozen shapes are checked rather +// than asserted. +import { isDeepStrictEqual } from "node:util"; +import { describe, expect, it } from "vitest"; +import type { Waitpoint } from "@trigger.dev/database"; +import { BatchId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import type { CompletedWaitpoint } from "@trigger.dev/core/v3"; +import type { + CompletedWaitpointRecord, + CompletedWaitpointResolver, + CompletedWaitpointsPointer, + ResolveCompletedWaitpointsArgs, +} from "@internal/run-store"; + +// The frozen key sets, pinned exactly and bidirectionally. Renames, removals, widenings and +// required-to-optional all already break compilation through the usage sites below; an ADDED +// OPTIONAL field does not, and on a jointly-owned frozen type that is the change neither lane +// may make unilaterally. These fail on it. +type Exact = [A] extends [B] ? ([B] extends [A] ? true : never) : never; + +const _recordKeys: Exact< + keyof CompletedWaitpointRecord, + | "id" + | "friendlyId" + | "type" + | "completedAt" + | "outputType" + | "outputIsError" + | "output" + | "completedByTaskRunId" + | "completedByBatchId" + | "completedAfter" + | "idempotencyKey" +> = true; + +const _pointerKeys: Exact = true; + +const _argsKeys: Exact< + keyof ResolveCompletedWaitpointsArgs, + "runId" | "batchId" | "pointer" | "order" | "records" +> = true; +import { enhanceExecutionSnapshotWithWaitpoints } from "./executionSnapshotSystem.js"; + +function makeWaitpoint(overrides: Partial): Waitpoint { + return { + id: "wp_default", + friendlyId: "waitpoint_default", + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date("2026-01-01T00:00:00.000Z"), + idempotencyKey: "idem_generated", + userProvidedIdempotencyKey: false, + inactiveIdempotencyKey: null, + idempotencyKeyExpiresAt: null, + completedByTaskRunId: null, + completedByBatchId: null, + completedAfter: null, + output: null, + outputType: "application/json", + outputIsError: false, + projectId: "proj_1", + environmentId: "env_1", + tags: [], + createdAt: new Date("2026-01-01T00:00:00.000Z"), + updatedAt: new Date("2026-01-01T00:00:00.000Z"), + ...overrides, + } as Waitpoint; +} + +// The WRITE side of the freeze: one Waitpoint row becomes one record. +function toRecord(w: Waitpoint): CompletedWaitpointRecord { + return { + id: w.id, + friendlyId: w.friendlyId, + type: w.type, + completedAt: (w.completedAt ?? new Date()).toISOString(), + outputType: w.outputType, + outputIsError: w.outputIsError, + output: recordOutputFor(w), + completedByTaskRunId: w.completedByTaskRunId ?? undefined, + completedByBatchId: w.completedByBatchId ?? undefined, + completedAfter: w.completedAfter?.toISOString(), + idempotencyKey: + w.userProvidedIdempotencyKey && !w.inactiveIdempotencyKey ? w.idempotencyKey : undefined, + }; +} + +function recordOutputFor(w: Waitpoint): CompletedWaitpointRecord["output"] { + if (w.output === null) return null; + // A RUN success re-derives byte-identically from TaskRun.output. A RUN error cannot, + // because TaskRun.error is jsonb, so it carries inline. + // + // This branch is deliberately BEFORE the application/store branch. An offloaded RUN + // success is still deriveFromRun, and that is correct: completeAttemptSuccess receives + // the same `output` and `outputType` the waitpoint got, so TaskRun.output holds the + // same ref string. The re-read stays byte-identical either way. + if (w.type === "RUN" && !w.outputIsError && w.completedByTaskRunId) + return { deriveFromRun: true }; + if (w.outputType === "application/store") return { ref: w.output }; + return { inline: w.output }; +} + +// The READ side of the freeze. Iterates `records`, never `order`. +async function referenceResolver( + args: ResolveCompletedWaitpointsArgs, + lookupRunOutput: (runId: string) => Promise +): Promise { + const out: CompletedWaitpoint[] = []; + for (const record of args.records) { + const indexes: (number | undefined)[] = []; + for (let i = 0; i < args.order.length; i++) { + if (args.order[i] === record.id) indexes.push(i); + } + if (indexes.length === 0) indexes.push(undefined); + + let output: string | undefined; + if (record.output === null) { + output = undefined; + } else if ("inline" in record.output) { + output = record.output.inline; + } else if ("ref" in record.output) { + output = record.output.ref; + } else if ("deriveFromRun" in record.output) { + output = record.completedByTaskRunId + ? await lookupRunOutput(record.completedByTaskRunId) + : undefined; + } else { + const _never: never = record.output; + throw new Error(`unknown record output variant: ${JSON.stringify(_never)}`); + } + + for (const index of indexes) { + out.push({ + id: record.id, + // Unreachable: the oracle's own loop pushes a non-negative integer or undefined. + // Reproduced because the frozen index-expansion rule names it. + index: index === -1 ? undefined : index, + friendlyId: record.friendlyId, + type: record.type, + completedAt: new Date(record.completedAt), + idempotencyKey: record.idempotencyKey, + completedByTaskRun: record.completedByTaskRunId + ? { + id: record.completedByTaskRunId, + friendlyId: RunId.toFriendlyId(record.completedByTaskRunId), + batch: args.batchId + ? { id: args.batchId, friendlyId: BatchId.toFriendlyId(args.batchId) } + : undefined, + } + : undefined, + completedAfter: record.completedAfter ? new Date(record.completedAfter) : undefined, + completedByBatch: record.completedByBatchId + ? { + id: record.completedByBatchId, + friendlyId: BatchId.toFriendlyId(record.completedByBatchId), + } + : undefined, + output, + outputType: record.outputType, + outputIsError: record.outputIsError, + }); + } + } + return out; +} + +// Proves the frozen hook signature is implementable exactly as declared. The reference +// resolver takes its TaskRun lookup as a second parameter, so the production shape is +// the curried form -- which is what the waitpoint lane will bind to a Prisma client. +// If this assignment stops compiling, the frozen signature has drifted. This file is +// typechecked by tsconfig.freeze-test.json, wired into this package's `typecheck` +// script, so that drift is caught -- vitest's esbuild transform alone would not catch it. +const resolverUnderTest: CompletedWaitpointResolver = (args) => + referenceResolver(args, async () => undefined); + +// The oracle spreads the snapshot, so it needs the two fields the mapping reads. +function makeSnapshot(batchId: string | null) { + return { id: "snap_1", runId: "run_1", batchId, checkpoint: null } as never; +} + +async function assertParity( + waitpoints: Waitpoint[], + order: string[], + batchId: string | null, + runOutputs: Record = {} +) { + const enhanced = enhanceExecutionSnapshotWithWaitpoints(makeSnapshot(batchId), waitpoints, order); + const args: ResolveCompletedWaitpointsArgs = { + runId: "run_1", + batchId: batchId ?? undefined, + pointer: { cycleSeq: 1, count: order.length }, + order, + records: waitpoints.map(toRecord), + }; + // count-carried-forward behaviour (order.length, not the record count) is covered by + // the run-store Redis suite, not here -- this line only constructs `args`, not asserts. + const resolved = await referenceResolver(args, async (id) => runOutputs[id]); + expect(resolved).toEqual(enhanced.completedWaitpoints); + return { enhanced, resolved }; +} + +describe("the frozen record shape", () => { + // Literals, not a mirror of the writer. A field rename or an encoding change must + // fail HERE, because the parity suite cannot see it. + it("pins the RUN record", () => { + expect( + toRecord( + makeWaitpoint({ + id: "wp_run", + friendlyId: "waitpoint_run", + type: "RUN", + completedByTaskRunId: "run_child", + output: '{"value":42}', + }) + ) + ).toEqual({ + id: "wp_run", + friendlyId: "waitpoint_run", + type: "RUN", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { deriveFromRun: true }, + completedByTaskRunId: "run_child", + completedByBatchId: undefined, + completedAfter: undefined, + idempotencyKey: undefined, + }); + }); + + it("pins the BATCH record", () => { + expect( + toRecord( + makeWaitpoint({ + id: "wp_batch", + friendlyId: "waitpoint_batch", + type: "BATCH", + completedByBatchId: "batch_child", + output: "Batch waitpoint completed", + }) + ) + ).toEqual({ + id: "wp_batch", + friendlyId: "waitpoint_batch", + type: "BATCH", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "Batch waitpoint completed" }, + completedByTaskRunId: undefined, + completedByBatchId: "batch_child", + completedAfter: undefined, + idempotencyKey: undefined, + }); + }); + + it("pins the DATETIME record", () => { + expect( + toRecord( + makeWaitpoint({ + id: "wp_dt", + friendlyId: "waitpoint_dt", + type: "DATETIME", + completedAfter: new Date("2026-02-02T00:00:00.000Z"), + }) + ) + ).toEqual({ + id: "wp_dt", + friendlyId: "waitpoint_dt", + type: "DATETIME", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: null, + completedByTaskRunId: undefined, + completedByBatchId: undefined, + completedAfter: "2026-02-02T00:00:00.000Z", + idempotencyKey: undefined, + }); + }); + + it("pins the MANUAL record, with a user idempotency key and an offloaded output", () => { + expect( + toRecord( + makeWaitpoint({ + id: "wp_manual", + friendlyId: "waitpoint_manual", + type: "MANUAL", + idempotencyKey: "idem_user", + userProvidedIdempotencyKey: true, + output: "s3://bucket/key", + outputType: "application/store", + }) + ) + ).toEqual({ + id: "wp_manual", + friendlyId: "waitpoint_manual", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/store", + outputIsError: false, + output: { ref: "s3://bucket/key" }, + completedByTaskRunId: undefined, + completedByBatchId: undefined, + completedAfter: undefined, + idempotencyKey: "idem_user", + }); + }); + + it("keeps an offloaded RUN success on deriveFromRun, not ref", () => { + // Branch precedence. TaskRun.output holds the same ref string, so the re-read is + // still byte-identical. A later edit that reorders the branches must fail here. + const record = toRecord( + makeWaitpoint({ + id: "wp_run_offloaded", + type: "RUN", + completedByTaskRunId: "run_child", + output: "s3://bucket/key", + outputType: "application/store", + }) + ); + expect(record.output).toEqual({ deriveFromRun: true }); + }); + + it("carries a RUN error inline, never deriveFromRun", () => { + const record = toRecord( + makeWaitpoint({ + id: "wp_run_err", + type: "RUN", + completedByTaskRunId: "run_child", + output: '{"type":"BUILT_IN_ERROR"}', + outputIsError: true, + }) + ); + expect(record.output).toEqual({ inline: '{"type":"BUILT_IN_ERROR"}' }); + }); +}); + +// The pointer's shape is pinned by CompletedWaitpointsPointer and tsconfig.freeze-test.json, +// not by a runtime assertion here -- a value that only echoes its own construction can't fail. + +describe("the completed-waitpoints freeze", () => { + it("expands a repeated id at each of its positions", async () => { + const w = makeWaitpoint({ id: "wp_a", type: "RUN", completedByTaskRunId: "run_child" }); + const { resolved } = await assertParity([w], ["wp_a", "wp_other", "wp_a"], "batch_1"); + expect(resolved.map((r) => r.index)).toEqual([0, 2]); + }); + + it("yields one entry with an undefined index for a record absent from order", async () => { + const w = makeWaitpoint({ id: "wp_absent", type: "MANUAL" }); + const { resolved } = await assertParity([w], ["wp_other"], null); + expect(resolved).toHaveLength(1); + expect(resolved[0]!.index).toBeUndefined(); + }); + + it("resolves a non-batch wait, where order is empty and one record exists", async () => { + // The commonest resume. Postgres's join holds the id while order does not, which is + // why `records` is authoritative and the mint comparison never reads `order`. + // batchId is null here on purpose: a triggerAndWait outside a batch is the shape + // this case is named for, and it exercises the oracle's `batchId ? ... : undefined` + // false branch, which no other case reaches. + // output stays null, so no TaskRun lookup is involved: both halves yield undefined. + const w = makeWaitpoint({ id: "wp_single", type: "RUN", completedByTaskRunId: "run_child" }); + const { resolved } = await assertParity([w], [], null); + expect(resolved).toHaveLength(1); + expect(resolved[0]!.index).toBeUndefined(); + expect(resolved[0]!.completedByTaskRun?.id).toBe("run_child"); + expect(resolved[0]!.completedByTaskRun?.batch).toBeUndefined(); + }); + + it("keys completedByBatch on the id alone, not on the type", async () => { + // The oracle checks completedByBatchId without looking at `type`, at + // executionSnapshotSystem.ts:107-113. A resolver keyed on type would pass every + // other case here and diverge in production. + const w = makeWaitpoint({ + id: "wp_manual_with_batch", + type: "MANUAL", + completedByBatchId: "batch_child", + }); + const { resolved } = await assertParity([w], ["wp_manual_with_batch"], null); + expect(resolved[0]!.completedByBatch?.id).toBe("batch_child"); + }); + + it("carries outputIsError on a non-RUN type", async () => { + const w = makeWaitpoint({ + id: "wp_manual_err", + type: "MANUAL", + output: '{"type":"STRING_ERROR"}', + outputIsError: true, + }); + const { resolved } = await assertParity([w], ["wp_manual_err"], null); + expect(resolved[0]!.outputIsError).toBe(true); + expect(resolved[0]!.output).toBe('{"type":"STRING_ERROR"}'); + }); + + it("resolves through the frozen hook signature", async () => { + // Exercises resolverUnderTest, so the declared CompletedWaitpointResolver type is + // proved implementable at runtime, on top of the compile-time proof at its + // declaration above (checked by tsconfig.freeze-test.json). + const w = makeWaitpoint({ id: "wp_hook", type: "MANUAL" }); + const resolved = await resolverUnderTest({ + runId: "run_1", + batchId: undefined, + pointer: { cycleSeq: 1, count: 1 }, + order: ["wp_hook"], + records: [toRecord(w)], + }); + expect(resolved).toHaveLength(1); + expect(resolved[0]!.id).toBe("wp_hook"); + expect(resolved[0]!.index).toBe(0); + }); + + it("round-trips all four waitpoint types", async () => { + const waitpoints = [ + makeWaitpoint({ id: "wp_run", type: "RUN", completedByTaskRunId: "run_child" }), + makeWaitpoint({ id: "wp_batch", type: "BATCH", completedByBatchId: "batch_child" }), + makeWaitpoint({ + id: "wp_dt", + type: "DATETIME", + completedAfter: new Date("2026-02-02T00:00:00.000Z"), + }), + makeWaitpoint({ id: "wp_manual", type: "MANUAL" }), + ]; + const { resolved } = await assertParity(waitpoints, ["wp_run", "wp_batch"], "batch_1"); + expect(resolved.map((r) => r.type)).toEqual(["RUN", "BATCH", "DATETIME", "MANUAL"]); + }); + + it("applies the idempotency-key rule in all four combinations", async () => { + const combos: Array<[boolean, string | null, string | undefined]> = [ + [true, null, "idem_user"], + [true, "cleared", undefined], + [false, null, undefined], + [false, "cleared", undefined], + ]; + for (const [userProvided, inactive, expected] of combos) { + const w = makeWaitpoint({ + id: "wp_idem", + idempotencyKey: "idem_user", + userProvidedIdempotencyKey: userProvided, + inactiveIdempotencyKey: inactive, + }); + const { resolved } = await assertParity([w], ["wp_idem"], null); + expect(resolved[0]!.idempotencyKey).toBe(expected); + } + }); + + it("forwards completedAfter on a MANUAL waitpoint with a timeout", async () => { + // The plan comment scopes completedAfter to DATETIME. The oracle forwards it for any + // type, so the resolver must too. + const w = makeWaitpoint({ + id: "wp_timeout", + type: "MANUAL", + completedAfter: new Date("2026-03-03T00:00:00.000Z"), + }); + const { resolved } = await assertParity([w], ["wp_timeout"], null); + expect(resolved[0]!.completedAfter).toEqual(new Date("2026-03-03T00:00:00.000Z")); + }); + + it("falls back off deriveFromRun when the completing run was deleted", async () => { + const w = makeWaitpoint({ + id: "wp_orphan", + type: "RUN", + completedByTaskRunId: null, + output: '{"value":42}', + }); + const { resolved } = await assertParity([w], ["wp_orphan"], null); + expect(resolved[0]!.output).toBe('{"value":42}'); + }); + + it("maps every output variant", async () => { + const runSuccess = makeWaitpoint({ + id: "wp_run_ok", + type: "RUN", + completedByTaskRunId: "run_ok", + output: '{"value":42}', + }); + const runError = makeWaitpoint({ + id: "wp_run_err", + type: "RUN", + completedByTaskRunId: "run_err", + output: '{"type":"BUILT_IN_ERROR"}', + outputIsError: true, + }); + const offloaded = makeWaitpoint({ + id: "wp_ref", + type: "MANUAL", + output: "s3://bucket/key", + outputType: "application/store", + }); + const empty = makeWaitpoint({ id: "wp_none", type: "MANUAL", output: null }); + + const { resolved } = await assertParity( + [runSuccess, runError, offloaded, empty], + ["wp_run_ok", "wp_run_err", "wp_ref", "wp_none"], + "batch_1", + // deriveFromRun: the same string TaskRun.output holds verbatim. + { run_ok: '{"value":42}' } + ); + expect(resolved.map((r) => r.output)).toEqual([ + '{"value":42}', + '{"type":"BUILT_IN_ERROR"}', + "s3://bucket/key", + undefined, + ]); + }); +}); + +describe("the exhaustive parity grid", () => { + // Dimensions mirror every Waitpoint column the oracle reads (type, output, outputType, + // outputIsError, completedByTaskRunId, completedByBatchId, completedAfter, + // userProvidedIdempotencyKey, inactiveIdempotencyKey), plus order-membership and the + // reading entry's batchId. A new column the oracle reads must widen a dimension here, + // so the pinned combination count below fails instead of coverage silently shrinking. + const TYPES: Waitpoint["type"][] = ["RUN", "BATCH", "DATETIME", "MANUAL"]; + const OUTPUTS: (string | null)[] = [null, '{"value":42}']; + const OUTPUT_TYPES = ["application/json", "application/store"]; + const OUTPUT_IS_ERRORS = [false, true]; + const TASK_RUN_IDS: (string | null)[] = [null, "run_child"]; + const BATCH_IDS: (string | null)[] = [null, "batch_child"]; + const COMPLETED_AFTERS: (Date | null)[] = [null, new Date("2026-02-02T00:00:00.000Z")]; + const IDEMPOTENCY_COMBOS: Array<[boolean, string | null]> = [ + [false, null], + [false, "cleared"], + [true, null], + [true, "cleared"], + ]; + const ORDER_MEMBERSHIPS = ["absent", "once", "twice"] as const; + const READING_BATCH_IDS: (string | null)[] = [null, "batch_reading_entry"]; + + // Only reached when type is RUN, output is set, outputIsError is false, and + // completedByTaskRunId is "run_child": the deriveFromRun branch. The value matches + // OUTPUTS' non-null entry so a correct resolver is byte-identical to the oracle. + const RUN_OUTPUT_LOOKUP: Record = { run_child: '{"value":42}' }; + + it("agrees with the oracle across every combination", async () => { + type Combo = { + type: Waitpoint["type"]; + output: string | null; + outputType: string; + outputIsError: boolean; + completedByTaskRunId: string | null; + completedByBatchId: string | null; + completedAfter: Date | null; + userProvidedIdempotencyKey: boolean; + inactiveIdempotencyKey: string | null; + orderMembership: (typeof ORDER_MEMBERSHIPS)[number]; + readingBatchId: string | null; + }; + const failures: Array<{ combo: Combo; oracle: unknown; resolver: unknown }> = []; + let cases = 0; + + for (const type of TYPES) { + for (const output of OUTPUTS) { + for (const outputType of OUTPUT_TYPES) { + for (const outputIsError of OUTPUT_IS_ERRORS) { + for (const completedByTaskRunId of TASK_RUN_IDS) { + for (const completedByBatchId of BATCH_IDS) { + for (const completedAfter of COMPLETED_AFTERS) { + for (const [ + userProvidedIdempotencyKey, + inactiveIdempotencyKey, + ] of IDEMPOTENCY_COMBOS) { + for (const orderMembership of ORDER_MEMBERSHIPS) { + for (const readingBatchId of READING_BATCH_IDS) { + cases++; + const combo: Combo = { + type, + output, + outputType, + outputIsError, + completedByTaskRunId, + completedByBatchId, + completedAfter, + userProvidedIdempotencyKey, + inactiveIdempotencyKey, + orderMembership, + readingBatchId, + }; + + const id = "wp_grid"; + const w = makeWaitpoint({ + id, + type, + output, + outputType, + outputIsError, + completedByTaskRunId, + completedByBatchId, + completedAfter, + idempotencyKey: "idem_user", + userProvidedIdempotencyKey, + inactiveIdempotencyKey, + }); + const order = + orderMembership === "absent" + ? ["wp_other"] + : orderMembership === "once" + ? [id] + : [id, id]; + + const enhanced = enhanceExecutionSnapshotWithWaitpoints( + makeSnapshot(readingBatchId), + [w], + order + ); + const args: ResolveCompletedWaitpointsArgs = { + runId: "run_1", + batchId: readingBatchId ?? undefined, + pointer: { cycleSeq: 1, count: order.length }, + order, + records: [toRecord(w)], + }; + const resolved = await referenceResolver( + args, + async (runId) => RUN_OUTPUT_LOOKUP[runId] + ); + + if (!isDeepStrictEqual(resolved, enhanced.completedWaitpoints)) { + failures.push({ + combo, + oracle: enhanced.completedWaitpoints, + resolver: resolved, + }); + } + } + } + } + } + } + } + } + } + } + } + + expect(cases).toBe(6144); + expect( + failures.length, + failures.length > 0 + ? `${failures.length}/${cases} combinations diverged. First: ${JSON.stringify(failures[0], null, 2)}` + : undefined + ).toBe(0); + }); +}); + +describe("the freeze's two deliberate divergences", () => { + it("pins completedAt at write time, where the oracle samples the clock", () => { + // The oracle applies `w.completedAt ?? new Date()`, so a null value changes on every + // read. No deterministic record can match that. The record pins it once instead. + const w = makeWaitpoint({ id: "wp_null_at", completedAt: null }); + const record = toRecord(w); + expect(Math.abs(Date.now() - new Date(record.completedAt).getTime())).toBeLessThan(60_000); + }); + + it("takes batch{} from the reading entry, not the completing run's own batch", async () => { + // A known, deliberate conflation in the oracle. Byte-compatibility requires it. + const w = makeWaitpoint({ id: "wp_run", type: "RUN", completedByTaskRunId: "run_child" }); + const { resolved } = await assertParity([w], ["wp_run"], "batch_reading_entry"); + expect(resolved[0]!.completedByTaskRun?.batch?.id).toBe("batch_reading_entry"); + }); +}); diff --git a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts index e79383a8bb6..81c41d2c2ae 100644 --- a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts @@ -58,7 +58,7 @@ function enhanceExecutionSnapshot( * Transforms a snapshot (with checkpoint but without waitpoints) into an EnhancedExecutionSnapshot * by combining it with pre-fetched waitpoints. */ -function enhanceExecutionSnapshotWithWaitpoints( +export function enhanceExecutionSnapshotWithWaitpoints( snapshot: ExecutionSnapshotWithCheckpoint, waitpoints: Waitpoint[], completedWaitpointOrder: string[] diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts index 5d5a80772a6..3dbed999445 100644 --- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts @@ -1,5 +1,4 @@ -import { timeoutError, tryCatch } from "@trigger.dev/core/v3"; -import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { timeoutError } from "@trigger.dev/core/v3"; import type { PrismaClientOrTransaction, TaskRun, @@ -7,13 +6,11 @@ import type { TaskRunExecutionStatus, Waitpoint, } from "@trigger.dev/database"; -import { Prisma, boundedIn } from "@trigger.dev/database"; -import type { RunStore } from "@internal/run-store"; import { assertNever } from "assert-never"; -import { nanoid } from "nanoid"; -import { UnclassifiableWaitpointId } from "../errors.js"; import { sendNotificationToWorker } from "../eventBus.js"; import { isFinalRunStatus } from "../statuses.js"; +import { LegacyPostgresWaitpointCoordinator } from "../waitpointCoordinator/legacyPostgresCoordinator.js"; +import type { WaitpointCoordinator } from "../waitpointCoordinator/types.js"; import type { EnqueueSystem } from "./enqueueSystem.js"; import type { ExecutionSnapshotSystem } from "./executionSnapshotSystem.js"; import { getLatestExecutionSnapshot } from "./executionSnapshotSystem.js"; @@ -45,11 +42,17 @@ export class WaitpointSystem { private readonly $: SystemResources; private readonly executionSnapshotSystem: ExecutionSnapshotSystem; private readonly enqueueSystem: EnqueueSystem; + private readonly coordinator: WaitpointCoordinator; constructor(private readonly options: WaitpointSystemOptions) { this.$ = options.resources; this.executionSnapshotSystem = options.executionSnapshotSystem; this.enqueueSystem = options.enqueueSystem; + this.coordinator = new LegacyPostgresWaitpointCoordinator({ + runStore: this.$.runStore, + prisma: this.$.prisma, + logger: this.$.logger, + }); } public async clearBlockingWaitpoints({ @@ -59,14 +62,7 @@ export class WaitpointSystem { runId: string; tx?: PrismaClientOrTransaction; }) { - // A run's edges co-locate with the run (the edge write routes by runId), so the router routes this - // taskRunId-keyed delete to the run's store rather than fanning out. The caller's `tx` is not - // forwarded — the delete runs on the owning store's own client (the router never threads a - // control-plane tx into a routed write). - const deleted = await this.$.runStore.deleteManyTaskRunWaitpoints( - { where: { taskRunId: runId } }, - tx - ); + const deleted = await this.coordinator.clearRunBlockState({ runId, tx }); return deleted.count; } @@ -84,86 +80,19 @@ export class WaitpointSystem { isError: boolean; }; }): Promise { - // Residency store-selection guard. completeWaitpoint arrives with only - // (waitpointId, output) — no run id — so the owning run-ops store is selected - // by the waitpoint's own residency. In single-DB this is the one store - // (no classification). An unclassifiable id throws loud — never default-routes. - let store: RunStore; - try { - store = await this.$.runStore.forWaitpointCompletion(id, { routeKind: "MANUAL" }); - } catch (error) { - this.$.logger.error("completeWaitpoint: unclassifiable waitpointId", { - waitpointId: id, - error, - }); - throw new UnclassifiableWaitpointId(id, { cause: error }); - } - - // 1. Complete the Waitpoint (if not completed) - const [updateError, updateResult] = await tryCatch( - store.updateManyWaitpoints({ - where: { id, status: "PENDING" }, - data: { - status: "COMPLETED", - completedAt: new Date(), - output: output?.value, - outputType: output?.type, - outputIsError: output?.isError, - }, - }) - ); - - if (updateError) { - this.$.logger.error("completeWaitpoint: error updating waitpoint:", { updateError }); - throw updateError; - } - - if (updateResult.count === 0) { - this.$.logger.info( - "completeWaitpoint: attempted to complete a waitpoint that is not PENDING", - { waitpointId: id } - ); - } - - // Re-read the just-written row from the RESOLVED store's PRIMARY: the replica (findWaitpoint's - // default) can miss it under lag → false "not found" → the parent hangs; this.$.prisma would - // instead hit the wrong DB. findWaitpointOnPrimary reads the owning store's primary. - const waitpoint = await store.findWaitpointOnPrimary({ - where: { id }, + const { waitpoint, blockedRuns } = await this.coordinator.complete({ + waitpointId: id, + output, }); - if (!waitpoint) { - this.$.logger.error("completeWaitpoint: waitpoint not found", { waitpointId: id }); - throw new Error("Waitpoint not found"); - } - - if (waitpoint.status !== "COMPLETED") { - this.$.logger.error(`completeWaitpoint: waitpoint is not completed`, { - waitpointId: id, - }); - throw new Error("Waitpoint not completed"); - } - - // 2. Find the TaskRuns blocked by this waitpoint. The edge (TaskRunWaitpoint) co-locates - // with its RUN, not this token, so it can live on the OTHER run-ops DB: read via the router - // (which fans the waitpointId lookup across both DBs) rather than the token's own `store`, - // or a cross-DB blocked run is never found and hangs forever. - const affectedTaskRuns = await this.$.runStore.findManyTaskRunWaitpoints( - { - where: { waitpointId: id }, - select: { taskRunId: true, spanIdToComplete: true, createdAt: true }, - }, - this.$.prisma - ); - - if (affectedTaskRuns.length === 0) { + if (blockedRuns.length === 0) { this.$.logger.debug(`completeWaitpoint: no TaskRunWaitpoints found for waitpoint`, { waitpointId: id, }); } // 3. Schedule trying to continue the runs - for (const run of affectedTaskRuns) { + for (const run of blockedRuns) { const jobId = `continueRunIfUnblocked:${run.taskRunId}`; //50ms in the future const availableAt = new Date(Date.now() + 50); @@ -220,81 +149,27 @@ export class WaitpointSystem { idempotencyKey?: string; idempotencyKeyExpiresAt?: Date; }) { - // Co-location invariant: a DATETIME wait waitpoint lives on the same run-ops DB as the run that - // blocks on it (so the block edge's local `Waitpoint` join resolves and completion/resume stay - // local). The minted waitpoint id is always a cuid, so without `coLocateWithRunId` the upsert - // would always route to LEGACY and a run-ops run on NEW would hang. The (env,idempotencyKey) dedup - // is within the owning run/tree (co-resident on one DB), so the dedup probe + rotation target the - // SAME store. With no run id (a standalone token has no owning run yet) the lookup falls back to - // a cross-DB NEW-then-LEGACY scan and the upsert routes by id-shape. Always routed through the - // run store (never a caller tx) so it can never bypass residency onto the wrong DB. - const colocate = runId ? { coLocateWithRunId: runId } : undefined; - const existingWaitpoint = idempotencyKey - ? await this.$.runStore.findWaitpoint( - { - where: { - environmentId, - idempotencyKey, - }, - }, - undefined, - colocate - ) - : undefined; - - if (existingWaitpoint) { - if ( - existingWaitpoint.idempotencyKeyExpiresAt && - new Date() > existingWaitpoint.idempotencyKeyExpiresAt - ) { - //the idempotency key has expired - //remove the waitpoint idempotencyKey - const rotateArgs = { - where: { - id: existingWaitpoint.id, - }, - data: { - idempotencyKey: nanoid(24), - inactiveIdempotencyKey: existingWaitpoint.idempotencyKey, - }, - }; - await this.$.runStore.updateWaitpoint(rotateArgs, undefined, colocate); + const result = await this.coordinator.createDateTimeWaitpoint({ + runId, + projectId, + environmentId, + completedAfter, + idempotencyKey, + idempotencyKeyExpiresAt, + }); - //let it fall through to create a new waitpoint - } else { - return { waitpoint: existingWaitpoint, isCached: true }; - } + if (result.kind === "cached") { + return { waitpoint: result.waitpoint, isCached: true }; } - const upsertArgs = { - where: { - environmentId_idempotencyKey: { - environmentId, - idempotencyKey: idempotencyKey ?? nanoid(24), - }, - }, - create: { - ...WaitpointId.generate(), - type: "DATETIME" as const, - idempotencyKey: idempotencyKey ?? nanoid(24), - idempotencyKeyExpiresAt, - userProvidedIdempotencyKey: !!idempotencyKey, - environmentId, - projectId, - completedAfter, - }, - update: {}, - }; - const waitpoint = await this.$.runStore.upsertWaitpoint(upsertArgs, undefined, colocate); - await this.$.worker.enqueue({ - id: `finishWaitpoint.${waitpoint.id}`, + id: `finishWaitpoint.${result.waitpoint.id}`, job: "finishWaitpoint", - payload: { waitpointId: waitpoint.id }, + payload: { waitpointId: result.waitpoint.id }, availableAt: completedAfter, }); - return { waitpoint, isCached: false }; + return { waitpoint: result.waitpoint, isCached: false }; } /** This creates a MANUAL waitpoint, that can be explicitly completed (or failed). @@ -322,117 +197,35 @@ export class WaitpointSystem { // to LEGACY by its cuid id-shape. Ignored when `runId` is set (co-location wins). standaloneResidency?: "NEW" | "LEGACY"; }): Promise<{ waitpoint: Waitpoint; isCached: boolean }> { - // Co-location invariant (see createDateTimeWaitpoint): when a `runId` is supplied the waitpoint - // co-locates with that run's DB and the (env,idempotencyKey) dedup is per-run (co-resident). A - // standalone token (api.v1.waitpoints.tokens.ts) passes no run id — it is created without an - // owner, blocked later by whichever run waits on it (possibly cross-DB, resolved by the - // run-co-resident block edge + completion fan-out). With no owner it reads the env mint kind via - // `standaloneResidency` so a minted-new env keeps its tokens on NEW; unset, it routes by id-shape. No tx here. - const colocate = runId - ? { coLocateWithRunId: runId } - : standaloneResidency - ? { residency: standaloneResidency } - : undefined; - const existingWaitpoint = idempotencyKey - ? await this.$.runStore.findWaitpoint( - { - where: { - environmentId, - idempotencyKey, - }, - }, - undefined, - colocate - ) - : undefined; - - if (existingWaitpoint) { - if ( - existingWaitpoint.idempotencyKeyExpiresAt && - new Date() > existingWaitpoint.idempotencyKeyExpiresAt - ) { - //the idempotency key has expired - //remove the waitpoint idempotencyKey - await this.$.runStore.updateWaitpoint( - { - where: { - id: existingWaitpoint.id, - }, - data: { - idempotencyKey: nanoid(24), - inactiveIdempotencyKey: existingWaitpoint.idempotencyKey, - }, - }, - undefined, - colocate - ); + const result = await this.coordinator.createManualWaitpoint({ + runId, + environmentId, + projectId, + idempotencyKey, + idempotencyKeyExpiresAt, + timeout, + tags, + standaloneResidency, + }); - //let it fall through to create a new waitpoint - } else { - return { waitpoint: existingWaitpoint, isCached: true }; - } + if (result.kind === "cached") { + return { waitpoint: result.waitpoint, isCached: true }; } - const maxRetries = 5; - let attempts = 0; - - while (attempts < maxRetries) { - try { - const waitpoint = await this.$.runStore.upsertWaitpoint( - { - where: { - environmentId_idempotencyKey: { - environmentId, - idempotencyKey: idempotencyKey ?? nanoid(24), - }, - }, - create: { - ...WaitpointId.generate(), - type: "MANUAL", - idempotencyKey: idempotencyKey ?? nanoid(24), - idempotencyKeyExpiresAt, - userProvidedIdempotencyKey: !!idempotencyKey, - environmentId, - projectId, - completedAfter: timeout, - tags, - }, - update: {}, - }, - undefined, - colocate - ); - - //schedule the timeout - if (timeout) { - await this.$.worker.enqueue({ - id: `finishWaitpoint.${waitpoint.id}`, - job: "finishWaitpoint", - payload: { - waitpointId: waitpoint.id, - error: JSON.stringify(timeoutError(timeout)), - }, - availableAt: timeout, - }); - } - - return { waitpoint, isCached: false }; - } catch (error) { - if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { - // Handle unique constraint violation (conflict) - attempts++; - if (attempts >= maxRetries) { - throw new Error( - `Failed to create waitpoint after ${maxRetries} attempts due to conflicts.` - ); - } - } else { - throw error; // Re-throw other errors - } - } + //schedule the timeout + if (timeout) { + await this.$.worker.enqueue({ + id: `finishWaitpoint.${result.waitpoint.id}`, + job: "finishWaitpoint", + payload: { + waitpointId: result.waitpoint.id, + error: JSON.stringify(timeoutError(timeout)), + }, + availableAt: timeout, + }); } - throw new Error(`Failed to create waitpoint after ${maxRetries} attempts due to conflicts.`); + return { waitpoint: result.waitpoint, isCached: false }; } /** @@ -489,25 +282,19 @@ export class WaitpointSystem { this.$.runStore ); - // Insert the blocking + historical connections via the run-ops store, routed by the owning - // run id so the edge co-resides with the run. Never pinned to the caller's control-plane tx: - // that joined `Waitpoint` on the wrong DB and wrote 0 edges. The pending check stays a - // SEPARATE store call so it gets its own READ COMMITTED snapshot (see the doc comment above). - await this.$.runStore.blockRunWithWaitpointEdges({ + // Insert the blocking + historical connections and re-check the pending count. The + // coordinator keeps these as two separate store statements, in this order, for the READ + // COMMITTED reason documented on the method and in the doc comment above. + const { pendingCount } = await this.coordinator.registerBlocks({ runId, waitpointIds: $waitpoints, projectId, spanIdToComplete, batchId: batch?.id, batchIndex: batch?.index, + client: prisma, }); - // Check if the run is actually blocked using a separate query (see above). Pass the writer so the - // pending re-read is read-your-writes on the owning PRIMARY (a lagging replica can strand the run). - // Route by the blocked run id: its blocking waitpoints co-locate with the run, so the router - // counts on the run's store and only falls back to the other DB for a cross-tree token. - const pendingCount = await this.$.runStore.countPendingWaitpoints($waitpoints, prisma, runId); - const isRunBlocked = pendingCount > 0; let newStatus: TaskRunExecutionStatus = "SUSPENDED"; @@ -605,10 +392,10 @@ export class WaitpointSystem { }): Promise { const $waitpoints = typeof waitpoints === "string" ? [waitpoints] : waitpoints; - // Same routed edge write as blockRunWithWaitpoint, routed by the owning run id. No lock - // needed: ON CONFLICT DO NOTHING makes concurrent inserts safe, and the parent snapshot is - // already EXECUTING_WITH_WAITPOINTS from blockRunWithCreatedBatch. - await this.$.runStore.blockRunWithWaitpointEdges({ + // Same routed edge write as blockRunWithWaitpoint. No lock needed: ON CONFLICT DO NOTHING + // makes concurrent inserts safe, and the parent snapshot is already + // EXECUTING_WITH_WAITPOINTS from blockRunWithCreatedBatch. No pending count here. + await this.coordinator.registerBlocksLockless({ runId, waitpointIds: $waitpoints, projectId, @@ -682,20 +469,7 @@ export class WaitpointSystem { return await this.$.runLock.lock("continueRunIfUnblocked", [runId], async () => { // 1. Get the any blocking waitpoints - const blockingWaitpoints = await this.$.runStore.findManyTaskRunWaitpoints( - { - where: { taskRunId: runId }, - select: { - id: true, - batchId: true, - batchIndex: true, - waitpoint: { - select: { id: true, status: true, type: true, completedAfter: true }, - }, - }, - }, - this.$.prisma - ); + const blockingWaitpoints = await this.coordinator.readRunBlockState(runId); // 2. There are blockers still, so do nothing if (blockingWaitpoints.some((w) => w.waitpoint.status !== "COMPLETED")) { @@ -926,11 +700,9 @@ export class WaitpointSystem { if (blockingWaitpoints.length > 0) { //5. Remove the blocking waitpoints - await this.$.runStore.deleteManyTaskRunWaitpoints({ - where: { - taskRunId: runId, - id: { in: boundedIn(blockingWaitpoints.map((b) => b.id)) }, - }, + await this.coordinator.clearRunBlockState({ + runId, + edgeIds: blockingWaitpoints.map((b) => b.id), }); this.$.logger.debug(`continueRunIfUnblocked: removed blocking waitpoints`, { @@ -953,15 +725,7 @@ export class WaitpointSystem { projectId: string; environmentId: string; }) { - return { - ...WaitpointId.generate(), - type: "RUN" as const, - status: "PENDING" as const, - idempotencyKey: nanoid(24), - userProvidedIdempotencyKey: false, - projectId, - environmentId, - }; + return this.coordinator.mintAssociatedWaitpointData({ projectId, environmentId }); } /** @@ -1045,12 +809,9 @@ export class WaitpointSystem { // Create waitpoint and link to run atomically const waitpointData = this.buildRunAssociatedWaitpoint({ projectId, environmentId }); - // RUN-type within-tree waitpoint that belongs to runId; routes by owning run id. - const waitpoint = await this.$.runStore.createWaitpoint({ - data: { - ...waitpointData, - completedByTaskRunId: runId, - }, + const waitpoint = await this.coordinator.createAssociatedWaitpoint({ + runId, + data: waitpointData, }); // If run has already finished (per snapshot), complete the waitpoint immediately so the parent can resume diff --git a/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts new file mode 100644 index 00000000000..50eba9d3a04 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts @@ -0,0 +1,20 @@ +// The snapshot sweeper needs to know which run statuses are terminal, and it cannot import that +// list: run-engine depends on run-store, not the other way round. So the list is duplicated, and +// this is the only thing that keeps the copy honest. +// +// Without it, a status added here and not there makes the sweeper treat a finished run as live and +// never apply its completion expiry. A status removed here and not there makes it treat a live run +// as finished. The second one reaps state a run is still using. +import { describe, expect, it } from "vitest"; +import { FINAL_RUN_STATUSES } from "@internal/run-store"; +import { getFinalRunStatuses } from "../statuses.js"; + +describe("terminal run statuses", () => { + it("match between the engine and the snapshot sweeper", () => { + expect([...FINAL_RUN_STATUSES].sort()).toEqual([...getFinalRunStatuses()].sort()); + }); + + it("are not empty, so the comparison cannot pass vacuously", () => { + expect(FINAL_RUN_STATUSES.length).toBeGreaterThan(0); + }); +}); diff --git a/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts new file mode 100644 index 00000000000..c74aa86db92 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts @@ -0,0 +1,75 @@ +// Builds the snapshot-store decorator over a real PostgresRunStore, for injection through the +// engine's `store` option — the seam runStoreInjectability.test.ts already proves. +// +// The point of injecting it is that the engine suites keep their own assertions: the same flows, +// the same expectations, a different store underneath. +import { + PostgresRunStore, + RedisSnapshotStore, + TaskRunExecutionSnapshotStore, + type SnapshotFaultInjector, + type SnapshotRepairEnqueuer, + type SnapshotStoreMode, +} from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "@internal/redis"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +export type DecoratedStoreHarness = { + store: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + /** Every read the decorator served, and which store answered it. */ + reads: { method: string; source: "redis" | "postgres" }[]; + /** Every append outcome, keyed by the write site that produced it. */ + writes: { site: string; outcome: string }[]; + /** Runs handed to the repair job because their append was lost. */ + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + quit(): Promise; +}; + +export function buildDecoratedStore(opts: { + prisma: PrismaClient; + redisOptions: RedisOptions; + mode: SnapshotStoreMode; + readPercent?: number; + faults?: SnapshotFaultInjector; + onAppendFailure?: SnapshotRepairEnqueuer; +}): DecoratedStoreHarness { + const redis = new RedisSnapshotStore({ + redisOptions: opts.redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const reads: DecoratedStoreHarness["reads"] = []; + const writes: DecoratedStoreHarness["writes"] = []; + const repairs: DecoratedStoreHarness["repairs"] = []; + + const store = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma: opts.prisma as never, readOnlyPrisma: opts.prisma as never }), + { + store: redis, + mode: opts.mode, + readPercent: opts.readPercent ?? 100, + ...(opts.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + await opts.onAppendFailure?.(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { + store, + redis, + reads, + writes, + repairs, + quit: () => redis.quit(), + }; +} diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts new file mode 100644 index 00000000000..ea3447acc56 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts @@ -0,0 +1,402 @@ +// The correctness spine: kill the process at each write boundary and prove the run still converges. +// +// The write protocol's whole claim is that whatever a crash leaves behind is a state the existing +// stall-and-repair machinery heals. That claim is not checkable by reading the code, so each test +// here injects a fault at one named boundary and then asserts three things: the run converges, it +// does not hang, and it burns at most one attempt number per crash. +// +// The bound is PER CRASH, not a flat one. The plan records that TLC refuted a flat bound of one in +// seven states, and that the property which holds is pgAttempt - maxLoggedAttempt <= crashCount. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { + InjectedSnapshotFault, + type SnapshotFaultBoundary, + type SnapshotFaultInjector, +} from "@internal/run-store"; +import { setTimeout } from "timers/promises"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +/** + * A local stand-in for the shared fault harness being built alongside this ticket. Its surface is + * the agreed one — arm, disarm, hook, fired — so swapping the import in costs no test-body change. + * + * `fired` is the guard against a silent pass. A boundary can be armed and never reached, in which + * case the test would go green having proved nothing, so every test asserts its boundary fired. + */ +function createFaultInjector(opts: { error: (boundary: SnapshotFaultBoundary) => Error }) { + const armed = new Map(); + const counts = new Map(); + + return { + arm(boundary: SnapshotFaultBoundary, opts?: { times?: number; runId?: string }) { + armed.set(boundary, { times: opts?.times ?? 1, ...(opts?.runId && { runId: opts.runId }) }); + }, + disarm(boundary: SnapshotFaultBoundary) { + armed.delete(boundary); + }, + fired(boundary: SnapshotFaultBoundary): number { + return counts.get(boundary) ?? 0; + }, + hook: ((boundary, context) => { + const entry = armed.get(boundary); + if (!entry) return; + if (entry.runId && context.runId !== entry.runId) return; + + counts.set(boundary, (counts.get(boundary) ?? 0) + 1); + entry.times -= 1; + if (entry.times <= 0) armed.delete(boundary); + + throw opts.error(boundary); + }) satisfies SnapshotFaultInjector, + }; +} + +function engineOptions( + prisma: any, + redisOptions: any, + harness: DecoratedStoreHarness, + heartbeatMs: number +) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + retryOptions: { maxTimeoutInMs: 50 }, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { PENDING_EXECUTING: heartbeatMs }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any) => ({ + number: 1, + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_${generateInternalId().slice(-12)}`, + spanId: `s_${generateInternalId().slice(-12)}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store crash boundaries", () => { + containerTest( + "afterPgBeforeRedis: the run converges and burns at most one attempt", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + // Crash between the Postgres commit and the Redis append of ONE transition. + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + faults.disarm("afterPgBeforeRedis"); + + // The boundary was actually reached. Without this the test could pass having proved nothing. + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres committed the attempt bump; the run is not stuck and not lost. + expect(attempt.run.attemptNumber).toBe(1); + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber).toBe(1); + + // The gap handed the run to the repair job rather than failing the caller. + expect(harness.repairs).toHaveLength(1); + expect(harness.repairs[0]!.runId).toBe(run.id); + + // Reads still resolve: the run's state machine is readable, so nothing hangs. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The bound: one crash costs at most one attempt number. + expect(pgRun.attemptNumber! - 1).toBeLessThanOrEqual(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "afterRedisBirthBeforePg: no run is created, and the next trigger succeeds", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("afterRedisBirthBeforePg", { times: 1 }); + await expect( + engine.trigger(triggerArgs("chaos-task", environment), prisma) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + expect(faults.fired("afterRedisBirthBeforePg")).toBe(1); + + // The harmless state: no run row, so nothing can ever read a run that has no snapshot. + const runsAfterCrash = await prisma.taskRun.count({ + where: { runtimeEnvironmentId: environment.id }, + }); + expect(runsAfterCrash).toBe(0); + + // A crashed birth must not poison the path: the next trigger runs to completion. + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(finished.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "midFlushRetry: a crash during a retry still converges through the repair job", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // A dead port makes attempt 0 fail FOR REAL, which is the only way the retry boundary is + // reachable: an injected fault at attempt 0 is treated as a dead process and skips the + // retries entirely. Arming midFlushRetry alone would fire nothing and pass for the wrong + // reason, which is what the fired() assertion below catches. + const harness = buildDecoratedStore({ + prisma, + redisOptions: { ...(redisOptions as object), port: 1, retryStrategy: () => null } as never, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("midFlushRetry", { times: 1 }); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + // The birth append failed for real and, before redis-only, that is survivable: Postgres is + // authoritative and the run exists. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.id).toBe(run.id); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + + // The retry boundary was genuinely reached, not merely armed. + expect(faults.fired("midFlushRetry")).toBeGreaterThanOrEqual(1); + + // The run converges regardless: Postgres holds every snapshot at this dial position. + expect(attempt.run.attemptNumber).toBe(1); + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("EXECUTING"); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "a crash-stalled run rejects a stale snapshot rather than hanging", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + faults.disarm("afterPgBeforeRedis"); + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres advanced; the Redis head did not. Reads come from Redis, so the caller now holds + // a snapshot id that no longer matches what the read store reports as latest. + // + // The contract is that this SURFACES rather than corrupts: the next operation to validate + // against latest rejects with a stale-snapshot error, which is the same answer a caller gets + // from an ordinary lost race. It does not hang, and it does not silently execute against the + // wrong state. + await expect( + engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }) + ).rejects.toThrow(/Snapshot changed/); + + // The run is still readable and still has a coherent state machine. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The gap was handed to the repair job, which is the compensator the protocol names. + expect(harness.repairs.length).toBeGreaterThanOrEqual(1); + expect(harness.repairs.some((r) => r.runId === run.id)).toBe(true); + + // And no attempt was burned by the rejection itself: the bound is per crash, and the + // rejected call never reached the attempt bump. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber ?? 0).toBeLessThanOrEqual(faults.fired("afterPgBeforeRedis")); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "two crashes cost at most two attempts, and the divergence does not amplify", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // dual-write, so reads still come from Postgres and the run can be driven forward through the + // normal API. That isolates the property under test — how many attempts two crashes cost — + // from the stale-read rejection the previous test covers. + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 2, runId: run.id }); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const crashes = faults.fired("afterPgBeforeRedis"); + expect(crashes).toBeGreaterThanOrEqual(1); + + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + + // pgAttempt - maxLoggedAttempt <= crashCount. Each crash costs at most one attempt, and the + // divergence does not amplify: two crashes never cost three. A flat bound of one was + // refuted by the model check, so the assertion is against the crash count, not a constant. + expect((pgRun.attemptNumber ?? 0) - 1).toBeLessThanOrEqual(crashes); + + // Postgres holds every snapshot at this dial position, so the run still converges. + expect(pgRun.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(harness.repairs.length).toBe(crashes); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts new file mode 100644 index 00000000000..17c186d074d --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts @@ -0,0 +1,304 @@ +// The read gate: the engine's own snapshot flows, run against the decorator with reads served from +// Redis. Same flows, same expectations, different store underneath — the point is that nothing in +// the engine has to know, so no existing suite is modified to make this pass. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { setTimeout } from "timers/promises"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +function engineOptions(prisma: any, redisOptions: any, harness: DecoratedStoreHarness) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any, n: number) => ({ + number: n, + // A real minted friendly id: the engine converts it back with RunId.fromFriendlyId, which + // rejects anything that is not the prefix plus a cuid body. + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_gate_${n}`, + spanId: `s_gate_${n}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store read gate", () => { + containerTest( + "drives a run to completion with every snapshot read served from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 1), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + expect(attempt.run.status).toBe("EXECUTING"); + + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + + // The gate: the engine read its snapshots, and Redis is what answered. + const fromRedis = harness.reads.filter((r) => r.source === "redis"); + expect(fromRedis.length).toBeGreaterThan(0); + expect(harness.reads.filter((r) => r.source === "postgres")).toEqual([]); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves getRunExecutionData from Redis at every step", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 2), prisma); + await setTimeout(500); + + const queued = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(queued); + expect(queued.snapshot.executionStatus).toBe("QUEUED"); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + const pending = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(pending); + expect(pending.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + + await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + const executing = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executing); + expect(executing.snapshot.executionStatus).toBe("EXECUTING"); + expect(executing.run.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "keeps the environment boundary on a snapshot read", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 3), prisma); + await setTimeout(500); + + // Scoped to its own environment the run reads normally. + const own = await engine.getRunExecutionData({ + runId: run.id, + environmentId: environment.id, + }); + assertNonNullable(own); + + // Scoped to any other environment the run must not leak across the tenant boundary. The + // assertion is parity rather than a fixed shape: whatever Postgres answers for this call, + // Redis has to answer the same, or the boundary behaves differently once reads move over. + const foreignEnvironmentId = generateInternalId(); + + const viaRedis = await engine + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + + const postgresOnly = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, postgresOnly) as never); + let viaPostgres: unknown; + try { + viaPostgres = await engineOff + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + } finally { + await engineOff.quit(); + await postgresOnly.quit(); + } + + expect(viaRedis).toEqual(viaPostgres); + // And whatever that shape is, it must not be the run's data. + expect(viaRedis).not.toMatchObject({ run: { id: run.id } }); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves a since-window wider than the cap from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 4), prisma); + await setTimeout(500); + + const first = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(first); + + // More transitions than the 50-cap, so the window is exercised at its boundary. + for (let i = 0; i < 60; i++) { + await harness.store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: `filler ${i}` }, + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.project.id, + organizationId: environment.organization.id, + }); + } + + const since = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: first.snapshot.id, + }); + assertNonNullable(since); + + // The newest 50, ascending — the same window Postgres would have produced. + expect(since.length).toBe(50); + expect(since[since.length - 1]!.snapshot.description).toBe("filler 59"); + expect(harness.reads.some((r) => r.source === "redis")).toBe(true); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "falls back to Postgres for a pre-cutover run", + async ({ prisma, redisOptions }) => { + // A run created while the dial was off has no keyspace. Turning reads on must not lose it. + const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); + + let runId: string; + let environment: any; + try { + environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engineOff, environment, "gate-task"); + const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); + runId = run.id; + await setTimeout(500); + } finally { + await engineOff.quit(); + await off.quit(); + } + + const on = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); + try { + const data = await engineOn.getRunExecutionData({ runId }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("QUEUED"); + expect(on.reads.some((r) => r.source === "postgres")).toBe(true); + } finally { + await engineOn.quit(); + await on.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts new file mode 100644 index 00000000000..463e1ecd388 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { + WaitpointKeyTagError, + assertSingleSlot, + edgeField, + idempotencyKey, + runBlockKeys, + waitpointIdFromEdgeField, + waitpointKeys, + watcherField, +} from "./keys.js"; + +describe("waitpointKeys", () => { + it("puts the record and its watchers under one hash tag", () => { + const k = waitpointKeys("abc123w"); + expect(k.record).toBe("wp:{abc123w}"); + expect(k.watchers).toBe("wp:{abc123w}:w"); + }); +}); + +describe("runBlockKeys", () => { + it("puts all three run keys under one hash tag", () => { + const k = runBlockKeys("run_abc"); + expect(k.pend).toBe("wp:run:{run_abc}:pend"); + expect(k.done).toBe("wp:run:{run_abc}:done"); + expect(k.edge).toBe("wp:run:{run_abc}:edge"); + }); +}); + +describe("idempotencyKey", () => { + it("tags by environment, so one environment's reservations share a slot", () => { + expect(idempotencyKey("env_1", "my-key")).toBe("wp:idem:{env_1}:my-key"); + }); +}); + +describe("edgeField", () => { + it("keys by waitpoint id and batch index, matching the Postgres unique key", () => { + expect(edgeField("w_a", 3)).toBe("w_a#3"); + }); + + it("collapses a null or absent batch index onto one field", () => { + expect(edgeField("w_a")).toBe("w_a#"); + expect(edgeField("w_a", null)).toBe("w_a#"); + }); + + it("distinguishes index 0 from an absent index", () => { + expect(edgeField("w_a", 0)).not.toBe(edgeField("w_a")); + }); +}); + +describe("waitpointIdFromEdgeField", () => { + it("round-trips back to the waitpoint id", () => { + for (const index of [undefined, null, 0, 7]) { + expect(waitpointIdFromEdgeField(edgeField("w_a", index))).toBe("w_a"); + } + }); + + it("returns undefined for a field with no separator", () => { + expect(waitpointIdFromEdgeField("nope")).toBeUndefined(); + }); + + it("splits on the last separator, tolerating a '#' inside the waitpoint id", () => { + expect(waitpointIdFromEdgeField("a#b#3")).toBe("a#b"); + }); +}); + +describe("watcherField", () => { + it("keys by run id and batch index, so one run can watch at several indexes", () => { + expect(watcherField("run_a", 2)).toBe("run_a#2"); + expect(watcherField("run_a")).toBe("run_a#"); + expect(watcherField("run_a", 0)).not.toBe(watcherField("run_a")); + }); +}); + +describe("assertSingleSlot", () => { + it("accepts keys that share one tag", () => { + const k = runBlockKeys("run_abc"); + expect(() => assertSingleSlot("runReadBlockState", [k.pend, k.done, k.edge])).not.toThrow(); + }); + + it("accepts a single tagged key", () => { + expect(() => assertSingleSlot("wpIdemReserve", [idempotencyKey("env_1", "k")])).not.toThrow(); + }); + + it("accepts an empty key list", () => { + expect(() => assertSingleSlot("noKeys", [])).not.toThrow(); + }); + + it("rejects keys from two different tags", () => { + const wp = waitpointKeys("w_a"); + const run = runBlockKeys("run_abc"); + expect(() => assertSingleSlot("bad", [wp.record, run.pend])).toThrow(WaitpointKeyTagError); + }); + + it("rejects an untagged key", () => { + expect(() => assertSingleSlot("bad", ["wp:no-tag"])).toThrow(WaitpointKeyTagError); + }); + + it("rejects an empty tag", () => { + expect(() => assertSingleSlot("bad", ["wp:{}"])).toThrow(WaitpointKeyTagError); + }); + + it("rejects an empty first pair, matching Redis rather than skipping to a later one", () => { + // Redis stops at the first `{`/`}` pair. An empty one means no tag at all, so it hashes + // the whole key. A regex would have found `a` here and wrongly claimed a shared slot. + expect(() => assertSingleSlot("bad", ["wp:{}{a}", "wp:{}{a}"])).toThrow(WaitpointKeyTagError); + }); + + it("takes the first pair when several are present", () => { + expect(() => assertSingleSlot("ok", ["wp:{a}{b}", "wp:{a}:w"])).not.toThrow(); + expect(() => assertSingleSlot("bad", ["wp:{a}{b}", "wp:{b}:w"])).toThrow(WaitpointKeyTagError); + }); + + it("does not degrade on a key made of many opening braces", () => { + const started = performance.now(); + expect(() => assertSingleSlot("bad", ["{".repeat(50_000)])).toThrow(WaitpointKeyTagError); + expect(performance.now() - started).toBeLessThan(1_000); + }); + + it("names the operation and the offending key in the error", () => { + const wp = waitpointKeys("w_a"); + const run = runBlockKeys("run_abc"); + try { + assertSingleSlot("myOperation", [wp.record, run.pend]); + throw new Error("should have thrown"); + } catch (error) { + expect(error).toBeInstanceOf(WaitpointKeyTagError); + expect((error as Error).message).toContain("myOperation"); + expect((error as Error).message).toContain(run.pend); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts new file mode 100644 index 00000000000..28eac087b4b --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts @@ -0,0 +1,98 @@ +/** + * Waitpoint coordination keyspace. Two hash tags, deliberately: + * + * - `wp:{waitpointId}` — the record, its status and completion envelope, plus the + * watcher hash. A waitpoint has N watcher runs, so it cannot live under any single + * run's tag. + * - `wp:run:{runId}:*` — one run's pending set, delivered set and edge set. The pending + * set's cardinality is the blocked-versus-unblocked signal, so it has to be readable + * atomically, which means one slot. + * + * Every script therefore touches exactly one tag, and assertSingleSlot enforces it on + * every invocation. A cluster would reject a cross-slot script; a single-node test server + * would not, so this assertion is the only thing standing between a cross-slot bug and + * production. + */ + +export type WaitpointKeys = { record: string; watchers: string }; +export type RunBlockKeys = { pend: string; done: string; edge: string }; + +export function waitpointKeys(waitpointId: string): WaitpointKeys { + const base = `wp:{${waitpointId}}`; + return { record: base, watchers: `${base}:w` }; +} + +export function runBlockKeys(runId: string): RunBlockKeys { + const base = `wp:run:{${runId}}`; + return { pend: `${base}:pend`, done: `${base}:done`, edge: `${base}:edge` }; +} + +export function idempotencyKey(environmentId: string, key: string): string { + return `wp:idem:{${environmentId}}:${key}`; +} + +// "#" separates the id from the index. An absent index collapses onto the empty suffix, +// which is how the partial unique index on a null batchIndex behaves; index 0 keeps its +// own field, because "0" and "" are different strings. The split back to an id below is +// taken from the LAST "#", not the first, so this stays unambiguous even if a waitpoint id +// or a run id ever contains "#" itself. +const SEPARATOR = "#"; + +export function edgeField(waitpointId: string, batchIndex?: number | null): string { + return `${waitpointId}${SEPARATOR}${batchIndex ?? ""}`; +} + +export function watcherField(runId: string, batchIndex?: number | null): string { + return `${runId}${SEPARATOR}${batchIndex ?? ""}`; +} + +// The last-"#" rule here is re-implemented as a Lua pattern in runClear (scripts.ts). This +// function has no caller besides its own test, so that test is what pins the rule as a +// specification the Lua mirrors, not just documentation of this helper. +export function waitpointIdFromEdgeField(field: string): string | undefined { + const separator = field.lastIndexOf(SEPARATOR); + return separator === -1 ? undefined : field.slice(0, separator); +} + +export class WaitpointKeyTagError extends Error { + constructor(operation: string, keys: string[], offending: string) { + super( + `Waitpoint operation ${operation} would span more than one cluster slot: ` + + `key ${JSON.stringify(offending)} does not share the tag of ${JSON.stringify(keys)}` + ); + this.name = "WaitpointKeyTagError"; + } +} + +// Redis's own keyHashSlot rule: the FIRST `{`, then the FIRST `}` after it. A missing brace +// or an empty pair means no tag, and Redis hashes the whole key. A regex would instead find +// the first NON-empty pair, disagreeing with Redis on `wp:{}{a}`. +function hashTag(key: string): string | undefined { + const open = key.indexOf("{"); + if (open === -1) return undefined; + + const close = key.indexOf("}", open + 1); + if (close === -1 || close === open + 1) return undefined; + + return key.slice(open + 1, close); +} + +/** + * Throw unless every key carries the same non-empty hash tag. Called on every script + * invocation, because the keys embed ids and are only known at call time. + */ +export function assertSingleSlot(operation: string, keys: string[]): void { + let tag: string | undefined; + + for (const key of keys) { + const found = hashTag(key); + if (!found) { + throw new WaitpointKeyTagError(operation, keys, key); + } + if (tag === undefined) { + tag = found; + } else if (found !== tag) { + throw new WaitpointKeyTagError(operation, keys, key); + } + } +} diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts new file mode 100644 index 00000000000..d1e48fa4f8d --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts @@ -0,0 +1,437 @@ +import type { RunStore } from "@internal/run-store"; +import { tryCatch } from "@trigger.dev/core/v3"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import type { Logger } from "@trigger.dev/core/logger"; +import type { PrismaClient, Waitpoint } from "@trigger.dev/database"; +import { boundedIn, Prisma } from "@trigger.dev/database"; +import { nanoid } from "nanoid"; +import { UnclassifiableWaitpointId } from "../errors.js"; +import type { + AssociatedWaitpointData, + ClearRunBlockStateParams, + CompleteParams, + CompleteResult, + CreateDateTimeWaitpointParams, + CreateManualWaitpointParams, + CreateWaitpointResult, + RegisterBlocksLocklessParams, + RegisterBlocksParams, + RunBlockEdge, + WaitpointCoordinator, +} from "./types.js"; + +export type LegacyPostgresWaitpointCoordinatorOptions = { + runStore: RunStore; + prisma: PrismaClient; + logger: Logger; +}; + +/** + * Waitpoint coordination against Postgres, through the run-ops store. + * + * Dependencies are deliberately narrow: no run lock, no worker, no event bus. + * That makes "this owns waitpoint state only" structural rather than a convention. + */ +export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator { + private readonly runStore: RunStore; + private readonly prisma: PrismaClient; + private readonly logger: Logger; + + constructor(options: LegacyPostgresWaitpointCoordinatorOptions) { + this.runStore = options.runStore; + this.prisma = options.prisma; + this.logger = options.logger; + } + + async clearRunBlockState({ + runId, + edgeIds, + tx, + }: ClearRunBlockStateParams): Promise<{ count: number }> { + if (edgeIds) { + // Bounded delete of named edges, on the unblock path. No tx: that path is not inside a + // caller transaction, and boundedIn caps the id-list arity for Prisma. + return this.runStore.deleteManyTaskRunWaitpoints({ + where: { + taskRunId: runId, + id: { in: boundedIn(edgeIds) }, + }, + }); + } + + // A run's edges co-locate with the run (the edge write routes by runId), so the router routes + // this taskRunId-keyed delete to the run's store rather than fanning out. The caller's `tx` is + // passed through: a routing store strips it, and a single store joins it. + return this.runStore.deleteManyTaskRunWaitpoints({ where: { taskRunId: runId } }, tx); + } + + async readRunBlockState(runId: string): Promise { + return this.runStore.findManyTaskRunWaitpoints( + { + where: { taskRunId: runId }, + select: { + id: true, + batchId: true, + batchIndex: true, + waitpoint: { + select: { id: true, status: true, type: true, completedAfter: true }, + }, + }, + }, + this.prisma + ); + } + + async registerBlocks({ + client, + ...edge + }: RegisterBlocksParams): Promise<{ pendingCount: number }> { + await this.#writeBlockEdges(edge); + + // Check if the run is actually blocked using a separate query. The separate statement is the + // point: under PostgreSQL READ COMMITTED each statement gets its own snapshot, so a + // concurrent completion that commits between the edge write and this check is still seen. + // It queries ALL requested ids, not just inserted ones: a row that already existed (ON + // CONFLICT skipped the insert) but is still PENDING must still block. Pass the caller's + // client so the re-read is read-your-writes on the owning PRIMARY, and pass the run id so + // the router counts on the run's store instead of fanning out to both DBs. + const pendingCount = await this.runStore.countPendingWaitpoints( + edge.waitpointIds, + client, + edge.runId + ); + + return { pendingCount }; + } + + async registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise { + await this.#writeBlockEdges(params); + } + + async complete({ waitpointId, output }: CompleteParams): Promise { + // Residency store-selection guard. complete arrives with only (waitpointId, output) — no run + // id — so the owning run-ops store is selected by the waitpoint's own residency. In single-DB + // this is the one store (no classification). An unclassifiable id throws loud — never + // default-routes. The try wraps ONLY the resolve: widening it would swallow the + // "Waitpoint not found" path that a single store relies on. + let store: RunStore; + try { + store = await this.runStore.forWaitpointCompletion(waitpointId, { routeKind: "MANUAL" }); + } catch (error) { + this.logger.error("completeWaitpoint: unclassifiable waitpointId", { + waitpointId, + error, + }); + throw new UnclassifiableWaitpointId(waitpointId, { cause: error }); + } + + // 1. Complete the Waitpoint (if not completed) + const [updateError, updateResult] = await tryCatch( + store.updateManyWaitpoints({ + where: { id: waitpointId, status: "PENDING" }, + data: { + status: "COMPLETED", + completedAt: new Date(), + output: output?.value, + outputType: output?.type, + outputIsError: output?.isError, + }, + }) + ); + + if (updateError) { + this.logger.error("completeWaitpoint: error updating waitpoint:", { updateError }); + throw updateError; + } + + if (updateResult.count === 0) { + this.logger.info("completeWaitpoint: attempted to complete a waitpoint that is not PENDING", { + waitpointId, + }); + } + + // Re-read the just-written row from the RESOLVED store's PRIMARY: the replica (findWaitpoint's + // default) can miss it under lag → false "not found" → the parent hangs. Going back through + // the router would re-resolve the store and change the routing, so use the handle. + const waitpoint = await store.findWaitpointOnPrimary({ + where: { id: waitpointId }, + }); + + if (!waitpoint) { + this.logger.error("completeWaitpoint: waitpoint not found", { waitpointId }); + throw new Error("Waitpoint not found"); + } + + if (waitpoint.status !== "COMPLETED") { + this.logger.error(`completeWaitpoint: waitpoint is not completed`, { waitpointId }); + throw new Error("Waitpoint not completed"); + } + + // 2. Find the TaskRuns blocked by this waitpoint. The edge (TaskRunWaitpoint) co-locates + // with its RUN, not this token, so it can live on the OTHER run-ops DB: read via the router + // (which fans the waitpointId lookup across both DBs) rather than the token's own `store`, + // or a cross-DB blocked run is never found and hangs forever. + const blockedRuns = await this.runStore.findManyTaskRunWaitpoints( + { + where: { waitpointId }, + select: { taskRunId: true, spanIdToComplete: true, createdAt: true }, + }, + this.prisma + ); + + return { waitpoint, blockedRuns }; + } + + async createDateTimeWaitpoint({ + runId, + projectId, + environmentId, + completedAfter, + idempotencyKey, + idempotencyKeyExpiresAt, + }: CreateDateTimeWaitpointParams): Promise { + // Co-location invariant: a DATETIME wait waitpoint lives on the same run-ops DB as the run that + // blocks on it (so the block edge's local `Waitpoint` join resolves and completion/resume stay + // local). The minted waitpoint id is always a cuid, so without `coLocateWithRunId` the upsert + // would always route to LEGACY and a run-ops run on NEW would hang. The (env,idempotencyKey) dedup + // is within the owning run/tree (co-resident on one DB), so the dedup probe + rotation target the + // SAME store. With no run id (a standalone token has no owning run yet) the lookup falls back to + // a cross-DB NEW-then-LEGACY scan and the upsert routes by id-shape. Always routed through the + // run store (never a caller tx) so it can never bypass residency onto the wrong DB. + const colocate = runId ? { coLocateWithRunId: runId } : undefined; + const existingWaitpoint = idempotencyKey + ? await this.runStore.findWaitpoint( + { + where: { + environmentId, + idempotencyKey, + }, + }, + undefined, + colocate + ) + : undefined; + + if (existingWaitpoint) { + if ( + existingWaitpoint.idempotencyKeyExpiresAt && + new Date() > existingWaitpoint.idempotencyKeyExpiresAt + ) { + //the idempotency key has expired + //remove the waitpoint idempotencyKey + const rotateArgs = { + where: { + id: existingWaitpoint.id, + }, + data: { + idempotencyKey: nanoid(24), + inactiveIdempotencyKey: existingWaitpoint.idempotencyKey, + }, + }; + await this.runStore.updateWaitpoint(rotateArgs, undefined, colocate); + + //let it fall through to create a new waitpoint + } else { + return { kind: "cached", waitpoint: existingWaitpoint }; + } + } + + // The two `nanoid(24)` calls below are deliberately separate and produce DIFFERENT values: + // the upsert `where` key must not match the `create` key, or a guaranteed-miss upsert becomes + // a possible update. Do not hoist either to a shared constant. + const upsertArgs = { + where: { + environmentId_idempotencyKey: { + environmentId, + idempotencyKey: idempotencyKey ?? nanoid(24), + }, + }, + create: { + ...WaitpointId.generate(), + type: "DATETIME" as const, + idempotencyKey: idempotencyKey ?? nanoid(24), + idempotencyKeyExpiresAt, + userProvidedIdempotencyKey: !!idempotencyKey, + environmentId, + projectId, + completedAfter, + }, + update: {}, + }; + const waitpoint = await this.runStore.upsertWaitpoint(upsertArgs, undefined, colocate); + + return { kind: "created", waitpoint }; + } + + async createManualWaitpoint({ + runId, + environmentId, + projectId, + idempotencyKey, + idempotencyKeyExpiresAt, + timeout, + tags, + standaloneResidency, + }: CreateManualWaitpointParams): Promise { + // Co-location invariant (see createDateTimeWaitpoint): when a `runId` is supplied the waitpoint + // co-locates with that run's DB and the (env,idempotencyKey) dedup is per-run (co-resident). A + // standalone token (api.v1.waitpoints.tokens.ts) passes no run id — it is created without an + // owner, blocked later by whichever run waits on it (possibly cross-DB, resolved by the + // run-co-resident block edge + completion fan-out). With no owner it reads the env mint kind via + // `standaloneResidency` so a minted-new env keeps its tokens on NEW; unset, it routes by id-shape. No tx here. + const colocate = runId + ? { coLocateWithRunId: runId } + : standaloneResidency + ? { residency: standaloneResidency } + : undefined; + const existingWaitpoint = idempotencyKey + ? await this.runStore.findWaitpoint( + { + where: { + environmentId, + idempotencyKey, + }, + }, + undefined, + colocate + ) + : undefined; + + if (existingWaitpoint) { + if ( + existingWaitpoint.idempotencyKeyExpiresAt && + new Date() > existingWaitpoint.idempotencyKeyExpiresAt + ) { + //the idempotency key has expired + //remove the waitpoint idempotencyKey + await this.runStore.updateWaitpoint( + { + where: { + id: existingWaitpoint.id, + }, + data: { + idempotencyKey: nanoid(24), + inactiveIdempotencyKey: existingWaitpoint.idempotencyKey, + }, + }, + undefined, + colocate + ); + + //let it fall through to create a new waitpoint + } else { + return { kind: "cached", waitpoint: existingWaitpoint }; + } + } + + const maxRetries = 5; + let attempts = 0; + + while (attempts < maxRetries) { + try { + // As in createDateTimeWaitpoint, the two `nanoid(24)` calls are deliberately separate and + // differ. Both, and `WaitpointId.generate()`, are re-evaluated on every attempt: that is + // what makes a retry after a unique-constraint conflict try a fresh key. + const waitpoint = await this.runStore.upsertWaitpoint( + { + where: { + environmentId_idempotencyKey: { + environmentId, + idempotencyKey: idempotencyKey ?? nanoid(24), + }, + }, + create: { + ...WaitpointId.generate(), + type: "MANUAL", + idempotencyKey: idempotencyKey ?? nanoid(24), + idempotencyKeyExpiresAt, + userProvidedIdempotencyKey: !!idempotencyKey, + environmentId, + projectId, + completedAfter: timeout, + tags, + }, + update: {}, + }, + undefined, + colocate + ); + + return { kind: "created", waitpoint }; + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { + // Handle unique constraint violation (conflict) + attempts++; + if (attempts >= maxRetries) { + throw new Error( + `Failed to create waitpoint after ${maxRetries} attempts due to conflicts.` + ); + } + } else { + throw error; // Re-throw other errors + } + } + } + + throw new Error(`Failed to create waitpoint after ${maxRetries} attempts due to conflicts.`); + } + + mintAssociatedWaitpointData({ + projectId, + environmentId, + }: { + projectId: string; + environmentId: string; + }): AssociatedWaitpointData { + return { + ...WaitpointId.generate(), + type: "RUN" as const, + status: "PENDING" as const, + idempotencyKey: nanoid(24), + userProvidedIdempotencyKey: false, + projectId, + environmentId, + }; + } + + async createAssociatedWaitpoint({ + runId, + data, + }: { + runId: string; + data: AssociatedWaitpointData; + }): Promise { + // RUN-type within-tree waitpoint that belongs to runId; routes by owning run id. + return this.runStore.createWaitpoint({ + data: { + ...data, + completedByTaskRunId: runId, + }, + }); + } + + /** + * The edge write, shared by both register paths so they cannot drift. + * + * Routed by the owning run id so the edge co-resides with the run. Never pinned to a caller + * transaction: that joined `Waitpoint` on the wrong DB, wrote 0 edges, and silently never + * suspended the parent. The write is idempotent (ON CONFLICT DO NOTHING). + */ + #writeBlockEdges({ + runId, + waitpointIds, + projectId, + spanIdToComplete, + batchId, + batchIndex, + }: RegisterBlocksLocklessParams): Promise { + return this.runStore.blockRunWithWaitpointEdges({ + runId, + waitpointIds, + projectId, + spanIdToComplete, + batchId, + batchIndex, + }); + } +} diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts new file mode 100644 index 00000000000..820b4145f86 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts @@ -0,0 +1,388 @@ +import type { Callback, Redis, Result } from "@internal/redis"; + +/** + * Lua for the waitpoint coordination protocol. Three rules hold throughout: + * + * 1. Every key a script touches is declared in KEYS. No script builds a key name inside + * Lua. ioredis prefixes only the KEYS array, so a key minted in Lua would be + * unprefixed while the client wrote a prefixed one — and a script with a single + * declared key gives the caller's single-slot assertion nothing to compare. + * 2. Lua never parses JSON. Each script branches only on a short status string and moves + * opaque blobs, so every encoding decision stays in TypeScript. + * 3. A missing HGET returns Lua `false`, not `nil` — measured directly against a live + * Redis: `EVAL "return {'a', false, 'c'}"` and a table holding a missing-field HGET + * result both come back as 3 elements; only `EVAL "return {'a', nil, 'c'}"` comes back + * as 1. A `false` element converts to a reply-array null and does NOT shorten anything + * after it — only a genuine Lua nil truncates. Every returned slot is still coerced + * with `or ''` regardless, not to prevent truncation, but so an absent value arrives + * as `''` rather than `null`, giving the TypeScript one shape to decode instead of + * two. + * + * STORED_COMPLETED is the value written into the record's `status` field and is + * UPPERCASE. The outcome tokens below are lowercase and are a separate vocabulary: they + * name what a script DID, not what a record IS. Sharing one constant between the two + * makes an already-completed record invisible to every script. + */ + +const STORED_COMPLETED = "COMPLETED"; + +const MISSING = "missing"; +const CREATED = "created"; +const EXISTS = "exists"; +const REGISTERED = "registered"; +const DID_COMPLETE = "completed"; +const ALREADY = "already"; +const RESERVED = "reserved"; +const CLEARED = "cleared"; +const DRAINED = "drained"; +const DISCARDED = "discarded"; + +export function registerWaitpointCommands(redis: Redis): void { + // KEYS: record. ARGV: recordJson, status ('PENDING'|'COMPLETED'), completionJson (''). + redis.defineCommand("wpCreateIfAbsent", { + numberOfKeys: 1, + lua: ` + local record = KEYS[1] + + -- EXISTS-then-HSET inside one script, rather than a field-by-field HSETNX: the + -- record and its status must appear together or not at all. + if redis.call('EXISTS', record) == 1 then + local vals = redis.call('HMGET', record, 'r', 'status', 'c') + return { '${EXISTS}', vals[1] or '', vals[2] or '', vals[3] or '' } + end + + redis.call('HSET', record, 'r', ARGV[1], 'status', ARGV[2]) + if ARGV[3] ~= '' then + redis.call('HSET', record, 'c', ARGV[3]) + end + + return { '${CREATED}' } + `, + }); + + // KEYS: record, watchers. ARGV: watcherField, watcherJson. + redis.defineCommand("wpRegisterOrReport", { + numberOfKeys: 2, + lua: ` + local record, watchers = KEYS[1], KEYS[2] + + -- A missing waitpoint is never a silent no-op: the caller throws. Defaulting to + -- "not blocked" here would resume a run whose waitpoint never completed. + if redis.call('EXISTS', record) == 0 then + return { '${MISSING}' } + end + + if redis.call('HGET', record, 'status') == '${STORED_COMPLETED}' then + return { '${DID_COMPLETE}', redis.call('HGET', record, 'c') or '' } + end + + -- The watcher lands before any flip can read the watcher hash, because this script + -- and wpComplete are both atomic on this same shard. So a register either appears + -- in the flip's watcher list, or it observes COMPLETED above. + -- + -- HSETNX: the first registration wins, mirroring the edge's ON CONFLICT DO NOTHING. + redis.call('HSETNX', watchers, ARGV[1], ARGV[2]) + return { '${REGISTERED}' } + `, + }); + + // KEYS: record, watchers. ARGV: completionJson. + redis.defineCommand("wpComplete", { + numberOfKeys: 2, + lua: ` + local record, watchers = KEYS[1], KEYS[2] + + if redis.call('EXISTS', record) == 0 then + return { '${MISSING}' } + end + + local outcome = '${DID_COMPLETE}' + if redis.call('HGET', record, 'status') == '${STORED_COMPLETED}' then + -- Double completion is not an error, and the FIRST completion wins. This is the + -- guard a conditional UPDATE ... WHERE status = 'PENDING' used to provide. + outcome = '${ALREADY}' + else + redis.call('HSET', record, 'status', '${STORED_COMPLETED}', 'c', ARGV[1]) + end + + -- Returning the watchers here is what removes the reverse fan-out query. The + -- envelope comes back too, because delivery runs on each watcher's own shard and + -- cannot read this key. + local out = { outcome, redis.call('HGET', record, 'c') or '' } + local entries = redis.call('HVALS', watchers) + for i = 1, #entries do + out[#out + 1] = entries[i] + end + + return out + `, + }); + + // KEYS: idempotency key. ARGV: waitpointId, expiresAtMs ('' for no expiry). + redis.defineCommand("wpIdemReserve", { + numberOfKeys: 1, + lua: ` + local key = KEYS[1] + + -- Guard before the SET: a non-numeric expiry must not land a reservation that can + -- never expire because PEXPIREAT then errors out after the write already happened. + if ARGV[2] ~= '' and tonumber(ARGV[2]) == nil then + return redis.error_reply('wpIdemReserve: ARGV[2] must be numeric or empty') + end + + -- SET NX returns a status reply on success and false on conflict. + if redis.call('SET', key, ARGV[1], 'NX') then + -- Expiry only when the caller has one. A reservation with no expiry is the common + -- case and must never grow one here. + if ARGV[2] ~= '' then + redis.call('PEXPIREAT', key, tonumber(ARGV[2])) + end + return { '${RESERVED}', ARGV[1] } + end + + return { '${EXISTS}', redis.call('GET', key) or '' } + `, + }); + + // KEYS: record, watchers. No ARGV. Discards a losing reservation's orphan record. + redis.defineCommand("wpDiscard", { + numberOfKeys: 2, + lua: ` + redis.call('DEL', KEYS[1], KEYS[2]) + return { '${DISCARDED}' } + `, + }); + + // KEYS: pend, done, edge. + // ARGV: n, then n groups of 5 — waitpointId, edgeField, edgeJson, reportedFlag + // ('1'|'0'), reportedJson (''). reportedFlag, not the emptiness of reportedJson, is what + // decides the branch: a waitpoint can be reported COMPLETED with no completion envelope + // (see the FINISHED-healing path), and that case must still take the reported branch — + // flag '1', reportedJson '' — or the run would block forever on something already done. + redis.defineCommand("runAbsorbBlockers", { + numberOfKeys: 3, + lua: ` + local pend, done, edge = KEYS[1], KEYS[2], KEYS[3] + local n = tonumber(ARGV[1]) + + -- Guard before any write: a wrong n must not half-apply the script. HDEL/HSETNX below + -- are irreversible mid-script, and Redis does not roll back a script that errors. + if #ARGV ~= 1 + n * 5 then + return redis.error_reply('runAbsorbBlockers: arity mismatch') + end + + -- seenDelivered makes the delivered-pair output DISTINCT BY ID: two edges for one + -- waitpoint must contribute one pair, not two. + local requestedIds = {} + local seenDelivered = {} + local out = { '0', '0' } + + for i = 0, n - 1 do + local id = ARGV[2 + i * 5] + local field = ARGV[3 + i * 5] + local edgeJson = ARGV[4 + i * 5] + local reportedFlag = ARGV[5 + i * 5] + local reported = ARGV[6 + i * 5] + + -- HSETNX is the ON CONFLICT DO NOTHING of the edge write: a retry must not + -- overwrite the first attempt's metadata. + redis.call('HSETNX', edge, field, edgeJson) + requestedIds[id] = true + + if reportedFlag == '1' then + -- Already COMPLETED when the watcher registered. It never becomes pending, even + -- when reported ('' here) carries no envelope. + redis.call('HSET', done, id, reported) + redis.call('SREM', pend, id) + if not seenDelivered[id] then + seenDelivered[id] = true + out[#out + 1] = id + out[#out + 1] = reported + end + else + -- Check the delivered set FIRST. A completion that landed between register and + -- absorb has already delivered here, and that delivery wins. + local delivered = redis.call('HGET', done, id) + if delivered then + if not seenDelivered[id] then + seenDelivered[id] = true + out[#out + 1] = id + out[#out + 1] = delivered + end + else + redis.call('SADD', pend, id) + end + end + end + + -- Computed AFTER every write in this batch, as the count of distinct requested ids + -- with no entry in done. Counting incrementally during the loop is order-dependent: + -- a later group's completion for an id already counted as pending would leave the + -- count stale, reporting a waitpoint as both pending and delivered. + local pendingOfRequested = 0 + for id in pairs(requestedIds) do + if redis.call('HEXISTS', done, id) == 0 then + pendingOfRequested = pendingOfRequested + 1 + end + end + + out[1] = tostring(pendingOfRequested) + out[2] = tostring(redis.call('SCARD', pend)) + return out + `, + }); + + // KEYS: pend, done. ARGV: waitpointId, completionJson. + redis.defineCommand("runDeliverCompletion", { + numberOfKeys: 2, + lua: ` + local pend, done = KEYS[1], KEYS[2] + + redis.call('HSET', done, ARGV[1], ARGV[2]) + redis.call('SREM', pend, ARGV[1]) + + -- The caller treats this as a wakeup trigger, not as the resume decision: the + -- resume is decided under the run lock, and this count covers store-resident + -- blockers only. + return { tostring(redis.call('SCARD', pend)) } + `, + }); + + // KEYS: pend, done, edge. + redis.defineCommand("runReadBlockState", { + numberOfKeys: 3, + lua: ` + local pend, done, edge = KEYS[1], KEYS[2], KEYS[3] + + local pendIds = redis.call('SMEMBERS', pend) + -- HKEYS, never HGETALL: the delivered set's values are completion envelopes with + -- inline outputs, and materializing those inside a single-threaded script would + -- block the shard. + local doneIds = redis.call('HKEYS', done) + local edges = redis.call('HGETALL', edge) + + local out = { tostring(#pendIds), tostring(#doneIds), tostring(#edges) } + for i = 1, #pendIds do out[#out + 1] = pendIds[i] end + for i = 1, #doneIds do out[#out + 1] = doneIds[i] end + for i = 1, #edges do out[#out + 1] = edges[i] end + return out + `, + }); + + // KEYS: pend, done, edge. ARGV: n, then n edge fields. n = 0 clears everything. + redis.defineCommand("runClear", { + numberOfKeys: 3, + lua: ` + local pend, done, edge = KEYS[1], KEYS[2], KEYS[3] + local n = tonumber(ARGV[1]) + + -- Guard before any write, same reasoning as runAbsorbBlockers. + if #ARGV ~= 1 + n then + return redis.error_reply('runClear: arity mismatch') + end + + if n == 0 then + redis.call('DEL', pend, done, edge) + return { '${CLEARED}' } + end + + for i = 1, n do + redis.call('HDEL', edge, ARGV[1 + i]) + end + + -- Reconcile rather than delete by name. The edge set is the authority: after the + -- drain, pend and done may only hold ids that some surviving edge still references. + -- + -- Two reasons this is a superset of "remove the drained ids". First, one waitpoint + -- can hold several edges at different batch indexes, so a drained field must not + -- evict a delivery another edge still needs. Second, runDeliverCompletion writes + -- done[id] unconditionally, so a crash between register and absorb can leave a + -- delivered entry with no edge at all, which no name-derived drain could reach. + local remaining = {} + local fields = redis.call('HKEYS', edge) + for i = 1, #fields do + local sep = string.find(fields[i], '#[^#]*$') + if sep then + remaining[string.sub(fields[i], 1, sep - 1)] = true + end + end + + local doneIds = redis.call('HKEYS', done) + for i = 1, #doneIds do + if not remaining[doneIds[i]] then + redis.call('HDEL', done, doneIds[i]) + end + end + + local pendIds = redis.call('SMEMBERS', pend) + for i = 1, #pendIds do + if not remaining[pendIds[i]] then + redis.call('SREM', pend, pendIds[i]) + end + end + + return { '${DRAINED}' } + `, + }); +} + +declare module "@internal/redis" { + interface RedisCommander { + wpCreateIfAbsent( + recordKey: string, + recordJson: string, + status: string, + completionJson: string, + callback?: Callback + ): Result; + wpRegisterOrReport( + recordKey: string, + watchersKey: string, + watcherField: string, + watcherJson: string, + callback?: Callback + ): Result; + wpComplete( + recordKey: string, + watchersKey: string, + completionJson: string, + callback?: Callback + ): Result; + wpIdemReserve( + key: string, + waitpointId: string, + expiresAtMs: string, + callback?: Callback + ): Result; + wpDiscard( + recordKey: string, + watchersKey: string, + callback?: Callback + ): Result; + runAbsorbBlockers( + pendKey: string, + doneKey: string, + edgeKey: string, + ...args: Array> + ): Result; + runDeliverCompletion( + pendKey: string, + doneKey: string, + waitpointId: string, + completionJson: string, + callback?: Callback + ): Result; + runReadBlockState( + pendKey: string, + doneKey: string, + edgeKey: string, + callback?: Callback + ): Result; + runClear( + pendKey: string, + doneKey: string, + edgeKey: string, + ...args: Array> + ): Result; + } +} diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts new file mode 100644 index 00000000000..f0e9c0c297d --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts @@ -0,0 +1,1837 @@ +// Redis-only suite: the coordinator holds no Prisma reference, so no Postgres container +// is needed. redisTest FLUSHALLs before every test, so ids may be reused across describes. +import { createRedisClient, type RedisOptions } from "@internal/redis"; +import { redisTest } from "@internal/testcontainers"; +import { + deriveWaitpointIdFromAnchor, + generateRunOpsId, + generateWaitpointId, +} from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { + edgeField, + idempotencyKey, + runBlockKeys, + watcherField, + WaitpointKeyTagError, +} from "./keys.js"; +import { registerWaitpointCommands } from "./scripts.js"; +import { + WaitpointNotFoundError, + WaitpointStoreCoordinator, + type BlockEdge, + type WaitpointCompletion, + type WaitpointRecordInput, + type WatcherEntry, +} from "./storeCoordinator.js"; + +const ENV_ID = "env_1"; +const PROJECT_ID = "proj_1"; +const NOW = "2026-08-21T12:00:00.000Z"; + +function coordinator(redisOptions: RedisOptions) { + return new WaitpointStoreCoordinator({ redisOptions }); +} + +function record(id: string, overrides: Partial = {}): WaitpointRecordInput { + return { + id, + friendlyId: `waitpoint_${id}`, + type: "MANUAL", + environmentId: ENV_ID, + projectId: PROJECT_ID, + createdAt: NOW, + updatedAt: NOW, + userProvidedIdempotencyKey: false, + tags: [], + ...overrides, + }; +} + +function completion(overrides: Partial = {}): WaitpointCompletion { + return { + completedAt: NOW, + outputType: "application/json", + outputIsError: false, + output: { inline: '{"ok":true}' }, + ...overrides, + }; +} + +describe("createIfAbsent", () => { + redisTest("creates a PENDING record and reports created", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + expect(result.outcome).toBe("created"); + } finally { + await store.quit(); + } + }); + + redisTest("returns the existing record on a second call", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + const second = await store.createIfAbsent({ + record: record("w_a", { friendlyId: "waitpoint_DIFFERENT" }), + status: "PENDING", + }); + + expect(second.outcome).toBe("exists"); + if (second.outcome !== "exists") throw new Error("unreachable"); + // The first write wins: a retry must not overwrite the stored record. + expect(second.record.friendlyId).toBe("waitpoint_w_a"); + expect(second.status).toBe("PENDING"); + expect(second.completion).toBeUndefined(); + } finally { + await store.quit(); + } + }); + + redisTest("preserves every record field through a round trip", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const full = record("w_a", { + type: "RUN", + idempotencyKey: "key-1", + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: NOW, + completedAfter: NOW, + completedByTaskRunId: "run_child", + completedByBatchId: "batch_1", + tags: ["one", "two"], + }); + + await store.createIfAbsent({ record: full, status: "PENDING" }); + const read = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + expect(read.outcome).toBe("exists"); + if (read.outcome !== "exists") throw new Error("unreachable"); + // Every field the frozen return shapes need must survive the blob round trip. + expect(read.record).toEqual(full); + } finally { + await store.quit(); + } + }); + + redisTest( + "can create an already-COMPLETED record with no completion envelope", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // This is the shape that catches a status-casing mismatch: the record is stored + // COMPLETED, and a register must see it as completed rather than pending. + await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" }); + + const reported = await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + createdAt: NOW, + }); + + expect(reported.outcome).toBe("completed"); + if (reported.outcome !== "completed") throw new Error("unreachable"); + expect(reported.completion).toBeUndefined(); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "can create an already-COMPLETED record with a completion", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ + record: record("w_a", { type: "RUN" }), + status: "COMPLETED", + completion: completion(), + }); + + const reported = await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + createdAt: NOW, + }); + + expect(reported.outcome).toBe("completed"); + if (reported.outcome !== "completed") throw new Error("unreachable"); + expect(reported.completion?.output).toEqual({ inline: '{"ok":true}' }); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "reads a COMPLETED record back through createIfAbsent, with an envelope", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ + record: record("w_a"), + status: "COMPLETED", + completion: completion(), + }); + + const second = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + expect(second.outcome).toBe("exists"); + if (second.outcome !== "exists") throw new Error("unreachable"); + expect(second.status).toBe("COMPLETED"); + expect(second.completion?.output).toEqual({ inline: '{"ok":true}' }); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "reads a COMPLETED record back through createIfAbsent, with no envelope", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" }); + + const second = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + expect(second.outcome).toBe("exists"); + if (second.outcome !== "exists") throw new Error("unreachable"); + expect(second.status).toBe("COMPLETED"); + expect(second.completion).toBeUndefined(); + } finally { + await store.quit(); + } + } + ); +}); + +describe("registerOrReport", () => { + redisTest("registers a watcher against a PENDING waitpoint", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + const result = await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + createdAt: NOW, + }); + expect(result.outcome).toBe("registered"); + } finally { + await store.quit(); + } + }); + + redisTest("reports the completion inline for a COMPLETED waitpoint", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.complete({ waitpointId: "w_a", completion: completion() }); + + const result = await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + createdAt: NOW, + }); + + expect(result.outcome).toBe("completed"); + if (result.outcome !== "completed") throw new Error("unreachable"); + expect(result.completion?.output).toEqual({ inline: '{"ok":true}' }); + } finally { + await store.quit(); + } + }); + + redisTest("throws for a waitpoint that does not exist", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await expect( + store.registerOrReport({ waitpointId: "w_missing", runId: "run_1", createdAt: NOW }) + ).rejects.toThrow(WaitpointNotFoundError); + } finally { + await store.quit(); + } + }); + + redisTest("keeps one watcher entry per batch index", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + batchIndex: 0, + createdAt: NOW, + }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + batchIndex: 2, + createdAt: NOW, + }); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(completed.watchers).toHaveLength(2); + expect(completed.watchers.map((w) => w.batchIndex).sort((a, b) => a! - b!)).toEqual([0, 2]); + } finally { + await store.quit(); + } + }); + + redisTest("carries spanIdToComplete through to the watcher entry", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: "span_abc", + createdAt: NOW, + }); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(completed.watchers[0]!.spanIdToComplete).toBe("span_abc"); + expect(completed.watchers[0]!.runId).toBe("run_1"); + expect(completed.watchers[0]!.createdAt).toBe(NOW); + } finally { + await store.quit(); + } + }); + + redisTest("keeps the first registration's watcher on a re-register", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: "span_first", + createdAt: NOW, + }); + // Same run, same (absent) batch index, so the watcher field collides. HSETNX must + // not let this second registration overwrite the first one's span. + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: "span_second", + createdAt: NOW, + }); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(completed.watchers).toHaveLength(1); + expect(completed.watchers[0]!.spanIdToComplete).toBe("span_first"); + } finally { + await store.quit(); + } + }); +}); + +describe("complete", () => { + redisTest("flips PENDING to COMPLETED and returns the watchers", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_2", createdAt: NOW }); + + const result = await store.complete({ waitpointId: "w_a", completion: completion() }); + + expect(result.outcome).toBe("completed"); + expect(result.watchers.map((w) => w.runId).sort()).toEqual(["run_1", "run_2"]); + } finally { + await store.quit(); + } + }); + + redisTest("is idempotent and returns the watchers again", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW }); + + const first = await store.complete({ waitpointId: "w_a", completion: completion() }); + const second = await store.complete({ + waitpointId: "w_a", + completion: completion({ output: { inline: '{"second":true}' } }), + }); + + expect(first.outcome).toBe("completed"); + expect(second.outcome).toBe("already"); + // The FIRST completion wins, matching the guard on status = PENDING. + expect(second.completion?.output).toEqual({ inline: '{"ok":true}' }); + expect(second.watchers.map((w) => w.runId)).toEqual(["run_1"]); + } finally { + await store.quit(); + } + }); + + redisTest("throws for a waitpoint that does not exist", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await expect( + store.complete({ waitpointId: "w_missing", completion: completion() }) + ).rejects.toThrow(WaitpointNotFoundError); + } finally { + await store.quit(); + } + }); + + redisTest("returns an empty watcher list when nobody is blocked", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + const result = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(result.watchers).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest( + "keeps the watcher list intact when the completion field is absent on an already-completed record", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + // registerOrReport never lets a watcher land once status is COMPLETED, so this + // shape is forced by hand: it pins that an absent 'c' field decodes to an + // undefined completion without disturbing the watchers that follow it in the + // reply array. + await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" }); + const watcher: WatcherEntry = { runId: "run_1", createdAt: NOW }; + await probe.hset("wp:{w_a}:w", watcherField("run_1"), JSON.stringify(watcher)); + + const result = await store.complete({ waitpointId: "w_a", completion: completion() }); + + expect(result.outcome).toBe("already"); + expect(result.completion).toBeUndefined(); + expect(result.watchers).toHaveLength(1); + expect(result.watchers[0]!.runId).toBe("run_1"); + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); + + redisTest("sets no TTL on the record or the watcher key", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW }); + await store.complete({ waitpointId: "w_a", completion: completion() }); + + // -1 means the key exists with no expiry. Anything >= 0 breaks the retention rule. + expect(await probe.pttl("wp:{w_a}")).toBe(-1); + expect(await probe.pttl("wp:{w_a}:w")).toBe(-1); + } finally { + probe.disconnect(); + await store.quit(); + } + }); +}); + +// A coordinator method now drives each of these scripts, but this block stays: it is the +// only place asserting the RAW reply shape, so a Lua/TypeScript framing change made on +// both sides at once would still fail here even though every class-level test passed. +describe("reply framing (direct Lua — pins the wire shape the coordinator decodes)", () => { + const envelope = JSON.stringify(completion()); + + redisTest( + "does not double-count a waitpoint reported pending then delivered in the same batch", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + const fieldA = edgeField("w_solo", 0); + const fieldB = edgeField("w_solo", 1); + + // Group 0 arrives unreported (still pending); group 1 for the SAME waitpoint + // arrives already reported. This is the straddle that broke pendingOfRequested. + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_solo", + fieldA, + "{}", + "0", + "", + "w_solo", + fieldB, + "{}", + "1", + envelope + ); + + expect(reply).toEqual(["0", "0", "w_solo", envelope]); + expect(await client.scard(keys.pend)).toBe(0); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "produces the identical result when the same two groups arrive in reverse order", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + const fieldA = edgeField("w_solo", 0); + const fieldB = edgeField("w_solo", 1); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_solo", + fieldB, + "{}", + "1", + envelope, + "w_solo", + fieldA, + "{}", + "0", + "" + ); + + expect(reply).toEqual(["0", "0", "w_solo", envelope]); + expect(await client.scard(keys.pend)).toBe(0); + } finally { + client.disconnect(); + } + } + ); + + redisTest("counts two distinct unreported ids as fully pending", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_a", + edgeField("w_a", 0), + "{}", + "0", + "", + "w_b", + edgeField("w_b", 0), + "{}", + "0", + "" + ); + + expect(reply).toEqual(["2", "2"]); + expect(await client.scard(keys.pend)).toBe(2); + } finally { + client.disconnect(); + } + }); + + redisTest( + "counts one reported and one unreported id as one pending, one delivered", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_a", + edgeField("w_a", 0), + "{}", + "0", + "", + "w_b", + edgeField("w_b", 0), + "{}", + "1", + envelope + ); + + expect(reply).toEqual(["1", "1", "w_b", envelope]); + expect(await client.scard(keys.pend)).toBe(1); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "reported flag '1' with an empty envelope still delivers, not pends", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + + // The bug this task fixed: COMPLETED-with-no-envelope must take the reported + // branch on the flag alone, not on the envelope being non-empty. + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "1", + "w_a", + edgeField("w_a", 0), + "{}", + "1", + "" + ); + + expect(reply).toEqual(["0", "0", "w_a", ""]); + expect(await client.scard(keys.pend)).toBe(0); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "counts the same unreported id passed twice as one pending, not two", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_a", + edgeField("w_a", 0), + "{}", + "0", + "", + "w_a", + edgeField("w_a", 1), + "{}", + "0", + "" + ); + + expect(reply).toEqual(["1", "1"]); + expect(await client.scard(keys.pend)).toBe(1); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "counts an id already in done, passed unreported, as delivered rather than pending", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + // A completion that landed between register and absorb — the delivered set + // already has this id before the absorb call ever sees it. + await client.hset(keys.done, "w_a", envelope); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "1", + "w_a", + edgeField("w_a", 0), + "{}", + "0", + "" + ); + + expect(reply).toEqual(["0", "0", "w_a", envelope]); + expect(await client.scard(keys.pend)).toBe(0); + } finally { + client.disconnect(); + } + } + ); + + redisTest("rejects an arity mismatch before writing anything", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + const field = edgeField("w_solo", 0); + + // n says 2 groups (1 + 2 * 5 = 11 ARGV entries expected) but only one group (5 + // ARGV entries) is supplied. + await expect( + client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_solo", + field, + "{}", + "0", + "" + ) + ).rejects.toThrow(); + + expect(await client.exists(keys.pend)).toBe(0); + expect(await client.exists(keys.done)).toBe(0); + expect(await client.exists(keys.edge)).toBe(0); + } finally { + client.disconnect(); + } + }); + + redisTest( + "runClear rejects an arity mismatch before writing anything", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + const field = edgeField("w_solo", 0); + await client.hset(keys.edge, field, "{}"); + await client.sadd(keys.pend, "w_solo"); + + // n says 2 fields but only one field is supplied. + await expect( + client.runClear(keys.pend, keys.done, keys.edge, "2", field) + ).rejects.toThrow(); + + expect(await client.hexists(keys.edge, field)).toBe(1); + expect(await client.sismember(keys.pend, "w_solo")).toBe(1); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "wpIdemReserve rejects a non-numeric expiry and does not create the reservation", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const key = idempotencyKey(ENV_ID, "key-1"); + + await expect(client.wpIdemReserve(key, "w_a", "not-a-number")).rejects.toThrow(); + + expect(await client.exists(key)).toBe(0); + } finally { + client.disconnect(); + } + } + ); +}); + +describe("createWithIdempotencyKey", () => { + // Real minted ids. The method rejects anything but a standalone DATETIME/MANUAL id, because + // its loser-discard is only safe for an id that was never handed out. + const idA = generateWaitpointId("MANUAL"); + const idB = generateWaitpointId("MANUAL"); + redisTest("creates the waitpoint and wins the reservation", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.createWithIdempotencyKey({ + record: record(idA, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + expect(result).toEqual({ waitpointId: idA, created: true }); + } finally { + await store.quit(); + } + }); + + redisTest("returns the winner's id and deletes the loser", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.createWithIdempotencyKey({ + record: record(idA, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + const second = await store.createWithIdempotencyKey({ + record: record(idB, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + expect(second).toEqual({ waitpointId: idA, created: false }); + // The loser cleans up after itself: nothing ever referenced its id. + expect(await probe.exists(`wp:{${idB}}`)).toBe(0); + expect(await probe.exists(`wp:{${idA}}`)).toBe(1); + } finally { + probe.disconnect(); + await store.quit(); + } + }); + + redisTest( + "the original creator's own retry does not discard its own record", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + const withKey = record(idA, { + idempotencyKey: "key-1", + userProvidedIdempotencyKey: true, + }); + + const first = await store.createWithIdempotencyKey({ + record: withKey, + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + expect(first).toEqual({ waitpointId: idA, created: true }); + + // The SAME caller, retrying with the SAME record id and the SAME key — not a + // different id racing for the same reservation. + const retry = await store.createWithIdempotencyKey({ + record: withKey, + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + expect(retry).toEqual({ waitpointId: idA, created: false }); + // The record must survive: a wrongly-discarded record would delete this too. + expect(await probe.exists(`wp:{${idA}}`)).toBe(1); + + // The real proof: something usable is still there for every later caller that + // blocks on this id. + const registered = await store.registerOrReport({ + waitpointId: idA, + runId: "run_1", + createdAt: NOW, + }); + expect(registered.outcome).toBe("registered"); + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); + + redisTest("sets no expiry when the record carries none", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.createWithIdempotencyKey({ + record: record(idA, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + // The common case. An expiry appearing here would be a retention rule violation. + expect(await probe.pttl(`wp:idem:{${ENV_ID}}:key-1`)).toBe(-1); + } finally { + probe.disconnect(); + await store.quit(); + } + }); + + redisTest("sets the expiry the record carries", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.createWithIdempotencyKey({ + record: record(idA, { + idempotencyKey: "key-1", + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + const ttl = await probe.pttl(`wp:idem:{${ENV_ID}}:key-1`); + // Wide band, deliberately: the deadline is computed from the test process's clock + // and applied as an absolute PEXPIREAT, while PTTL is computed against the Redis + // server's own clock. A few ms of disagreement between those two clocks is normal + // and shows up as overshoot on this read, not as a bug in the reservation. The + // band still catches every failure worth catching — wrong units, no expiry + // applied, a negative TTL — without re-asserting that two independent clocks + // agree to the millisecond. + expect(ttl).toBeGreaterThan(55_000); + expect(ttl).toBeLessThanOrEqual(65_000); + } finally { + probe.disconnect(); + await store.quit(); + } + }); + + redisTest("scopes reservations by environment", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createWithIdempotencyKey({ + record: record(idA, { idempotencyKey: "key-1" }), + environmentId: "env_1", + idempotencyKey: "key-1", + }); + + const other = await store.createWithIdempotencyKey({ + record: record(idB, { idempotencyKey: "key-1", environmentId: "env_2" }), + environmentId: "env_2", + idempotencyKey: "key-1", + }); + + expect(other).toEqual({ waitpointId: idB, created: true }); + } finally { + await store.quit(); + } + }); +}); + +redisTest( + "rejects a derived RUN id, whose loser-discard would be unsafe", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // A derived id is recomputable from its anchor, so another caller can register a + // watcher on it. Discarding one could delete a record already in use. + const derived = deriveWaitpointIdFromAnchor(`run_${generateRunOpsId()}`, "RUN")!; + await expect( + store.createWithIdempotencyKey({ + record: record(derived, { type: "RUN", idempotencyKey: "key-1" }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }) + ).rejects.toThrow(/freshly minted DATETIME or MANUAL/); + } finally { + await store.quit(); + } + } +); + +describe("the single-slot guard", () => { + redisTest("rejects an invocation whose keys span two tags", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // Reaches the same wrapper every operation goes through, so this proves the guard + // is live at the call path and not only in the pure unit test. + expect(() => + store.assertKeysForTest("wpComplete", ["wp:{w_a}", "wp:run:{run_1}:pend"]) + ).toThrow(WaitpointKeyTagError); + } finally { + await store.quit(); + } + }); +}); + +const RUN_ID = "run_1"; + +function edge(waitpointId: string, overrides: Partial = {}): BlockEdge { + return { waitpointId, createdAt: NOW, type: "MANUAL", ...overrides }; +} + +describe("absorbBlockers", () => { + redisTest("counts pending blockers and reports the store total", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a"), edge("w_b")], + }); + + expect(result.pendingOfRequested).toBe(2); + expect(result.storePendingTotal).toBe(2); + expect(result.alreadyDelivered).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest( + "counts a repeated waitpoint id once, matching a count over distinct rows", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { batchIndex: 0 }), edge("w_a", { batchIndex: 2 })], + }); + + // The count this replaces was a COUNT(*) over waitpoint rows, so two edges for + // one waitpoint contributed one. Both numbers must say 1, not 2. + expect(result.pendingOfRequested).toBe(1); + expect(result.storePendingTotal).toBe(1); + + const state = await store.readBlockState(RUN_ID); + expect(state.edges).toHaveLength(2); + expect(state.edges.map((e) => e.batchIndex).sort()).toEqual([0, 2]); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "does not add a reported-complete blocker to the pending set", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { reported: { completion: completion() } }), edge("w_b")], + }); + + expect(result.pendingOfRequested).toBe(1); + expect(result.storePendingTotal).toBe(1); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "a later absorb reads back the stored envelope, not a bare flag", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const envelope = completion({ output: { inline: '{"first":true}' } }); + + // Reported once, with an envelope — this write is what's under test. + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { reported: { completion: envelope } })], + }); + + // Same waitpoint id, arriving unreported this time: takes the "read `done` back" + // path, exposing whatever the first call actually stored under that id. + const second = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(second.alreadyDelivered).toHaveLength(1); + expect(second.alreadyDelivered[0]!.completion).toEqual(envelope); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "a later absorb for a no-envelope delivery reads back no completion", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { reported: {} })], + }); + + const second = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(second.alreadyDelivered).toHaveLength(1); + expect(second.alreadyDelivered[0]!.completion).toBeUndefined(); + } finally { + await store.quit(); + } + } + ); + + redisTest("reports a repeated already-delivered id once", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [ + edge("w_a", { batchIndex: 0, reported: { completion: completion() } }), + edge("w_a", { batchIndex: 1, reported: { completion: completion() } }), + ], + }); + + expect(result.alreadyDelivered).toHaveLength(1); + } finally { + await store.quit(); + } + }); + + redisTest("lets a delivery that raced ahead of the absorb win", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + const result = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(0); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + } finally { + await store.quit(); + } + }); + + redisTest("is idempotent when run twice", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const first = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + const second = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(first.storePendingTotal).toBe(1); + expect(second.storePendingTotal).toBe(1); + expect((await store.readBlockState(RUN_ID)).edges).toHaveLength(1); + } finally { + await store.quit(); + } + }); + + redisTest("keeps the first edge's metadata on a retry", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { spanIdToComplete: "span_first" })], + }); + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { spanIdToComplete: "span_second" })], + }); + + expect((await store.readBlockState(RUN_ID)).edges[0]!.spanIdToComplete).toBe("span_first"); + } finally { + await store.quit(); + } + }); + + redisTest("reports the run's real total for an empty edge list", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + const result = await store.absorbBlockers({ runId: RUN_ID, edges: [] }); + + // pendingOfRequested is 0 because nothing was requested. storePendingTotal is the + // run's whole store-resident set, which is NOT empty. + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(1); + expect(result.alreadyDelivered).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest( + "reports a smaller pendingOfRequested than storePendingTotal when an unrelated blocker is already pending", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // w_x is a live blocker from an earlier absorb, unrelated to this call's request. + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_x")] }); + + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { reported: { completion: completion() } })], + }); + + // Nothing THIS call requested is pending (w_a arrived already delivered), but the + // run's whole store-resident set still holds w_x — a divergence for a different + // reason than an empty request list, so a reply[0]/reply[1] swap or a + // re-derived-in-TypeScript pendingOfRequested would both be caught here too. + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(1); + } finally { + await store.quit(); + } + } + ); + + redisTest("sets no TTL on any run key", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + // -1 is "exists, no expiry"; -2 is "no key". Neither is a TTL. `pend` is emptied by + // the delivery, and Redis deletes an empty set, so -2 is expected there. + for (const key of [ + `wp:run:{${RUN_ID}}:pend`, + `wp:run:{${RUN_ID}}:done`, + `wp:run:{${RUN_ID}}:edge`, + ]) { + expect(await probe.pttl(key)).toBeLessThan(0); + } + } finally { + probe.disconnect(); + await store.quit(); + } + }); +}); + +describe("deliverCompletion", () => { + redisTest("removes the blocker and returns the new store total", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] }); + + expect( + ( + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }) + ).storePendingTotal + ).toBe(1); + + expect( + ( + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_b", + completion: completion(), + }) + ).storePendingTotal + ).toBe(0); + } finally { + await store.quit(); + } + }); + + redisTest("is idempotent", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + const again = await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + expect(again.storePendingTotal).toBe(0); + } finally { + await store.quit(); + } + }); +}); + +describe("readBlockState", () => { + redisTest( + "returns the pending ids, the delivered ids and the edges", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ + runId: RUN_ID, + edges: [ + edge("w_a", { batchIndex: 0, completedAfter: NOW, type: "DATETIME" }), + edge("w_b"), + ], + }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + const state = await store.readBlockState(RUN_ID); + + expect(state.pendingIds).toEqual(["w_b"]); + expect(state.deliveredIds).toEqual(["w_a"]); + expect(state.edges).toHaveLength(2); + + const datetime = state.edges.find((e) => e.waitpointId === "w_a"); + // type and completedAfter must ride the edge: a frozen return type needs them, and + // they live on the waitpoint's own shard, which this read cannot touch. + expect(datetime?.type).toBe("DATETIME"); + expect(datetime?.completedAfter).toBe(NOW); + expect(datetime?.edgeId).toBe("w_a#0"); + } finally { + await store.quit(); + } + } + ); + + redisTest("returns empty collections for a run with no blockers", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + expect(await store.readBlockState("run_unknown")).toEqual({ + pendingIds: [], + deliveredIds: [], + edges: [], + }); + } finally { + await store.quit(); + } + }); +}); + +describe("clearBlockState", () => { + redisTest("drains the named edges and reconciles", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + expect((await store.clearBlockState({ runId: RUN_ID, edgeIds: ["w_a#"] })).outcome).toBe( + "drained" + ); + + const state = await store.readBlockState(RUN_ID); + expect(state.edges.map((e) => e.waitpointId)).toEqual(["w_b"]); + expect(state.deliveredIds).toEqual([]); + expect(state.pendingIds).toEqual(["w_b"]); + } finally { + await store.quit(); + } + }); + + redisTest( + "keeps a waitpoint's delivery while another edge for it survives", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { batchIndex: 0 }), edge("w_a", { batchIndex: 1 })], + }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + await store.clearBlockState({ runId: RUN_ID, edgeIds: ["w_a#0"] }); + + // One edge remains, so the delivery must remain too — dropping it would make the + // surviving edge look undelivered. + const state = await store.readBlockState(RUN_ID); + expect(state.edges.map((e) => e.edgeId)).toEqual(["w_a#1"]); + expect(state.deliveredIds).toEqual(["w_a"]); + } finally { + await store.quit(); + } + } + ); + + redisTest("reaps a delivered entry that no edge references", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // The register-before-absorb window: a delivery can land for a waitpoint whose edge + // was never written. A name-derived drain could never reach it. + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_kept")] }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_orphan", + completion: completion(), + }); + + expect((await store.readBlockState(RUN_ID)).deliveredIds).toEqual(["w_orphan"]); + + await store.clearBlockState({ runId: RUN_ID, edgeIds: ["w_nothing#"] }); + + const state = await store.readBlockState(RUN_ID); + expect(state.deliveredIds).toEqual([]); + expect(state.edges.map((e) => e.waitpointId)).toEqual(["w_kept"]); + } finally { + await store.quit(); + } + }); + + redisTest("clears everything when no edge ids are given", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] }); + + expect((await store.clearBlockState({ runId: RUN_ID })).outcome).toBe("cleared"); + expect(await store.readBlockState(RUN_ID)).toEqual({ + pendingIds: [], + deliveredIds: [], + edges: [], + }); + } finally { + await store.quit(); + } + }); + + redisTest( + "is a no-op for an explicitly empty edge id list, unlike an omitted one", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] }); + + // Omitting edgeIds reaches the Lua's n === 0 branch and clears everything (proven + // above). A caller-computed EMPTY array must not collapse onto that: it means + // "nothing to drain", not "clear the run". + expect((await store.clearBlockState({ runId: RUN_ID, edgeIds: [] })).outcome).toBe("noop"); + + const state = await store.readBlockState(RUN_ID); + expect(state.edges.map((e) => e.waitpointId).sort()).toEqual(["w_a", "w_b"]); + expect(state.pendingIds.sort()).toEqual(["w_a", "w_b"]); + } finally { + await store.quit(); + } + } + ); +}); + +describe("registerBlocks: a COMPLETED waitpoint with no envelope never blocks (regression)", () => { + redisTest( + "created COMPLETED with no envelope: registerBlocks does not block the run", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // No `completion` at all — the FINISHED-healing shape from Task 4's "can create an + // already-COMPLETED record with no completion envelope" test. + await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" }); + + const result = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(0); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + // The whole point: no fabricated envelope, and the delivery is real on the run + // shard, not just absent from pending. + expect(result.alreadyDelivered[0]!.completion).toBeUndefined(); + expect((await store.readBlockState(RUN_ID)).deliveredIds).toEqual(["w_a"]); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "created COMPLETED with an envelope: behaves identically with respect to blocking", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ + record: record("w_a"), + status: "COMPLETED", + completion: completion(), + }); + + const result = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(0); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + } finally { + await store.quit(); + } + } + ); +}); + +describe("registerBlocks: the two orderings", () => { + redisTest("block first, then complete: the run blocks, then wakes", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + const blocked = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + expect(blocked.pendingOfRequested).toBe(1); + expect(blocked.storePendingTotal).toBe(1); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(completed.watchers.map((w) => w.runId)).toEqual([RUN_ID]); + + const delivered = await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completed.completion!, + }); + expect(delivered.storePendingTotal).toBe(0); + } finally { + await store.quit(); + } + }); + + redisTest("complete first, then block: the run never goes pending", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.complete({ waitpointId: "w_a", completion: completion() }); + + const result = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(0); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + + const state = await store.readBlockState(RUN_ID); + expect(state.pendingIds).toEqual([]); + expect(state.deliveredIds).toEqual(["w_a"]); + } finally { + await store.quit(); + } + }); + + redisTest("throws when a blocking waitpoint does not exist", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await expect( + store.registerBlocks({ runId: RUN_ID, edges: [edge("w_missing")] }) + ).rejects.toThrow(WaitpointNotFoundError); + } finally { + await store.quit(); + } + }); + + redisTest( + "a throw mid-loop leaves the earlier watcher registered, and that residue is safe", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_ok"), status: "PENDING" }); + + await expect( + store.registerBlocks({ runId: RUN_ID, edges: [edge("w_ok"), edge("w_missing")] }) + ).rejects.toThrow(WaitpointNotFoundError); + + // registerBlocks throws before absorbBlockers ever runs, so the run's own shard + // is untouched. + const state = await store.readBlockState(RUN_ID); + expect(state.pendingIds).toEqual([]); + expect(state.edges).toEqual([]); + + // But w_ok's watcher WAS registered on w_ok's own shard before the throw. + const completed = await store.complete({ waitpointId: "w_ok", completion: completion() }); + expect(completed.watchers.map((w) => w.runId)).toEqual([RUN_ID]); + + // Delivering it writes a `done` entry for a run that was never blocked on it — + // inert residue, not a false resume: no edge ever named it, and clearBlockState's + // reconcile would drop it the moment this run's block state is next drained. + const delivered = await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_ok", + completion: completed.completion!, + }); + expect(delivered.storePendingTotal).toBe(0); + expect((await store.readBlockState(RUN_ID)).deliveredIds).toEqual(["w_ok"]); + } finally { + await store.quit(); + } + } + ); + + redisTest("is idempotent when run twice", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + const first = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + const second = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(first.storePendingTotal).toBe(1); + expect(second.storePendingTotal).toBe(1); + expect((await store.readBlockState(RUN_ID)).edges).toHaveLength(1); + } finally { + await store.quit(); + } + }); + + redisTest( + "mixed set: one pending and one already complete blocks the run once", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_pending"), status: "PENDING" }); + await store.createIfAbsent({ record: record("w_done"), status: "PENDING" }); + await store.complete({ waitpointId: "w_done", completion: completion() }); + + const result = await store.registerBlocks({ + runId: RUN_ID, + edges: [edge("w_pending"), edge("w_done")], + }); + + expect(result.pendingOfRequested).toBe(1); + expect(result.storePendingTotal).toBe(1); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_done"]); + } finally { + await store.quit(); + } + } + ); +}); + +describe("multi-index merge, end to end into the executor shape", () => { + redisTest( + "a run blocked on one waitpoint at two indexes resolves to two entries", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ + record: record("w_child", { type: "RUN", completedByTaskRunId: "run_child" }), + status: "PENDING", + }); + + await store.registerBlocks({ + runId: RUN_ID, + edges: [ + edge("w_child", { batchIndex: 0, batchId: "batch_1", type: "RUN" }), + edge("w_child", { batchIndex: 2, batchId: "batch_1", type: "RUN" }), + ], + }); + + const completed = await store.complete({ + waitpointId: "w_child", + completion: completion({ output: null }), + }); + // The cross-shard fact this test claims to prove: two registers for the same + // waitpoint at different indexes fanned out into two distinct watcher entries. + expect( + completed.watchers.map((w) => w.batchIndex).sort((a, b) => (a ?? 0) - (b ?? 0)) + ).toEqual([0, 2]); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_child", + completion: completed.completion!, + }); + + const state = await store.readBlockState(RUN_ID); + expect(state.pendingIds).toEqual([]); + expect(state.deliveredIds).toEqual(["w_child"]); + + // Derive the cycle's ordered id list the way the read path does: keep only edges + // that carry a batch index, sort ascending, map to id. Derived inline on purpose — + // another lane owns the order rule and its resolver, and this test's job is to + // prove the COORDINATOR preserved the edge multiplicity across two shards, not to + // own that rule. + const order = state.edges + .filter((e) => e.batchIndex !== undefined && e.batchIndex !== null) + .sort((a, b) => a.batchIndex! - b.batchIndex!) + .map((e) => e.waitpointId); + + // One waitpoint, two edges, so the id repeats — that repeat is what expands into + // two entries for the executor, and losing it would silently drop a batch item. + expect(order).toEqual(["w_child", "w_child"]); + expect(state.edges.map((e) => e.edgeId).sort()).toEqual(["w_child#0", "w_child#2"]); + expect(state.edges.every((e) => e.batchId === "batch_1")).toBe(true); + } finally { + await store.quit(); + } + } + ); +}); + +describe("the resume cycle drains and can start again", () => { + redisTest("a second wait on the same waitpoint blocks nothing", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completed.completion!, + }); + + const first = await store.readBlockState(RUN_ID); + await store.clearBlockState({ runId: RUN_ID, edgeIds: first.edges.map((e) => e.edgeId) }); + + // Cycle two. The waitpoint is COMPLETED for good, so the register reports it and the + // run is never blocked. + const second = await store.registerBlocks({ + runId: RUN_ID, + edges: [edge("w_a", { batchIndex: 5 })], + }); + + expect(second.storePendingTotal).toBe(0); + expect(second.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + expect((await store.readBlockState(RUN_ID)).edges.map((e) => e.edgeId)).toEqual(["w_a#5"]); + } finally { + await store.quit(); + } + }); +}); + +// Every test above is a sequence of awaits. Redis guarantees atomicity WITHIN a script, so +// those tests can only ever prove single-script invariants. These races drive real +// concurrent calls (Promise.all over N copies) against the multi-script TypeScript +// sequences, and assert an invariant that holds regardless of who wins — never a timing. +describe("genuine concurrency", () => { + const CONCURRENCY = 8; + + redisTest( + "exactly one of N concurrent completers wins, and every caller sees its completion", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_2", createdAt: NOW }); + + const results = await Promise.all( + Array.from({ length: CONCURRENCY }, (_, i) => + store.complete({ + waitpointId: "w_a", + completion: completion({ output: { inline: `{"racer":${i}}` } }), + }) + ) + ); + + const winners = results.filter((r) => r.outcome === "completed"); + const losers = results.filter((r) => r.outcome === "already"); + expect(winners).toHaveLength(1); + expect(losers).toHaveLength(CONCURRENCY - 1); + + // Every caller, winner and losers alike, reads back the SAME stored completion. + const stored = winners[0]!.completion; + for (const r of results) { + expect(r.completion).toEqual(stored); + } + + // And every caller returns the full watcher list — a race must never truncate it. + for (const r of results) { + expect(r.watchers.map((w) => w.runId).sort()).toEqual(["run_1", "run_2"]); + } + } finally { + await store.quit(); + } + } + ); + + redisTest( + "a pre-existing registration survives N concurrent attempts to re-register its field", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: "span_first", + createdAt: NOW, + }); + + // Same run, same (absent) batch index as the registration above, so every one of + // these collides on the exact same watcher field. + await Promise.all( + Array.from({ length: CONCURRENCY }, (_, i) => + store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: `span_racer_${i}`, + createdAt: NOW, + }) + ) + ); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + const forRun1 = completed.watchers.filter((w) => w.runId === "run_1"); + expect(forRun1).toHaveLength(1); + expect(forRun1[0]!.spanIdToComplete).toBe("span_first"); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "exactly one of N concurrent idempotency-keyed creators wins, and every loser cleans up", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + const ids = Array.from({ length: CONCURRENCY }, () => generateWaitpointId("MANUAL")); + + const results = await Promise.all( + ids.map((id) => + store.createWithIdempotencyKey({ + record: record(id, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }) + ) + ); + + const winners = results.filter((r) => r.created); + expect(winners).toHaveLength(1); + + const winnerId = winners[0]!.waitpointId; + for (const r of results) { + expect(r.waitpointId).toBe(winnerId); + } + expect(await probe.exists(`wp:{${winnerId}}`)).toBe(1); + + for (const id of ids) { + if (id === winnerId) continue; + expect(await probe.exists(`wp:{${id}}`)).toBe(0); + expect(await probe.exists(`wp:{${id}}:w`)).toBe(0); + } + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); + + redisTest( + "registerBlocks racing complete never leaves a waitpoint double-booked or the pending count negative", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + for (let i = 0; i < 30; i++) { + const waitpointId = `w_race_${i}`; + const runId = `run_race_${i}`; + await store.createIfAbsent({ record: record(waitpointId), status: "PENDING" }); + + // Two edges for the SAME waitpoint: registerBlocks registers them one at a + // time, so a concurrent complete() has a real window to land between the two + // registrations — the exact straddle that makes absorbBlockers' per-group + // reported/unreported split matter, rather than racing a single all-or-nothing + // group. + const [blocked] = await Promise.all([ + store.registerBlocks({ + runId, + edges: [edge(waitpointId, { batchIndex: 0 }), edge(waitpointId, { batchIndex: 1 })], + }), + store.complete({ waitpointId, completion: completion() }), + ]); + + const state = await store.readBlockState(runId); + const delivered = state.deliveredIds.includes(waitpointId); + const pending = state.pendingIds.includes(waitpointId); + + expect(delivered && pending).toBe(false); + expect(blocked.storePendingTotal).toBeGreaterThanOrEqual(0); + expect(blocked.storePendingTotal).toBeLessThanOrEqual(1); + } + } finally { + await store.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts new file mode 100644 index 00000000000..723552c57ab --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts @@ -0,0 +1,538 @@ +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { Logger } from "@trigger.dev/core/logger"; +import { parseWaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { + assertSingleSlot, + edgeField, + idempotencyKey, + runBlockKeys, + waitpointKeys, + watcherField, +} from "./keys.js"; +import { registerWaitpointCommands } from "./scripts.js"; + +/** The values written into a record's `status` field. Uppercase, and never a token. */ +export type WaitpointStatus = "PENDING" | "COMPLETED"; + +/** Every script this coordinator may invoke. The wrapper below is the only entry point. */ +type ScriptName = + | "wpCreateIfAbsent" + | "wpRegisterOrReport" + | "wpComplete" + | "wpIdemReserve" + | "wpDiscard" + | "runAbsorbBlockers" + | "runDeliverCompletion" + | "runReadBlockState" + | "runClear"; + +/** + * The immutable half of a waitpoint, written once at creation. Carries every field the + * legacy-shaped return types need, including the two that gate the executor-visible + * idempotency key and the token surface. + */ +export type WaitpointRecordInput = { + id: string; + friendlyId: string; + type: "RUN" | "BATCH" | "DATETIME" | "MANUAL"; + environmentId: string; + projectId: string; + createdAt: string; + updatedAt: string; + userProvidedIdempotencyKey: boolean; + tags: string[]; + idempotencyKey?: string; + idempotencyKeyExpiresAt?: string; + completedAfter?: string; + completedByTaskRunId?: string; + completedByBatchId?: string; +}; + +/** + * A stored output: a small inline value, an already-offloaded reference, or null when the + * value is re-derivable from a business fact and is therefore never copied forward. + */ +export type WaitpointCompletionOutput = { inline: string } | { ref: string } | null; + +/** + * The completion half of a waitpoint, written at the flip. + * + * This is the coordinator's OWN type, deliberately not a projection of any frozen record + * type. The store treats a completion as an opaque blob: it writes it, returns it, and + * never inspects a field. Whoever owns the read-time resolver maps between this and the + * frozen record shape, so the two can evolve without a type dependency in either + * direction. + */ +export type WaitpointCompletion = { + /** ISO 8601. */ + completedAt: string; + outputType: string; + outputIsError: boolean; + output: WaitpointCompletionOutput; +}; + +export type WatcherEntry = { + runId: string; + batchIndex?: number; + spanIdToComplete?: string; + createdAt: string; +}; + +export type CreateIfAbsentResult = + | { outcome: "created" } + | { + outcome: "exists"; + record: WaitpointRecordInput; + status: WaitpointStatus; + completion?: WaitpointCompletion; + }; + +export type RegisterOrReportResult = + | { outcome: "registered" } + | { outcome: "completed"; completion?: WaitpointCompletion }; + +export type CompleteResult = { + outcome: "completed" | "already"; + completion?: WaitpointCompletion; + watchers: WatcherEntry[]; +}; + +/** + * One run-to-waitpoint edge. The metadata a frozen return type — an existing API response + * shape this store must keep reproducing — needs travels here. + */ +export type BlockEdge = { + waitpointId: string; + batchIndex?: number | null; + batchId?: string; + spanIdToComplete?: string; + createdAt: string; + type: WaitpointRecordInput["type"]; + completedAfter?: string; + // Set when the register step already reported this waitpoint COMPLETED. The box, not + // `completion`, carries the "reported" fact: box present + no completion means + // COMPLETED-with-no-envelope, box absent means never reported. + reported?: { completion?: WaitpointCompletion }; +}; + +export type AbsorbResult = { + /** + * How many DISTINCT requested ids were still pending. Equivalent to the count the + * previous path took over this call's ids, which was a COUNT over waitpoint rows — so + * two edges for one waitpoint contribute one. This is the number a caller should use to + * keep today's block-time gate unchanged. + */ + pendingOfRequested: number; + /** + * The run's whole pending set, counting STORE-RESIDENT blockers only. A run can also be + * blocked by a legacy waitpoint, which this number cannot see, so it is never on its own + * a decision to resume. + */ + storePendingTotal: number; + alreadyDelivered: Array<{ waitpointId: string; completion?: WaitpointCompletion }>; +}; + +// absorbBlockers strips `reported` before writing the edge blob, so a value read back +// here can never carry it — Omit says so instead of inheriting a field that is always +// undefined. +export type BlockStateEdge = Omit & { edgeId: string }; + +export type BlockState = { + pendingIds: string[]; + deliveredIds: string[]; + edges: BlockStateEdge[]; +}; + +export class WaitpointNotFoundError extends Error { + constructor(waitpointId: string) { + super(`Waitpoint ${waitpointId} is not present in the store`); + this.name = "WaitpointNotFoundError"; + } +} + +export type WaitpointStoreCoordinatorOptions = { + redisOptions: RedisOptions; + logger?: Logger; +}; + +// Lua returns '' for an absent value, never nil, because every reply slot is coerced to +// keep the array from truncating. So a nullish check would not fire and JSON.parse('') +// throws. One helper, used at every decode site. +function parseJson(raw: string | undefined): T | undefined { + return raw ? (JSON.parse(raw) as T) : undefined; +} + +export class WaitpointStoreCoordinator { + private readonly redis: Redis; + private readonly logger: Logger; + #quit?: Promise; + + constructor(options: WaitpointStoreCoordinatorOptions) { + this.logger = options.logger ?? new Logger("WaitpointStoreCoordinator", "debug"); + this.redis = createRedisClient(options.redisOptions, { + onError: (error) => + this.logger.error("WaitpointStoreCoordinator redis client error", { error }), + }); + registerWaitpointCommands(this.redis); + } + + // Idempotent and error-swallowing: every test calls this in a finally, and a double quit + // must never mask the real assertion failure. + async quit(): Promise { + if (!this.#quit) { + this.#quit = this.redis.quit().then( + () => undefined, + () => undefined + ); + } + await this.#quit; + } + + /** + * The ONLY way this class invokes a script. Routing every call through one place is what + * makes the single-slot guard un-forgettable: a method added later cannot reach a script + * without passing its keys through this assertion. + * + * Every script's signature is (...keys, ...argv) => string[], so one cast covers them + * all. The typed RedisCommander augmentation in scripts.ts documents each shape. + */ + #call(script: ScriptName, keys: string[], ...argv: string[]): Promise { + assertSingleSlot(script, keys); + const command = this.redis[script] as (...args: string[]) => Promise; + return command.call(this.redis, ...keys, ...argv); + } + + /** + * Exposed for the guard's own test. Delegates through #call rather than calling + * assertSingleSlot directly, so a mutation to the guard inside #call fails this test too + * — not only the tests that happen to exercise a real script. + * + * With cross-tag (invalid) keys, assertSingleSlot throws synchronously inside #call, + * before any promise exists, and that throw propagates straight out of this method. With + * same-tag (valid) keys, #call would go on to dispatch a real script call; this method + * never returns or awaits that promise, and swallows whatever it eventually settles to, + * so a valid-key call here can never surface as an unhandled rejection in the caller. + */ + assertKeysForTest(operation: string, keys: string[]): void { + this.#call(operation as ScriptName, keys).catch(() => undefined); + } + + async createIfAbsent(args: { + record: WaitpointRecordInput; + status: WaitpointStatus; + completion?: WaitpointCompletion; + }): Promise { + const keys = waitpointKeys(args.record.id); + + const reply = await this.#call( + "wpCreateIfAbsent", + [keys.record], + JSON.stringify(args.record), + args.status, + args.completion ? JSON.stringify(args.completion) : "" + ); + + if (reply[0] === "created") { + return { outcome: "created" }; + } + + // reply[1] is '' only if the record hash exists with no 'r' field, which should never + // happen — but ?? never fires on '', so a bare JSON.parse('') would throw an + // undiagnosable SyntaxError instead of naming the waitpoint. + const record = parseJson(reply[1]); + if (!record) { + throw new Error(`Waitpoint ${args.record.id} exists in the store with no record blob`); + } + + return { + outcome: "exists", + record, + status: reply[2] === "COMPLETED" ? "COMPLETED" : "PENDING", + completion: parseJson(reply[3]), + }; + } + + async registerOrReport(args: { + waitpointId: string; + runId: string; + batchIndex?: number | null; + spanIdToComplete?: string; + createdAt: string; + }): Promise { + const keys = waitpointKeys(args.waitpointId); + + // batchIndex is nullable at the boundary (matching the column) and undefined inside, + // because JSON.stringify drops an undefined field but keeps a null one. + const watcher: WatcherEntry = { + runId: args.runId, + batchIndex: args.batchIndex ?? undefined, + spanIdToComplete: args.spanIdToComplete, + createdAt: args.createdAt, + }; + + const reply = await this.#call( + "wpRegisterOrReport", + [keys.record, keys.watchers], + watcherField(args.runId, args.batchIndex), + JSON.stringify(watcher) + ); + + if (reply[0] === "missing") { + throw new WaitpointNotFoundError(args.waitpointId); + } + if (reply[0] === "completed") { + return { outcome: "completed", completion: parseJson(reply[1]) }; + } + + return { outcome: "registered" }; + } + + async complete(args: { + waitpointId: string; + completion: WaitpointCompletion; + }): Promise { + const keys = waitpointKeys(args.waitpointId); + + const reply = await this.#call( + "wpComplete", + [keys.record, keys.watchers], + JSON.stringify(args.completion) + ); + + if (reply[0] === "missing") { + throw new WaitpointNotFoundError(args.waitpointId); + } + + return { + outcome: reply[0] as "completed" | "already", + completion: parseJson(reply[1]), + watchers: reply.slice(2).map((entry) => JSON.parse(entry) as WatcherEntry), + }; + } + + /** + * Create a waitpoint under an idempotency key. + * + * The reservation and the record sit under different hash tags, so no script spans + * them. That makes the ORDER load-bearing: create first, then reserve. + * + * Reserve-first would mean a crash between the two steps leaves a reservation naming a + * waitpoint that does not exist. Every later request with that key loses the + * reservation, blocks on the winner's id, and throws when it registers — correctly, but + * forever, because an idempotency key commonly carries no expiry to clear it. + * + * Create-first inverts the failure: a crash leaves an orphan record that nothing ever + * referenced, because its id is random and unpublished. No caller hangs, but nothing + * currently reclaims that record either: the backstop collector the wider plan + * describes is keyed off a run's status, and this orphan has no owning run, so that + * collector never sees it. The record is harmless — inert, unreferenced, never + * returned to anyone — but it is a real leak until a later ticket adds a reaper for + * standalone idempotency-keyed orphans specifically. + */ + async createWithIdempotencyKey(args: { + record: WaitpointRecordInput; + environmentId: string; + idempotencyKey: string; + // `created` means THIS CALL won the reservation, not that the id is new. A retry by the + // original creator reports false, because the reservation it is losing to is its own. A + // caller must not gate one-time side effects on it without handling that. + }): Promise<{ waitpointId: string; created: boolean }> { + // Standalone ids only. The discard below deletes this call's own record, and that is + // only safe because a freshly minted id was never handed out, so nothing can reference + // it. A RUN or BATCH id is DERIVED from its anchor, so any caller can recompute it and + // register a watcher on it — discarding one could delete a record already in use. + const parsed = parseWaitpointId(args.record.id); + if (parsed.format !== "b32hexW" || (parsed.type !== "DATETIME" && parsed.type !== "MANUAL")) { + throw new Error( + `createWithIdempotencyKey requires a freshly minted DATETIME or MANUAL id, got ${args.record.id}` + ); + } + + await this.createIfAbsent({ record: args.record, status: "PENDING" }); + + const expiresAtMs = args.record.idempotencyKeyExpiresAt + ? String(new Date(args.record.idempotencyKeyExpiresAt).getTime()) + : ""; + + const reply = await this.#call( + "wpIdemReserve", + [idempotencyKey(args.environmentId, args.idempotencyKey)], + args.record.id, + expiresAtMs + ); + + if (reply[0] === "reserved") { + return { waitpointId: args.record.id, created: true }; + } + + const winner = reply[1]; + if (winner !== args.record.id) { + // Safe to discard: this id is random and was never handed to any caller, so no + // watcher can reference it. Both keys share the record's tag. + const keys = waitpointKeys(args.record.id); + await this.#call("wpDiscard", [keys.record, keys.watchers]); + } + + return { waitpointId: winner, created: false }; + } + + async absorbBlockers(args: { runId: string; edges: BlockEdge[] }): Promise { + const keys = runBlockKeys(args.runId); + + // No fast path for an empty list: storePendingTotal is defined as the run's WHOLE + // store-resident pending set, so it has to be read even when nothing is requested. + const argv: string[] = [String(args.edges.length)]; + for (const item of args.edges) { + const { reported, ...stored } = item; + const reportedFlag = reported !== undefined ? "1" : "0"; + const reportedJson = reported?.completion ? JSON.stringify(reported.completion) : ""; + argv.push( + item.waitpointId, + edgeField(item.waitpointId, item.batchIndex), + JSON.stringify(stored), + reportedFlag, + reportedJson + ); + } + + const reply = await this.#call("runAbsorbBlockers", [keys.pend, keys.done, keys.edge], ...argv); + + const alreadyDelivered: AbsorbResult["alreadyDelivered"] = []; + for (let i = 2; i < reply.length; i += 2) { + alreadyDelivered.push({ + waitpointId: reply[i]!, + completion: parseJson(reply[i + 1]), + }); + } + + return { + pendingOfRequested: Number(reply[0]), + storePendingTotal: Number(reply[1]), + alreadyDelivered, + }; + } + + /** + * Block a run on a set of waitpoints. + * + * Register on every waitpoint's own shard FIRST, then absorb on the run's shard. The + * order is the protocol: a completion that lands in between finds the watcher already + * registered, so it delivers onto the run's shard, and the absorb sees that delivery and + * never marks the waitpoint pending. + * + * The register keys the decision to skip the pending set on OUTCOME, never on whether a + * completion envelope came back — a waitpoint can be reported COMPLETED with none. + * + * A throw partway through (a missing waitpoint) intentionally leaves any + * already-registered watchers in place rather than unwinding them. That's safe: a later + * `complete` on one of those waitpoints still delivers correctly, and if it lands before + * this run ever retries `registerBlocks`, the stray `done` entry it writes is inert until + * a future absorb or `clearBlockState`'s reconcile reads it — never a false resume. + */ + async registerBlocks(args: { runId: string; edges: BlockEdge[] }): Promise { + const registered: BlockEdge[] = []; + + for (const item of args.edges) { + const result = await this.registerOrReport({ + waitpointId: item.waitpointId, + runId: args.runId, + batchIndex: item.batchIndex, + spanIdToComplete: item.spanIdToComplete, + createdAt: item.createdAt, + }); + + registered.push( + result.outcome === "completed" + ? { ...item, reported: { completion: result.completion } } + : item + ); + } + + return this.absorbBlockers({ runId: args.runId, edges: registered }); + } + + async deliverCompletion(args: { + runId: string; + waitpointId: string; + completion: WaitpointCompletion; + }): Promise<{ storePendingTotal: number }> { + const keys = runBlockKeys(args.runId); + + const reply = await this.#call( + "runDeliverCompletion", + [keys.pend, keys.done], + args.waitpointId, + JSON.stringify(args.completion) + ); + + return { storePendingTotal: Number(reply[0]) }; + } + + async readBlockState(runId: string): Promise { + const keys = runBlockKeys(runId); + const reply = await this.#call("runReadBlockState", [keys.pend, keys.done, keys.edge]); + + // Slots 0 and 1 are true element counts, but slot 2 is the FLAT length of the edge + // HGETALL — two entries per edge, field then value. The cursor arithmetic below relies + // on that asymmetry, so do not "normalise" it without changing the Lua too. + const pendCount = Number(reply[0]); + const doneCount = Number(reply[1]); + const edgeCount = Number(reply[2]); + + let cursor = 3; + const pendingIds = reply.slice(cursor, cursor + pendCount); + cursor += pendCount; + const deliveredIds = reply.slice(cursor, cursor + doneCount); + cursor += doneCount; + + const edges: BlockStateEdge[] = []; + for (let i = 0; i < edgeCount; i += 2) { + const edgeId = reply[cursor + i]!; + // An edge value is always a non-empty JSON.stringify, so a missing slot here means + // the cursor walked off the end of the reply. That must fail loudly, not decode a + // BlockEdge with no waitpointId — the exact off-by-one this task's arithmetic guards + // against. + const edgeJson = reply[cursor + i + 1]; + if (!edgeJson) { + throw new Error( + `readBlockState(${runId}): missing edge payload at reply index ${cursor + i + 1}` + ); + } + const stored = JSON.parse(edgeJson) as BlockEdge; + edges.push({ ...stored, edgeId }); + } + + return { pendingIds, deliveredIds, edges }; + } + + /** + * Drain one cycle's edges, or clear the run entirely when no edge ids are given. + * + * The selective form RECONCILES: any pending or delivered entry that no surviving edge + * references goes too, not only the named ones. See runClear in scripts.ts for why. + */ + async clearBlockState(args: { + runId: string; + edgeIds?: string[]; + }): Promise<{ outcome: "cleared" | "drained" | "noop" }> { + // `omitted` and `explicitly empty` must not collapse onto each other: the Lua's + // n === 0 means "clear the whole run", so an omitted edgeIds stays the terminal clear, + // but a caller that computed zero edges to drain gets a genuine no-op that never + // reaches Redis. + if (args.edgeIds && args.edgeIds.length === 0) { + return { outcome: "noop" }; + } + + const keys = runBlockKeys(args.runId); + const edgeIds = args.edgeIds ?? []; + + const reply = await this.#call( + "runClear", + [keys.pend, keys.done, keys.edge], + String(edgeIds.length), + ...edgeIds + ); + + return { outcome: reply[0] as "cleared" | "drained" }; + } +} diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts new file mode 100644 index 00000000000..8a50abb7d1c --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts @@ -0,0 +1,145 @@ +import type { ReadClient } from "@internal/run-store"; +import type { PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database"; + +/** + * The waitpoint and edge state operations that `WaitpointSystem` delegates. + * + * Orchestration stays in `WaitpointSystem`: the run lock, snapshot transitions, + * worker-job enqueues, event emissions and racepoints. This owns waitpoint and + * edge state only, so a non-Postgres implementation can replace it without any + * caller learning that it changed. + * + * The residency hints and `tx` are opaque pass-throughs. Opaque does not mean + * type-free — a Prisma type appears here — it means a non-Postgres implementation + * never reads the value. + */ +export type WaitpointCoordinator = { + clearRunBlockState(params: ClearRunBlockStateParams): Promise<{ count: number }>; + readRunBlockState(runId: string): Promise; + registerBlocks(params: RegisterBlocksParams): Promise<{ pendingCount: number }>; + registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise; + complete(params: CompleteParams): Promise; + createDateTimeWaitpoint(params: CreateDateTimeWaitpointParams): Promise; + createManualWaitpoint(params: CreateManualWaitpointParams): Promise; + mintAssociatedWaitpointData(params: { + projectId: string; + environmentId: string; + }): AssociatedWaitpointData; + createAssociatedWaitpoint(params: { + runId: string; + data: AssociatedWaitpointData; + }): Promise; +}; + +export type ClearRunBlockStateParams = { + runId: string; + /** Edge ids to delete. Omit to clear every edge for the run. */ + edgeIds?: string[]; + /** + * Forwarded verbatim on the full-clear leg only, and never on the bounded leg + * or an edge write. A routing store strips it; a single store joins it. + */ + tx?: PrismaClientOrTransaction; +}; + +/** + * One block edge, with the fields the unblock decision reads. + * + * `batchId` is read by no logic. It rides inside the two `logger.debug` payloads in + * `continueRunIfUnblocked`, so removing it changes log output. + */ +export type RunBlockEdge = { + id: string; + batchId: string | null; + batchIndex: number | null; + waitpoint: Pick; +}; + +export type RegisterBlocksParams = { + runId: string; + waitpointIds: string[]; + projectId: string; + spanIdToComplete?: string; + batchId?: string; + batchIndex?: number; + /** + * Read client for the pending count only. The caller resolves `tx ?? prisma` once + * and passes the result, so the writer is used when the caller is inside a + * transaction and the pending re-read is read-your-writes on the owning primary. + * Never forwarded to the edge write. + */ + client: ReadClient; +}; + +/** + * The lockless variant writes the edge and does not count. Two methods rather than + * one method with a flag, so "the batch path issues no extra query" is structural. + */ +export type RegisterBlocksLocklessParams = Omit; + +export type CompleteParams = { + waitpointId: string; + output?: { + value: string; + type?: string; + isError: boolean; + }; +}; + +/** One run blocked by the completed waitpoint, with the fields the caller's fan-out loop reads. */ +type BlockedRun = { + taskRunId: string; + spanIdToComplete: string | null; + createdAt: Date; +}; + +export type CompleteResult = { + waitpoint: Waitpoint; + blockedRuns: BlockedRun[]; +}; + +/** + * Discriminated on purpose. The caller enqueues the `finishWaitpoint` job only in the + * `created` branch, because today's create methods return before their enqueue on the + * cached path. A boolean would let a later edit enqueue on both branches. + */ +export type CreateWaitpointResult = + | { kind: "cached"; waitpoint: Waitpoint } + | { kind: "created"; waitpoint: Waitpoint }; + +export type CreateDateTimeWaitpointParams = { + /** When set, the waitpoint co-locates with this run's DB and the dedup probe targets it. */ + runId?: string; + projectId: string; + environmentId: string; + completedAfter: Date; + idempotencyKey?: string; + idempotencyKeyExpiresAt?: Date; +}; + +export type CreateManualWaitpointParams = { + runId?: string; + environmentId: string; + projectId: string; + idempotencyKey?: string; + idempotencyKeyExpiresAt?: Date; + timeout?: Date; + tags?: string[]; + /** + * See the `standaloneResidency` param doc on `WaitpointSystem.createManualWaitpoint` for the + * full rationale. Only a Postgres implementation reads this. + */ + standaloneResidency?: "NEW" | "LEGACY"; +}; + +/** The RUN-waitpoint row data. Pure — no store touch — so the mint is coordinator-owned. */ +export type AssociatedWaitpointData = { + id: string; + friendlyId: string; + type: "RUN"; + status: "PENDING"; + idempotencyKey: string; + userProvidedIdempotencyKey: false; + projectId: string; + environmentId: string; +}; diff --git a/internal-packages/run-engine/src/index.ts b/internal-packages/run-engine/src/index.ts index 2c54e4c20c0..2c98edf6866 100644 --- a/internal-packages/run-engine/src/index.ts +++ b/internal-packages/run-engine/src/index.ts @@ -38,3 +38,26 @@ export type { ProcessBatchItemCallback, BatchCompletionCallback, } from "./batch-queue/types.js"; + +// Waitpoint store coordinator. Exported but not yet wired: a later ticket routes +// WaitpointSystem onto it behind a per-organisation flag. +export { + WaitpointStoreCoordinator, + WaitpointNotFoundError, +} from "./engine/waitpointCoordinator/storeCoordinator.js"; +export type { + AbsorbResult, + BlockEdge, + BlockState, + BlockStateEdge, + CompleteResult, + CreateIfAbsentResult, + RegisterOrReportResult, + WaitpointCompletion, + WaitpointCompletionOutput, + WaitpointRecordInput, + WaitpointStatus, + WaitpointStoreCoordinatorOptions, + WatcherEntry, +} from "./engine/waitpointCoordinator/storeCoordinator.js"; +export { WaitpointKeyTagError } from "./engine/waitpointCoordinator/keys.js"; diff --git a/internal-packages/run-engine/tsconfig.freeze-test.json b/internal-packages/run-engine/tsconfig.freeze-test.json new file mode 100644 index 00000000000..284a132e293 --- /dev/null +++ b/internal-packages/run-engine/tsconfig.freeze-test.json @@ -0,0 +1,19 @@ +// Typechecks completedWaitpointFreeze.test.ts, which tsconfig.build.json otherwise excludes +// (src/**/*.test.ts) and vitest's esbuild transform never checks. This config has no +// "@triggerdotdev/source" customCondition, so it resolves @internal/run-store from its built +// `dist`, not from source -- same as tsconfig.build.json. That means this gate only sees a +// source change in run-store once run-store has been rebuilt, so it MUST be run through turbo +// (`pnpm run typecheck --filter @internal/run-engine`), whose `typecheck` task declares +// `dependsOn: ["^build"]`. Running `tsc -p tsconfig.freeze-test.json` (or `pnpm run typecheck`) +// directly inside this package, against a stale dist/, passes green while the frozen type has +// already drifted in source. Do not "fix" this with customConditions: that pulls +// @trigger.dev/core's source in too, which fails to typecheck here on `lib: ES2020`. +{ + "extends": "./tsconfig.build.json", + "include": ["src/engine/systems/completedWaitpointFreeze.test.ts"], + "exclude": [], + "compilerOptions": { + "composite": false, + "declaration": false + } +} diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts new file mode 100644 index 00000000000..07948670b28 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts @@ -0,0 +1,150 @@ +// A caller-supplied snapshot id must survive into Postgres, so the decorator can own the id and both +// stores hold the same one under dual-write. Absent, Prisma's @default(cuid()) still supplies it. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { setupSnapshotIdFixture } from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore caller-supplied snapshot id", () => { + postgresTest("completeAttemptSuccess writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id, + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } }); + expect(snapshot).not.toBeNull(); + expect(snapshot!.runId).toBe(run.id); + }); + + postgresTest("expireRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("expireParkedRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("rescheduleRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("createExecutionSnapshot writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const created = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toBe(id); + }); + + postgresTest("an absent id still gets a generated one", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const created = await store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toMatch(/^c[a-z0-9]{24}$/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts new file mode 100644 index 00000000000..8026f2f99ec --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts @@ -0,0 +1,132 @@ +// The caller's instant must land in BOTH timestamp columns, on BOTH schema variants. +// +// `updatedAt` is declared `@updatedAt`, which Prisma manages itself, so whether an explicit value +// survives a create is a property of the client rather than of the schema. The two variants are +// separately generated clients over separately declared schemas, so agreeing declarations are not +// evidence that they agree in behaviour. This asserts it on each. +// +// It matters because the decorator writes one instant to both stores. If Prisma overrode it here, +// Postgres and Redis would hold different values for a column the comparator checks for equality, +// on every snapshot. +import { heteroPostgresTest, heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +/** Five minutes in the past, so a column default could never coincide with it. */ +const STAMP = new Date(Date.now() - 5 * 60 * 1000); + +async function writeSnapshot( + prisma: AnyClient, + schemaVariant: "legacy" | "dedicated", + suffix: string +) { + const scope = + schemaVariant === "dedicated" + ? { + environmentId: `env_${suffix}`, + projectId: `proj_${suffix}`, + organizationId: `org_${suffix}`, + } + : await seedLegacyScope(prisma as PrismaClient, suffix); + + const store = new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + schemaVariant, + }); + + const runId = generateInternalId(); + const id = generateInternalId(); + + await (prisma as PrismaClient).taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${suffix}`, + runtimeEnvironmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + organizationId: scope.organizationId, + projectId: scope.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + } as never, + }); + + await store.createExecutionSnapshot({ + id, + createdAt: STAMP, + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + projectId: scope.projectId, + organizationId: scope.organizationId, + }); + + return (prisma as PrismaClient).taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); +} + +async function seedLegacyScope(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { + environmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + }; +} + +describe("snapshot timestamps are the caller's, on both schema variants", () => { + heteroPostgresTest("legacy client honours the supplied instant", async ({ prisma14 }) => { + const row = await writeSnapshot(prisma14, "legacy", "tsleg"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + // The one Prisma manages. If it overrode the value, the two stores would disagree here on + // every snapshot. + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + }); + + heteroRunOpsPostgresTest( + "dedicated client honours the supplied instant", + async ({ prisma17 }) => { + const row = await writeSnapshot(prisma17, "dedicated", "tsded"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + } + ); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts new file mode 100644 index 00000000000..dda0f483bf5 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts @@ -0,0 +1,312 @@ +// snapshotWrites: false is the redis-only dial position. Every run mutation still lands; no snapshot +// row is written and no completed-waitpoint join row is inserted. The default stays true, so nothing +// changes for any existing caller. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, +} from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore snapshotWrites flag", () => { + postgresTest("defaults to writing snapshots", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + }); + + postgresTest("writes the run mutation but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("createCancelledRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("expireRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "EXPIRED" + ); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("expireParkedRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("rescheduleRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const delayUntil = new Date(Date.now() + 60_000); + + await store.rescheduleRun(run.id, { + delayUntil, + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.delayUntil?.toISOString()).toBe(delayUntil.toISOString()); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("lockRunToWorker writes the lock but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "DEQUEUED" + ); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createExecutionSnapshot echoes the input when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const echoed = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.id).toBe(id); + expect(echoed.runId).toBe(run.id); + expect(echoed.executionStatus).toBe("EXECUTING"); + expect(echoed.attemptNumber).toBe(2); + expect(echoed.isValid).toBe(true); + expect(echoed.checkpoint).toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("the echoed row rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.runStatus).toBe("PENDING"); + }); + + postgresTest("the echoed row reports an errored snapshot as invalid", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.isValid).toBe(false); + expect(echoed.error).toBe("snapshot is not the latest"); + }); + + postgresTest("createExecutionSnapshot needs an id when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await expect( + store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }) + ).rejects.toThrow(/snapshotWrites is off/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index b7d5086431b..df718b4a1af 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -30,6 +30,7 @@ import type { TaskRunWithWaitpoint, } from "./types.js"; import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; // Loose delegate method shape: each generated client types delegate methods as // `(args: PackageLocalArgs) => PrismaPromise<…>` against its own nominal @@ -118,6 +119,13 @@ export type PostgresRunStoreOptions = { maxWait?: number; /** Env-driven P2028-at-acquisition retry config, threaded from the app boundary (IoC). */ transactionStartRetry?: TransactionStartRetryConfig; + /** + * When false the store writes no execution-snapshot rows: every nested `executionSnapshots.create` + * is omitted and `createExecutionSnapshot` echoes its input instead of inserting. Only the + * redis-only dial position sets this, once the Redis store is the sole snapshot writer. + * Defaults to true, so the store behaves exactly as it always has. + */ + snapshotWrites?: boolean; }; // A caller sub-select for a relation: `{ select?, include? }` or `true` for a bare `key: true`. @@ -638,6 +646,7 @@ export class PostgresRunStore implements RunStore { private readonly prisma: RunOpsCapableClient; private readonly readOnlyPrisma: RunOpsCapableClient; private readonly schemaVariant: RunStoreSchemaVariant; + private readonly snapshotWrites: boolean; private readonly maxWait?: number; private readonly transactionStartRetry?: TransactionStartRetryConfig; @@ -650,6 +659,22 @@ export class PostgresRunStore implements RunStore { this.schemaVariant = options.schemaVariant ?? "legacy"; this.maxWait = options.maxWait; this.transactionStartRetry = options.transactionStartRetry; + this.snapshotWrites = options.snapshotWrites ?? true; + } + + /** + * Wraps a nested snapshot create so a single flag removes it everywhere. Prisma treats an absent + * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather + * than sending an empty one. + */ + #nestedSnapshot(create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput): + | { + executionSnapshots: { + create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput; + }; + } + | Record { + return this.snapshotWrites ? { executionSnapshots: { create } } : {}; } // The writer handle in read-client form, so the routing layer can honor a caller-passed client @@ -726,6 +751,8 @@ export class PostgresRunStore implements RunStore { const snapshotCreate = { id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -743,7 +770,7 @@ export class PostgresRunStore implements RunStore { const run = (await this.#writeClientWithoutTransaction(tx).taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; return { ...run, associatedWaitpoint: null }; @@ -755,7 +782,7 @@ export class PostgresRunStore implements RunStore { const run = (await c.taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; @@ -772,9 +799,7 @@ export class PostgresRunStore implements RunStore { }, data: { ...params.data, - executionSnapshots: { - create: snapshotCreate, - }, + ...this.#nestedSnapshot(snapshotCreate), associatedWaitpoint: params.associatedWaitpoint ? { create: params.associatedWaitpoint, @@ -813,23 +838,26 @@ export class PostgresRunStore implements RunStore { ): Promise { const client = tx ?? this.prisma; + const snapshotCreate = { + id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, + engine: params.snapshot.engine, + executionStatus: params.snapshot.executionStatus, + description: params.snapshot.description, + runStatus: params.snapshot.runStatus, + environmentId: params.snapshot.environmentId, + environmentType: params.snapshot.environmentType, + projectId: params.snapshot.projectId, + organizationId: params.snapshot.organizationId, + workerId: params.snapshot.workerId, + runnerId: params.snapshot.runnerId, + }; + return client.taskRun.create({ data: { ...params.data, - executionSnapshots: { - create: { - engine: params.snapshot.engine, - executionStatus: params.snapshot.executionStatus, - description: params.snapshot.description, - runStatus: params.snapshot.runStatus, - environmentId: params.snapshot.environmentId, - environmentType: params.snapshot.environmentType, - projectId: params.snapshot.projectId, - organizationId: params.snapshot.organizationId, - workerId: params.snapshot.workerId, - runnerId: params.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot(snapshotCreate), }, }); } @@ -923,20 +951,21 @@ export class PostgresRunStore implements RunStore { outputType: data.outputType, usageDurationMs: data.usageDurationMs, costInCents: data.costInCents, - executionSnapshots: { - create: { - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - attemptNumber: data.snapshot.attemptNumber, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - workerId: data.snapshot.workerId, - runnerId: data.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + attemptNumber: data.snapshot.attemptNumber, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + workerId: data.snapshot.workerId, + runnerId: data.snapshot.runnerId, + }), }, { select: args.select } ) as Promise>; @@ -1129,18 +1158,19 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, { select: args.select } ) as Promise>; @@ -1260,42 +1290,46 @@ export class PostgresRunStore implements RunStore { cliVersion: data.cliVersion ?? undefined, maxDurationInSeconds: data.maxDurationInSeconds ?? undefined, maxAttempts: data.maxAttempts ?? undefined, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: "V2", - executionStatus: "PENDING_EXECUTING", - description: "Run was dequeued for execution", - runStatus: "PENDING", - attemptNumber: data.snapshot.attemptNumber ?? undefined, - previousSnapshotId: data.snapshot.previousSnapshotId, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - checkpointId: data.snapshot.checkpointId ?? undefined, - batchId: data.snapshot.batchId ?? undefined, - // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. - completedWaitpointOrder: data.snapshot.completedWaitpointOrder, - workerId: data.snapshot.workerId ?? undefined, - runnerId: data.snapshot.runnerId ?? undefined, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: "V2", + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + attemptNumber: data.snapshot.attemptNumber ?? undefined, + previousSnapshotId: data.snapshot.previousSnapshotId, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + checkpointId: data.snapshot.checkpointId ?? undefined, + batchId: data.snapshot.batchId ?? undefined, + // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. + completedWaitpointOrder: data.snapshot.completedWaitpointOrder, + workerId: data.snapshot.workerId ?? undefined, + runnerId: data.snapshot.runnerId ?? undefined, + }), }, }); - if (dedicated) { - await this.#connectCompletedWaitpoints( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); - } else { - await this.#connectCompletedWaitpointsLegacy( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); + // The join rows link to the snapshot row above. With snapshot writes off there is no such row, + // so inserting them would leave dangling links for a snapshot that only the Redis store holds. + if (this.snapshotWrites) { + if (dedicated) { + await this.#connectCompletedWaitpoints( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } else { + await this.#connectCompletedWaitpointsLegacy( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } } return result; @@ -1363,18 +1397,19 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, }); } catch (error) { @@ -1435,21 +1470,21 @@ export class PostgresRunStore implements RunStore { data: { delayUntil: data.delayUntil, ...(data.queueTimestamp !== undefined && { queueTimestamp: data.queueTimestamp }), - ...(data.snapshot && { - executionSnapshots: { - create: { - engine: "V2", - executionStatus: data.snapshot.executionStatus ?? "DELAYED", - description: - data.snapshot.description ?? "Delayed run was rescheduled to a future date", - runStatus: data.snapshot.runStatus ?? "DELAYED", - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, - }), + ...(data.snapshot && + this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: "V2", + executionStatus: data.snapshot.executionStatus ?? "DELAYED", + description: + data.snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: data.snapshot.runStatus ?? "DELAYED", + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + })), }, }); } @@ -1826,7 +1861,7 @@ export class PostgresRunStore implements RunStore { const branches = args.idempotencyKeys.map((key) => { const base = params.length; params.push(args.runtimeEnvironmentId, args.taskIdentifier, key); - return `SELECT "friendlyId", "idempotencyKey", "idempotencyKeyExpiresAt" FROM "TaskRun" WHERE "runtimeEnvironmentId" = $${base + 1} AND "taskIdentifier" = $${base + 2} AND "idempotencyKey" = $${base + 3}`; + return `SELECT "id", "createdAt", "friendlyId", "idempotencyKey", "idempotencyKeyExpiresAt" FROM "TaskRun" WHERE "runtimeEnvironmentId" = $${base + 1} AND "taskIdentifier" = $${base + 2} AND "idempotencyKey" = $${base + 3}`; }); return prisma.$queryRawUnsafe( branches.join(" UNION ALL "), @@ -1969,6 +2004,8 @@ export class PostgresRunStore implements RunStore { prisma: PrismaClientOrTransaction ): Promise> { const { + id, + createdAt, run, snapshot, previousSnapshotId, @@ -1984,10 +2021,65 @@ export class PostgresRunStore implements RunStore { error, } = input; + // Left possibly-undefined ON PURPOSE. Prisma omits an undefined key, so the column keeps taking + // whatever it took before this method was touched: the schema declares `completedWaitpointOrder + // String[]` with no default and the column is nullable, so an omitted key stores NULL, not `{}`. + // Defaulting here would send `{}` instead and change what a live write stores. + // + // The redis-only echo below DOES need a concrete array, because it returns the row shape to the + // caller and that field is not nullable in the payload type. That default belongs to the echo, + // not to the write, so the two are kept apart. + const completedWaitpointOrder = completedWaitpoints + ?.filter((c) => c.index !== undefined) + .sort((a, b) => a.index! - b.index!) + .map((w) => w.id); + + // Redis-only: no row is written and the decorator owns the document. Echo the input in the shape + // the caller expects, so every caller of this method keeps working while Postgres holds nothing. + if (!this.snapshotWrites) { + if (!id) { + throw new Error( + "PostgresRunStore.createExecutionSnapshot: snapshotWrites is off, so the caller must supply the snapshot id" + ); + } + + const now = createdAt ?? new Date(); + return { + id, + engine: "V2", + executionStatus: snapshot.executionStatus, + description: snapshot.description, + previousSnapshotId: previousSnapshotId ?? null, + runId: run.id, + runStatus: run.status === "DEQUEUED" ? "PENDING" : run.status, + attemptNumber: run.attemptNumber ?? null, + batchId: batchId ?? null, + environmentId, + environmentType, + projectId, + organizationId, + checkpointId: checkpointId ?? null, + workerId: workerId ?? null, + runnerId: runnerId ?? null, + metadata: snapshot.metadata ?? null, + completedWaitpointOrder: completedWaitpointOrder ?? [], + isValid: !error, + error: error ?? null, + createdAt: now, + updatedAt: now, + checkpoint: null, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { checkpoint: true }; + }>; + } + const dedicated = this.schemaVariant === "dedicated"; const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ data: { + id, + createdAt, + updatedAt: createdAt, engine: "V2", executionStatus: snapshot.executionStatus, description: snapshot.description, @@ -2007,10 +2099,7 @@ export class PostgresRunStore implements RunStore { metadata: snapshot.metadata ?? undefined, // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas, so a // cross-DB (NEW-resident) token can be recorded without a Prisma `connect` existence check. - completedWaitpointOrder: completedWaitpoints - ?.filter((c) => c.index !== undefined) - .sort((a, b) => a.index! - b.index!) - .map((w) => w.id), + completedWaitpointOrder, isValid: !error, error, }, @@ -2669,7 +2758,7 @@ export class PostgresRunStore implements RunStore { data: { environmentId: string; name: string; projectId: string; id?: string }, tx?: PrismaClientOrTransaction, // `residency` selects the store at the router; a single store has one client and ignores it. - _residency?: "NEW" | "LEGACY" + _residency?: ShardKey ): Promise { const prisma = tx ?? this.prisma; diff --git a/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts new file mode 100644 index 00000000000..081e9efe1d4 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts @@ -0,0 +1,188 @@ +// Every declared parameter must actually reach the delegate. +// +// The compiler cannot check this. A forwarder that omits a trailing OPTIONAL argument compiles +// cleanly, and the effect is silent: `findLatestExecutionSnapshot` would stop applying its tenant +// scope, and `upsertWaitpointTag` would stop applying its residency hint, so a write would land on +// the wrong database. Both of those shipped in this file before this test existed. +// +// So this reads the source of the base against the source of the interface and asserts that each +// forward passes exactly the parameters its signature declares, in order. Source-level, because +// that is the only place the property is visible. +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const dir = join(import.meta.dirname); +const interfaceSource = readFileSync(join(dir, "types.ts"), "utf8"); +const baseSource = readFileSync(join(dir, "delegatingRunStore.ts"), "utf8"); + +/** Replaces comments and string bodies with spaces, so neither can shift a brace depth. */ +function blank(text: string): string { + let out = ""; + let i = 0; + while (i < text.length) { + const two = text.slice(i, i + 2); + if (two === "//") { + const end = text.indexOf("\n", i); + const stop = end === -1 ? text.length : end; + out += " ".repeat(stop - i); + i = stop; + } else if (two === "/*") { + const end = text.indexOf("*/", i + 2); + const stop = end === -1 ? text.length : end + 2; + out += text.slice(i, stop).replace(/[^\n]/g, " "); + i = stop; + } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { + const quote = text[i]; + let j = i + 1; + while (j < text.length && text[j] !== quote) j += text[j] === "\\" ? 2 : 1; + out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); + i = j + 1; + } else { + out += text[i]; + i += 1; + } + } + return out; +} + +function interfaceBody(source: string): string { + const blanked = blank(source); + const decl = "export interface RunStore {"; + const start = blanked.indexOf(decl) + decl.length; + let depth = 1; + let end = start; + while (depth > 0 && end < blanked.length) { + if (blanked[end] === "{") depth += 1; + else if (blanked[end] === "}") depth -= 1; + if (depth > 0) end += 1; + } + return blanked.slice(start, end); +} + +/** Splits a balanced parameter list on top-level commas. */ +function splitParams(signature: string): string[] { + // A generic member reads `name(...)`, so the parameter list starts after the + // balanced angle block, not at the first parenthesis. + let searchFrom = 0; + const angle = signature.indexOf("<"); + const paren = signature.indexOf("("); + if (angle !== -1 && angle < paren) { + let angleDepth = 0; + for (let i = angle; i < signature.length; i++) { + if (signature[i] === "<") angleDepth += 1; + else if (signature[i] === ">") { + angleDepth -= 1; + if (angleDepth === 0) { + searchFrom = i; + break; + } + } + } + } + + const open = signature.indexOf("(", searchFrom); + let depth = 0; + let close = open; + for (let i = open; i < signature.length; i++) { + if ("([{<".includes(signature[i]!)) depth += 1; + else if (")]}>".includes(signature[i]!)) { + depth -= 1; + if (depth === 0) { + close = i; + break; + } + } + } + const inner = signature.slice(open + 1, close); + const parts: string[] = []; + let level = 0; + let current = ""; + for (const ch of inner) { + if ("([{<".includes(ch)) level += 1; + else if (")]}>".includes(ch)) level -= 1; + if (ch === "," && level === 0) { + parts.push(current); + current = ""; + } else { + current += ch; + } + } + if (current.trim()) parts.push(current); + return parts; +} + +function paramNames(signature: string): string[] { + return splitParams(signature) + .map((p) => /^\s*([A-Za-z_$][\w$]*)\s*\??\s*:/.exec(p)?.[1]) + .filter((n): n is string => Boolean(n)); +} + +/** Member name to its declared parameter names, for members with a single signature. */ +function declaredParams(): Map { + const body = interfaceBody(interfaceSource); + const spans: string[] = []; + let level = 0; + let from = 0; + for (let i = 0; i < body.length; i++) { + const ch = body[i]!; + if ("{([".includes(ch)) level += 1; + else if ("})]".includes(ch)) level -= 1; + else if (ch === ";" && level === 0) { + spans.push(body.slice(from, i)); + from = i + 1; + } + } + + const seen = new Map(); + for (const span of spans) { + const match = /^\s*(?:readonly\s+)?([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(span); + if (!match) continue; + const name = match[1]!; + seen.set(name, [...(seen.get(name) ?? []), paramNames(span)]); + } + + // Overloaded members forward through a cast and apply the whole argument list, so they are not + // subject to this check. + return new Map( + [...seen].filter(([, sigs]) => sigs.length === 1).map(([n, sigs]) => [n, sigs[0]!]) + ); +} + +describe("the pass-through forwards every declared parameter", () => { + const declared = declaredParams(); + + it("parsed the interface, so a parse failure cannot pass this suite", () => { + expect(declared.size).toBeGreaterThan(50); + expect(declared.get("expireParkedRun")).toEqual(["runId", "data", "tx"]); + expect(declared.get("findLatestExecutionSnapshot")).toEqual([ + "runId", + "client", + "environmentId", + ]); + }); + + it("passes exactly the declared parameters, in order, for every single-signature member", () => { + const wrong: string[] = []; + + for (const [name, params] of declared) { + const forward = new RegExp(`return this\\.delegate\\.${name}\\(([^;]*)\\);`).exec(baseSource); + + if (!forward) { + wrong.push(`${name}: no forward found`); + continue; + } + + const passed = forward[1]! + .split(",") + .map((a) => a.trim()) + .filter(Boolean); + + if (passed.join(",") !== params.join(",")) { + wrong.push(`${name}: declares (${params.join(", ")}) but forwards (${passed.join(", ")})`); + } + } + + expect(wrong).toEqual([]); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts new file mode 100644 index 00000000000..7fdcd9ac07b --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -0,0 +1,118 @@ +// What this suite covers, and why it is now small. +// +// `DelegatingRunStore` restates every interface signature and forwards its arguments by name, so +// most of what a pass-through can get wrong is a compile error rather than a test failure: +// +// member of the interface missing -> `implements RunStore`, TS2420 +// public member the interface lacks -> the parity assertion in the base +// forwarded to the wrong delegate member -> argument types do not match, TS2345/TS2322 +// arguments reordered -> same +// +// Three things remain invisible to the compiler, and they are what is left here. +// +// First, the seven overloaded members. TypeScript cannot express one body that satisfies an overload +// set, so their single implementation forwards through a cast, and the cast is exactly where a +// wrong-member forward would stop being a type error. +// +// Second, a dropped OPTIONAL argument. Omitting a trailing `tx` compiles cleanly and silently stops +// forwarding the caller's transaction. +// +// Third, whether a data property is read live or captured once at construction. Both typecheck; only +// one is correct. +// +// No database is involved in whether a pass-through passes through, so none is started. Behaviour +// against a real store is covered by the container suites for the decorator built on this base. +import { describe, expect, it } from "vitest"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; +import type { RunStore } from "./types.js"; + +/** + * The members whose implementation forwards through a cast, because they are overloaded. These are + * the only methods where the compiler is not already checking the forward. + */ +const OVERLOADED_MEMBERS = [ + "finalizeRun", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", +] as const; + +type ProbedCall = { name: string; args: unknown[] }; + +/** + * Records what was called and answers with a per-member sentinel, so a forward to the wrong member + * returns the wrong value rather than merely returning something. + */ +function forwardingProbe(): { store: RunStore; calls: ProbedCall[] } { + const calls: ProbedCall[] = []; + + const store = new Proxy({} as Record, { + get(_target, prop: string) { + if ((RUN_STORE_PROPERTY_NAMES as readonly string[]).includes(prop)) { + return `property:${prop}`; + } + return (...args: unknown[]) => { + calls.push({ name: prop, args }); + return `result:${prop}`; + }; + }, + }); + + return { store: store as unknown as RunStore, calls }; +} + +describe("DelegatingRunStore", () => { + it("forwards every method to the member of the same name", () => { + const { store, calls } = forwardingProbe(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + + for (const name of RUN_STORE_METHOD_NAMES) { + expect(base[name]()).toBe(`result:${name}`); + } + + expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); + }); + + it("covers every overloaded member, so the list cannot rot", () => { + // If a member gains or loses overloads, the cast set changes and this suite should follow. + for (const name of OVERLOADED_MEMBERS) { + expect(RUN_STORE_METHOD_NAMES).toContain(name); + } + }); + + it("forwards arguments untouched through an overloaded member's cast", () => { + const { store, calls } = forwardingProbe(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + const args = ["first", { second: true }, undefined, 4]; + + for (const name of OVERLOADED_MEMBERS) { + base[name](...args); + } + + // The overloaded implementations apply the whole argument list, so every argument survives, + // including a trailing optional the typed members would legitimately drop. + for (const call of calls) { + expect(call.args).toEqual(args); + } + expect(calls.map((c) => c.name)).toEqual([...OVERLOADED_MEMBERS]); + }); + + it("reads a data property live, so a delegate that changes is not cached", () => { + const store = { primaryReadClient: "first" } as unknown as RunStore; + const base = new DelegatingRunStore(store); + + expect(base.primaryReadClient).toBe("first" as unknown); + (store as unknown as Record).primaryReadClient = "second"; + expect(base.primaryReadClient).toBe("second" as unknown); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts new file mode 100644 index 00000000000..c7fe3225c76 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -0,0 +1,750 @@ +// A pass-through over another RunStore. +// +// It exists so a decorator can override the handful of methods it cares about and inherit the rest. +// +// Every member restates the interface signature and forwards its arguments BY NAME, so the +// forwarding is itself type-checked: a body that called the wrong delegate method, or dropped an +// argument, does not compile. That is the whole point of the shape. An untyped forwarder would let +// both mistakes through, because a pass-through has no other behaviour to catch them. +// +// Seven members are overloaded. Their overloads are declared so callers keep the full contract, and +// their single implementation signature is the one place a cast appears: TypeScript cannot express +// one body that satisfies an overload set without it. +// +// Keeping this in step with the interface is not a matter of memory. `implements RunStore` rejects a +// member that is missing, and the assertion at the foot of the file rejects one the interface never +// declared. + +import type { + BatchTaskRun, + BatchTaskRunItemStatus, + Prisma, + PrismaClientOrTransaction, + TaskRun, + TaskRunStatus, + WaitpointTag, +} from "@trigger.dev/database"; +import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; +import type { Residency } from "@trigger.dev/core/v3/isomorphic"; +import type { + ClearIdempotencyKeyInput, + CompletionSnapshotInput, + CreateBatchTaskRunData, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateFailedRunInput, + CreateRunInput, + ExpireSnapshotInput, + FinalizeRunData, + ForWaitpointCompletionContext, + IdempotencyKeyRunMatch, + LockRunData, + PromotePendingVersionArgs, + ReadClient, + RescheduleSnapshotInput, + RewriteDebouncedRunData, + RunStore, + TaskRunWithWaitpoint, + WaitpointColocationOptions, +} from "./types.js"; + +export class DelegatingRunStore implements RunStore { + constructor(protected readonly delegate: RunStore) {} + + runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + return this.delegate.runInTransaction(runId, fn); + } + + createRun(params: CreateRunInput, tx?: PrismaClientOrTransaction): Promise { + return this.delegate.createRun(params, tx); + } + + createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createCancelledRun(params, tx); + } + + createFailedRun( + params: CreateFailedRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createFailedRun(params, tx); + } + + startAttempt( + runId: string, + data: { attemptNumber: number; executedAt?: Date; isWarmStart: boolean }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.startAttempt(runId, data, args, tx); + } + + completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + recordRetryOutcome( + runId: string, + data: { machinePreset?: string; usageDurationMs: number; costInCents: number }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.recordRetryOutcome(runId, data, args, tx); + } + + requeueRun( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.requeueRun(runId, args, tx); + } + + recordBulkActionMembership( + runId: string, + bulkActionId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.recordBulkActionMembership(runId, bulkActionId, tx); + } + + cancelRun( + runId: string, + data: { + completedAt?: Date; + error: TaskRunError; + bulkActionId?: string; + usageDurationMs?: number; + costInCents?: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.cancelRun(runId, data, args, tx); + } + + failRunPermanently( + runId: string, + data: { + status: TaskRunStatus; + completedAt: Date; + error: TaskRunError; + usageDurationMs: number; + costInCents: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.failRunPermanently(runId, data, args, tx); + } + + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + tx?: PrismaClientOrTransaction + ): Promise; + finalizeRun(...args: unknown[]): unknown { + return (this.delegate.finalizeRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + expireRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + snapshot: ExpireSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.expireRun(runId, data, args, tx); + } + + expireRunsBatch( + runIds: string[], + data: { error: TaskRunError; now: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.expireRunsBatch(runIds, data, tx); + } + + lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + parkPendingVersion( + runId: string, + data: { statusReason: string }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.parkPendingVersion(runId, data, args, tx); + } + + promotePendingVersionRuns( + runId: string, + args?: PromotePendingVersionArgs, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.promotePendingVersionRuns(runId, args, tx); + } + + expireParkedRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.expireParkedRun(runId, data, tx); + } + + suspendForCheckpoint( + runId: string, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.suspendForCheckpoint(runId, args, tx); + } + + resumeFromCheckpoint( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.resumeFromCheckpoint(runId, args, tx); + } + + rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rescheduleRun(runId, data, tx); + } + + enqueueDelayedRun( + runId: string, + data: { queuedAt: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.enqueueDelayedRun(runId, data, tx); + } + + rewriteDebouncedRun( + runId: string, + data: RewriteDebouncedRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rewriteDebouncedRun(runId, data, tx); + } + + updateMetadata( + runId: string, + data: { + metadata: string | null; + metadataType?: string; + metadataVersion: { increment: number }; + updatedAt: Date; + }, + options: { expectedMetadataVersion?: number }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.updateMetadata(runId, data, options, tx); + } + + clearIdempotencyKey( + params: ClearIdempotencyKeyInput, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.clearIdempotencyKey(params, tx); + } + + pushTags( + runId: string, + tags: string[], + where: { runtimeEnvironmentId: string }, + tx?: PrismaClientOrTransaction + ): Promise<{ updatedAt: Date }> { + return this.delegate.pushTags(runId, tags, where, tx); + } + + pushRealtimeStream( + runId: string, + streamId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.pushRealtimeStream(runId, streamId, tx); + } + + get primaryReadClient(): ReadClient { + return this.delegate.primaryReadClient; + } + + findRun( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise | null>; + findRun( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise | null>; + findRun(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRun(...args: unknown[]): unknown { + return (this.delegate.findRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise>; + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise>; + findRunOrThrow(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRunOrThrow(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrow as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise | null>; + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise | null>; + findRunOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise>; + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise>; + findRunOrThrowOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOrThrowOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrowOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + select: S; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + include: I; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise; + findRuns(...args: unknown[]): unknown { + return (this.delegate.findRuns as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIds( + ids: string[], + args: { select: S }, + client?: ReadClient + ): Promise>>; + findRunsByIds( + ids: string[], + args: { include: I }, + client?: ReadClient + ): Promise>>; + findRunsByIds(ids: string[], client?: ReadClient): Promise>; + findRunsByIds(...args: unknown[]): unknown { + return (this.delegate.findRunsByIds as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIdempotencyKeys( + args: { runtimeEnvironmentId: string; taskIdentifier: string; idempotencyKeys: string[] }, + client?: ReadClient + ): Promise { + return this.delegate.findRunsByIdempotencyKeys(args, client); + } + + createBatchTaskRunItem( + data: { batchTaskRunId: string; taskRunId: string; status: BatchTaskRunItemStatus }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRunItem(data, tx); + } + + findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + // When set, scopes the read to this environment (tenant boundary); a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string + ): Promise | null> { + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findExecutionSnapshot(args, client); + } + + findManyExecutionSnapshots( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createExecutionSnapshot(input, tx); + } + + findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + findWaitpointConnectedRunIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointConnectedRunIds(waitpointId, client); + } + + findWaitpointCompletedSnapshotIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointCompletedSnapshotIds(waitpointId, client); + } + + blockRunWithWaitpointEdges(params: { + runId: string; + waitpointIds: string[]; + projectId: string; + spanIdToComplete?: string; + batchId?: string; + batchIndex?: number; + tx?: PrismaClientOrTransaction; + }): Promise { + return this.delegate.blockRunWithWaitpointEdges(params); + } + + countPendingWaitpoints( + waitpointIds: string[], + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.countPendingWaitpoints(waitpointIds, client, runId); + } + + countPendingWaitpointsWithPresence( + waitpointIds: string[], + client?: ReadClient + ): Promise<{ pendingIds: string[]; presentIds: string[] }> { + return this.delegate.countPendingWaitpointsWithPresence(waitpointIds, client); + } + + createWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.createWaitpoint(args, tx, opts); + } + + upsertWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.upsertWaitpoint(args, tx, opts); + } + + findWaitpoint( + args: Prisma.SelectSubset, + client?: ReadClient, + opts?: WaitpointColocationOptions + ): Promise | null> { + return this.delegate.findWaitpoint(args, client, opts); + } + + findWaitpointOnPrimary( + args: Prisma.SelectSubset + ): Promise | null> { + return this.delegate.findWaitpointOnPrimary(args); + } + + findManyWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient, + runId?: string + ): Promise[]> { + return this.delegate.findManyWaitpoints(args, client, runId); + } + + updateWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.updateWaitpoint(args, tx, opts); + } + + updateManyWaitpoints( + args: Prisma.WaitpointUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyWaitpoints(args, tx); + } + + forWaitpointCompletion( + waitpointId: string, + context: ForWaitpointCompletionContext + ): Promise { + return this.delegate.forWaitpointCompletion(waitpointId, context); + } + + findManyTaskRunWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyTaskRunWaitpoints(args, client); + } + + deleteManyTaskRunWaitpoints( + args: Prisma.TaskRunWaitpointDeleteManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.deleteManyTaskRunWaitpoints(args, tx); + } + + findTaskRunAttempt( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findTaskRunAttempt(args, client); + } + + createTaskRunCheckpoint( + args: Prisma.SelectSubset, + ownerRunId?: string, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createTaskRunCheckpoint(args, ownerRunId, tx); + } + + createBatchTaskRun( + data: CreateBatchTaskRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRun(data, tx); + } + + updateBatchTaskRun( + args: { + where: Prisma.BatchTaskRunWhereUniqueInput; + data: Prisma.BatchTaskRunUpdateInput; + select: S; + }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.updateBatchTaskRun(args, tx); + } + + findBatchTaskRunById( + id: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunById(id, args, client); + } + + findBatchTaskRunByFriendlyId( + friendlyId: string, + environmentId: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByFriendlyId(friendlyId, environmentId, args, client); + } + + findBatchTaskRunByIdempotencyKey( + environmentId: string, + idempotencyKey: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByIdempotencyKey( + environmentId, + idempotencyKey, + args, + client + ); + } + + updateManyBatchTaskRun( + args: Prisma.BatchTaskRunUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRun(args, tx); + } + + countBatchTaskRunItems( + where: { batchTaskRunId: string; status?: BatchTaskRunItemStatus }, + client?: ReadClient + ): Promise { + return this.delegate.countBatchTaskRunItems(where, client); + } + + updateManyBatchTaskRunItems( + args: Prisma.BatchTaskRunItemUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRunItems(args, tx); + } + + findManyBatchTaskRunItems( + where: { taskRunId?: string; batchTaskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyBatchTaskRunItems(where, args, client); + } + + findBatchTaskRunItem( + where: { batchTaskRunId: string; taskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunItem(where, args, client); + } + + upsertWaitpointTag( + data: { environmentId: string; name: string; projectId: string; id?: string }, + tx?: PrismaClientOrTransaction, + // A tag has no owning run to co-locate with; when no minted `id` pins it by id-shape, a + // minted-new env's tags read this residency (NEW) so they land with the env's tokens/runs + // instead of defaulting to LEGACY. Single-store impls ignore it. + residency?: Residency + ): Promise { + return this.delegate.upsertWaitpointTag(data, tx, residency); + } + + findManyWaitpointTags( + args: { + where: Prisma.WaitpointTagWhereInput; + orderBy?: + | Prisma.WaitpointTagOrderByWithRelationInput + | Prisma.WaitpointTagOrderByWithRelationInput[]; + take?: number; + skip?: number; + }, + client?: ReadClient + ): Promise { + return this.delegate.findManyWaitpointTags(args, client); + } +} + +// `implements` above rejects a member of the interface that is missing here. It says nothing about a +// member that should not exist, so the reverse direction is asserted too: a public member this class +// declares and the interface does not is a build failure. +// +// `protected delegate` is correctly absent from `keyof`, so the constructor parameter does not trip +// this. +type _ClassDeclaresNoExtraMembers = [Exclude] extends [ + never, +] + ? true + : never; +const _classParity: _ClassDeclaresNoExtraMembers = true; +void _classParity; diff --git a/internal-packages/run-store/src/index.ts b/internal-packages/run-store/src/index.ts index 3717dc01527..07be12da94e 100644 --- a/internal-packages/run-store/src/index.ts +++ b/internal-packages/run-store/src/index.ts @@ -3,3 +3,10 @@ export * from "./PostgresRunStore.js"; export * from "./runOpsStore.js"; export * from "./readReplicaClient.js"; export * from "./redisSnapshotStore.js"; +export * from "./routingStoreMetrics.js"; +export * from "./snapshotComparator.js"; +export * from "./delegatingRunStore.js"; +export * from "./taskRunExecutionSnapshotStore.js"; +export * from "./snapshotEntry.js"; +export * from "./snapshotFaultInjection.js"; +export * from "./snapshotOrphanSweeper.js"; diff --git a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts new file mode 100644 index 00000000000..f1394be597e --- /dev/null +++ b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts @@ -0,0 +1,193 @@ +// getExecutionSnapshotsSince resolves its cursor to a createdAt before it asks for the window, so +// the snapshot id is gone by then and getSince cannot serve it. This read takes the cursor instead, +// and has to agree with the Postgres read it stands in for — same-millisecond blind spot included. +import { describe, expect } from "vitest"; +import { redisTest } from "@internal/testcontainers"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function entry(runId: string, id: string, createdAt: string): SnapshotEntryInput { + return { + id, + engine: "V2", + executionStatus: "EXECUTING", + description: "d", + runId, + runStatus: "EXECUTING", + createdAt, + environmentId: "env_1", + environmentType: "DEVELOPMENT", + projectId: "proj_1", + organizationId: "org_1", + }; +} + +const at = (seconds: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); + +async function seed( + store: RedisSnapshotStore, + runId: string, + stamps: { id: string; createdAt: string }[] +): Promise { + for (const [index, stamp] of stamps.entries()) { + await store.append({ + entry: entry(runId, stamp.id, stamp.createdAt), + kind: index === 0 ? "birth" : "transition", + isTerminal: false, + }); + } +} + +describe("getSinceCreatedAt", () => { + redisTest( + "returns only entries newer than the cursor, oldest first", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_window"; + await seed( + store, + runId, + [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(1)); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + // Ascending, matching what the engine hands its caller after its own reverse(). + expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); + } finally { + await store.quit(); + } + } + ); + + redisTest("misses when the run has no keyspace", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + // A miss is the coexistence path: the caller falls back to Postgres for a pre-cutover run. + expect((await store.getSinceCreatedAt("run_absent", at(0))).kind).toBe("miss"); + } finally { + await store.quit(); + } + }); + + redisTest("returns an empty hit when nothing is newer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_nothing_newer"; + await seed(store, runId, [{ id: "snap_0", createdAt: at(0) }]); + + const result = await store.getSinceCreatedAt(runId, at(5)); + + // A hit, not a miss: Redis owns this run, so the caller must not fall back and re-read + // Postgres for a window it already answered. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest("drops a same-millisecond neighbour, as Postgres does", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_same_ms"; + const shared = at(1); + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1a", createdAt: shared }, + { id: "snap_1b", createdAt: shared }, + { id: "snap_2", createdAt: at(2) }, + ]); + + const result = await store.getSinceCreatedAt(runId, shared); + + // Postgres serves this window with `createdAt: { gt: cursor }`, which drops both same-ms + // entries. Returning snap_1b here would be more correct than Postgres and would therefore + // read as divergence in compare mode. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual(["snap_2"]); + } finally { + await store.quit(); + } + }); + + redisTest("caps the window at the limit, keeping the newest", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_capped"; + await seed( + store, + runId, + Array.from({ length: 60 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(0), { limit: 50 }); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toHaveLength(50); + // The engine takes the NEWEST 50 and reverses, so the window ends at the newest entry. + expect(result.entries[result.entries.length - 1]!.id).toBe("snap_59"); + expect(result.entries[0]!.id).toBe("snap_10"); + } finally { + await store.quit(); + } + }); + + redisTest("scans no further than the answer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_deep_history"; + await seed( + store, + runId, + Array.from({ length: 400 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const started = Date.now(); + const result = await store.getSinceCreatedAt(runId, at(394), { limit: 50 }); + const elapsed = Date.now() - started; + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual([ + "snap_395", + "snap_396", + "snap_397", + "snap_398", + "snap_399", + ]); + // The walk stops at the cursor rather than reading the run's history. The bound is generous + // on purpose: it fails on a full scan of 400 entries, not on ordinary timing noise. + expect(elapsed).toBeLessThan(1_000); + } finally { + await store.quit(); + } + }); + + redisTest("scopes the window to an environment", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_env_scoped"; + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1", createdAt: at(1) }, + ]); + + const foreign = await store.getSinceCreatedAt(runId, at(0), { environmentId: "env_other" }); + + expect(foreign.kind).toBe("hit"); + if (foreign.kind !== "hit") return; + expect(foreign.entries).toEqual([]); + } finally { + await store.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts index 369d79e5338..a9db5a0790d 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.test.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts @@ -1,7 +1,7 @@ // Unit suite for the raw Redis execution-snapshot store. Redis-only: the store holds no Prisma // reference, so no Postgres container is needed. import { expect, describe, vi } from "vitest"; -import { redisTest } from "@internal/testcontainers"; +import { redisTest, slotOf } from "@internal/testcontainers"; import { createRedisClient } from "@internal/redis"; import { Logger } from "@trigger.dev/core/logger"; import { @@ -10,6 +10,8 @@ import { isValidFor, RedisSnapshotStore, type SnapshotEntryInput, + type CompletedWaitpointsPointer, + type CompletedWaitpointRecord, } from "./redisSnapshotStore.js"; describe("snapshotKeys", () => { @@ -236,6 +238,145 @@ describe("append", () => { } ); + redisTest( + "round-trips a typed records array through the cycle hash's records field", + async ({ redisOptions }) => { + // The only place CompletedWaitpointRecord[] physically enters Redis. If the writer ever + // serializes a different envelope, this is where that would show up as a broken round trip. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 }); + const raw = createRedisClient(redisOptions); + const records: CompletedWaitpointRecord[] = [ + { + id: "w_a", + friendlyId: "waitpoint_a", + type: "RUN", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { deriveFromRun: true }, + completedByTaskRunId: "run_child", + }, + ]; + try { + await store.append({ + entry: entry({ id: "snap_1" }), + kind: "birth", + isTerminal: false, + cycle: { + kind: "new", + completedWaitpoints: [{ id: "w_a", index: 0 }], + records, + }, + }); + + const storedRaw = await raw.hget("snap:{run_1}:wp:1", "records"); + expect(JSON.parse(storedRaw!)).toEqual(records); + } finally { + raw.disconnect(); + await store.quit(); + } + } + ); + + redisTest( + "a recordless new cycle clears another cycle's records off a reused key", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 }); + const raw = createRedisClient(redisOptions); + try { + await store.append({ + entry: entry({ id: "snap_1" }), + kind: "birth", + isTerminal: false, + cycle: { + kind: "new", + completedWaitpoints: [{ id: "w_a", index: 0 }], + records: [ + { + id: "w_a", + friendlyId: "waitpoint_a", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "stale" }, + }, + ], + }, + }); + expect(await raw.hget("snap:{run_1}:wp:1", "records")).not.toBeNull(); + + // Only the counter is lost, as under maxmemory eviction. A birth does not check seq, so + // the next new cycle re-mints cycleSeq 1 onto the surviving key. + await raw.del("snap:{run_1}:seq"); + + await store.append({ + entry: entry({ id: "snap_2" }), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_b", index: 0 }] }, + }); + + expect(await raw.hget("snap:{run_1}:wp:1", "order")).toBe(JSON.stringify(["w_b"])); + expect(await raw.hget("snap:{run_1}:wp:1", "records")).toBeNull(); + } finally { + raw.disconnect(); + await store.quit(); + } + } + ); + + redisTest( + "a carry-forward refuses a cycle this incarnation never minted", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + const raw = createRedisClient(redisOptions); + try { + await store.append({ + entry: entry({ id: "snap_1" }), + kind: "birth", + isTerminal: false, + cycle: { + kind: "new", + completedWaitpoints: [{ id: "w_old", index: 0 }], + records: [ + { + id: "w_old", + friendlyId: "waitpoint_old", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "stale" }, + }, + ], + }, + }); + + // Lose the whole keyspace except the cycle key, as under maxmemory eviction. + await raw.del("snap:{run_1}:e", "snap:{run_1}:idx", "snap:{run_1}:cur", "snap:{run_1}:seq"); + + const carried = await store.append({ + entry: entry({ id: "snap_2" }), + kind: "birth", + isTerminal: false, + cycle: { kind: "carryForward", cycleSeq: 1 }, + }); + + // Written, flagged, and carrying NO pointer: the dead incarnation's waitpoints must not + // be served to a fresh run under a count that agrees with them. + expect(carried).toMatchObject({ outcome: "written", cycleMismatch: true }); + expect(carried).not.toHaveProperty("cycleSeq"); + const read = await store.getLatest("run_1"); + expect(read?.cycle).toBeUndefined(); + expect(read?.completedWaitpointIds).toBeUndefined(); + } finally { + raw.disconnect(); + await store.quit(); + } + } + ); + redisTest( "reports a duplicate id without overwriting the original entry", async ({ redisOptions }) => { @@ -384,6 +525,129 @@ describe("cycle keys", () => { }); }); +describe("cycle key deletion sweep", () => { + const KEY_SUFFIXES = ["e", "idx", "cur", "seq", "wp:1", "wp:2"] as const; + const INJECTION_POINTS = ["beforeCarryForward", "beforeSecondNewCycle"] as const; + + function powerset(items: readonly T[]): T[][] { + let out: T[][] = [[]]; + for (const item of items) { + out = out.concat(out.map((s) => [...s, item])); + } + return out; + } + + // Mechanized replacement for the two hand-picked eviction regressions above: replays the same + // birth/carryForward/new-cycle/terminal shape once per element of the powerset of key deletions, + // at two points in the sequence, and checks that a cycle key's records never leak into a pointer + // naming a different cycle than the one the key's own order field currently describes. + redisTest( + "records read back for a pointer never mention an id outside that pointer's own order", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + const raw = createRedisClient(redisOptions); + type Violation = { + subset: string[]; + injectionPoint: string; + runId: string; + pointer: CompletedWaitpointsPointer; + recordsFound: unknown; + }; + const violations: Violation[] = []; + let replays = 0; + + try { + for (const injectionPoint of INJECTION_POINTS) { + for (const subset of powerset(KEY_SUFFIXES)) { + replays++; + const runId = `run_sweep_${injectionPoint}_${replays}`; + const base = `snap:{${runId}}`; + const damage = async () => { + if (subset.length > 0) { + await raw.del(...subset.map((s) => `${base}:${s}`)); + } + }; + + await store.append({ + entry: entry({ id: "snap_1", runId }), + kind: "birth", + isTerminal: false, + cycle: { + kind: "new", + completedWaitpoints: [{ id: "w_c1", index: 0 }], + records: [ + { + id: "w_c1", + friendlyId: "waitpoint_c1", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "payload_c1" }, + }, + ], + }, + }); + + if (injectionPoint === "beforeCarryForward") await damage(); + await store.append({ + entry: entry({ id: "snap_2", runId }), + kind: "transition", + isTerminal: false, + cycle: { kind: "carryForward", cycleSeq: 1 }, + }); + + if (injectionPoint === "beforeSecondNewCycle") await damage(); + // birth, not transition: both eviction regressions above only reproduce past a birth's + // liveness bypass -- a transition here would just report skippedNoKeyspace once e or seq + // is gone, exempting the replay before the cycle-mint branch ever ran. + await store.append({ + entry: entry({ id: "snap_3", runId }), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_c2", index: 0 }] }, + }); + + await store.append({ + entry: entry({ id: "snap_4", runId, executionStatus: "FINISHED" }), + kind: "transition", + isTerminal: true, + }); + + for (const id of ["snap_1", "snap_2", "snap_3", "snap_4"]) { + const read = await store.getById(runId, id); + if (!read?.cycle) continue; + const orderIds = new Set(read.completedWaitpointIds?.order ?? []); + const recordsRaw = await raw.hget(`${base}:wp:${read.cycle.cycleSeq}`, "records"); + if (recordsRaw === null) continue; + const recordIds = (JSON.parse(recordsRaw) as { id: string }[]).map((r) => r.id); + if (recordIds.some((rid) => !orderIds.has(rid))) { + violations.push({ + subset, + injectionPoint, + runId, + pointer: read.cycle, + recordsFound: recordIds, + }); + } + } + } + } + } finally { + raw.disconnect(); + await store.quit(); + } + + if (violations.length > 0) { + throw new Error( + `${violations.length} violation(s) across ${replays} replays. First: ` + + JSON.stringify(violations[0]) + ); + } + } + ); +}); + describe("read-side cycle mismatch", () => { redisTest( "warns and records a metric when a cycle's count disagrees with its order", @@ -1019,45 +1283,23 @@ describe("expectedCur compare-and-set", () => { ); }); -// CRC16/XMODEM over a key's hash tag, per Redis's cluster hashing rule. CLUSTER KEYSLOT is -// unavailable on this standalone container ("cluster support disabled"), so the slot is computed -// here instead. Verified against the `cluster-key-slot` package's output for our key shapes. -function crc16(str: string): number { - let crc = 0; - for (let i = 0; i < str.length; i++) { - crc ^= str.charCodeAt(i) << 8; - for (let j = 0; j < 8; j++) { - crc = crc & 0x8000 ? (crc << 1) ^ 0x1021 : crc << 1; - crc &= 0xffff; - } - } - return crc; -} - -function hashSlot(key: string): number { - const start = key.indexOf("{"); - const end = start === -1 ? -1 : key.indexOf("}", start + 1); - const tag = start !== -1 && end !== -1 && end > start + 1 ? key.slice(start + 1, end) : key; - return crc16(tag) % 16384; -} - describe("hash tag and keyPrefix", () => { it("every key for one run lands in one cluster slot", () => { // Keys come from snapshotKeys() plus the wp: suffix the Lua prelude derives the same way, // with a keyPrefix prepended by hand as ioredis would. A dropped hash tag would split the slots. - // Pin the helper itself before trusting it: the published XMODEM check value, and two known + // Pin the shared helper before trusting it: the published XMODEM check value, and two known // slots (one matching cluster-key-slot, one a different run's tag as a negative control -- // otherwise a constant-valued crc16 would satisfy slots.size === 1 for the wrong reason). - expect(crc16("123456789")).toBe(0x31c3); - expect(hashSlot("engine:snap:{run_1}:e")).toBe(8108); - expect(hashSlot("engine:snap:{run_2}:e")).toBe(12239); + expect(slotOf("123456789")).toBe(0x31c3); + expect(slotOf("engine:snap:{run_1}:e")).toBe(8108); + expect(slotOf("engine:snap:{run_2}:e")).toBe(12239); const k = snapshotKeys("run_1"); const base = k.e.slice(0, -2); const keys = [k.e, k.idx, k.cur, k.seq, `${base}:wp:1`, `${base}:wp:2`].map( (key) => `engine:${key}` ); - const slots = new Set(keys.map(hashSlot)); + const slots = new Set(keys.map(slotOf)); expect(slots.size).toBe(1); }); @@ -1113,6 +1355,53 @@ describe("hash tag and keyPrefix", () => { }); describe("observability", () => { + redisTest("cycle-key bytes cover records, not just order", async ({ redisOptions }) => { + // The plan puts a metric on the wp: KEY size. records dominates that key once + // populated, so measuring order alone understates it by orders of magnitude. + const calls: Array<[string, number]> = []; + const metrics = { + recordAppend: () => {}, + recordEntryBytes: () => {}, + recordCycleKeyBytes: (b: number) => calls.push(["cycleBytes", b]), + recordCycleCount: () => {}, + recordSkippedNoKeyspace: () => {}, + recordCycleMismatch: () => {}, + recordLatency: () => {}, + }; + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000, metrics }); + try { + const records: CompletedWaitpointRecord[] = [ + { + id: "w_a", + friendlyId: "waitpoint_a", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "y".repeat(20_000) }, + }, + ]; + const orderJson = JSON.stringify(["w_a"]); + const recordsJson = JSON.stringify(records); + + await store.append({ + entry: entry({ id: "snap_1" }), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }], records }, + }); + + expect(calls).toEqual([ + [ + "cycleBytes", + Buffer.byteLength(orderJson, "utf8") + Buffer.byteLength(recordsJson, "utf8"), + ], + ]); + } finally { + await store.quit(); + } + }); + redisTest("records sizes and outcomes without ever rejecting", async ({ redisOptions }) => { const calls: unknown[][] = []; const metrics = { @@ -1224,3 +1513,40 @@ describe("observability", () => { } ); }); + +describe("the reserved completedWaitpoints field", () => { + redisTest("append rejects an entry that sets it", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + try { + const runId = "run_reserved_throw"; + const pointer: CompletedWaitpointsPointer = { cycleSeq: 1, count: 0 }; + await expect( + store.append({ + entry: { ...entry({ id: "snap_1", runId }), completedWaitpoints: pointer }, + kind: "birth", + isTerminal: false, + }) + ).rejects.toThrow(/reserved/i); + } finally { + await store.quit(); + } + }); + + redisTest("a stored entry never holds the key", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + try { + const runId = "run_reserved_absent"; + await store.append({ + entry: entry({ id: "snap_1", runId }), + kind: "birth", + isTerminal: false, + }); + const read = await store.getLatest(runId); + expect(read).not.toBeNull(); + expect(read!.raw).not.toContain("completedWaitpoints"); + expect(read!.entry).not.toHaveProperty("completedWaitpoints"); + } finally { + await store.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 7b60843e2b4..2339ab0dd5e 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -1,11 +1,12 @@ import { createRedisClient, type Callback, - type Redis, + type RedisClient, type RedisOptions, type Result, } from "@internal/redis"; import { Logger } from "@trigger.dev/core/logger"; +import type { CompletedWaitpoint } from "@trigger.dev/core/v3/schemas"; export type SnapshotKeys = { e: string; idx: string; cur: string; seq: string }; @@ -28,11 +29,106 @@ export function deriveOrder(completedWaitpoints: CompletedWaitpointRef[]): strin .map((w) => w.id); } +/** + * The COMPLETE distinct set of completed-waitpoint ids, including those with no batch index. + * + * This is deliberately not `deriveOrder` deduped. `order` is the index oracle and carries only + * batch-indexed ids, because its positions ARE the indexes. A wait with no batch index (every + * `wait.for`, every single `triggerAndWait`, every token) has no position and is absent from it, + * while Postgres records it in the completed-waitpoint join like any other. Reading the id set back + * from `order` therefore loses exactly those waits, and a run resumed from Redis loses their results. + */ +export function deriveDistinctIds(completedWaitpoints: CompletedWaitpointRef[]): string[] { + return [...new Set(completedWaitpoints.map((w) => w.id))]; +} + // isValid is derived, never stored, so the entry JSON stays byte-identical to the caller's document. export function isValidFor(entry: { error?: unknown }): boolean { return !entry.error; } +// --------------------------------------------------------------------------- +// The completed-waitpoints freeze. Frozen jointly with the waitpoint lane. +// Do not change a field here without re-agreeing the contract with that lane. +// --------------------------------------------------------------------------- + +/** + * The once-per-wait-cycle pointer. `cycleSeq` names the snap:{runId}:wp: + * key. `count` is order.length -- NOT the record count -- so it is zero for any + * wait that carries no batch index. + */ +export type CompletedWaitpointsPointer = { + cycleSeq: number; + count: number; +}; + +/** + * A record's output. + * - `inline` holds the literal value. MANUAL and DATETIME are bounded by the offload + * thresholds; error outputs are not (only BUILT_IN_ERROR truncates), so the bound is + * the completion body limit. Postgres holds the same strings, so this is a copy. + * - `ref` holds an application/store reference that was already offloaded. + * - `deriveFromRun` means the resolver reads TaskRun.output for completedByTaskRunId. + * Only a RUN record with outputIsError false AND a non-null completedByTaskRunId uses + * it: TaskRun.output is a String column holding the same string verbatim, so the + * re-read is byte-identical. A RUN error cannot use it, because TaskRun.error is + * jsonb and never round-trips. Waitpoint.completedByTaskRun is onDelete: SetNull, so + * an orphaned RUN waitpoint (the completing run row was deleted) has no run left to + * derive from -- its output carries inline instead. + */ +export type CompletedWaitpointRecordOutput = + | { inline: string } + | { ref: string } + | { deriveFromRun: true } + | null; + +/** + * One completed waitpoint, one per DISTINCT id in a wait cycle. The resolver expands + * this into one CompletedWaitpoint per position of the id in the cycle's order list. + */ +export type CompletedWaitpointRecord = { + id: string; + friendlyId: string; + type: "RUN" | "BATCH" | "DATETIME" | "MANUAL"; + /** ISO. The writer pins it, applying the null fallback once. */ + completedAt: string; + /** Defaults to "application/json" at source. */ + outputType: string; + outputIsError: boolean; + output: CompletedWaitpointRecordOutput; + /** RUN. The resolver derives friendlyId, and batch{} from the READING entry's batchId. */ + completedByTaskRunId?: string; + /** BATCH. The resolver derives friendlyId. */ + completedByBatchId?: string; + /** ISO. Any type may set it: a MANUAL waitpoint with a timeout does. */ + completedAfter?: string; + /** Already resolved: userProvidedIdempotencyKey && !inactiveIdempotencyKey. */ + idempotencyKey?: string; +}; + +/** + * What the store hands the resolver. The store owns the keyspace, so the store reads + * and parses the cycle hash. The resolver never touches Redis and never derives a key. + */ +export type ResolveCompletedWaitpointsArgs = { + runId: string; + /** The batchId of the entry being READ, never the entry that minted the cycle. */ + batchId?: string; + pointer: CompletedWaitpointsPointer; + /** Index oracle only. A SUBSET of the record ids. Repeats preserved. */ + order: string[]; + /** The authoritative, complete set. Iterate this, never `order`. */ + records: CompletedWaitpointRecord[]; +}; + +/** + * This lane owns the signature. The waitpoint lane owns the implementation, which + * lives in run-engine because a deriveFromRun record needs a Postgres read. + */ +export type CompletedWaitpointResolver = ( + args: ResolveCompletedWaitpointsArgs +) => Promise; + export type SnapshotEntryInput = { id: string; engine: "V2"; @@ -53,6 +149,16 @@ export type SnapshotEntryInput = { runnerId?: string; metadata?: unknown; error?: string; + /** + * RESERVED. Always unset. `append` rejects a set value. + * + * The pointer's physical form is the `#c` sidecar field on the `e` hash, + * because the append Lua mints both halves after the client serializes the entry. + * The entry JSON must stay byte-identical to the caller's document, and the Postgres + * snapshot row has no pointer column, so a pointer inside the JSON would stop the two + * documents from being comparable for the dual-write comparator. + */ + completedWaitpoints?: CompletedWaitpointsPointer; }; export type WaitpointIds = { present: boolean; distinctIds: string[]; order: string[] }; @@ -67,8 +173,14 @@ export type SnapshotRead = { isValid: boolean; entry: Record; raw: string; - cycle?: { cycleSeq: number; count: number }; + cycle?: CompletedWaitpointsPointer; completedWaitpointIds?: WaitpointIds; + /** + * The entry points at a cycle key that no longer exists, so its waitpoints are unreachable rather + * than absent. A caller must not treat this as an empty set: it has to fall back to Postgres, + * which still holds the join rows. + */ + danglingCycle?: boolean; }; export type AppendResult = @@ -93,8 +205,20 @@ export type SnapshotStoreMetrics = { recordLatency(op: string, ms: number): void; }; -export type RedisSnapshotStoreOptions = { - redisOptions: RedisOptions; +/** + * How the store reaches Redis. Exactly one of the two, enforced by the type rather than a runtime + * check: `never` on the opposite member makes both "neither" and "both" a compile error. + * + * `client` exists because production points at a Valkey/Redis CLUSTER, and cluster topology is not + * this package's business. Every command the store issues is key-addressed and every key carries a + * `{runId}` hashtag, so one slot serves a whole run and both endpoint shapes behave identically. + * A caller-supplied client is owned by the caller: `quit()` leaves it open. + */ +export type RedisSnapshotStoreConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type RedisSnapshotStoreOptions = RedisSnapshotStoreConnection & { completedTtlMs: number; sinceLimit?: number; highWater?: { entryBytes?: number; cycleKeyBytes?: number; cycleCount?: number }; @@ -102,13 +226,25 @@ export type RedisSnapshotStoreOptions = { logger?: Logger; }; +/** + * Both window scripts return four leading slots before the first row: the id-cursor variant's + * `sinceRaw`, the head's order, the head's distinct set, and the head's dangling flag. Rows follow + * in four-element groups, so the head row is the group at this offset. + * + * Named because the offset drifted out of the comments describing it twice, and the second drift + * arrived in the change that fixed the first. + */ +const WINDOW_HEAD_ROW_INDEX = 4; + const SKIPPED = "skipped"; const FORKED = "forked"; const WRITTEN = "written"; const DUPLICATE = "duplicate"; export class RedisSnapshotStore { - private readonly redis: Redis; + private readonly redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + private readonly ownsClient: boolean; private readonly logger: Logger; private readonly completedTtlMs: number; private readonly sinceLimit: number; @@ -122,15 +258,22 @@ export class RedisSnapshotStore { this.sinceLimit = options.sinceLimit ?? 50; this.metrics = options.metrics; this.highWater = options.highWater ?? {}; - this.redis = createRedisClient(options.redisOptions, { - onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), - }); + this.ownsClient = options.client === undefined; + this.redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), + }); this.#registerCommands(); } async quit(): Promise { // Idempotent and error-swallowing: every test calls this in a `finally`, and a double quit() // (or one after a failed connect) must never mask the real assertion failure. + // + // An injected client is the caller's. Closing it here would take down a connection shared with + // the sweeper or with another component, so a borrowed client is left open. + if (!this.ownsClient) return; if (!this.#quit) { this.#quit = this.redis.quit().then( () => undefined, @@ -155,9 +298,32 @@ export class RedisSnapshotStore { isTerminal: boolean; expectedCur?: string; cycle?: - | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[]; records?: string } - | { kind: "carryForward"; cycleSeq: number }; + | { + kind: "new"; + completedWaitpoints: CompletedWaitpointRef[]; + records?: CompletedWaitpointRecord[]; + } + | { + kind: "carryForward"; + cycleSeq: number; + /** + * The same refs a `new` cycle would carry. A carry the store refuses falls back to + * minting inside the same call, and it cannot do that without them: with no refs there is + * nothing to mint from, so the entry is written with no pointer, as before. + * + * Every production caller supplies them. Omitting them gives up the fallback. + */ + completedWaitpoints?: CompletedWaitpointRef[]; + records?: CompletedWaitpointRecord[]; + }; }): Promise { + if (args.entry.completedWaitpoints !== undefined) { + throw new Error( + "completedWaitpoints is a reserved entry field and must stay unset. The pointer's " + + "physical form is the `#c` sidecar field, which the append script mints. " + + "Writing it into the entry JSON breaks byte-comparability with the Postgres row." + ); + } return this.#timed("append", async () => { const k = snapshotKeys(args.entry.runId); const raw = JSON.stringify(args.entry); @@ -166,17 +332,29 @@ export class RedisSnapshotStore { let cycleMode = "none"; let cycleSeqIn = "0"; let orderJson = ""; + let distinctJson = ""; let records = ""; let orderCount = "0"; if (args.cycle?.kind === "new") { const order = deriveOrder(args.cycle.completedWaitpoints); cycleMode = "new"; orderJson = JSON.stringify(order); - records = args.cycle.records ?? ""; + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); + records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; orderCount = String(order.length); } else if (args.cycle?.kind === "carryForward") { cycleMode = "carry"; cycleSeqIn = String(args.cycle.cycleSeq); + + // Carried for the refusal path only. The script uses these solely when it declines the + // pointer and mints a replacement, and can only do that when the caller supplied them. + if (args.cycle.completedWaitpoints) { + const order = deriveOrder(args.cycle.completedWaitpoints); + orderJson = JSON.stringify(order); + records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; + orderCount = String(order.length); + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); + } } const reply = (await this.redis.appendSnapshotEntry( @@ -196,14 +374,21 @@ export class RedisSnapshotStore { records, orderCount, args.expectedCur ?? "", - args.expectedCur !== undefined ? "1" : "0" + args.expectedCur !== undefined ? "1" : "0", + distinctJson )) as string[]; - return this.#interpretAppend(reply, raw, orderJson, args.entry.runId); + return this.#interpretAppend(reply, raw, orderJson, records, args.entry.runId); }); } - #interpretAppend(reply: string[], raw: string, orderJson: string, runId: string): AppendResult { + #interpretAppend( + reply: string[], + raw: string, + orderJson: string, + records: string, + runId: string + ): AppendResult { if (reply[0] === SKIPPED) { this.metrics?.recordSkippedNoKeyspace(); this.metrics?.recordAppend("skippedNoKeyspace", "none"); @@ -224,7 +409,7 @@ export class RedisSnapshotStore { if (cycleMismatch) { this.metrics?.recordCycleMismatch(); } - this.#observeSizes(raw, orderJson, cycleSeq, runId); + this.#observeSizes(raw, orderJson, records, cycleSeq, runId); this.metrics?.recordAppend("written", ttl); return { outcome: "written", @@ -235,14 +420,21 @@ export class RedisSnapshotStore { }; } - #observeSizes(raw: string, orderJson: string, cycleSeq: number, runId: string): void { + #observeSizes( + raw: string, + orderJson: string, + records: string, + cycleSeq: number, + runId: string + ): void { const entryBytes = Buffer.byteLength(raw, "utf8"); this.metrics?.recordEntryBytes(entryBytes); if (this.highWater.entryBytes !== undefined && entryBytes > this.highWater.entryBytes) { this.logger.warn("RedisSnapshotStore entry above high-water mark", { runId, entryBytes }); } if (orderJson !== "") { - const cycleBytes = Buffer.byteLength(orderJson, "utf8"); + // The whole wp: key, not just its order field: records dominates it once populated. + const cycleBytes = Buffer.byteLength(orderJson, "utf8") + Buffer.byteLength(records, "utf8"); this.metrics?.recordCycleKeyBytes(cycleBytes); if (this.highWater.cycleKeyBytes !== undefined && cycleBytes > this.highWater.cycleKeyBytes) { this.logger.warn("RedisSnapshotStore cycle key above high-water mark", { @@ -290,7 +482,17 @@ export class RedisSnapshotStore { return this.#timed("getSnapshotWaitpointIds", async () => { const k = snapshotKeys(runId); const reply = await this.redis.readSnapshotWaitpointIds(k.e, k.idx, k.cur, k.seq, snapshotId); - return decodeWaitpointIds(reply[0] === "1", reply[1] ?? ""); + // A dangling pointer means this entry's waitpoints are unreachable, not absent. Reporting + // `present: false` is what sends the caller to Postgres, which still holds the join rows. + if (reply[3] === "1") { + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore snapshot points at a cycle key that is gone", { + runId, + snapshotId, + }); + return { present: false, distinctIds: [], order: [] }; + } + return decodeWaitpointIds(reply[0] === "1", reply[1] ?? "", reply[2] ?? ""); }); } @@ -324,11 +526,13 @@ export class RedisSnapshotStore { } const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; const rows: SnapshotRead[] = []; - // Tracks whether the Lua-chosen head row (always the first, i === 2) itself survives the + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the // env filter below -- headOrder must never be attributed to a different, surviving row. let headSurvived = false; - for (let i = 2; i + 3 < reply.length; i += 4) { + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { // orderKnown is false here: headOrder covers only the head row, resolved separately below. const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], @@ -338,15 +542,97 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 2) headSurvived = true; + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; } } rows.reverse(); const head = headSurvived ? rows[rows.length - 1] : undefined; - const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); if (head) { head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } + if (head.cycle) { + this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); + } + } + return { kind: "hit", entries: rows, headWaitpointIds }; + }); + } + + /** + * The same window as {@link getSince}, addressed by a createdAt cursor instead of a snapshot id. + * + * `getExecutionSnapshotsSince` resolves its cursor to a createdAt before it asks for the window, + * so the snapshot id is gone by the time this call is made and `getSince` cannot serve it. The + * cursor is exclusive and keeps Postgres's same-millisecond blind spot, so the two reads agree. + */ + async getSinceCreatedAt( + runId: string, + createdAt: Date | string, + opts?: { environmentId?: string; limit?: number } + ): Promise { + return this.#timed("getSinceCreatedAt", async () => { + const k = snapshotKeys(runId); + const limit = opts?.limit ?? this.sinceLimit; + const cursor = typeof createdAt === "string" ? createdAt : createdAt.toISOString(); + + const reply = await this.redis.readSnapshotsSinceCreatedAt( + k.e, + k.idx, + k.cur, + k.seq, + cursor, + String(limit) + ); + if (reply === null) return { kind: "miss" }; + + const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; + const rows: SnapshotRead[] = []; + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the env filter, + // so headOrder is never attributed to a different, surviving row. + let headSurvived = false; + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { + const decoded = this.#decode( + [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], + opts?.environmentId, + runId, + false + ); + if (decoded) { + rows.push(decoded); + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; + } + } + + rows.reverse(); + const head = headSurvived ? rows[rows.length - 1] : undefined; + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); + if (head) { + head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } if (head.cycle) { this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); } @@ -377,7 +663,7 @@ export class RedisSnapshotStore { orderKnown: boolean ): SnapshotRead | null { if (!reply || reply.length === 0) return null; - const [id, raw, seqStr, pointer, orderJson] = reply; + const [id, raw, seqStr, pointer, orderJson, distinctJson, dangling] = reply; const entry = JSON.parse(raw) as Record; if (environmentId !== undefined && entry.environmentId !== environmentId) return null; const read: SnapshotRead = { @@ -390,8 +676,16 @@ export class RedisSnapshotStore { if (pointer) { const [cs, count] = pointer.split(":"); read.cycle = { cycleSeq: Number(cs), count: Number(count) }; + if (dangling === "1") { + read.danglingCycle = true; + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore entry points at a cycle key that is gone", { + runId, + snapshotId: id, + }); + } if (orderKnown) { - const ids = decodeWaitpointIds(true, orderJson); + const ids = decodeWaitpointIds(true, orderJson, distinctJson ?? ""); read.completedWaitpointIds = ids; this.#checkCycleMismatch(runId, Number(count), ids.order.length); } @@ -413,6 +707,24 @@ export class RedisSnapshotStore { if not cs then return '' end return redis.call('HGET', wpKey(cs), 'order') or '' end + -- A pointer whose cycle key is GONE. Not the same as having no pointer: this entry should + -- have waitpoints and cannot produce them, so a read must refuse rather than answer empty. + -- Reachable by eviction, and by the completion TTL, which is applied to every key for a run + -- at the same moment but lets them expire independently. + local function danglingFor(pointer) + if not pointer then return '0' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '0' end + if redis.call('EXISTS', wpKey(cs)) == 0 then return '1' end + return '0' + end + -- The complete id set, which is NOT the order deduped: order holds only batch-indexed ids. + local function distinctFor(pointer) + if not pointer then return '' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '' end + return redis.call('HGET', wpKey(cs), 'distinct') or '' + end `; this.redis.defineCommand("appendSnapshotEntry", { @@ -432,6 +744,9 @@ export class RedisSnapshotStore { local orderCount = ARGV[11] local expectedCur = ARGV[12] local casEnabled = ARGV[13] == '1' + -- The COMPLETE distinct id set. Not the order deduped: order omits every id with no batch + -- index, and those ids still have to come back on a read. + local distinctJson = ARGV[14] -- Liveness is TWO anchors: e and seq. All keys get the same PEXPIRE but expire independently -- (or seq can vanish under maxmemory eviction while e survives), so checking e alone lets a @@ -462,20 +777,45 @@ export class RedisSnapshotStore { local cycleSeq = 0 local mismatch = 0 - if cycleMode == 'new' then - -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal - -- PEXPIRE loop from 1..c is correct. - cycleSeq = redis.call('HINCRBY', seqKey, 'c', 1) - redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount) + + -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal + -- PEXPIRE loop from 1..c is correct. + local function mintCycle() + local minted = redis.call('HINCRBY', seqKey, 'c', 1) + redis.call('HSET', wpKey(minted), 'order', orderJson, 'count', orderCount, 'distinct', distinctJson) if records ~= '' then - redis.call('HSET', wpKey(cycleSeq), 'records', records) + redis.call('HSET', wpKey(minted), 'records', records) + else + -- A new cycle owns the whole key: a lost seq counter can re-mint a cycleSeq whose key + -- still holds another cycle's records, and order/count stay mutually consistent so the + -- mismatch check cannot see it. No-op on a fresh key. + redis.call('HDEL', wpKey(minted), 'records') end + return minted + end + + if cycleMode == 'new' then + cycleSeq = mintCycle() elseif cycleMode == 'carry' then - cycleSeq = cycleSeqIn - local c = redis.call('HGET', wpKey(cycleSeq), 'count') - if not c then + -- Attach the CARRIED pointer only if this incarnation actually minted that cycle. seq can + -- be evicted while a wp: key survives, so a bare key-exists check would adopt a dead + -- incarnation's order and records under a consistent count, invisibly. + local minted = tonumber(redis.call('HGET', seqKey, 'c') or '0') + local c = redis.call('HGET', wpKey(cycleSeqIn), 'count') + if not c or minted < cycleSeqIn then + -- Refusing the pointer is right. Writing the entry WITHOUT one is not: it becomes the + -- head with no waitpoints, and a read of it answers present-with-nothing, which is the + -- one answer that tells the engine's repair it need not look. Mint a fresh cycle from + -- the refs the caller carried, in this same atomic call, so the entry always has a + -- pointer that can be trusted. The mismatch is still reported, for the metric. mismatch = 1 + -- Only possible when the caller carried the refs. With none there is nothing to mint + -- from, and the entry is written with no pointer, which is the older behaviour. + if distinctJson ~= '' then + cycleSeq = mintCycle() + end else + cycleSeq = cycleSeqIn orderCount = c end end @@ -527,7 +867,7 @@ export class RedisSnapshotStore { local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') if not vals[1] then return nil end -- Coerce every element: a Lua false TRUNCATES the returned array at that position. - return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -539,7 +879,7 @@ export class RedisSnapshotStore { if not cur then return nil end local vals = redis.call('HMGET', eKey, cur, cur .. '#s', cur .. '#c') if not vals[1] then return nil end - return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -552,7 +892,78 @@ export class RedisSnapshotStore { return { '0', '' } end local pointer = redis.call('HGET', eKey, id .. '#c') - return { '1', orderFor(pointer) } + return { '1', orderFor(pointer), distinctFor(pointer), danglingFor(pointer) } + `, + }); + + this.redis.defineCommand("readSnapshotsSinceCreatedAt", { + numberOfKeys: 4, + lua: ` + ${PRELUDE} + local cursor = ARGV[1] + local limit = tonumber(ARGV[2]) + + -- A run with no keyspace is a MISS, so the caller falls back to Postgres. A run that has one + -- and nothing newer is an empty HIT, so it does not fall back for a window it owns. + -- + -- Both anchors, for the reason the append script gives: keys expire independently, and an + -- index lost to eviction while the entry hash survives would otherwise report an empty HIT + -- on every poll for the rest of the run's life, with Postgres holding the transitions. + if redis.call('EXISTS', eKey) == 0 or redis.call('EXISTS', idxKey) == 0 then return nil end + + -- STRICTLY greater than the cursor, and same-millisecond entries are dropped. Postgres + -- serves this window with createdAt > cursor and drops them too; a Redis read that is more + -- correct than the Postgres read shows up as divergence in compare mode. + -- + -- createdAt is always toISOString() output, one fixed-width UTC format, so a lexicographic + -- compare is a chronological compare. Walking newest-first lets the scan stop at the first + -- entry at or before the cursor, which makes its length the length of the ANSWER rather + -- than the length of the run's history. + local out = { '', '', '', '' } + local headId = nil + local offset = 0 + local page = limit + local done = false + + while not done do + local ids = redis.call('ZREVRANGE', idxKey, offset, offset + page - 1) + if #ids == 0 then break end + + for i = 1, #ids do + local id = ids[i] + local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') + if vals[1] then + local createdAt = cjson.decode(vals[1])['createdAt'] + if not createdAt or createdAt <= cursor then + done = true + break + end + if not headId then headId = id end + out[#out + 1] = id + out[#out + 1] = vals[1] + out[#out + 1] = vals[2] or '' + out[#out + 1] = vals[3] or '' + if (#out - 2) / 4 >= limit then + done = true + break + end + end + end + + offset = offset + page + end + + if headId then + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) + end + return out `, }); @@ -582,7 +993,7 @@ export class RedisSnapshotStore { -- The head is the newest SURVIVING entry, and it is the only one whose cycle key is read. -- Deriving the order after the loop keeps it paired with the row it is attached to: a row -- dropped for a missing body must not donate its cycle data to the next one. - local out = { sinceRaw, '' } + local out = { sinceRaw, '', '', '' } local headId = nil for i = 1, #ids do local id = ids[i] @@ -596,7 +1007,14 @@ export class RedisSnapshotStore { end end if headId then - out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) end return out `, @@ -604,9 +1022,26 @@ export class RedisSnapshotStore { } } -export function decodeWaitpointIds(present: boolean, orderJson: string): WaitpointIds { +export function decodeWaitpointIds( + present: boolean, + orderJson: string, + distinctJson = "" +): WaitpointIds { const order: string[] = orderJson === "" ? [] : (JSON.parse(orderJson) as string[]); - return { present, distinctIds: [...new Set(order)], order }; + + // The complete set is stored separately, because `order` omits every id with no batch index, so + // deduping the order to recover it silently drops every wait that has none. + // + // A cycle key always holds both fields, written by one command, so a missing `distinct` beside a + // NON-EMPTY `order` means the invariant is broken. Reconstructing from the order there would be + // the same lossy shortcut this field exists to remove, and the loss would be silent. Report the + // entry as not present instead, which sends the caller to Postgres. + if (distinctJson === "" && order.length > 0) { + return { present: false, distinctIds: [], order: [] }; + } + + const distinctIds: string[] = distinctJson === "" ? [] : (JSON.parse(distinctJson) as string[]); + return { present, distinctIds, order }; } declare module "@internal/redis" { @@ -629,6 +1064,7 @@ declare module "@internal/redis" { orderCount: string, expectedCur: string, casEnabled: string, + distinctJson: string, callback?: Callback ): Result; readSnapshotById( @@ -654,6 +1090,15 @@ declare module "@internal/redis" { id: string, callback?: Callback ): Result; + readSnapshotsSinceCreatedAt( + eKey: string, + idxKey: string, + curKey: string, + seqKey: string, + createdAtCursor: string, + limit: string, + callback?: Callback + ): Result; readSnapshotsSince( eKey: string, idxKey: string, diff --git a/internal-packages/run-store/src/routingStoreMetrics.ts b/internal-packages/run-store/src/routingStoreMetrics.ts new file mode 100644 index 00000000000..d4aa454d903 --- /dev/null +++ b/internal-packages/run-store/src/routingStoreMetrics.ts @@ -0,0 +1,16 @@ +/** + * Counters the routing store emits. Injected the same way RedisSnapshotStore takes its metrics, + * so the package stays free of a metrics dependency and a test can assert on a fake. + * + * runops_shard_duplicate_id_total — one id returned by two shards that should be disjoint + * runops_waitpoint_probe_fallback_total — a waitpoint was not on the store its id named + */ +export type RoutingStoreMetrics = { + recordDuplicateId(shardKeys: string[]): void; + recordWaitpointProbeFallback(from: string, to: string): void; +}; + +export const noopRoutingStoreMetrics: RoutingStoreMetrics = { + recordDuplicateId() {}, + recordWaitpointProbeFallback() {}, +}; diff --git a/internal-packages/run-store/src/runOpsStore.nShardMatrix.test.ts b/internal-packages/run-store/src/runOpsStore.nShardMatrix.test.ts new file mode 100644 index 00000000000..3f42e36171a --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.nShardMatrix.test.ts @@ -0,0 +1,305 @@ +// FOUR-STORE MATRIX — proves RoutingRunStore is correct across legacy + new + two gen-2 shards +// (a, b) against REAL databases (makeNShardRunOpsPostgresTest). NEVER mocked. This is where the +// §3.4 disjoint-sum fix is proven end-to-end: a double count here strands a blocked run forever. +// +// runOpsStore.mixedResidency.test.ts is the TWO-store invariant lock and stays byte-identical; this +// file is the N-store extension and lives separately. + +import { makeNShardRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { resolveShard } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +const matrixTest = makeNShardRunOpsPostgresTest(2); + +// A gen-2 id: 24-char base32hex core, the shard char at index 24, version "2" at index 25. +// resolveShard(gen2("a", ...)) === "a". A bare cuid-length id classifies "legacy". +function gen2(shardChar: string, seed: string): string { + const core = (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24); + return `${core}${shardChar}2`; +} +function cuid(seed: string): string { + return (seed + "c".repeat(25)).slice(0, 25); // 25 chars → LEGACY +} + +function makeStore(prisma: AnyClient, variant: "legacy" | "dedicated") { + return new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + schemaVariant: variant, + }); +} + +// The real four-store split: legacy (full schema) + new + gen-2 a + gen-2 b (dedicated subset), +// routed by the REAL core resolveShard. +function makeMatrixRouter( + legacyPrisma: PrismaClient, + newPrisma: RunOpsPrismaClient, + shardPrismas: RunOpsPrismaClient[] +) { + return new RoutingRunStore({ + new: makeStore(newPrisma, "dedicated"), + legacy: makeStore(legacyPrisma, "legacy"), + shards: [ + { key: "a", store: makeStore(shardPrismas[0]!, "dedicated") }, + { key: "b", store: makeStore(shardPrismas[1]!, "dedicated") }, + ], + resolveShard, + }); +} + +async function seedLegacyEnv(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { + organizationId: organization.id, + projectId: project.id, + runtimeEnvironmentId: environment.id, + environmentId: environment.id, + }; +} + +function buildRun(params: { + runId: string; + runtimeEnvironmentId: string; + organizationId: string; + projectId: string; + createdAt?: Date; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${params.runId}`, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: params.createdAt ?? new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +async function seedPendingWaitpoint( + prisma: AnyClient, + params: { id: string; projectId: string; environmentId: string } +) { + await (prisma as PrismaClient).waitpoint.create({ + data: { + id: params.id, + friendlyId: `wp_${params.id}`, + type: "MANUAL", + status: "PENDING", + idempotencyKey: `idem_${params.id}`, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + }, + }); +} + +describe("RoutingRunStore four-store matrix — disjoint sum on real databases", () => { + matrixTest( + "countPendingWaitpoints unions across a gen-2 shard and the gen-1 pair with no double count", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "disjoint"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + + // The blocked run lives on shard a. + const runId = gen2("a", "run"); + // Its blocking waitpoints: a gen-2 waitpoint on shard b, a cuid on legacy, and a cuid MIRRORED + // onto both gen-1 stores (the drain-mirror case that must count once). + const wpB = gen2("b", "wpb"); + const wpCuid = cuid("wpcuid"); + const wpMirror = cuid("wpmirror"); + + const dedicatedEnv = { projectId: env.projectId, environmentId: env.environmentId }; + await seedPendingWaitpoint(shardPrismas[1]!, { id: wpB, ...dedicatedEnv }); + await seedPendingWaitpoint(legacyPrisma, { id: wpCuid, ...dedicatedEnv }); + await seedPendingWaitpoint(legacyPrisma, { id: wpMirror, ...dedicatedEnv }); + await seedPendingWaitpoint(newPrisma, { id: wpMirror, ...dedicatedEnv }); + + // b:wpB (1) + legacy:wpCuid (1) + wpMirror (once, though on both gen-1 stores) = 3. + const count = await router.countPendingWaitpoints([wpB, wpCuid, wpMirror], undefined, runId); + expect(count).toBe(3); + } + ); + + matrixTest( + "a gen-2 waitpoint on the run's own shard contributes exactly once, not twice", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "ownshard"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + const runId = gen2("a", "run2"); + const wpA = gen2("a", "wpa"); + await seedPendingWaitpoint(shardPrismas[0]!, { + id: wpA, + projectId: env.projectId, + environmentId: env.environmentId, + }); + // wpA lives on the run's own shard a → found by the presence query, never re-queried elsewhere. + expect(await router.countPendingWaitpoints([wpA], undefined, runId)).toBe(1); + } + ); +}); + +describe("RoutingRunStore four-store matrix — alias topology", () => { + matrixTest( + "an aliased gen-2 shard counts its database ONCE in a sum (declaration, not identity)", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "alias"); + // Shard "a" aliases "new" over the SAME database, but via a SEPARATE store object built over + // the same client — exactly how the wiring layer will construct it. Identity dedupe would see + // two objects and double-count; declaration dedupe counts the database once. + const newStore = makeStore(newPrisma, "dedicated"); + const aStoreSameDb = makeStore(newPrisma, "dedicated"); // distinct object, same DB + const router = new RoutingRunStore({ + new: newStore, + legacy: makeStore(legacyPrisma, "legacy"), + shards: [{ key: "a", store: aStoreSameDb, aliasOf: "new" }], + resolveShard, + }); + + const wp = cuid("aliaswp"); + await seedPendingWaitpoint(newPrisma, { + id: wp, + projectId: env.projectId, + environmentId: env.environmentId, + }); + // No runId → the id-less sum fans over DISTINCT databases. The aliased "a" must not add a + // second leg over the "new" database, or the one pending waitpoint counts twice. + expect(await router.countPendingWaitpoints([wp])).toBe(1); + } + ); +}); + +describe("RoutingRunStore four-store matrix — mixed gen-1 and gen-2 reads", () => { + matrixTest( + "findRunsByIds hydrates a mixed id set across legacy, new and both gen-2 shards", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "mixed"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + + const legacyId = cuid("mixleg"); + // A v1 run-ops id (version "1") routes to "new"; gen-2 ids route to their shard char. + const newId = ("mixnew".replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01"; + const aId = gen2("a", "mixa"); + const bId = gen2("b", "mixb"); + const all = [legacyId, newId, aId, bId]; + + for (const runId of all) { + await router.createRun(buildRun({ runId, ...env })); + } + + const found = await router.findRunsByIds(all, { select: { id: true } }); + expect(new Set([...found.keys()])).toEqual(new Set(all)); + } + ); +}); + +describe("RoutingRunStore four-store matrix — cross-tree completion across gen-2 shards", () => { + matrixTest( + "a gen-2 waitpoint completes on its own shard even under the cross-tree legacy pin", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "crosstree"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + // The waitpoint is owned by a run on shard b; the blocked run is on shard a (cross-tree). + const wpB = gen2("b", "ctwp"); + await seedPendingWaitpoint(shardPrismas[1]!, { + id: wpB, + projectId: env.projectId, + environmentId: env.environmentId, + }); + // isCrossTreeIdempotency pins gen-1 flows to legacy; a gen-2 id must OVERRIDE that pin, or the + // completion write lands on legacy, matches zero rows, and strands the run. + const store = await router.forWaitpointCompletion(wpB, { + isCrossTreeIdempotency: true, + } as never); + // The returned store finds wpB on its primary — only shard b holds it, so the override worked. + const found = await store.findWaitpoint({ where: { id: wpB } }, store.primaryReadClient); + expect(found?.id).toBe(wpB); + } + ); +}); + +describe("RoutingRunStore four-store matrix — pagination merge", () => { + matrixTest( + "findRuns merges an open-predicate page across all four stores in orderBy order", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "paginate"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + // One run per store, distinct createdAt so the global sort order is unambiguous. + const rows = [ + { id: cuid("pgleg"), at: new Date("2024-01-01T00:00:00Z") }, + { + id: ("pgnew".replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01", + at: new Date("2024-01-02T00:00:00Z"), + }, + { id: gen2("a", "pga"), at: new Date("2024-01-03T00:00:00Z") }, + { id: gen2("b", "pgb"), at: new Date("2024-01-04T00:00:00Z") }, + ]; + for (const r of rows) { + await router.createRun(buildRun({ runId: r.id, ...env, createdAt: r.at })); + } + // Open predicate (no id set) → fan out + merge; take 2 skip 1 over createdAt desc. + const page = (await router.findRuns({ + where: { runtimeEnvironmentId: env.runtimeEnvironmentId }, + select: { id: true, createdAt: true }, + orderBy: { createdAt: "desc" }, + take: 2, + skip: 1, + })) as Array<{ id: string }>; + // Global desc order is b, a, new, legacy; skip 1 take 2 → [a, new]. + expect(page.map((r) => r.id)).toEqual([rows[2]!.id, rows[1]!.id]); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.runKeyedRouting.test.ts b/internal-packages/run-store/src/runOpsStore.runKeyedRouting.test.ts index e29ecb11cb8..e5cb229ef06 100644 --- a/internal-packages/run-store/src/runOpsStore.runKeyedRouting.test.ts +++ b/internal-packages/run-store/src/runOpsStore.runKeyedRouting.test.ts @@ -319,8 +319,9 @@ describe("RoutingRunStore.countPendingWaitpoints — route by runId then partiti "legacy_run" ); expect(count).toBe(1); - // Fallback queried the other store with ONLY the id missing on the run's store. - expect(newStore.calls.map((c) => c.method)).toEqual(["countPendingWaitpoints"]); + // Fallback queried the other store with ONLY the id missing on the run's store. It uses the + // presence variant so the results can be unioned by id (a drain mirror counts once at N). + expect(newStore.calls.map((c) => c.method)).toEqual(["countPendingWaitpointsWithPresence"]); expect(newStore.calls[0]?.args[0]).toEqual(["waitpoint_crosstree"]); }); diff --git a/internal-packages/run-store/src/runOpsStore.shardMap.test.ts b/internal-packages/run-store/src/runOpsStore.shardMap.test.ts index c25574e7561..8f2cc8c6485 100644 --- a/internal-packages/run-store/src/runOpsStore.shardMap.test.ts +++ b/internal-packages/run-store/src/runOpsStore.shardMap.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { RoutingRunStore } from "./runOpsStore.js"; +import { RoutingRunStore, UnknownShardKey } from "./runOpsStore.js"; import type { ReadClient, RunStore } from "./types.js"; // Pins the routing ALGEBRA: probe order, merge precedence, and the two id-less fallbacks that @@ -10,7 +10,7 @@ import type { ReadClient, RunStore } from "./types.js"; // MUST NOT assert invocation order for a PARALLEL fan-out: both legs are issued before either // resolves, so the order they are created in is not a behaviour. -type Slot = "new" | "legacy"; +type Slot = string; type Call = { slot: Slot; method: string }; @@ -19,8 +19,14 @@ type FakeConfig = { runs?: Array>; // Edge rows this store returns from findManyTaskRunWaitpoints, regardless of filter. edges?: Array>; + // Batch row this store returns from findBatchTaskRunById, regardless of filter. + batch?: Record | null; // Waitpoint rows this store returns from findWaitpoint, regardless of filter. waitpoint?: Record | null; + // Rows this store returns from findRunsByIdempotencyKeys, regardless of filter. + idempotencyMatches?: Array>; + // Waitpoint ids this store reports as pending (present = pending here) for count/collect probes. + pendingWaitpointIds?: string[]; }; type FakeStore = RunStore & { @@ -79,10 +85,42 @@ function fakeStore(slot: Slot, log: Call[], config: FakeConfig = {}): FakeStore return Promise.resolve({ slot } as never); }) as FakeStore["updateWaitpoint"], + createWaitpoint: ((_args: unknown) => { + record("createWaitpoint"); + return Promise.resolve({ slot } as never); + }) as FakeStore["createWaitpoint"], + + runInTransaction: ((_runId: unknown, fn: (store: unknown, tx: unknown) => unknown) => { + record("runInTransaction"); + return Promise.resolve(fn(store, {})); + }) as FakeStore["runInTransaction"], + findManyTaskRunWaitpoints: ((_args: unknown, _client?: ReadClient) => { record("findManyTaskRunWaitpoints"); return Promise.resolve((config.edges ?? []) as never); }) as FakeStore["findManyTaskRunWaitpoints"], + + updateManyWaitpoints: ((_args: unknown) => { + record("updateManyWaitpoints"); + return Promise.resolve({ count: 1 } as never); + }) as FakeStore["updateManyWaitpoints"], + + findRunsByIdempotencyKeys: ((_args: unknown, _client?: ReadClient) => { + record("findRunsByIdempotencyKeys"); + return Promise.resolve((config.idempotencyMatches ?? []) as never); + }) as FakeStore["findRunsByIdempotencyKeys"], + + findBatchTaskRunById: ((_id: unknown, _args?: unknown, _client?: ReadClient) => { + record("findBatchTaskRunById"); + return Promise.resolve((config.batch ?? null) as never); + }) as FakeStore["findBatchTaskRunById"], + + countPendingWaitpointsWithPresence: ((waitpointIds: string[], _client?: ReadClient) => { + record("countPendingWaitpointsWithPresence"); + const pending = new Set(config.pendingWaitpointIds ?? []); + const found = waitpointIds.filter((id) => pending.has(id)); + return Promise.resolve({ pendingIds: found, presentIds: found } as never); + }) as FakeStore["countPendingWaitpointsWithPresence"], }; return store as unknown as FakeStore; @@ -103,6 +141,12 @@ function buildRouter(newConfig: FakeConfig = {}, legacyConfig: FakeConfig = {}) const trace = (log: Call[]) => log.map((c) => `${c.slot}:${c.method}`); +// A parallel fan-out issues every leg before any resolves, so the ORDER legs appear in the log is +// NOT a behaviour and MUST NOT be asserted. Compare the multiset of slots instead. Order assertions +// via `trace` are valid only for the sequential (two-or-fewer-store) probe. +const slots = (log: Call[], method?: string) => + (method ? log.filter((c) => c.method === method) : log).map((c) => c.slot).sort(); + describe("RoutingRunStore #probeOrder — new then legacy, sequential", () => { it("probes new BEFORE legacy for an unrouted findRun", async () => { const { router, log } = buildRouter(); @@ -166,10 +210,19 @@ describe("RoutingRunStore id-less fallbacks — the two defaults differ by role" expect(trace(log)).toEqual(["new:createRun"]); }); - it("routes an id-less checkpoint create to new (#idlessRouteShard)", async () => { + it("throws for an id-less checkpoint create rather than defaulting to new", async () => { const { router, log } = buildRouter(); - await router.createTaskRunCheckpoint({ data: {} } as never); - expect(trace(log)).toEqual(["new:createTaskRunCheckpoint"]); + await expect(router.createTaskRunCheckpoint({ data: {} } as never)).rejects.toThrow( + "createTaskRunCheckpoint requires ownerRunId to route" + ); + expect(trace(log)).toEqual([]); + }); + + it("throws for a batch create with no id rather than defaulting to new", async () => { + const { router } = buildRouter(); + await expect(router.createBatchTaskRun({} as never)).rejects.toThrow( + "createBatchTaskRun requires data.id to route" + ); }); it("routes an id-less waitpoint update to legacy (#idlessWaitpointShard)", async () => { @@ -178,3 +231,696 @@ describe("RoutingRunStore id-less fallbacks — the two defaults differ by role" expect(trace(log)).toEqual(["legacy:updateWaitpoint"]); }); }); + +describe("RoutingRunStore id-to-shard-key seam", () => { + it("defaults to the core resolveShard when neither seam is injected", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + }); + // A gen-1 v1 body (version "1" at index 25) routes to new. + await router.findRun({ id: "a".repeat(24) + "01" }); + expect(trace(log)).toEqual(["new:findRun"]); + }); + + it("keeps the legacy classify seam working, so the corpus stays green", async () => { + const { router, log } = buildRouter(); + await router.findRun({ id: "new_run_1" }); + expect(trace(log)).toEqual(["new:findRun"]); + }); + + it("prefers an injected resolveShard over classify", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + classify: () => "NEW", + resolveShard: () => "legacy", + }); + await router.findRun({ id: "anything" }); + expect(trace(log)).toEqual(["legacy:findRun"]); + }); + + // An id naming a shard nobody configured must fail loud rather than fall back to a default + // store, which would be a silent read against the wrong database. The throw is SYNCHRONOUS: + // routing happens before any query is issued, and `await store.findRun(...)` propagates it + // identically. Only a `.catch()`-style caller would see the difference. + it("throws for an id resolving to an unconfigured shard key", () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + resolveShard: () => "a", + }); + expect(() => router.findRun({ id: "anything" })).toThrow( + 'no store is configured for shard key "a"' + ); + expect(trace(log)).toEqual([]); + }); + + // The case above injects a resolver. This one does NOT: it uses the real `resolveShard`, which + // the compat constructor defaults to. `resolveShard` is pure id-shape, so a gen-2 shaped id + // names its shard char whatever the topology holds — the two-store compat router therefore + // reaches this throw for any gen-2 id, with no shard configured anywhere. + // + // That matters beyond this class: these ids reach read routes as URL parameters, so whatever + // sits above the router must translate this throw into a 4xx rather than let it surface as a + // 5xx that any caller can induce. + it("reaches the unconfigured-shard throw for a real gen-2 id, even on the compat pair", () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + }); + + const genTwoId = `${"0".repeat(24)}a2`; + + expect(() => router.findRun({ id: genTwoId })).toThrow( + 'no store is configured for shard key "a"' + ); + expect(trace(log)).toEqual([]); + }); + + // Typed, not a bare Error: the API boundary matches on it to answer 404 instead of 500, and + // the operator needs the key and the configured set to tell a forged id from a dropped shard. + it("throws a typed UnknownShardKey carrying the key and the configured set", () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + }); + + let thrown: unknown; + try { + router.findRun({ id: `${"0".repeat(24)}a2` }); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(UnknownShardKey); + const error = thrown as UnknownShardKey; + expect(error.name).toBe("UnknownShardKey"); + expect(error.shardKey).toBe("a"); + expect([...error.configured].sort()).toEqual(["legacy", "new"]); + }); + + it("still routes gen-1 shapes on the compat pair with the real resolver", () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + }); + + router.findRun({ id: `${"0".repeat(24)}01` }); + router.findRun({ id: "c".repeat(25) }); + + expect(trace(log)).toEqual(["new:findRun", "legacy:findRun"]); + }); +}); + +function buildNShardRouter(shardKeys: string[], opts: { aliasOf?: Record } = {}) { + const log: Call[] = []; + const newStore = fakeStore("new", log); + const legacyStore = fakeStore("legacy", log); + const byKey: Record = { new: newStore, legacy: legacyStore }; + const shards = shardKeys.map((key) => { + const aliasOf = opts.aliasOf?.[key]; + const store = aliasOf ? byKey[aliasOf]! : fakeStore(key as Slot, log); + byKey[key] = store; + return aliasOf ? { key, store, aliasOf } : { key, store }; + }); + const router = new RoutingRunStore({ + new: newStore, + legacy: legacyStore, + shards, + resolveShard: (id: string) => id.split(":")[0]!, + }); + return { router, log, byKey }; +} + +describe("RoutingRunStore #distinctStores — one entry per database", () => { + // findRunsByIds reaches #fanOutPartitioned, the third unconfigured-shard guard. It must throw + // the typed error too, or this read path answers 500 where the boundary would give a 404. + it("throws a typed UnknownShardKey from the partitioned id fan-out", async () => { + const { router } = buildNShardRouter(["a"]); + + await expect(router.findRunsByIds(["a:r1", "z:r2"])).rejects.toBeInstanceOf(UnknownShardKey); + }); + + it("routes an id to its gen-2 shard", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.findRun({ id: "a:run_1" }); + expect(trace(log)).toEqual(["a:findRun"]); + }); + + it("counts an aliased shard's database ONCE in a sum", async () => { + // "a" aliases "new": two keys, one database. + const { router, log } = buildNShardRouter(["a"], { aliasOf: { a: "new" } }); + const result = await router.updateManyWaitpoints({ + where: { status: "PENDING" }, + data: {}, + } as never); + expect(trace(log)).toEqual(["new:updateManyWaitpoints", "legacy:updateManyWaitpoints"]); + expect(result.count).toBe(2); + }); + + it("still routes an id whose key is an alias", async () => { + const { router, log, byKey } = buildNShardRouter(["a"], { aliasOf: { a: "new" } }); + expect(byKey.a).toBe(byKey.new); + await router.findRun({ id: "a:run_1" }); + expect(trace(log)).toEqual(["new:findRun"]); + }); + + it("keeps #fanOutPartitioned key-driven so an aliased bucket is not dropped", async () => { + const { router, log } = buildNShardRouter(["a"], { aliasOf: { a: "new" } }); + await router.findRunsByIds(["a:r1", "legacy:r2"]); + // Both buckets get a leg. The aliased bucket routes onto the shared store. + expect(log.filter((c) => c.method === "findRuns")).toHaveLength(2); + }); + + it("rejects an aliasOf naming an unconfigured key", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: fakeStore("a" as Slot, log), aliasOf: "nope" }], + }) + ).toThrow('aliasOf "nope"'); + }); + + it("rejects a shard key that reuses a reserved key (new/legacy)", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "new", store: fakeStore("shadow" as Slot, log) }], + }) + ).toThrow("must be unique"); + }); + + it("rejects a duplicate custom shard key", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a1" as Slot, log) }, + { key: "a", store: fakeStore("a2" as Slot, log) }, + ], + }) + ).toThrow("must be unique"); + }); + + it("rejects a self-alias (a -> a), which would drop its database from every fan-out", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: fakeStore("a" as Slot, log), aliasOf: "a" }], + }) + ).toThrow("must name a non-aliased store"); + }); + + it("rejects an alias chain (a -> b where b is itself aliased)", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a" as Slot, log), aliasOf: "b" }, + { key: "b", store: fakeStore("b" as Slot, log), aliasOf: "new" }, + ], + }) + ).toThrow("must name a non-aliased store"); + }); + + it("rejects an alias cycle (a -> b, b -> a), which would drop both databases", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a" as Slot, log), aliasOf: "b" }, + { key: "b", store: fakeStore("b" as Slot, log), aliasOf: "a" }, + ], + }) + ).toThrow("must name a non-aliased store"); + }); +}); + +describe("RoutingRunStore probe at N", () => { + it("keeps the sequential short circuit at two distinct stores", async () => { + const { router, log } = buildRouter({ runs: [{ id: "r1" }] }); + await router.findRun({ spanId: "span_x" }); + expect(trace(log)).toEqual(["new:findRun"]); + }); + + it("issues every leg in parallel above two distinct stores", async () => { + // "b" carries the only hit. A sequential probe would stop the moment it found a result; a + // true parallel fan-out queries every OTHER leg too, since all legs are issued before any + // resolves. Miss-path (every leg misses) throw semantics are covered separately below. + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a", log) }, + { key: "b", store: fakeStore("b", log, { runs: [{ id: "r1" }] }) }, + ], + resolveShard: (id: string) => id.split(":")[0]!, + }); + await router.findRun({ spanId: "span_x" }); + expect(slots(log)).toEqual(["a", "b", "legacy", "new"]); + }); + + it("gives the legacy leg the canonical throw when every leg misses", async () => { + const { router } = buildNShardRouter(["a"]); + await expect(router.findRunOrThrow({ spanId: "span_x" })).rejects.toThrow("no run on legacy"); + }); + + it("tolerates a failing leg when another leg wins", async () => { + const log: Call[] = []; + const broken = fakeStore("a", log); + (broken as { findRun: unknown }).findRun = () => Promise.reject(new Error("shard a is down")); + const router = new RoutingRunStore({ + new: fakeStore("new", log, { runs: [{ id: "r1" }] }), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: broken }], + resolveShard: (id: string) => id.split(":")[0]!, + }); + await expect(router.findRun({ spanId: "span_x" })).resolves.toMatchObject({ id: "r1" }); + }); + + it("surfaces a leg failure when no leg wins", async () => { + const log: Call[] = []; + const broken = fakeStore("a", log); + (broken as { findRun: unknown }).findRun = () => Promise.reject(new Error("shard a is down")); + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: broken }], + resolveShard: (id: string) => id.split(":")[0]!, + }); + await expect(router.findRun({ spanId: "span_x" })).rejects.toThrow("shard a is down"); + }); +}); + +describe("RoutingRunStore merge precedence and duplicate alarm", () => { + const spy = () => { + const seen: string[][] = []; + return { + metrics: { + recordDuplicateId: (k: string[]) => seen.push(k), + recordWaitpointProbeFallback() {}, + }, + seen, + }; + }; + + it("stays silent for a duplicate run id across the gen-1 pair", async () => { + const { metrics, seen } = spy(); + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { runs: [{ id: "dup", from: "new" }] }), + legacy: fakeStore("legacy", log, { runs: [{ id: "dup", from: "legacy" }] }), + metrics, + }); + const rows = (await router.findRuns({ + where: { runtimeEnvironmentId: "env_1" }, + select: { id: true, from: true }, + })) as Array<{ from: string }>; + expect(rows).toHaveLength(1); + expect(rows[0]?.from).toBe("new"); // NEW wins the precedence merge + expect(seen).toEqual([]); + }); + + it("alarms for a duplicate involving a gen-2 shard and still picks deterministically", async () => { + const { metrics, seen } = spy(); + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a", log, { runs: [{ id: "dup", from: "a" }] }) }, + { key: "b", store: fakeStore("b", log, { runs: [{ id: "dup", from: "b" }] }) }, + ], + resolveShard: (id: string) => id.split(":")[0]!, + metrics, + }); + const rows = (await router.findRuns({ + where: { runtimeEnvironmentId: "env_1" }, + select: { id: true, from: true }, + })) as Array<{ from: string }>; + expect(rows).toHaveLength(1); + expect(rows[0]?.from).toBe("b"); // last in #precedence [legacy, new, a, b] wins + expect(seen).toEqual([["a", "b"]]); + }); + + it("passes through an edge row whose projection omits id", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { edges: [{ taskRunId: "r1" }] }), + legacy: fakeStore("legacy", log, { edges: [{ taskRunId: "r2" }] }), + }); + const edges = (await router.findManyTaskRunWaitpoints({ + where: { waitpointId: "w" }, + select: { taskRunId: true }, + })) as Array<{ taskRunId: string }>; + expect(edges).toHaveLength(2); + }); +}); + +describe("RoutingRunStore findRunsByIdempotencyKeys tiebreak", () => { + const older = new Date("2026-01-01T00:00:00Z"); + const newer = new Date("2026-01-02T00:00:00Z"); + const match = (id: string, createdAt: Date) => ({ + id, + createdAt, + friendlyId: `run_${id}`, + idempotencyKey: "k", + idempotencyKeyExpiresAt: null, + }); + + it("keeps NEW-wins across the gen-1 pair even when legacy is older", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { idempotencyMatches: [match("n1", newer)] }), + legacy: fakeStore("legacy", log, { idempotencyMatches: [match("l1", older)] }), + }); + const rows = await router.findRunsByIdempotencyKeys({ + runtimeEnvironmentId: "env", + taskIdentifier: "t", + idempotencyKeys: ["k"], + }); + expect(rows.map((r) => r.id)).toEqual(["n1"]); + }); + + it("takes the earliest createdAt once a gen-2 shard is involved", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { idempotencyMatches: [match("n1", newer)] }), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a", log, { idempotencyMatches: [match("a1", older)] }) }, + ], + resolveShard: (id: string) => id.split(":")[0]!, + }); + const rows = await router.findRunsByIdempotencyKeys({ + runtimeEnvironmentId: "env", + taskIdentifier: "t", + idempotencyKeys: ["k"], + }); + expect(rows.map((r) => r.id)).toEqual(["a1"]); + }); +}); + +describe("RoutingRunStore countPendingWaitpoints — disjoint-sum partition", () => { + // resolveShard: "x:..." -> "x" (a gen-2 shard); a bare cuid -> "legacy". + function partitionRouter(pending: Record, spy?: (k: string[]) => void) { + const log: Call[] = []; + const mk = (slot: string) => fakeStore(slot, log, { pendingWaitpointIds: pending[slot] ?? [] }); + const router = new RoutingRunStore({ + new: mk("new"), + legacy: mk("legacy"), + shards: [ + { key: "a", store: mk("a") }, + { key: "b", store: mk("b") }, + ], + resolveShard: (id: string) => (id.includes(":") ? id.split(":")[0]! : "legacy"), + ...(spy ? { metrics: { recordDuplicateId: spy, recordWaitpointProbeFallback() {} } } : {}), + }); + return { router, log }; + } + const countCalls = (log: Call[]) => slots(log, "countPendingWaitpointsWithPresence"); + + it("sends a gen-2 absent id to its own shard only", async () => { + const { router, log } = partitionRouter({ b: ["b:w1"] }); + expect(await router.countPendingWaitpoints(["b:w1"], undefined, "a:run")).toBe(1); + expect(countCalls(log)).toEqual(["a", "b"]); // run shard a (presence) + fallback b + }); + + it("contributes zero for a gen-2 id whose shard IS the run's shard", async () => { + const { router, log } = partitionRouter({}); + expect(await router.countPendingWaitpoints(["a:w1"], undefined, "a:run")).toBe(0); + expect(countCalls(log)).toEqual(["a"]); // absent on a; no fallback leg (b-bucket empty, a skipped) + }); + + it("probes BOTH gen-1 stores for a cuid absent id when the run is on a gen-2 shard", async () => { + const { router, log } = partitionRouter({ legacy: ["cuid_w1"] }); + expect(await router.countPendingWaitpoints(["cuid_w1"], undefined, "a:run")).toBe(1); + expect(countCalls(log)).toEqual(["a", "legacy", "new"]); + }); + + it("probes only legacy for a cuid when the run is on new", async () => { + const { router, log } = partitionRouter({ legacy: ["cuid_w1"] }); + expect(await router.countPendingWaitpoints(["cuid_w1"], undefined, "new:run")).toBe(1); + // run shard resolves "new:run" -> "new" (presence); cuid -> {legacy, new} minus new = legacy + expect(countCalls(log)).toEqual(["legacy", "new"]); + }); + + it("probes only new for a cuid when the run is on legacy", async () => { + const { router, log } = partitionRouter({ new: ["cuid_w1"] }); + expect(await router.countPendingWaitpoints(["cuid_w1"], undefined, "cuid_run")).toBe(1); + // run shard resolves cuid -> "legacy"; cuid -> {legacy, new} minus legacy = new only + expect(countCalls(log)).toEqual(["legacy", "new"]); + }); + + it("counts a drain-mirrored cuid ONCE and stays silent", async () => { + const seen: string[][] = []; + const { router } = partitionRouter({ new: ["cuid_w1"], legacy: ["cuid_w1"] }, (k) => + seen.push(k) + ); + expect(await router.countPendingWaitpoints(["cuid_w1"], undefined, "a:run")).toBe(1); + expect(seen).toEqual([]); // the gen-1 mirror is expected, never alarmed + }); + + it("fails loud when an absent id resolves to an unconfigured shard key", async () => { + // "c:w1" resolves to shard "c", which is not configured. Silently dropping it would under-count + // a pending waitpoint and prematurely unblock the run — so it must throw, not skip. + const { router } = partitionRouter({}); + await expect(router.countPendingWaitpoints(["c:w1"], undefined, "a:run")).rejects.toThrow( + 'unconfigured shard key "c"' + ); + }); + + // The API boundary answers a non-retryable 404 by matching on the TYPE, so every + // unconfigured-shard guard has to throw the typed error and not a bare Error. Two other guards + // besides #shardStore reach an unconfigured key: this partition, and #fanOutPartitioned below. + it("throws a typed UnknownShardKey from the absent-id partition", async () => { + const { router } = partitionRouter({}); + + await expect( + router.countPendingWaitpoints(["c:w1"], undefined, "a:run") + ).rejects.toBeInstanceOf(UnknownShardKey); + }); + + it("returns zero for an id absent everywhere", async () => { + const { router } = partitionRouter({}); + expect(await router.countPendingWaitpoints(["b:w9"], undefined, "a:run")).toBe(0); + }); + + it("id-less count unions a drain-mirrored cuid to one, never sums it to two", async () => { + // No runId: fans over distinct stores. A cuid pending on BOTH gen-1 stores is one waitpoint. + const { router } = partitionRouter({ new: ["cuid_w1"], legacy: ["cuid_w1"] }); + expect(await router.countPendingWaitpoints(["cuid_w1"])).toBe(1); + }); + + it("returns the true total for a mixed gen-2 and cuid set with no double count", async () => { + const { router } = partitionRouter({ + b: ["b:w1"], + legacy: ["cuid_w1"], + new: ["cuid_w1", "cuid_w2"], + }); + const count = await router.countPendingWaitpoints( + ["b:w1", "cuid_w1", "cuid_w2", "b:w9"], + undefined, + "a:run" + ); + expect(count).toBe(3); // b:w1 + cuid_w1 (mirror, once) + cuid_w2; b:w9 absent + }); +}); + +describe("RoutingRunStore waitpoint probes at N", () => { + it("routes a gen-2 waitpoint directly, with no probe", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.updateWaitpoint({ where: { id: "b:w1" }, data: {} } as never); + expect(trace(log)).toEqual(["b:updateWaitpoint"]); + }); + + it("keeps a cuid waitpoint on the gen-1 pair and never probes a gen-2 shard", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { waitpoint: { id: "cuid_w1" } }), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: fakeStore("a", log) }], + resolveShard: (id: string) => (id.includes(":") ? id.split(":")[0]! : "legacy"), + }); + await router.updateWaitpoint({ where: { id: "cuid_w1" }, data: {} } as never); + expect(log.map((c) => c.slot)).not.toContain("a"); + }); + + it("records a probe fallback when the waitpoint is not on the store its id names", async () => { + const falls: Array<[string, string]> = []; + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { waitpoint: { id: "cuid_w1" } }), + legacy: fakeStore("legacy", log, { waitpoint: null }), + metrics: { + recordDuplicateId() {}, + recordWaitpointProbeFallback: (from, to) => falls.push([from, to]), + }, + }); + await router.updateWaitpoint({ where: { id: "cuid_w1" }, data: {} } as never); + expect(falls).toEqual([["legacy", "new"]]); + }); + + it("lets a gen-2 waitpoint id beat the cross-tree legacy pin", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + const store = await router.forWaitpointCompletion("b:w1", { + isCrossTreeIdempotency: true, + } as never); + expect((store as FakeStore).slot).toBe("b"); + expect(log.map((c) => c.slot)).not.toContain("legacy"); + }); + + it("keeps the legacy pin for a cuid waitpoint in a cross-tree completion", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log, { waitpoint: { id: "cuid_w1" } }), + shards: [{ key: "a", store: fakeStore("a", log) }], + resolveShard: (id: string) => (id.includes(":") ? id.split(":")[0]! : "legacy"), + }); + const store = await router.forWaitpointCompletion("cuid_w1", { + isCrossTreeIdempotency: true, + } as never); + expect((store as FakeStore).slot).toBe("legacy"); + expect(log.map((c) => c.slot)).not.toContain("a"); + }); +}); + +describe("RoutingRunStore gen-2 shard refuses a co-located cuid waitpoint", () => { + // resolveShard: "x:..." -> "x" (gen-2 shard); a bare id (no colon) -> "legacy". + function coLocateRouter() { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: fakeStore("a", log) }], + resolveShard: (id: string) => (id.includes(":") ? id.split(":")[0]! : "legacy"), + }); + return { router, log }; + } + + it("throws when a cuid waitpoint is co-located onto a gen-2 shard", () => { + const { router, log } = coLocateRouter(); + // Routing throws SYNCHRONOUSLY, before any write is issued. + expect(() => + router.createWaitpoint({ data: { id: "cuid_w1" } } as never, undefined, { + coLocateWithRunId: "a:run_1", + }) + ).toThrow('onto gen-2 shard "a"'); + expect(trace(log)).toEqual([]); + }); + + it("throws when an id-less waitpoint is co-located onto a gen-2 shard", () => { + // Prisma's @default(cuid()) would otherwise mint a cuid on the gen-2 shard AFTER the write, + // leaving it unroutable for its own completion (CodeRabbit finding). Reject it up front. + const { router, log } = coLocateRouter(); + expect(() => + router.createWaitpoint({ data: {} } as never, undefined, { coLocateWithRunId: "a:run_1" }) + ).toThrow('onto gen-2 shard "a"'); + expect(trace(log)).toEqual([]); + }); + + it("allows a gen-2 waitpoint co-located onto its own shard", async () => { + const { router, log } = coLocateRouter(); + await router.createWaitpoint({ data: { id: "a:w1" } } as never, undefined, { + coLocateWithRunId: "a:run_1", + }); + expect(trace(log)).toEqual(["a:createWaitpoint"]); + }); + + it("allows a cuid waitpoint co-located onto a gen-1 store", async () => { + const { router, log } = coLocateRouter(); + await router.createWaitpoint({ data: { id: "cuid_w1" } } as never, undefined, { + coLocateWithRunId: "legacy_run", + }); + expect(trace(log)).toEqual(["legacy:createWaitpoint"]); + }); +}); + +describe("RoutingRunStore id-less read/route defaults hold at N (never a gen-2 shard)", () => { + // With gen-2 shards a and b configured, each id-less default must still resolve to its named + // gen-1 store, never leak to a gen-2 shard. + it("#routeOrNew falls back to new for an id-less create", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.createRun({ data: {} } as never); + expect(trace(log)).toEqual(["new:createRun"]); + }); + + it("#routeOrNew falls back to new for an id-less runInTransaction", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.runInTransaction(undefined, async () => undefined); + expect(trace(log)).toEqual(["new:runInTransaction"]); + }); + + it("#resolveWaitpointStore(undefined) falls back to legacy for an id-less update", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.updateWaitpoint({ where: { idempotencyKey: "k" }, data: {} } as never); + expect(trace(log)).toEqual(["legacy:updateWaitpoint"]); + }); + + it("#waitpointWriteStore with no owner and no residency falls back to legacy", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.createWaitpoint({ data: {} } as never); + expect(trace(log)).toEqual(["legacy:createWaitpoint"]); + }); +}); + +describe("RoutingRunStore batch probe tolerates legitimate dual-residency", () => { + it("does NOT alarm when a batch is found on a gen-2 shard AND legacy", async () => { + const seen: string[][] = []; + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log, { batch: { id: "batch_dup", from: "legacy" } }), + shards: [{ key: "a", store: fakeStore("a", log, { batch: { id: "batch_dup", from: "a" } }) }], + resolveShard: (id: string) => id.split(":")[0]!, + metrics: { recordDuplicateId: (k) => seen.push(k), recordWaitpointProbeFallback() {} }, + }); + const batch = (await router.findBatchTaskRunById("batch_dup")) as { from: string } | null; + // batchTriggerV3 writes raw to the control plane while runEngine routes by id, so this is a + // legitimate dual-residency, not a routing-invariant violation — no alarm. + expect(seen).toEqual([]); + // Precedence still picks deterministically (gen-2 shard 'a' outranks legacy). + expect(batch?.from).toBe("a"); + }); + + it("still alarms when a RUN is found on a gen-2 shard AND legacy (real violation)", async () => { + const seen: string[][] = []; + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log, { runs: [{ id: "dup", from: "legacy" }] }), + shards: [{ key: "a", store: fakeStore("a", log, { runs: [{ id: "dup", from: "a" }] }) }], + resolveShard: (id: string) => id.split(":")[0]!, + metrics: { recordDuplicateId: (k) => seen.push(k), recordWaitpointProbeFallback() {} }, + }); + await router.findRun({ spanId: "span_x" }); + expect(seen).toEqual([["legacy", "a"]]); + }); +}); diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index 27bd78866d4..7551e552b34 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -7,7 +7,11 @@ import type { TaskRunStatus, WaitpointTag, } from "@trigger.dev/database"; -import { ownerEngine, type Residency, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { + resolveShard as coreResolveShard, + type Residency, + type ShardKey, +} from "@trigger.dev/core/v3/isomorphic"; import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; import type { ClearIdempotencyKeyInput, @@ -30,8 +34,10 @@ import type { TaskRunWithWaitpoint, WaitpointColocationOptions, } from "./types.js"; +import { Logger } from "@trigger.dev/core/logger"; import { isReadReplicaClient } from "./readReplicaClient.js"; import { CONNECTED_RUNS_LIMIT } from "./PostgresRunStore.js"; +import { noopRoutingStoreMetrics, type RoutingStoreMetrics } from "./routingStoreMetrics.js"; import { boundedIn } from "@trigger.dev/database"; @@ -42,21 +48,42 @@ const LEGACY_SHARD: ShardKey = "legacy"; /** * Run-ops routing substrate for the TaskRun-core method group. Implements {@link RunStore} over a - * map from shard key to store, selecting one by the residency classifier (`ownerEngine`: run-ops - * id→NEW, cuid→LEGACY). The compat constructor holds the two gen-1 shards — a NEW store (the + * map from shard key to store, selecting one by `resolveShard` (gen-2 id→its shard char, gen-1 + * run-ops id→NEW, cuid→LEGACY). The compat constructor holds the two gen-1 shards — a NEW store (the * dedicated run-ops DB, where new runs are born) and a LEGACY store (the control-plane DB). * Inert until the injecting seam wires it in under `isSplitEnabled()`; reads no flag here. * - * Every shard MUST be a distinct database. Single-DB does not construct this class at all — the - * injecting seam returns a bare PostgresRunStore — and split mode requires two configured run-ops - * URLs whose distinctness the boot sentinel enforces fail-closed. Two shard keys that resolve to - * ONE store would make the sum sites (#sumCounts, the counting fan-outs) count that store twice. + * Two shard keys MAY resolve to one store only through a declared `aliasOf`. #distinctStores then + * holds one entry per database, so a sum never counts a database twice. An undeclared duplicate + * store is still a configuration error. * - * Three policies are held as data rather than implied by statement order: {@link #probeOrder} for a - * lookup with no routable id, {@link #precedence} for a merge, and the two id-less fallbacks. A - * merge MUST iterate #precedence and a probe MUST iterate #probeOrder — the two are the reverse of - * each other, so swapping them changes behaviour. + * #probeOrder and #precedence each list one key per distinct store. At two shards they are exact + * reverses. At N they are not: both put gen-2 shards last, so a probe finds the gen-1 pair first + * and a merge lets a gen-2 shard win. A probe MUST iterate #probeOrder and a merge MUST iterate + * #precedence. */ +/** + * An id resolved to a shard key the topology has no store for. Typed so a caller above the + * router can answer a 4xx instead of letting a routing failure surface as a 5xx: these ids + * arrive as URL parameters, and `resolveShard` is pure id-shape, so any gen-2 shaped id names + * a shard char whether or not one is configured. + */ +export class UnknownShardKey extends Error { + readonly shardKey: string; + readonly configured: string[]; + + constructor(shardKey: string, configured: string[], subject?: string) { + super( + subject === undefined + ? `RoutingRunStore: no store is configured for shard key "${shardKey}"` + : `RoutingRunStore: ${subject} resolves to unconfigured shard key "${shardKey}"` + ); + this.name = "UnknownShardKey"; + this.shardKey = shardKey; + this.configured = configured; + } +} + export class RoutingRunStore implements RunStore { readonly #shards: ReadonlyMap; // Sequential probe for a lookup with no routable id. The first non-null result wins, and the LAST @@ -65,25 +92,90 @@ export class RoutingRunStore implements RunStore { // Ascending authority for a merge. The last write wins, so the highest-authority shard wins a // duplicate id. Every merge in this class MUST use this order. readonly #precedence: readonly ShardKey[]; + // One entry per distinct database, in precedence order. A fan-out sum iterates this, never #shards. + readonly #distinctStores: ReadonlyArray<{ key: ShardKey; store: RunStore }>; // The two id-less defaults. They differ by role on purpose: a route with no id lands on the // steady-state home, a waitpoint read with no id lands on the legacy store. readonly #idlessRouteShard: ShardKey; readonly #idlessWaitpointShard: ShardKey; - readonly #classify: (id: string) => Residency; + // Id to shard key. `resolveShard` is the gen-2 seam. `classify` is the gen-1 seam, kept because + // five sites inject it and three of those are the regression corpus. A gen-1 classifier can + // only ever name the two reserved keys, so it can never reach a gen-2 shard. + readonly #resolveShardKey: (id: string) => ShardKey; + readonly #metrics: RoutingStoreMetrics; + readonly #logger: Logger; // Compat constructor: the two gen-1 stores, keyed by their reserved shard keys. The options type // MUST stay closed — a union arm loosens the excess-property check and retires the // `@ts-expect-error onLegacyRead` lock in the test corpus. - constructor(options: { new: RunStore; legacy: RunStore; classify?: (id: string) => Residency }) { + constructor(options: { + new: RunStore; + legacy: RunStore; + classify?: (id: string) => Residency; + resolveShard?: (id: string) => ShardKey; + shards?: ReadonlyArray<{ key: ShardKey; store: RunStore; aliasOf?: ShardKey }>; + metrics?: RoutingStoreMetrics; + logger?: Logger; + }) { + const shards = options.shards ?? []; + // Keys must be unique across the reserved pair and every configured shard. A key of "new" or + // "legacy" would overwrite the reserved #shards entry; a repeated custom key would appear twice + // in #precedence and #distinctStores, so a fan-out would query one database twice. + const shardKeys = [NEW_SHARD, LEGACY_SHARD, ...shards.map((s) => s.key)]; + if (new Set(shardKeys).size !== shardKeys.length) { + throw new Error( + "RoutingRunStore: shard keys must be unique and cannot reuse the reserved 'new' or 'legacy' keys" + ); + } + const configured = new Set(shardKeys); + const aliasedKeys = new Set(shards.filter((s) => s.aliasOf !== undefined).map((s) => s.key)); + for (const shard of shards) { + if (shard.aliasOf === undefined) { + continue; + } + // An alias must name a REAL root store, so #distinctStores keeps exactly one entry per + // database. A target that is itself aliased (a chain or a cycle) would drop every key in the + // cycle from #distinctStores, and that database would vanish from every read and write. + if (!configured.has(shard.aliasOf)) { + throw new Error( + `RoutingRunStore: shard "${shard.key}" declares aliasOf "${shard.aliasOf}", which is not configured` + ); + } + if (shard.aliasOf === shard.key || aliasedKeys.has(shard.aliasOf)) { + throw new Error( + `RoutingRunStore: shard "${shard.key}" aliasOf "${shard.aliasOf}" must name a non-aliased store; chains and cycles are not allowed` + ); + } + } + this.#shards = new Map([ [NEW_SHARD, options.new], [LEGACY_SHARD, options.legacy], + ...shards.map((s) => [s.key, s.store] as const), ]); - this.#probeOrder = [NEW_SHARD, LEGACY_SHARD]; - this.#precedence = [LEGACY_SHARD, NEW_SHARD]; + + const gen2Keys = shards.map((s) => s.key); + this.#probeOrder = [NEW_SHARD, LEGACY_SHARD, ...gen2Keys]; + this.#precedence = [LEGACY_SHARD, NEW_SHARD, ...gen2Keys]; this.#idlessRouteShard = NEW_SHARD; this.#idlessWaitpointShard = LEGACY_SHARD; - this.#classify = options.classify ?? ownerEngine; + const classify = options.classify; + this.#resolveShardKey = + options.resolveShard ?? + (classify !== undefined + ? (id: string) => (classify(id) === "NEW" ? NEW_SHARD : LEGACY_SHARD) + : coreResolveShard); + + // One entry per physical database, in precedence order. A declared alias contributes none: + // it shares its target's database, and a second leg over one database double-counts a sum. + // The discriminator is the DECLARATION, not object identity — the wiring layer may build a + // second store object over a shared client. + this.#distinctStores = this.#precedence + .filter((key) => !aliasedKeys.has(key)) + .map((key) => ({ key, store: this.#shardStore(key) })); + + this.#metrics = options.metrics ?? noopRoutingStoreMetrics; + this.#logger = options.logger ?? new Logger("RoutingRunStore", "warn"); } // A routing store spans two databases and has no single primary — routed reads resolve the @@ -103,19 +195,109 @@ export class RoutingRunStore implements RunStore { return client != null && !isReadReplicaClient(client) ? store.primaryReadClient : undefined; } - // The store for a shard key. Unreachable with the compat constructor — #shardKeyOfSafe yields only - // the two reserved keys — so this throw fires only if a caller wires a partial map. + // The store for a shard key. REACHABLE with the compat constructor: it defaults to the real + // `resolveShard`, which is pure id-shape, so any gen-2 shaped id names a shard char even when + // no shard is configured. Fails loud rather than reading the wrong database; the API boundary + // turns `UnknownShardKey` into a 404 so a caller-supplied id cannot induce a 5xx. #shardStore(key: ShardKey): RunStore { const store = this.#shards.get(key); if (store === undefined) { - throw new Error(`RoutingRunStore: no store is configured for shard key "${key}"`); + throw new UnknownShardKey(key, [...this.#shards.keys()]); } return store; } - // The shard that owns an existing id. Throws only when an injected classifier throws. + // A duplicate id is EXPECTED across the gen-1 pair (drain mirrors a token onto both). Any other + // combination breaks id-determinism: alarm, but keep the deterministic pick. + #reportDuplicateId(id: string, shardKeys: ShardKey[]): void { + if (shardKeys.every((key) => key === NEW_SHARD || key === LEGACY_SHARD)) { + return; + } + this.#metrics.recordDuplicateId(shardKeys); + this.#logger.error("RoutingRunStore: one id returned by two shards", { id, shardKeys }); + } + + // Merge key-tagged legs, keeping one row per id. Legs MUST arrive in #precedence order, so the + // highest-authority copy is written last and wins. A winner keeps the POSITION of its first + // sighting: callers observe row order whenever `orderBy` is absent. Rows whose projection omits + // `id` cannot be deduped and pass through unchanged. A duplicate whose reporting keys leave the + // gen-1 pair alarms via #reportDuplicateId. + #mergeById>(legs: Array<{ key: ShardKey; rows: R[] }>): R[] { + const byId = new Map(); + const keysById = new Map(); + const passthrough: R[] = []; + for (const { key, rows } of legs) { + for (const row of rows) { + const id = row.id; + if (typeof id !== "string") { + passthrough.push(row); + continue; + } + byId.set(id, row); + const keys = keysById.get(id); + if (keys) keys.push(key); + else keysById.set(id, [key]); + } + } + for (const [id, keys] of keysById) { + if (keys.length > 1) this.#reportDuplicateId(id, keys); + } + return [...byId.values(), ...passthrough]; + } + + // Where to look for waitpoint ids the run's own shard did not return. A gen-2 id names exactly + // one shard, so it goes there and nowhere else — that is what keeps the legs disjoint and the sum + // sound. A cuid names no shard: drain can mirror it onto NEW while it keeps its id, and a gen-2 + // run can block on a pre-gen-2 cuid token, so there is no single gen-1 partner. Both gen-1 stores + // are probed and the results are de-duped by id. A target equal to `runKey` is skipped (already + // probed); an id resolving to an unconfigured shard fails loud rather than being dropped. + #partitionAbsentIds(runKey: ShardKey, ids: string[]): Array<{ key: ShardKey; ids: string[] }> { + const byKey = new Map(); + const push = (key: ShardKey, id: string) => { + // The run's own shard was already probed, so skip it. But an id resolving to a shard nobody + // configured is UnknownShardKey: silently dropping it here would UNDER-count a pending + // waitpoint and prematurely unblock the run — the exact failure this method guards against. + // Fail loud instead (§7 append-only rule). + if (key === runKey) return; + if (!this.#shards.has(key)) { + throw new UnknownShardKey(key, [...this.#shards.keys()], `waitpoint "${id}"`); + } + const bucket = byKey.get(key); + if (bucket) bucket.push(id); + else byKey.set(key, [id]); + }; + for (const id of ids) { + const home = this.#shardKeyOfSafe(id); + if (home === LEGACY_SHARD) { + push(LEGACY_SHARD, id); + push(NEW_SHARD, id); + } else { + push(home, id); + } + } + return this.#precedence + .filter((key) => byKey.has(key)) + .map((key) => ({ key, ids: byKey.get(key)! })); + } + + // A cuid is deliberately probed on BOTH gen-1 stores, so the same id from both is the expected + // drain mirror, not a violation. Any other id maps to one shard, so a two-leg return is a bug. + #isGen1MirrorProbe(id: string): boolean { + return this.#shardKeyOfSafe(id) === LEGACY_SHARD; + } + + // The gen-1 pair members other than `key`. A cuid waitpoint can only ever be drain-relocated + // BETWEEN the two gen-1 stores, so its "where does it really live" probe stays confined here and + // never touches a gen-2 shard. + #gen1PairExcept(key: ShardKey): Array<{ key: ShardKey; store: RunStore }> { + return [NEW_SHARD, LEGACY_SHARD] + .filter((k) => k !== key) + .map((k) => ({ key: k, store: this.#shardStore(k) })); + } + + // The shard that owns an existing id. Throws only when an injected resolver throws. #shardKeyOf(id: string): ShardKey { - return this.#classify(id) === "NEW" ? NEW_SHARD : LEGACY_SHARD; + return this.#resolveShardKey(id); } // An unclassifiable id is treated as LEGACY (probe the control-plane DB rather than drop a @@ -128,32 +310,88 @@ export class RoutingRunStore implements RunStore { } } - // Sequential probe over #probeOrder, returning the first non-null result. `isLast` marks the leg - // that owns the canonical not-found throw, so a caller can swap in its throwing variant there. + // #distinctStores sorted by `order`. Shared by #probeFirst and #fanOut so every ordered walk + // over the distinct-store set goes through one sort. + #orderedLegs(order: readonly ShardKey[]): ReadonlyArray<{ key: ShardKey; store: RunStore }> { + const rank = new Map(order.map((key, i) => [key, i])); + return [...this.#distinctStores].sort( + (a, b) => (rank.get(a.key) ?? 0) - (rank.get(b.key) ?? 0) + ); + } + + // A lookup with no routable id. At two distinct stores this is the sequential short circuit, + // byte-identical to before: the second leg is never queried when the first answers. Above two, + // a sequential walk would cost N round trips, so every leg is issued in parallel and the winner + // comes from #precedence. `isLast` marks the leg that owns the canonical not-found throw. async #probeFirst( - fn: (store: RunStore, key: ShardKey, isLast: boolean) => Promise + fn: (store: RunStore, key: ShardKey, isLast: boolean) => Promise, + opts?: { alarmOnDuplicate?: boolean } ): Promise { - const last = this.#probeOrder.length - 1; - for (let i = 0; i < last; i++) { - const key = this.#probeOrder[i]!; - const found = await fn(this.#shardStore(key), key, false); - if (found != null) { - return found; + const legs = this.#orderedLegs(this.#probeOrder); + const lastIndex = legs.length - 1; + + if (legs.length <= 2) { + for (let i = 0; i < lastIndex; i++) { + const { store, key } = legs[i]!; + const found = await fn(store, key, false); + if (found != null) { + return found; + } + } + const { store, key } = legs[lastIndex]!; + return fn(store, key, true); + } + + // Parallel. Every leg takes the NON-throwing arm, so one leg cannot reject a lookup another + // leg answers. A rejection is held and only surfaces when nothing was found. + const settled = await Promise.allSettled(legs.map(({ store, key }) => fn(store, key, false))); + + const hits: Array<{ key: ShardKey; value: Awaited }> = []; + let firstRejection: unknown; + settled.forEach((outcome, i) => { + if (outcome.status === "rejected") { + firstRejection ??= outcome.reason; + return; + } + if (outcome.value != null) { + hits.push({ key: legs[i]!.key, value: outcome.value }); } + }); + + // A row on two stores is a routing-invariant violation for entities that live on exactly one + // store (runs, waitpoints, attempts, snapshots). Batches are the exception: `batchTriggerV3` + // writes raw to the control plane while runEngine routes by id, so a batch is legitimately + // dual-resident. Those callers pass `alarmOnDuplicate: false` so a batch on legacy + a gen-2 + // shard is not mistaken for a violation. + if (hits.length > 1 && opts?.alarmOnDuplicate !== false) { + this.#reportDuplicateId( + String((hits[0]!.value as { id?: unknown })?.id ?? "unknown"), + hits.map((h) => h.key) + ); + } + if (hits.length > 0) { + // Highest authority wins: #precedence ascends, so take the last hit in that order. + const rank = new Map(this.#precedence.map((key, i) => [key, i])); + hits.sort((a, b) => (rank.get(a.key) ?? 0) - (rank.get(b.key) ?? 0)); + return hits[hits.length - 1]!.value; + } + if (firstRejection !== undefined) { + throw firstRejection; } - const key = this.#probeOrder[last]!; - return fn(this.#shardStore(key), key, true); + // Nothing found anywhere. The LEGACY leg owns the canonical not-found throw, so give it the + // throwing arm. One extra query, on the miss path only. + const legacy = this.#shardStore(LEGACY_SHARD); + return fn(legacy, LEGACY_SHARD, true); } - // Run `fn` on every shard in parallel, returning the results in `order`. Pass #probeOrder where - // the result-array order is observable; pass #precedence where a duplicate id's winner decides - // the value. The two orders are the reverse of each other, so passing the wrong one is a - // behaviour change. + // Run `fn` on every DISTINCT store in parallel. `order` selects the ordering; membership is + // always one entry per database, so an aliased key never contributes a second leg. #fanOut( order: readonly ShardKey[], fn: (store: RunStore, key: ShardKey) => Promise ): Promise { - return Promise.all(order.map((key) => fn(this.#shardStore(key), key))); + const legs = this.#orderedLegs(order); + return Promise.all(legs.map(({ store, key }) => fn(store, key))); } // Apply `fn` to every shard and sum the counts. A sum is order-independent, so this takes no order. @@ -174,6 +412,11 @@ export class RoutingRunStore implements RunStore { const byShard = new Map(); for (const id of ids) { const key = this.#shardKeyOfSafe(id); + // An id resolving to a shard nobody configured is UnknownShardKey. Dropping it would silently + // omit a row from the hydrated set, so fail loud (§7 append-only rule). + if (!this.#shards.has(key)) { + throw new UnknownShardKey(key, [...this.#shards.keys()], `id "${id}"`); + } const bucket = byShard.get(key); if (bucket) bucket.push(id); else byShard.set(key, [id]); @@ -188,13 +431,11 @@ export class RoutingRunStore implements RunStore { return Promise.all(legs); } - // Every shard other than `key`, in probe order. With the compat constructor this yields exactly - // one entry, which is why each caller may take the first. At more than two shards a caller MUST - // fan out over all of them instead. + // Every distinct store other than `key`'s, in precedence order. With the compat constructor this + // yields exactly one entry, which is why each caller may take the first. At more than two shards + // a caller MUST fan out over all of them instead. #shardsExcept(key: ShardKey): Array<{ key: ShardKey; store: RunStore }> { - return this.#probeOrder - .filter((k) => k !== key) - .map((k) => ({ key: k, store: this.#shardStore(k) })); + return this.#distinctStores.filter((s) => s.key !== key); } // A `findRuns` caller bound to the given store (preserves `this`; the overload set isn't @@ -270,21 +511,27 @@ export class RoutingRunStore implements RunStore { if (typeof id !== "string") { return home; } + // A gen-2 waitpoint carries its shard in its id and its row lives there. No probe. + if (homeKey !== NEW_SHARD && homeKey !== LEGACY_SHARD) { + return home; + } if ( await home.findWaitpoint({ where: { id } }, onPrimary ? home.primaryReadClient : undefined) ) { return home; } - const [other] = this.#shardsExcept(homeKey); - if (other === undefined) { - return home; + for (const { key, store } of this.#gen1PairExcept(homeKey)) { + if ( + await store.findWaitpoint( + { where: { id } }, + onPrimary ? store.primaryReadClient : undefined + ) + ) { + this.#metrics.recordWaitpointProbeFallback(homeKey, key); + return store; + } } - return (await other.store.findWaitpoint( - { where: { id } }, - onPrimary ? other.store.primaryReadClient : undefined - )) - ? other.store - : home; + return home; } static #waitpointId(clause: unknown): string | undefined { @@ -468,12 +715,11 @@ export class RoutingRunStore implements RunStore { async #findRunsOpen(args: FindRunsArgs, client?: ReadClient): Promise { const { args: selArgs, addedFields } = ensureProjected(args); const fan = widenForMerge(selArgs); - const legs = await this.#fanOut(this.#precedence, (store) => - this.#findManyOn(store, client)(fan) - ); - const byId = new Map>(); - for (const r of legs.flat()) byId.set(r.id as string, r); - return finalizeRows([...byId.values()], args, addedFields); + const legs = await this.#fanOut(this.#precedence, async (store, key) => ({ + key, + rows: (await this.#findManyOn(store, client)(fan)) as Record[], + })); + return finalizeRows(this.#mergeById(legs), args, addedFields); } // Canonical grouped replacement for `Promise.all(ids.map(id => readThroughRun(id)))`: reuses @@ -539,14 +785,45 @@ export class RoutingRunStore implements RunStore { if (args.idempotencyKeys.length === 0) { return []; } - const legs = await this.#fanOut(this.#precedence, (store) => - store.findRunsByIdempotencyKeys(args, RoutingRunStore.#ownPrimary(store, client)) + const legs = await this.#fanOut(this.#precedence, (store, key) => + store + .findRunsByIdempotencyKeys(args, RoutingRunStore.#ownPrimary(store, client)) + .then((rows) => ({ key, rows })) ); - const byKey = new Map(); - for (const row of legs.flat()) { - if (row.idempotencyKey != null) byKey.set(row.idempotencyKey, row); + // Dedupe by KEY, not id: two runs on two shards can legitimately share a global key. Across the + // gen-1 pair the winner stays today's precedence result (NEW wins), which the duplicate-guard + // contract depends on. Once a gen-2 shard supplies a candidate, order by creation instead, so + // the winner does not depend on configured shard order. + const byKey = new Map>(); + for (const { key, rows } of legs) { + for (const row of rows) { + if (row.idempotencyKey == null) continue; + const bucket = byKey.get(row.idempotencyKey); + if (bucket) bucket.push({ key, row }); + else byKey.set(row.idempotencyKey, [{ key, row }]); + } + } + const out: IdempotencyKeyRunMatch[] = []; + for (const candidates of byKey.values()) { + const gen1Only = candidates.every(({ key }) => key === NEW_SHARD || key === LEGACY_SHARD); + if (gen1Only) { + out.push(candidates[candidates.length - 1]!.row); + continue; + } + out.push( + [...candidates].sort((a, b) => { + const byCreated = a.row.createdAt.getTime() - b.row.createdAt.getTime(); + return byCreated !== 0 + ? byCreated + : a.row.id < b.row.id + ? -1 + : a.row.id > b.row.id + ? 1 + : 0; + })[0]!.row + ); } - return [...byKey.values()]; + return out; } // --------------------------------------------------------------------------- @@ -1153,10 +1430,20 @@ export class RoutingRunStore implements RunStore { runId?: string ): Promise { if (runId === undefined) { + // No run id to partition on: query every distinct store and UNION by id, matching the routed + // path below. A drain-mirrored cuid pending on both gen-1 stores must count once, not twice — + // summing raw counts here would reintroduce the double count this method exists to remove. const legs = await this.#fanOut(this.#probeOrder, (store) => - store.countPendingWaitpoints(waitpointIds, RoutingRunStore.#ownPrimary(store, client)) + store.countPendingWaitpointsWithPresence( + waitpointIds, + RoutingRunStore.#ownPrimary(store, client) + ) ); - return legs.reduce((sum, leg) => sum + leg, 0); + const union = new Set(); + for (const leg of legs) { + for (const id of leg.pendingIds) union.add(id); + } + return union.size; } if (waitpointIds.length === 0) { @@ -1173,15 +1460,41 @@ export class RoutingRunStore implements RunStore { if (missing.length === 0) { return pendingIds.length; } - const [other] = this.#shardsExcept(runKey); - if (other === undefined) { + const plan = this.#partitionAbsentIds(runKey, missing); + if (plan.length === 0) { return pendingIds.length; } - const otherPending = await other.store.countPendingWaitpoints( - missing, - RoutingRunStore.#ownPrimary(other.store, client) + const legs = await Promise.all( + plan.map(async ({ key, ids }) => { + const store = this.#shardStore(key); + const { pendingIds: found } = await store.countPendingWaitpointsWithPresence( + ids, + RoutingRunStore.#ownPrimary(store, client) + ); + return { key, found }; + }) ); - return pendingIds.length + otherPending; + // UNION by id, never a sum of counts. A cuid mirrored onto both gen-1 stores appears twice; + // summing it is exactly the double count that leaves pendingCount above zero forever and never + // unblocks the run. The run store's pending set seeds the union; missing ids are disjoint from + // it by construction. #reportDuplicateId is the tripwire for a NON-mirror two-leg return, which + // a consistent resolveShard makes unreachable — it guards a future partition bug. + const union = new Set(pendingIds); + const seenFrom = new Map(); + for (const { key, found } of legs) { + for (const id of found) { + const keys = seenFrom.get(id); + if (keys) keys.push(key); + else seenFrom.set(id, [key]); + union.add(id); + } + } + for (const [id, keys] of seenFrom) { + if (keys.length > 1 && !this.#isGen1MirrorProbe(id)) { + this.#reportDuplicateId(id, keys); + } + } + return union.size; } // Fan out and union: an id lives on exactly one store in steady state (a drain-mirror can put it on @@ -1238,7 +1551,19 @@ export class RoutingRunStore implements RunStore { waitpointId: string | undefined ): RunStore { if (ownerId !== undefined) { - return this.#shardStore(this.#shardKeyOfSafe(ownerId)); + const key = this.#shardKeyOfSafe(ownerId); + // A gen-2 shard holds only ids stamped for that shard, because a waitpoint completes on the + // shard its own id names. Anything else stranded the blocked run: a cuid (routes to the gen-1 + // pair on completion), an id for a DIFFERENT gen-2 shard, or NO id at all — Prisma's + // @default(cuid()) then mints a cuid on the gen-2 shard after the write. The mint layer must + // stamp the owner's shard onto the waitpoint id, so fail loud rather than write an orphan. + const isGen2 = key !== NEW_SHARD && key !== LEGACY_SHARD; + if (isGen2 && (waitpointId === undefined || this.#shardKeyOfSafe(waitpointId) !== key)) { + throw new Error( + `RoutingRunStore: refusing to co-locate waitpoint "${waitpointId ?? ""}" onto gen-2 shard "${key}"; its id must be stamped for that shard` + ); + } + return this.#shardStore(key); } if (residency !== undefined) { return this.#shardStore(residency === "NEW" ? NEW_SHARD : LEGACY_SHARD); @@ -1350,10 +1675,11 @@ export class RoutingRunStore implements RunStore { return rows; } - // Collect the scalar waitpoint rows (relation re-resolution happens in the caller). With a run id in - // scope and a bounded id set to partition on, route to the run's store and fall back to the other DB - // for ONLY the ids missing there (a rare cross-tree token) — the two legs are disjoint by - // construction, so no dedup is needed. Otherwise fan out to BOTH and dedup by id NEW-wins. + // Collect the scalar waitpoint rows (relation re-resolution happens in the caller). With a run id + // in scope and a bounded id set, route to the run's store and fall back for ONLY the ids missing + // there. The fallback targets come from #partitionAbsentIds: a gen-2 id to its own shard, a cuid + // to BOTH gen-1 stores. The cuid legs are not disjoint, so the fallback rows are merged by id. + // Otherwise (no bounded id set) fan out to every store and dedup by id NEW-wins. async #collectManyWaitpoints( scalarArgs: Record, client: ReadClient | undefined, @@ -1375,38 +1701,41 @@ export class RoutingRunStore implements RunStore { if (missing.length === 0) { return fromRun; } - const [other] = this.#shardsExcept(runKey); - if (other === undefined) { + // Same partition as countPendingWaitpoints: a gen-2 missing id goes to its own shard, a + // cuid to both gen-1 stores. The cuid legs are NOT disjoint, so merge by id (a drain mirror + // appears once) rather than concatenate. + const plan = this.#partitionAbsentIds(runKey, missing); + if (plan.length === 0) { return fromRun; } - const fromOther = (await other.store.findManyWaitpoints( - narrowArgsToIds(scalarArgs, missing) as Prisma.WaitpointFindManyArgs, - RoutingRunStore.#ownPrimary(other.store, client) - )) as Record[]; - return [...fromRun, ...fromOther]; + const legs = await Promise.all( + plan.map(async ({ key, ids }) => { + const store = this.#shardStore(key); + return { + key, + rows: (await store.findManyWaitpoints( + narrowArgsToIds(scalarArgs, ids) as Prisma.WaitpointFindManyArgs, + RoutingRunStore.#ownPrimary(store, client) + )) as Record[], + }; + }) + ); + return [...fromRun, ...this.#mergeById(legs)]; } // No bounded id set to partition on → fall through to the fan-out path. } - const legs = await this.#fanOut( - this.#precedence, - (store) => - store.findManyWaitpoints( - scalarArgs as Prisma.WaitpointFindManyArgs, - RoutingRunStore.#ownPrimary(store, client) - ) as Promise[]> - ); - // A token mirrored onto both DBs during drain appears in BOTH legs; dedup by id in #precedence - // order, so the highest-authority copy wins. Without this, edge-waitpoint hydration could read a - // stale LEGACY status and strand the run. Rows whose projection omits `id` pass through. - const byId = new Map>(); - const passthrough: Record[] = []; - for (const w of legs.flat()) { - const id = w.id; - if (typeof id === "string") byId.set(id, w); - else passthrough.push(w); - } - return [...byId.values(), ...passthrough]; + // A token mirrored onto both DBs during drain appears in BOTH legs; #mergeById dedups by id in + // #precedence order, so the highest-authority copy wins. Without this, edge-waitpoint hydration + // could read a stale LEGACY status and strand the run. + const legs = await this.#fanOut(this.#precedence, async (store, key) => ({ + key, + rows: (await store.findManyWaitpoints( + scalarArgs as Prisma.WaitpointFindManyArgs, + RoutingRunStore.#ownPrimary(store, client) + )) as Record[], + })); + return this.#mergeById(legs); } // Re-resolve a waitpoint's group-A relations across BOTH DBs and attach them to `row`. Each target @@ -1537,27 +1866,36 @@ export class RoutingRunStore implements RunStore { waitpointId: string, context: ForWaitpointCompletionContext ): Promise { - // Preferred store: explicit legacy-authority pins first, else the waitpoint's id-shape. + // A gen-2 waitpoint's row lives on the shard its id names. The three legacy pins encode "the one + // non-NEW store", a gen-1 idea, so a gen-2 id OVERRIDES them: honouring the pin would send the + // completion write to legacy, match zero rows, and strand the blocked run. A gen-2 id is also + // directly routable, so it takes no probe. + const idKey = this.#shardKeyOfSafe(waitpointId); + const isGen2 = idKey !== NEW_SHARD && idKey !== LEGACY_SHARD; + if (isGen2) { + return this.#shardStore(idKey); + } const preferredKey = context.treeOwnerResidency === "LEGACY" || context.isCrossTreeIdempotency === true || context.hasLegacyParent === true ? LEGACY_SHARD - : this.#shardKeyOfSafe(waitpointId); + : idKey; const preferred = this.#shardStore(preferredKey); - // Resolve to where the waitpoint ACTUALLY lives: a migrated run's waitpoint can be on NEW - // with a LEGACY-classified id (or vice versa), so verify and fall back rather than route - // by id-shape alone and miss it (which leaves the blocked run stuck forever). This guard - // selects the store a WRITE (updateManyWaitpoints) then lands on, so it must probe each - // store's PRIMARY (mirroring #resolveWaitpointStore's onPrimary): a just-created waitpoint the - // replica hasn't caught up on would otherwise mis-resolve the owner and strand the run. + // Resolve to where a CUID waitpoint ACTUALLY lives: a migrated run's waitpoint can be on NEW + // with a LEGACY-classified id (or vice versa), so verify and fall back across the gen-1 pair + // rather than route by id-shape alone and miss it (which leaves the blocked run stuck forever). + // This guard selects the store a WRITE (updateManyWaitpoints) then lands on, so it must probe + // each store's PRIMARY: a just-created waitpoint the replica has not caught up on would + // otherwise mis-resolve the owner and strand the run. if ( await preferred.findWaitpoint({ where: { id: waitpointId } }, preferred.primaryReadClient) ) { return preferred; } - for (const { store } of this.#shardsExcept(preferredKey)) { + for (const { key, store } of this.#gen1PairExcept(preferredKey)) { if (await store.findWaitpoint({ where: { id: waitpointId } }, store.primaryReadClient)) { + this.#metrics.recordWaitpointProbeFallback(preferredKey, key); return store; } } @@ -1596,13 +1934,14 @@ export class RoutingRunStore implements RunStore { RoutingRunStore.#ownPrimary(store, client) )) as Record[]; } else { - const legs = await this.#fanOut(this.#precedence, (store) => - store.findManyTaskRunWaitpoints( + const legs = await this.#fanOut(this.#precedence, async (store, key) => ({ + key, + rows: (await store.findManyTaskRunWaitpoints( scalarArgs as typeof args, RoutingRunStore.#ownPrimary(store, client) - ) - ); - edges = dedupeEdgesById(legs.flat()) as Record[]; + )) as Record[], + })); + edges = this.#mergeById(legs); } if (waitpoint) { @@ -1717,7 +2056,13 @@ export class RoutingRunStore implements RunStore { ownerRunId?: string, tx?: PrismaClientOrTransaction ): Promise> { - const store = this.#routeOrNew(ownerRunId); + // A create is a mint decision the mint layer owns. Defaulting to NEW was harmless with one + // dedicated store; at N it is a silent write to the wrong shard, and the run-routed snapshot's + // checkpointId FK then resolves on a different database. Fail loud instead. + if (ownerRunId === undefined) { + throw new Error("createTaskRunCheckpoint requires ownerRunId to route"); + } + const store = this.#route(ownerRunId); return store.createTaskRunCheckpoint(args, ownerRunId, undefined); } @@ -1731,9 +2076,12 @@ export class RoutingRunStore implements RunStore { ): Promise { // Route by the batch's classifiable internal id: run-ops id→NEW, cuid→LEGACY. The caller's // `tx` is never forwarded — the create runs on the owning store's own client so the batch and - // its co-resident child runs/items land on the same DB. Mirrors the by-id waitpoint-write routing / - // updateBatchTaskRun. - const store = await this.#routeOrNewForWrite(data.id); + // its co-resident child runs/items land on the same DB. A create with no id is a mint decision + // the mint layer must have made; at N a silent NEW default is a wrong-shard write, so fail loud. + if (data.id === undefined) { + throw new Error("createBatchTaskRun requires data.id to route"); + } + const store = await this.#routeForWrite(data.id); return store.createBatchTaskRun(data, undefined); } @@ -1763,8 +2111,9 @@ export class RoutingRunStore implements RunStore { ): Promise | null> { // Never forward the caller's client verbatim (a cross-DB probe with one shared client can // only reach one DB); its presence resolves each leg to that store's OWN primary. - return this.#probeFirst((store) => - store.findBatchTaskRunById(id, args, RoutingRunStore.#ownPrimary(store, client)) + return this.#probeFirst( + (store) => store.findBatchTaskRunById(id, args, RoutingRunStore.#ownPrimary(store, client)), + { alarmOnDuplicate: false } ); } @@ -1777,13 +2126,15 @@ export class RoutingRunStore implements RunStore { ): Promise | null> { // Never forward the caller's client verbatim; its presence resolves each leg to that // store's OWN primary. - return this.#probeFirst((store) => - store.findBatchTaskRunByFriendlyId( - friendlyId, - environmentId, - args, - RoutingRunStore.#ownPrimary(store, client) - ) + return this.#probeFirst( + (store) => + store.findBatchTaskRunByFriendlyId( + friendlyId, + environmentId, + args, + RoutingRunStore.#ownPrimary(store, client) + ), + { alarmOnDuplicate: false } ); } @@ -1802,13 +2153,15 @@ export class RoutingRunStore implements RunStore { ): Promise | null> { // Never forward the caller's client verbatim; its presence resolves each leg to that // store's OWN primary. - return this.#probeFirst((store) => - store.findBatchTaskRunByIdempotencyKey( - environmentId, - idempotencyKey, - args, - RoutingRunStore.#ownPrimary(store, client) - ) + return this.#probeFirst( + (store) => + store.findBatchTaskRunByIdempotencyKey( + environmentId, + idempotencyKey, + args, + RoutingRunStore.#ownPrimary(store, client) + ), + { alarmOnDuplicate: false } ); } @@ -1921,17 +2274,20 @@ export class RoutingRunStore implements RunStore { skip: 0, ...(args.take != null ? { take: skip + args.take } : {}), }; - const legs = await this.#fanOut(this.#precedence, (store) => - store.findManyWaitpointTags(perLeg, RoutingRunStore.#ownPrimary(store, client)) - ); - const byId = new Map(); - for (const tag of legs.flat()) byId.set(tag.id, tag); + const legs = await this.#fanOut(this.#precedence, async (store, key) => ({ + key, + rows: (await store.findManyWaitpointTags( + perLeg, + RoutingRunStore.#ownPrimary(store, client) + )) as unknown as Array>, + })); + const deduped = this.#mergeById(legs) as unknown as WaitpointTag[]; const merged = args.orderBy ? (sortByOrderBy( - [...byId.values()] as unknown as Array>, + deduped as unknown as Array>, args.orderBy as unknown as NonNullable ) as unknown as WaitpointTag[]) - : [...byId.values()]; + : deduped; return merged.slice(skip, args.take != null ? skip + args.take : undefined); } @@ -2055,20 +2411,6 @@ function narrowArgsToIds(args: Record, ids: string[]): Record(rows: R[]): R[] { - const byId = new Map(); - const passthrough: R[] = []; - for (const row of rows) { - const id = (row as { id?: unknown }).id; - if (typeof id === "string") byId.set(id, row); - else passthrough.push(row); - } - return [...byId.values(), ...passthrough]; -} - // A caller sub-select for an edge relation: `{ select?, include? }`, `true` for a bare `key: true`, // or undefined when not requested. type SubProjection = { select?: any; include?: any } | true | undefined; diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts new file mode 100644 index 00000000000..2b10bd766d0 --- /dev/null +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -0,0 +1,115 @@ +// The member names of RunStore, as data. +// +// The decorator suites enumerate this to drive one call per member. Member PRESENCE is not proved +// here: that is the compiler's job, through `implements RunStore` on the pass-through base and the +// assertions at the foot of this file. +// +import type { RunStore } from "./types.js"; + +// Every method the RunStore interface declares. The forwarding probe enumerates this to drive one +// call per member; member PRESENCE is proved by the compiler, in the assertions at the foot of this +// file and by `implements RunStore` on the generated class. +export const RUN_STORE_METHOD_NAMES = [ + "runInTransaction", + "createRun", + "createCancelledRun", + "createFailedRun", + "startAttempt", + "completeAttemptSuccess", + "recordRetryOutcome", + "requeueRun", + "recordBulkActionMembership", + "cancelRun", + "failRunPermanently", + "finalizeRun", + "expireRun", + "expireRunsBatch", + "lockRunToWorker", + "parkPendingVersion", + "promotePendingVersionRuns", + "expireParkedRun", + "suspendForCheckpoint", + "resumeFromCheckpoint", + "rescheduleRun", + "enqueueDelayedRun", + "rewriteDebouncedRun", + "updateMetadata", + "clearIdempotencyKey", + "pushTags", + "pushRealtimeStream", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", + "findRunsByIdempotencyKeys", + "createBatchTaskRunItem", + "findLatestExecutionSnapshot", + "findExecutionSnapshot", + "findManyExecutionSnapshots", + "createExecutionSnapshot", + "findSnapshotCompletedWaitpointIds", + "findSnapshotCompletedWaitpointIdsWithPresence", + "findWaitpointConnectedRunIds", + "findWaitpointCompletedSnapshotIds", + "blockRunWithWaitpointEdges", + "countPendingWaitpoints", + "countPendingWaitpointsWithPresence", + "createWaitpoint", + "upsertWaitpoint", + "findWaitpoint", + "findWaitpointOnPrimary", + "findManyWaitpoints", + "updateWaitpoint", + "updateManyWaitpoints", + "forWaitpointCompletion", + "findManyTaskRunWaitpoints", + "deleteManyTaskRunWaitpoints", + "findTaskRunAttempt", + "createTaskRunCheckpoint", + "createBatchTaskRun", + "updateBatchTaskRun", + "findBatchTaskRunById", + "findBatchTaskRunByFriendlyId", + "findBatchTaskRunByIdempotencyKey", + "updateManyBatchTaskRun", + "countBatchTaskRunItems", + "updateManyBatchTaskRunItems", + "findManyBatchTaskRunItems", + "findBatchTaskRunItem", + "upsertWaitpointTag", + "findManyWaitpointTags", +] as const; + +// Data properties the base exposes as getters over the delegate, not as forwarders. +export const RUN_STORE_PROPERTY_NAMES = ["primaryReadClient"] as const; + +// --------------------------------------------------------------------------- +// Parity with the interface, checked by the compiler. +// +// The lists above are produced by parsing types.ts. These assertions compare them +// against `keyof RunStore`, which the compiler derives from the interface itself, +// so a name this generator failed to parse, or invented, is a build failure rather +// than a silent gap. Both directions are checked: a missing name and an extra one. +// --------------------------------------------------------------------------- + +type RunStoreMemberName = + | (typeof RUN_STORE_METHOD_NAMES)[number] + | (typeof RUN_STORE_PROPERTY_NAMES)[number]; + +/** Fails when the interface declares a member the generator did not emit. */ +type _EveryInterfaceMemberIsListed = [Exclude] extends [never] + ? true + : never; +const _everyInterfaceMemberIsListed: _EveryInterfaceMemberIsListed = true; +void _everyInterfaceMemberIsListed; + +/** Fails when the generator emitted a name the interface does not declare. */ +type _EveryListedNameIsOnTheInterface = [Exclude] extends [ + never, +] + ? true + : never; +const _everyListedNameIsOnTheInterface: _EveryListedNameIsOnTheInterface = true; +void _everyListedNameIsOnTheInterface; diff --git a/internal-packages/run-store/src/snapshotComparator.isolation.test.ts b/internal-packages/run-store/src/snapshotComparator.isolation.test.ts new file mode 100644 index 00000000000..ae71d227b80 --- /dev/null +++ b/internal-packages/run-store/src/snapshotComparator.isolation.test.ts @@ -0,0 +1,57 @@ +// Proves the Frozen rule: the comparator's VALUE-import set is empty. Every import it has is +// `import type`, erased at runtime, so the compiled module pulls in no Redis or Prisma client and +// cannot read. Goes red the instant any value import is added — a client, the barrel, or a dynamic +// import(). The detector is pinned against redisSnapshotStore.ts (which value-imports a client) so +// this cannot pass as a tautology. +import { expect, it, describe } from "vitest"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, resolve } from "node:path"; + +const here = dirname(fileURLToPath(import.meta.url)); + +// Returns the module specifiers a file imports FOR VALUE (i.e. that survive to runtime). `import type` +// declarations and named blocks whose specifiers are all inline `type` are erased and excluded. +function valueImports(sourcePath: string): string[] { + const raw = readFileSync(sourcePath, "utf8"); + const out: string[] = []; + + // Statements are scanned on RAW source, anchored to line start (`^\s*import`), so a `//` comment + // line never matches and no stripping can hide a real import. Only the mid-line dynamic `import(` + // check runs on comment-stripped source. The pin test below guarantees the scan catches a real import. + const stripped = raw.replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/.*$/gm, ""); + if (/(^|[^.\w])import\s*\(/.test(stripped)) out.push(""); + + const importRe = /^\s*import\b([\s\S]*?)\bfrom\s*["']([^"']+)["']/gm; + for (let m = importRe.exec(raw); m !== null; m = importRe.exec(raw)) { + const clause = m[1]; + const spec = m[2]; + if (/^\s*type\b/.test(clause)) continue; // `import type ... from` + const named = clause.match(/\{([\s\S]*?)\}/); + // Strip inline `type Foo` specifiers, including an `as Bar` alias, before checking whether any + // value specifier remains. + const inlineType = /\btype\s+[A-Za-z_$][\w$]*(?:\s+as\s+[A-Za-z_$][\w$]*)?/g; + if (named && !/(^|,)\s*[A-Za-z_$]/.test(named[1].replace(inlineType, ""))) { + continue; // every named specifier is an inline `type` — nothing left for value + } + out.push(spec); + } + + // Bare side-effect imports (`import "x"`) run the module. + const bareRe = /^\s*import\s*["']([^"']+)["']/gm; + for (let m = bareRe.exec(raw); m !== null; m = bareRe.exec(raw)) out.push(m[1]); + + return out; +} + +describe("comparator read-isolation", () => { + it("the detector flags a real value import (pin against the store)", () => { + // redisSnapshotStore.ts value-imports @internal/redis, so a working detector MUST see it. + const storeImports = valueImports(resolve(here, "redisSnapshotStore.ts")); + expect(storeImports).toContain("@internal/redis"); + }); + + it("the comparator has no value imports — it is import-type-only and cannot read", () => { + expect(valueImports(resolve(here, "snapshotComparator.ts"))).toEqual([]); + }); +}); diff --git a/internal-packages/run-store/src/snapshotComparator.test.ts b/internal-packages/run-store/src/snapshotComparator.test.ts new file mode 100644 index 00000000000..ce61386c85a --- /dev/null +++ b/internal-packages/run-store/src/snapshotComparator.test.ts @@ -0,0 +1,254 @@ +import { expect, it, describe } from "vitest"; +import { + diffLatest, + diffSince, + normalizeFromRedis, + normalizeFromPg, + SnapshotComparator, + type DivergenceClass, + type NormalizedSnapshot, +} from "./snapshotComparator.js"; +import type { SnapshotRead } from "./redisSnapshotStore.js"; + +function norm(over: Partial = {}): NormalizedSnapshot { + const base: NormalizedSnapshot = { + id: "s1", + engine: "V2", + executionStatus: "RUN_CREATED", + description: "d", + isValid: true, + error: null, + previousSnapshotId: null, + runId: "r1", + runStatus: "PENDING", + batchId: null, + attemptNumber: null, + environmentId: "env", + environmentType: "DEVELOPMENT", + projectId: "p", + organizationId: "o", + checkpointId: null, + workerId: null, + runnerId: null, + createdAt: 1000, + updatedAt: 1000, + metadata: null, + completedWaitpointOrder: [], + waitpointIdSet: [], + }; + return { ...base, ...over }; +} + +describe("diffLatest", () => { + it("no divergence when the two sides match", () => { + expect(diffLatest(norm(), norm())).toEqual([]); + }); + + it("reports a scalar difference by field", () => { + expect(diffLatest(norm(), norm({ executionStatus: "EXECUTING" }))).toEqual([ + { field: "executionStatus", class: "scalar", pg: "RUN_CREATED", redis: "EXECUTING" }, + ]); + }); + + it("compares createdAt and updatedAt by strict equality", () => { + expect(diffLatest(norm(), norm({ createdAt: 1001 }))).toEqual([ + { field: "createdAt", class: "scalar", pg: 1000, redis: 1001 }, + ]); + }); + + it("classifies a validity mismatch", () => { + const d = diffLatest(norm({ isValid: true }), norm({ isValid: false, error: "boom" })); + expect(d.map((x) => x.field).sort()).toEqual(["error", "isValid"]); + expect(d.find((x) => x.field === "isValid")!.class).toBe("validity"); + }); + + it("classifies completedWaitpointOrder differences as order, repeats significant", () => { + expect( + diffLatest( + norm({ completedWaitpointOrder: ["a", "a", "b"] }), + norm({ completedWaitpointOrder: ["a", "b"] }) + ) + ).toEqual([ + { field: "completedWaitpointOrder", class: "order", pg: ["a", "a", "b"], redis: ["a", "b"] }, + ]); + }); + + it("classifies waitpoint id set differences, order-insensitive", () => { + expect( + diffLatest(norm({ waitpointIdSet: ["a", "b"] }), norm({ waitpointIdSet: ["a", "b"] })) + ).toEqual([]); + const d2 = diffLatest(norm({ waitpointIdSet: ["a", "b"] }), norm({ waitpointIdSet: ["a"] })); + expect(d2[0]).toMatchObject({ field: "waitpointIdSet", class: "waitpointIdSet" }); + }); + + it("does NOT emit a divergence for a rotated idempotency key — invisible at id-set granularity", () => { + expect(diffLatest(norm({ waitpointIdSet: ["w1"] }), norm({ waitpointIdSet: ["w1"] }))).toEqual( + [] + ); + }); + + it("missingInRedis when the row exists only in Postgres", () => { + expect(diffLatest(norm(), null)).toEqual([ + expect.objectContaining({ class: "missingInRedis" }), + ]); + }); + + it("missingInPg when the row exists only in Redis", () => { + expect(diffLatest(null, norm())).toEqual([expect.objectContaining({ class: "missingInPg" })]); + }); + + it("raises unknownField for a key on neither the compared nor excluded list", () => { + const d = diffLatest(norm(), { ...norm(), somethingNew: 1 } as NormalizedSnapshot); + expect(d).toEqual([expect.objectContaining({ field: "somethingNew", class: "unknownField" })]); + }); + + it("normalizeFromRedis carries an unrecognised entry field, so unknownField fires on real input", () => { + const read: SnapshotRead = { + id: "s1", + seq: 1, + isValid: true, + raw: "{}", + entry: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "d", + runId: "r1", + runStatus: "PENDING", + createdAt: "2026-08-24T00:00:00.000Z", + environmentId: "env", + environmentType: "DEVELOPMENT", + projectId: "p", + organizationId: "o", + mysteryField: "surprise", + }, + }; + const redis = normalizeFromRedis(read); + expect(redis.mysteryField).toBe("surprise"); // not dropped by normalization + const d = diffLatest( + norm({ id: "s1", createdAt: redis.createdAt, updatedAt: redis.updatedAt }), + redis + ); + expect(d).toEqual([ + expect.objectContaining({ field: "mysteryField", class: "unknownField", redis: "surprise" }), + ]); + }); + + it("surfaces an inherited-name key and does not pollute the prototype", () => { + // JSON.parse produces OWN keys for `toString` and `__proto__` (unlike an object literal). + const entry = JSON.parse( + '{"engine":"V2","executionStatus":"RUN_CREATED","description":"d","runId":"r1",' + + '"runStatus":"PENDING","createdAt":"2026-08-24T00:00:00.000Z","environmentId":"env",' + + '"environmentType":"DEVELOPMENT","projectId":"p","organizationId":"o",' + + '"toString":"surprise","__proto__":{"polluted":true}}' + ) as Record; + const read: SnapshotRead = { id: "s1", seq: 1, isValid: true, raw: "{}", entry }; + const n = normalizeFromRedis(read) as Record; + + expect(Object.keys(n)).toContain("toString"); // carried as an own key despite the inherited name + expect(n["toString"]).toBe("surprise"); + expect(Object.getPrototypeOf(n)).toBe(Object.prototype); // __proto__ skipped, no pollution + expect("polluted" in {}).toBe(false); + + const d = diffLatest( + norm({ id: "s1", createdAt: n.createdAt as number, updatedAt: n.updatedAt as number }), + n as NormalizedSnapshot + ); + expect(d.some((x) => x.field === "toString" && x.class === "unknownField")).toBe(true); + }); + + it("normalizeFromPg's waitpointIdSet is index-bearing only, matching the Redis read surface", () => { + // A non-indexed completed waitpoint is in the relation but not in completedWaitpointOrder; Redis's + // distinctIds (dedupe of order) does not expose it, so the PG side must not either. + const row = { + id: "s1", + engine: "V2", + executionStatus: "EXECUTING", + description: "d", + isValid: true, + error: null, + previousSnapshotId: null, + runId: "r1", + runStatus: "EXECUTING", + batchId: null, + attemptNumber: null, + environmentId: "env", + environmentType: "DEVELOPMENT", + projectId: "p", + organizationId: "o", + checkpointId: null, + workerId: null, + runnerId: null, + createdAt: new Date(1000), + updatedAt: new Date(1000), + metadata: null, + completedWaitpointOrder: ["w_indexed"], + completedWaitpoints: [{ id: "w_indexed" }, { id: "w_nonindexed" }], + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const n = normalizeFromPg(row as any); + expect(n.waitpointIdSet).toEqual(["w_indexed"]); + }); +}); + +describe("diffSince", () => { + const cursor = { id: "s1", createdAtMs: 1000 }; + + it("a Postgres-only entry at the cursor ms is a lost append (missingInRedis), never a tie", () => { + const pg = [norm({ id: "s2", createdAt: 1000, previousSnapshotId: "s1" })]; + expect(diffSince({ pg, redis: [], cursor })).toEqual([ + expect.objectContaining({ field: "s2", class: "missingInRedis" }), + ]); + }); + + it("a Redis-only chain-boundary surplus at the cursor ms is expected:redisSurplusAtCursorTie", () => { + const redis = [norm({ id: "s2", createdAt: 1000, previousSnapshotId: "s1" })]; + expect(diffSince({ pg: [], redis, cursor })).toEqual([ + expect.objectContaining({ field: "s2", class: "expected:redisSurplusAtCursorTie" }), + ]); + }); + + it("a Redis-only surplus that is NOT a chain boundary is a real missingInPg", () => { + const redis = [norm({ id: "s3", createdAt: 1000, previousSnapshotId: "s2" })]; + expect(diffSince({ pg: [], redis, cursor })).toEqual([ + expect.objectContaining({ field: "s3", class: "missingInPg" }), + ]); + }); + + it("a Redis-only surplus above the cursor ms is a real missingInPg", () => { + const redis = [norm({ id: "s2", createdAt: 1500, previousSnapshotId: "s1" })]; + expect(diffSince({ pg: [], redis, cursor })).toEqual([ + expect.objectContaining({ field: "s2", class: "missingInPg" }), + ]); + }); +}); + +describe("SnapshotComparator", () => { + it("shouldSample honours the injected rng and percent", () => { + expect(new SnapshotComparator({ samplePercent: 10, rng: () => 0.05 }).shouldSample()).toBe( + true + ); + expect(new SnapshotComparator({ samplePercent: 10, rng: () => 0.5 }).shouldSample()).toBe( + false + ); + }); + + it("record emits one metric per divergence, tagged by class and op, and returns void", () => { + const seen: Array<{ op: string; cls: DivergenceClass }> = []; + const cmp = new SnapshotComparator({ + samplePercent: 100, + metrics: { + recordDivergence: (op, cls) => seen.push({ op, cls }), + recordSample: () => {}, + }, + }); + const ret = cmp.record("getLatest", [ + { field: "executionStatus", class: "scalar" }, + { field: "idempotencyKey", class: "expected:rotatedIdempotencyKey" }, + ]); + expect(ret).toBeUndefined(); + expect(seen).toEqual([ + { op: "getLatest", cls: "scalar" }, + { op: "getLatest", cls: "expected:rotatedIdempotencyKey" }, + ]); + }); +}); diff --git a/internal-packages/run-store/src/snapshotComparator.ts b/internal-packages/run-store/src/snapshotComparator.ts new file mode 100644 index 00000000000..d9f0407c030 --- /dev/null +++ b/internal-packages/run-store/src/snapshotComparator.ts @@ -0,0 +1,291 @@ +// Compare-mode read comparator: PURE diff layer. It NEVER serves a read — it takes results the caller +// already obtained and reports how the two stores disagree, by field, with a class. Type-only imports +// of client types, so this module holds no Redis or Prisma client (proven by the isolation test). +import type { Prisma } from "@trigger.dev/database"; +import type { SnapshotRead } from "./redisSnapshotStore.js"; + +export type DivergenceClass = + | "missingInRedis" + | "missingInPg" + | "scalar" + | "order" + | "waitpointIdSet" + | "validity" + | "unknownField" + // Reserved shared vocabulary for the payload-comparing layer a later ticket adds. This module + // compares id sets and order, not record payloads, so it never emits this — a rotated idempotency + // key does not change a waitpoint id. Kept in the union so the metric tag space stays stable. + | "expected:rotatedIdempotencyKey" + | "expected:redisSurplusAtCursorTie"; + +export type SnapshotDivergence = { + field: string; + class: DivergenceClass; + pg?: unknown; + redis?: unknown; +}; + +// The read operations the comparator samples. Bounded so the `op` metric attribute cannot become a +// high-cardinality label (a caller cannot pass a run id or other unbounded value). +export type SnapshotReadOp = "getLatest" | "getById" | "getSince" | "getSnapshotWaitpointIds"; + +export type NormalizedSnapshot = { + [k: string]: unknown; + id: string; + createdAt: number; // ms + updatedAt: number; // ms + completedWaitpointOrder: string[]; + waitpointIdSet: string[]; + previousSnapshotId?: string | null; +}; + +// The 22 compared entry columns. EXCLUDED_FIELDS names the columns deliberately not compared; any key +// on a normalized entry that is on neither list raises `unknownField`, so a new column fails loudly. +export const COMPARED_FIELDS = [ + "id", + "engine", + "executionStatus", + "description", + "isValid", + "error", + "previousSnapshotId", + "runId", + "runStatus", + "batchId", + "attemptNumber", + "environmentId", + "environmentType", + "projectId", + "organizationId", + "checkpointId", + "workerId", + "runnerId", + "createdAt", + "updatedAt", + "metadata", +] as const; + +// lastHeartbeatAt: Postgres-only, never written by the current engine. Waitpoint/checkpoint payloads: +// expanded by the run-engine resolver, out of this module's scope. +export const EXCLUDED_FIELDS = ["lastHeartbeatAt", "checkpoint", "completedWaitpoints"] as const; + +const SCALAR_FIELDS = COMPARED_FIELDS.filter((f) => f !== "metadata"); + +const KNOWN_KEYS = new Set([ + ...COMPARED_FIELDS, + ...EXCLUDED_FIELDS, + "completedWaitpointOrder", + "waitpointIdSet", +]); + +// Carry a source key normalization does not recognise onto the normalized object, so the unknownField +// check sees it instead of it being silently dropped (a false clean comparison). Skips the +// prototype-pollution keys. No own-property guard is needed: the normalizer only ever sets KNOWN_KEYS, +// so a non-known source key is never already present and cannot overwrite a normalized value. +const DANGEROUS_KEYS = new Set(["__proto__", "constructor", "prototype"]); +function carryUnknownKeys(target: NormalizedSnapshot, source: Record): void { + for (const k of Object.keys(source)) { + if (DANGEROUS_KEYS.has(k)) continue; + if (!KNOWN_KEYS.has(k)) target[k] = source[k]; + } +} + +function canonicalJson(v: unknown): string { + if (v === null || typeof v !== "object") return JSON.stringify(v) ?? "null"; + if (Array.isArray(v)) return `[${v.map(canonicalJson).join(",")}]`; + const obj = v as Record; + const keys = Object.keys(obj).sort(); + return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(obj[k])}`).join(",")}}`; +} + +export function normalizeFromPg( + row: Prisma.TaskRunExecutionSnapshotGetPayload<{ include: { completedWaitpoints: true } }> +): NormalizedSnapshot { + const n: NormalizedSnapshot = { + id: row.id, + engine: row.engine, + executionStatus: row.executionStatus, + description: row.description, + isValid: row.isValid, + error: row.error ?? null, + previousSnapshotId: row.previousSnapshotId ?? null, + runId: row.runId, + runStatus: row.runStatus, + batchId: row.batchId ?? null, + attemptNumber: row.attemptNumber ?? null, + environmentId: row.environmentId, + environmentType: row.environmentType, + projectId: row.projectId, + organizationId: row.organizationId, + checkpointId: row.checkpointId ?? null, + workerId: row.workerId ?? null, + runnerId: row.runnerId ?? null, + createdAt: row.createdAt.getTime(), + updatedAt: row.updatedAt.getTime(), + metadata: row.metadata ?? null, + completedWaitpointOrder: [...(row.completedWaitpointOrder ?? [])], + // Index-bearing distinct set, from completedWaitpointOrder, to match the Redis read surface + // (distinctIds = dedupe of `order`). The full relation holds non-indexed ids Redis does not + // expose here (payload-layer, out of scope), so comparing it would fire a spurious divergence. + waitpointIdSet: [...new Set(row.completedWaitpointOrder ?? [])].sort(), + }; + carryUnknownKeys(n, row as unknown as Record); + return n; +} + +export function normalizeFromRedis(read: SnapshotRead): NormalizedSnapshot { + const e = read.entry as Record; + const createdAtMs = new Date(String(e.createdAt)).getTime(); + const order = read.completedWaitpointIds?.order ?? []; + const idSet = [...(read.completedWaitpointIds?.distinctIds ?? [])].sort(); + const n: NormalizedSnapshot = { + id: read.id, + engine: (e.engine ?? "V2") as string, + executionStatus: e.executionStatus as string, + description: e.description as string, + isValid: read.isValid, + error: (e.error ?? null) as string | null, + previousSnapshotId: (e.previousSnapshotId ?? null) as string | null, + runId: e.runId as string, + runStatus: e.runStatus as string, + batchId: (e.batchId ?? null) as string | null, + attemptNumber: (e.attemptNumber ?? null) as number | null, + environmentId: e.environmentId as string, + environmentType: e.environmentType as string, + projectId: e.projectId as string, + organizationId: e.organizationId as string, + checkpointId: (e.checkpointId ?? null) as string | null, + workerId: (e.workerId ?? null) as string | null, + runnerId: (e.runnerId ?? null) as string | null, + createdAt: createdAtMs, + updatedAt: createdAtMs, // write-once row: updatedAt equals createdAt + metadata: e.metadata ?? null, + completedWaitpointOrder: [...order], + waitpointIdSet: idSet, + }; + carryUnknownKeys(n, e); + return n; +} + +function sameArray(a: string[], b: string[]): boolean { + return a.length === b.length && a.every((x, i) => x === b[i]); +} + +function fieldDivergences(pg: NormalizedSnapshot, redis: NormalizedSnapshot): SnapshotDivergence[] { + const out: SnapshotDivergence[] = []; + + for (const f of SCALAR_FIELDS) { + if (pg[f] !== redis[f]) { + out.push({ + field: f, + class: f === "isValid" ? "validity" : "scalar", + pg: pg[f], + redis: redis[f], + }); + } + } + if (canonicalJson(pg.metadata) !== canonicalJson(redis.metadata)) { + out.push({ field: "metadata", class: "scalar", pg: pg.metadata, redis: redis.metadata }); + } + if (!sameArray(pg.completedWaitpointOrder, redis.completedWaitpointOrder)) { + out.push({ + field: "completedWaitpointOrder", + class: "order", + pg: pg.completedWaitpointOrder, + redis: redis.completedWaitpointOrder, + }); + } + if (!sameArray(pg.waitpointIdSet, redis.waitpointIdSet)) { + out.push({ + field: "waitpointIdSet", + class: "waitpointIdSet", + pg: pg.waitpointIdSet, + redis: redis.waitpointIdSet, + }); + } + // Unknown keys on EITHER side, so a new field in either store fails loudly. + for (const k of new Set([...Object.keys(pg), ...Object.keys(redis)])) { + if (!KNOWN_KEYS.has(k)) { + out.push({ field: k, class: "unknownField", pg: pg[k], redis: redis[k] }); + } + } + return out; +} + +export function diffLatest( + pg: NormalizedSnapshot | null, + redis: NormalizedSnapshot | null +): SnapshotDivergence[] { + if (pg && !redis) return [{ field: pg.id, class: "missingInRedis", pg }]; + if (redis && !pg) return [{ field: redis.id, class: "missingInPg", redis }]; + if (!pg || !redis) return []; + return fieldDivergences(pg, redis); +} + +export type SnapshotComparatorMetrics = { + recordDivergence(op: SnapshotReadOp, cls: DivergenceClass): void; + recordSample(op: SnapshotReadOp): void; +}; + +// Samples reads and records divergence metrics. Holds no store and returns nothing from record(), so +// it structurally cannot serve a read. samplePercent is injected, never read from env.server. +export class SnapshotComparator { + readonly #samplePercent: number; + readonly #metrics?: SnapshotComparatorMetrics; + readonly #rng: () => number; + + constructor(opts: { + samplePercent: number; + metrics?: SnapshotComparatorMetrics; + rng?: () => number; + }) { + this.#samplePercent = opts.samplePercent; + this.#metrics = opts.metrics; + this.#rng = opts.rng ?? Math.random; + } + + shouldSample(): boolean { + return this.#rng() * 100 < this.#samplePercent; + } + + record(op: SnapshotReadOp, divergences: SnapshotDivergence[]): void { + this.#metrics?.recordSample(op); + for (const d of divergences) this.#metrics?.recordDivergence(op, d.class); + } +} + +export function diffSince(args: { + pg: NormalizedSnapshot[]; + redis: NormalizedSnapshot[]; + cursor: { id: string; createdAtMs: number }; +}): SnapshotDivergence[] { + const { pg, redis, cursor } = args; + const byId = (xs: NormalizedSnapshot[]) => new Map(xs.map((x) => [x.id, x])); + const pgMap = byId(pg); + const redisMap = byId(redis); + const out: SnapshotDivergence[] = []; + + // Present on both: field-diff. + for (const [id, p] of pgMap) { + const r = redisMap.get(id); + if (r) out.push(...fieldDivergences(p, r)); + } + // Postgres-only: ALWAYS a lost append. A same-ms tie can never surface here, because Postgres's own + // window drops the same-ms entry too. So there is no "expected tie" on this side. + for (const [id, p] of pgMap) { + if (!redisMap.has(id)) out.push({ field: id, class: "missingInRedis", pg: p }); + } + // Redis-only: expected ONLY when it is a chain boundary sitting exactly on the cursor ms (the + // id-cursor getSince path keeps a same-ms entry that Postgres's `> cursor` drops). Anything else is + // a real surplus. + for (const [id, r] of redisMap) { + if (pgMap.has(id)) continue; + const isTie = r.createdAt === cursor.createdAtMs && r.previousSnapshotId === cursor.id; + out.push({ + field: id, + class: isTie ? "expected:redisSurplusAtCursorTie" : "missingInPg", + redis: r, + }); + } + return out; +} diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts new file mode 100644 index 00000000000..635160fd831 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -0,0 +1,522 @@ +// The entry is built from a write site's input while Postgres builds the row from the same input by +// a different code path. This suite is the only thing that keeps those two paths equal, so it covers +// every one of the ten physical snapshot-create sites in PostgresRunStore. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, +} from "./snapshotEntry.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +/** + * NOTE ON createdAt. An earlier version of this suite built the expected entry with + * `createdAt: row.createdAt`, reading the value off the row it was checking and then asserting the + * two matched. That can never fail, and it hid a real divergence: seven of the eight write sites + * stamped the entry from the app clock while Postgres stamped its own column default, so the two + * stores held different instants for one snapshot. + * + * Every case now mints ONE instant, passes it to the store call, and gives the builder the same + * value. The row must carry it because the write site forwards it. A write site that stops + * forwarding the caller's instant fails here. + * + * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and + * it holds `createdAt` as an ISO string, so those are checked separately or not at all. + */ +function assertParity(entry: SnapshotEntryInput, row: Record) { + expect(row.id).toBe(entry.id); + expect(row.runId).toBe(entry.runId); + expect(row.engine).toBe(entry.engine); + expect(row.executionStatus).toBe(entry.executionStatus); + expect(row.description).toBe(entry.description); + expect(row.runStatus).toBe(entry.runStatus); + expect(row.environmentId).toBe(entry.environmentId); + expect(row.environmentType).toBe(entry.environmentType); + expect(row.projectId).toBe(entry.projectId); + expect(row.organizationId).toBe(entry.organizationId); + expect(row.attemptNumber ?? undefined).toBe(entry.attemptNumber ?? undefined); + expect(row.previousSnapshotId ?? undefined).toBe(entry.previousSnapshotId ?? undefined); + expect(row.batchId ?? undefined).toBe(entry.batchId ?? undefined); + expect(row.checkpointId ?? undefined).toBe(entry.checkpointId ?? undefined); + expect(row.workerId ?? undefined).toBe(entry.workerId ?? undefined); + expect(row.runnerId ?? undefined).toBe(entry.runnerId ?? undefined); + expect(row.isValid).toBe(entry.error === undefined); + expect((row.createdAt as Date).toISOString()).toBe(entry.createdAt); + // Write-once rows: both columns hold the one instant, so a Prisma-stamped updatedAt would drift. + expect((row.updatedAt as Date).toISOString()).toBe(entry.createdAt); +} + +/** Five minutes in the past, so a database default could never coincide with it. */ +const independentStamp = new Date(Date.now() - 5 * 60 * 1000); + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("entry to Postgres row parity", () => { + postgresTest("createRun, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createRun with an associated waitpoint, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot, + associatedWaitpoint: { + id: generateInternalId(), + friendlyId: `waitpoint_${runId.slice(-12)}`, + type: "RUN", + status: "PENDING", + idempotencyKey: generateInternalId(), + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createRun carries the worker and runner ids", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const { workerId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { ...birthSnapshot(id, env), workerId, runnerId: "runner_1" }; + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createCancelledRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { + ...birthSnapshot(id, env), + executionStatus: "FINISHED" as const, + description: "Run was cancelled", + runStatus: "CANCELED" as const, + }; + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("completeAttemptSuccess", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromCompletion({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("expireRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("expireParkedRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Parked run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("rescheduleRun with every default applied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("rescheduleRun with every value supplied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + executionStatus: "QUEUED" as const, + runStatus: "PENDING" as const, + description: "custom reschedule", + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("lockRunToWorker", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const previous = await store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: "Run was queued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + previousSnapshotId: previous.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }; + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromLock({ id, runId: run.id, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createExecutionSnapshot, the standalone site", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const created = await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(created.id).toBe(id); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "DEQUEUED" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING" as const, description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.runStatus).toBe("PENDING"); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot with an error is invalid in both", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.isValid).toBe(false); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); +}); + +// The clock-provenance guard. Independent of the builders above: it asserts that what Postgres +// stores is the instant the CALLER supplied, not one the database chose. Without this, a snapshot +// has two different creation times depending on which store answers, the compared field can never +// reach zero divergence, and the since-window cursor resolved from one store misfilters the window +// walked in the other. +describe("createdAt provenance", () => { + postgresTest("Postgres stores the caller's instant, not its own", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + // Far enough from now that a database default could never coincide with it. + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.createExecutionSnapshot({ + id, + createdAt: stamp, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); + + postgresTest("an absent instant still takes the database default", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const before = new Date(Date.now() - 1000); + + // Mode off supplies nothing, so Postgres must behave exactly as it always has. This is what + // keeps the merge test true. + await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.getTime()).toBeGreaterThan(before.getTime()); + }); + + postgresTest("a nested write site stores the caller's instant too", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + createdAt: stamp, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.test.ts b/internal-packages/run-store/src/snapshotEntry.test.ts new file mode 100644 index 00000000000..cc624d690a9 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.test.ts @@ -0,0 +1,185 @@ +// These mappings are values Postgres derives rather than receives. If either side changes and the +// other does not, dual-write silently stores two different documents for one snapshot. The parity +// suite next to this file checks the same thing against a real Postgres row; this one pins the +// rules on their own, so a failure says which rule broke. +import { describe, expect, it } from "vitest"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; + +const ctx = { id: "snap_1", runId: "run_1", createdAt: new Date("2026-08-24T00:00:00.000Z") }; +const scope = { + environmentId: "env_1", + environmentType: "DEVELOPMENT" as const, + projectId: "proj_1", + organizationId: "org_1", +}; + +describe("snapshotEntry derived values", () => { + it("rewrites a DEQUEUED run status to PENDING", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("PENDING"); + }); + + it("keeps every other run status unchanged", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("EXECUTING"); + }); + + it("applies the lock site's hard-coded values", () => { + const entry = entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + attemptNumber: 2, + completedWaitpointIds: [], + completedWaitpointOrder: [], + ...scope, + }); + + expect(entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(entry.description).toBe("Run was dequeued for execution"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.engine).toBe("V2"); + expect(entry.previousSnapshotId).toBe("snap_0"); + expect(entry.attemptNumber).toBe(2); + }); + + it("applies the reschedule defaults", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.executionStatus).toBe("DELAYED"); + expect(entry.runStatus).toBe("DELAYED"); + expect(entry.description).toBe("Delayed run was rescheduled to a future date"); + }); + + it("prefers a supplied reschedule value over the default", () => { + const entry = entryFromReschedule(ctx, { + ...scope, + executionStatus: "QUEUED", + runStatus: "PENDING", + description: "custom", + }); + + expect(entry.executionStatus).toBe("QUEUED"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.description).toBe("custom"); + }); + + it("sets engine V2 on a completion, which Postgres leaves to the column default", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + + expect(entry.engine).toBe("V2"); + }); + + it("carries a null completion attemptNumber through as null", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: null, + ...scope, + }); + + expect(entry.attemptNumber).toBeNull(); + }); + + it("omits an absent optional rather than writing undefined into the document", () => { + const entry = entryFromExpire(ctx, { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + ...scope, + }); + + expect(Object.keys(entry)).not.toContain("workerId"); + expect(Object.keys(entry)).not.toContain("attemptNumber"); + expect(JSON.parse(JSON.stringify(entry))).toEqual(entry); + }); + + it("reports a FINISHED entry as terminal and any other as not", () => { + const finished = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + const running = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(isTerminalEntry(finished)).toBe(true); + expect(isTerminalEntry(running)).toBe(false); + }); + + it("serialises createdAt as an ISO string", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.createdAt).toBe("2026-08-24T00:00:00.000Z"); + }); + + it("carries the birth site's worker and runner ids", () => { + const entry = entryFromCreateRun(ctx, { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + workerId: "worker_1", + runnerId: "runner_1", + ...scope, + }); + + expect(entry.workerId).toBe("worker_1"); + expect(entry.runnerId).toBe("runner_1"); + expect(entry.executionStatus).toBe("RUN_CREATED"); + }); + + it("never sets the reserved completedWaitpoints field", () => { + const built = [ + entryFromReschedule(ctx, { ...scope }), + entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + completedWaitpointIds: ["w_1"], + completedWaitpointOrder: ["w_1"], + ...scope, + }), + entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + completedWaitpoints: [{ id: "w_1", index: 0 }], + ...scope, + }), + ]; + + // The append script mints the pointer as a sidecar field, and rejects an entry that carries one. + for (const entry of built) { + expect(entry.completedWaitpoints).toBeUndefined(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.ts b/internal-packages/run-store/src/snapshotEntry.ts new file mode 100644 index 00000000000..748a95f6d1a --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.ts @@ -0,0 +1,168 @@ +// Builds the Redis entry for each execution-snapshot write site, from that site's own INPUT. +// +// Not from the delegate's return value: no nested write site includes the snapshot in what it +// returns. `createRun` returns the run, `expireParkedRun` returns a count, and the rest return a +// selected `TaskRun`. That means every value Postgres derives rather than receives has to be +// reproduced here, and snapshotEntry.parity.test.ts is what keeps the two sides from drifting. +import type { TaskRunStatus } from "@trigger.dev/database"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import type { + CompletionSnapshotInput, + CreateExecutionSnapshotInput, + CreateRunSnapshotInput, + ExpireSnapshotInput, + LockSnapshotInput, + RescheduleSnapshotInput, +} from "./types.js"; + +export type EntryBuildContext = { id: string; runId: string; createdAt: Date }; + +/** + * PostgresRunStore.#createExecutionSnapshot rewrites DEQUEUED to PENDING, because older runners + * reject DEQUEUED on a snapshot. Every site that can carry that status must rewrite it identically. + */ +function snapshotRunStatus(status: TaskRunStatus): string { + return status === "DEQUEUED" ? "PENDING" : status; +} + +function base(ctx: EntryBuildContext) { + return { + id: ctx.id, + runId: ctx.runId, + createdAt: ctx.createdAt.toISOString(), + engine: "V2" as const, + }; +} + +export function entryFromCreateRun( + ctx: EntryBuildContext, + snapshot: CreateRunSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** + * `completeAttemptSuccess` writes no `engine` column, so Postgres applies the schema default of + * `V2`. The entry states it, because SnapshotEntryInput requires the field. + */ +export function entryFromCompletion( + ctx: EntryBuildContext, + snapshot: CompletionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + attemptNumber: snapshot.attemptNumber, + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** Serves both `expireRun` and `expireParkedRun`; the two write identical snapshot columns. */ +export function entryFromExpire( + ctx: EntryBuildContext, + snapshot: ExpireSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.rescheduleRun supplies these three defaults inline, so the entry repeats them. */ +export function entryFromReschedule( + ctx: EntryBuildContext, + snapshot: RescheduleSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus ?? "DELAYED", + description: snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: snapshotRunStatus(snapshot.runStatus ?? "DELAYED"), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.#lockRunToWorker hard-codes the status, description and run status. */ +export function entryFromLock( + ctx: EntryBuildContext, + snapshot: LockSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + previousSnapshotId: snapshot.previousSnapshotId, + ...(snapshot.attemptNumber !== undefined && { attemptNumber: snapshot.attemptNumber }), + ...(snapshot.batchId !== undefined && { batchId: snapshot.batchId }), + ...(snapshot.checkpointId !== undefined && { checkpointId: snapshot.checkpointId }), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +export function entryFromCreateExecutionSnapshot( + ctx: EntryBuildContext, + input: CreateExecutionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: input.snapshot.executionStatus, + description: input.snapshot.description, + runStatus: snapshotRunStatus(input.run.status), + ...(input.run.attemptNumber !== undefined && + input.run.attemptNumber !== null && { attemptNumber: input.run.attemptNumber }), + ...(input.previousSnapshotId !== undefined && { previousSnapshotId: input.previousSnapshotId }), + ...(input.batchId !== undefined && { batchId: input.batchId }), + environmentId: input.environmentId, + environmentType: input.environmentType, + projectId: input.projectId, + organizationId: input.organizationId, + ...(input.checkpointId !== undefined && { checkpointId: input.checkpointId }), + ...(input.workerId !== undefined && { workerId: input.workerId }), + ...(input.runnerId !== undefined && { runnerId: input.runnerId }), + ...(input.snapshot.metadata !== undefined && + input.snapshot.metadata !== null && { metadata: input.snapshot.metadata }), + ...(input.error !== undefined && { error: input.error }), + }; +} + +/** + * A terminal entry is what makes the append script apply the completion TTL. FINISHED is the only + * terminal execution status; the run-level status is not consulted, because a run reaches its + * terminal state through a FINISHED snapshot in every path. + */ +export function isTerminalEntry(entry: SnapshotEntryInput): boolean { + return entry.executionStatus === "FINISHED"; +} diff --git a/internal-packages/run-store/src/snapshotFaultInjection.ts b/internal-packages/run-store/src/snapshotFaultInjection.ts new file mode 100644 index 00000000000..83abf43b179 --- /dev/null +++ b/internal-packages/run-store/src/snapshotFaultInjection.ts @@ -0,0 +1,45 @@ +// Test-only seam for the execution-snapshot write protocol. +// +// The protocol's correctness claim is about crashes: whatever the write order leaves behind at each +// boundary must be a state the existing stall-and-repair machinery heals. Proving that needs a crash +// at an exact point, which is what an injector gives. Production never sets one, so each boundary +// costs one optional call. + +/** The three points a crash can land between the two stores' writes. */ +export type SnapshotFaultBoundary = + /** A transition: Postgres has committed and the Redis append has not started. */ + | "afterPgBeforeRedis" + /** A birth: the Redis append has landed and the Postgres insert has not started. */ + | "afterRedisBirthBeforePg" + /** Inside the append retry loop, after at least one attempt has failed. */ + | "midFlushRetry"; + +export type SnapshotFaultInjector = ( + boundary: SnapshotFaultBoundary, + context: { runId: string; snapshotId: string } +) => void; + +/** + * Thrown by a test injector. The write path tells this apart from a real append failure, because an + * injected fault models a process that died rather than a call that failed. The two write paths then + * do different things with it, and both differ from a real failure: + * + * - A transition skips its remaining retries, hands the run to the repair job, and does NOT rethrow. + * Postgres has already committed, so the caller must not see an error. + * - A birth rethrows, so the Postgres insert never runs and the crash leaves an orphaned keyspace + * with no run row, which is the harmless state that ordering exists to produce. + * - A real append failure is retried, and only then handed to the repair job. + */ +export class InjectedSnapshotFault extends Error { + readonly boundary: SnapshotFaultBoundary; + + constructor(boundary: SnapshotFaultBoundary) { + super(`injected snapshot fault at ${boundary}`); + this.name = "InjectedSnapshotFault"; + this.boundary = boundary; + } +} + +export function isInjectedFault(error: unknown): error is InjectedSnapshotFault { + return error instanceof InjectedSnapshotFault; +} diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts new file mode 100644 index 00000000000..90df7bfcc1d --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts @@ -0,0 +1,160 @@ +// Production points at a Valkey/Redis CLUSTER. SCAN carries no key, so a cluster cannot route it: +// one connection iterates ONE node's keyspace and then returns a completed cursor. A single-client +// sweep would therefore report {scanned, expired, deleted, skipped} looking exactly like a clean +// pass, having examined roughly 1/N of the keyspace, and the rest would leak with nothing left to +// revisit it. Both sweep rules close unbounded leaks, and TRI-13453 gates the rollout dial on an +// OBSERVED sweep pass, so a false green here is the worst failure this component has. +// +// Everything the sweep does after the scan is key-addressed and a cluster client routes it without +// help, so the node list is the whole of the exposure. These tests pin that decision directly. +// +// There is no Redis-cluster container fixture in the repo (@internal/testcontainers ships slot +// arithmetic, not a cluster), so the cluster cases drive a real ioredis `Cluster` object that has +// never connected and assert which method the code reaches for. That is a test of our branch, not +// a simulation of Redis. A true multi-node integration test wants a cluster fixture, and the ticket +// building the cluster client is the one placed to add it. +import { describe, expect, it } from "vitest"; +import { Cluster, Redis } from "@internal/redis"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { clientPrefixOf, scanTargetsOf, SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; + +/** An ioredis Cluster that is never connected. `lazyConnect` keeps the constructor from dialling. */ +function offlineCluster(options?: { keyPrefix?: string }): Cluster { + return new Cluster([{ host: "127.0.0.1", port: 7000 }], { + lazyConnect: true, + redisOptions: options?.keyPrefix ? { keyPrefix: options.keyPrefix } : undefined, + }); +} + +describe("scanTargetsOf", () => { + it("returns the one connection for a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + try { + expect(scanTargetsOf(client)).toEqual([client]); + } finally { + client.disconnect(); + } + }); + + it("returns every master for a cluster, and never a replica", async () => { + const cluster = offlineCluster(); + const masters = [ + new Redis({ lazyConnect: true, port: 65001 }), + new Redis({ lazyConnect: true, port: 65002 }), + new Redis({ lazyConnect: true, port: 65003 }), + ]; + const replica = new Redis({ lazyConnect: true, port: 65004 }); + + const asked: string[] = []; + // The assertion that matters: the sweep asks for "master" specifically. Asking for "all" would + // scan replicas too and act twice on one keyspace. + (cluster as unknown as { nodes: (role: string) => Redis[] }).nodes = (role: string) => { + asked.push(role); + return role === "master" ? masters : [...masters, replica]; + }; + + expect(scanTargetsOf(cluster)).toEqual(masters); + expect(asked).toEqual(["master"]); + expect(scanTargetsOf(cluster)).not.toContain(replica); + + for (const client of [...masters, replica]) client.disconnect(); + cluster.disconnect(); + }); + + it("resolves the node list per call, so a failover is picked up", () => { + const cluster = offlineCluster(); + let generation = 0; + (cluster as unknown as { nodes: () => Redis[] }).nodes = () => { + generation += 1; + return Array.from( + { length: generation }, + (_v, i) => new Redis({ lazyConnect: true, port: 65100 + i }) + ); + }; + + expect(scanTargetsOf(cluster)).toHaveLength(1); + expect(scanTargetsOf(cluster)).toHaveLength(2); + cluster.disconnect(); + }); +}); + +describe("clientPrefixOf", () => { + it("reads the top-level keyPrefix on a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000, keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(client)).toBe("engine:"); + } finally { + client.disconnect(); + } + }); + + it("reads the nested redisOptions.keyPrefix on a cluster", () => { + // On a Cluster the prefix lives under redisOptions. Reading the top level yields "", every + // SCAN MATCH then misses, and the pass reports a clean sweep of nothing. + const cluster = offlineCluster({ keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(cluster)).toBe("engine:"); + } finally { + cluster.disconnect(); + } + }); + + it("is empty when no prefix is configured, for either shape", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + const cluster = offlineCluster(); + try { + expect(clientPrefixOf(client)).toBe(""); + expect(clientPrefixOf(cluster)).toBe(""); + } finally { + client.disconnect(); + cluster.disconnect(); + } + }); +}); + +describe("SweepResult.nodes", () => { + containerTest( + "a standalone pass reports the one connection it covered", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + const result = await sweeper.sweep({ dryRun: true }); + // Without this field a pass that covered one node of six is indistinguishable from a + // complete one, which is exactly the false green the fan-out exists to prevent. + expect(result.nodes).toBe(1); + } finally { + await sweeper.quit(); + } + } + ); +}); + +describe("client ownership", () => { + containerTest("quit() leaves a borrowed client open", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const client = new Redis(redisOptions); + const sweeper = new SnapshotOrphanSweeper({ + client, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + await sweeper.quit(); + // The store and the sweep can share one cluster client. If quit() closed a client it did not + // open, the first component to shut down would take the other one's connection with it. + await client.set(`ownership:${generateInternalId()}`, "1"); + expect(await client.ping()).toBe("PONG"); + } finally { + client.disconnect(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts new file mode 100644 index 00000000000..40c90bc4774 --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts @@ -0,0 +1,360 @@ +// Rule 2 deletes a whole keyspace on the strength of "findRunsByIds returned no row for it". The +// catch in #sweepBatch covers a lookup that THROWS; it cannot see a lookup that succeeds and is +// incomplete, and a row that exists but did not come back reads exactly like a run that never +// existed. `findRunsByIds` partitions ids by residency and asks each store only for its own, and +// with no client passed it reads each store's replica — both sound today, but neither is something +// this delete path can verify. +// +// A false negative leaks keys, which is bounded and recoverable. A false positive destroys a live +// run's execution state. So deletion requires two sightings across the confirm window, and these +// tests pin that: a single pass never deletes, however old the keyspace. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("rule 2 requires a second sighting", () => { + containerTest( + "one pass marks an orphan and deletes nothing, however old the keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A thousand times the age gate. Age is not what holds the deletion back. + const ancient = new Date(Date.now() - 1000 * ORPHAN_AGE_MS); + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + + const first = await sweeper.sweep(); + + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const second = await sweeper.sweep(); + + expect(second.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a marked keyspace is not deleted until the confirm window has passed", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // An hour. Passes minutes apart must not convert a candidate. + confirmOrphanAfterMs: 60 * 60 * 1000, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + await sweeper.sweep(); + const second = await sweeper.sweep(); + const third = await sweeper.sweep(); + + expect(second.deleted).toBe(0); + expect(second.pendingDeletion).toBe(1); + expect(third.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a transient miss followed by a found run does not delete, and does not leave the keyspace pre-authorised", + async ({ prisma, redisOptions }) => { + // The case the guard exists for. Pass 1 gets an incomplete answer and marks the keyspace. + // Pass 2 sees the run, so it must clear the mark: were the mark to survive, a LATER genuine + // absence would delete on its own first sighting and the two-sighting rule would be gone. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? // Succeeds and is incomplete: exactly what the catch cannot see. + Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // The run is alive and terminal in Postgres the whole time. Only the lookup lies. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const first = await sweeper.sweep(); + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + + lie = false; + const second = await sweeper.sweep(); + expect(second.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + // The run vanishes for real. With the mark cleared this is a first sighting again. + lie = true; + const third = await sweeper.sweep(); + expect(third.deleted).toBe(0); + expect(third.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const fourth = await sweeper.sweep(); + expect(fourth.deleted).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("the marker cannot expire out from under a candidate", () => { + containerTest( + "a marked keyspace still carries its mark after the whole keyspace is re-read", + async ({ prisma, redisOptions }) => { + // The marker used to be a key with its own TTL derived from the confirm window, which could + // be shorter than the interval between passes: the marker written at T was gone by + // T+interval, every pass wrote a fresh one, and rule 2 deleted nothing while reporting clean. + // It is now a field on the run's `seq` hash, so it lives exactly as long as the keyspace and + // there is no lifetime left to misconfigure. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + // The mark is a field on seq, and it carries no expiry of its own. + expect(await probe.hget(snapshotKeys(runId).seq, "orph")).not.toBeNull(); + expect(await probe.pttl(snapshotKeys(runId).seq)).toBe(-1); + + expect((await sweeper.sweep()).deleted).toBe(1); + // And it went with the keyspace rather than outliving it. + expect(await probe.exists(snapshotKeys(runId).seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a run seen alive clears its marker", () => { + containerTest( + "a lie, then a LIVE run, then a lie again does not delete", + async ({ prisma, redisOptions }) => { + // The hole a long marker lifetime opens. Only terminal runs used to clear the marker, so a + // keyspace marked by an incomplete lookup and then seen ALIVE kept its mark. A later genuine + // absence would then find a mature marker and delete on what is really a first sighting. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // EXECUTING, not terminal. This run never reaches rule 1, so rule 1 cannot be what clears + // the mark; a SUSPENDED run can legitimately sit here for weeks. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "EXECUTING" }, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + lie = false; + const seenAlive = await sweeper.sweep(); + expect(seenAlive.skipped).toBeGreaterThan(0); + expect(seenAlive.deleted).toBe(0); + + lie = true; + const afterAlive = await sweeper.sweep(); + // A first sighting again, because being seen alive cleared the mark. + expect(afterAlive.deleted).toBe(0); + expect(afterAlive.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a pass can stop inside a budget", () => { + containerTest( + "an already-passed deadline yields a partial pass", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() - 1 }); + + // redis-worker redelivers a job that outlives its visibility timeout, and nothing extends it, + // so a pass that cannot stop on its own runs concurrently with itself. + expect(result.partial).toBe(true); + expect(result.scanned).toBe(0); + } finally { + await sweeper.quit(); + } + } + ); + + containerTest("an aborted signal yields a partial pass", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + const controller = new AbortController(); + controller.abort(); + + try { + const result = await sweeper.sweep({ signal: controller.signal }); + expect(result.partial).toBe(true); + } finally { + await sweeper.quit(); + } + }); + + containerTest("a pass with budget to spare is not partial", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() + 60_000 }); + expect(result.partial).toBe(false); + } finally { + await sweeper.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts new file mode 100644 index 00000000000..7d7eb2aca0d --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -0,0 +1,469 @@ +// The sweep deletes whole keyspaces, so most of these tests are about what it must NOT touch: a live +// run, a young orphan, and any batch whose Postgres lookup did not come back. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date, terminal = false) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: terminal ? ("FINISHED" as const) : ("RUN_CREATED" as const), + description: "Run was created", + runStatus: terminal ? ("CANCELED" as const) : ("PENDING" as const), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("SnapshotOrphanSweeper", () => { + containerTest( + "rule 1 expires a terminal run whose keyspace never got one", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Non-terminal append, so no expiry is ever set — the lost-TTL-set case. + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const keys = snapshotKeys(runId); + expect(await probe.pttl(keys.e)).toBe(-1); + + const result = await sweeper.sweep(); + + expect(result.expired).toBe(1); + expect(result.deleted).toBe(0); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + const ttl = await probe.pttl(key); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(COMPLETED_TTL_MS); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 1 leaves a keyspace that already has an expiry alone", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A healthy terminal append sets the completion TTL itself. + await store.append({ + entry: birthEntry(runId, env, new Date(), true), + kind: "birth", + isTerminal: true, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "CANCELED" }, + }); + + const before = await probe.pttl(snapshotKeys(runId).e); + const result = await sweeper.sweep(); + + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + const after = await probe.pttl(snapshotKeys(runId).e); + // Not extended: the sweep must not keep resetting a countdown that is already running. + expect(after).toBeLessThanOrEqual(before); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 2 deletes a keyspace with no run row, cycle keys included", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The crashed birth: an entry, no Postgres run, non-terminal so no expiry. + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "transition", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + const cyclesBefore = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cyclesBefore.length).toBeGreaterThan(0); + + await sweeper.sweep(); + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + const keys = snapshotKeys(runId); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + expect(await probe.exists(key)).toBe(0); + } + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("rule 2 spares a young orphan", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Written just now: the Postgres insert of a healthy birth may still be in flight. + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "never touches a live run, however old its keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const ancient = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); + + // A run waiting on an untimed token can sit non-terminal for weeks. Reaping it would drop + // live state, which is the failure this rule exists to avoid. + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "WAITING_TO_RESUME" }, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(runId).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a dry run reports but changes nothing", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const orphan = generateInternalId(); + const terminal = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ entry: birthEntry(orphan, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(terminal, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(terminal, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const result = await sweeper.sweep({ dryRun: true }); + + // A dry pass writes no marker, so an unconfirmed rule 2 candidate reports as pending rather + // than as a deletion. That is what a real pass would do at this instant, which is the honest + // answer for a preview: nothing is confirmed yet, so nothing would be deleted yet. + expect(result.deleted).toBe(0); + expect(result.pendingDeletion).toBe(1); + expect(result.expired).toBe(1); + expect(await probe.exists(snapshotKeys(orphan).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(terminal).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "skips a batch whose run lookup failed, and deletes nothing", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const failing = { + findRunsByIds: async () => { + throw new Error("run lookup unavailable"); + }, + } as unknown as RunStore; + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: failing, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + + // A failed lookup says nothing about whether the run exists, and rule 2 deletes a whole + // keyspace. The sweep must resolve rather than throw, and must reap nothing. + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBeGreaterThan(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "discovers and reaps a keyspace whose entries are all invalid", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The append script writes `cur` and indexes the entry only when it is valid, so a keyspace + // whose entries all carry an error has neither. A sweep that discovers keyspaces by their + // `cur` key would never see this one, and neither rule would ever apply to it. + await store.append({ + entry: { ...birthEntry(runId, env, old), error: "stale write" }, + kind: "birth", + isTerminal: false, + }); + + const keys = snapshotKeys(runId); + expect(await probe.exists(keys.e)).toBe(1); + expect(await probe.exists(keys.cur)).toBe(0); + + await sweeper.sweep(); + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + expect(await probe.exists(keys.e)).toBe(0); + expect(await probe.exists(keys.seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "still finds keyspaces when the client carries a key prefix", + async ({ prisma, redisOptions }) => { + // ioredis prepends its keyPrefix to keys for ordinary commands, but NOT to a SCAN MATCH + // pattern, and it returns matched keys with the prefix still on them. The engine sets a + // prefix on every other Redis client it builds, so a sweep that ignored this would match + // nothing and report a clean pass: a safety net that silently protects nothing. + const prefixed = { ...(redisOptions as object), keyPrefix: "engine:" } as never; + const store = new RedisSnapshotStore({ + redisOptions: prefixed, + completedTtlMs: COMPLETED_TTL_MS, + }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions: prefixed, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(prefixed, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + await sweeper.sweep(); + const result = await sweeper.sweep(); + + expect(result.scanned).toBe(1); + expect(result.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + expect(await probe.exists(`snap:{${runId}}:wp:1`)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + const orphans = Array.from({ length: 5 }, () => generateInternalId()); + for (const runId of orphans) { + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + } + + await sweeper.sweep({ batchSize: 2 }); + const result = await sweeper.sweep({ batchSize: 2 }); + + expect(result.deleted).toBe(5); + for (const runId of orphans) { + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts new file mode 100644 index 00000000000..86bb084d5c9 --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -0,0 +1,567 @@ +// Reaps snapshot keyspaces that no healthy path will ever clean up. +// +// Two rules, because neither can see what the other leaves behind: +// +// 1. The run is terminal in Postgres but its keyspace never got the completion expiry — a +// terminal append whose TTL-set was lost. Applying the expiry now reaps it on the same +// schedule a healthy terminal append would have. +// 2. The keyspace has no Postgres run row at all, and is older than a threshold — a crashed +// birth. It is non-terminal so it carries no expiry, and it has no run row, so rule 1 can +// never match it. Without this rule that leak has no bound. +// +// Nothing schedules this. The engine's worker is what has to run it, and run-store cannot reach the +// engine, so the wiring belongs to the ticket that owns production construction. +import { + Cluster, + createRedisClient, + type Redis, + type RedisClient, + type RedisOptions, +} from "@internal/redis"; +import { Logger } from "@trigger.dev/core/logger"; +import type { TaskRunStatus } from "@trigger.dev/database"; +import { snapshotKeys } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +/** + * Mirrors the engine's `finalStatuses`. run-store cannot import from run-engine — the dependency + * runs the other way — so the list is duplicated and a parity test in run-engine asserts the copy + * stays equal to the original. + */ +export const FINAL_RUN_STATUSES: readonly TaskRunStatus[] = [ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +]; + +const FINAL = new Set(FINAL_RUN_STATUSES); + +/** Comfortably above run-creation latency, so a birth in flight is never mistaken for an orphan. */ +const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; + +/** + * The keyspace prefix, owned by `snapshotKeys` in the store rather than configurable here. A sweep + * that could be pointed at a different prefix would be a fiction: the store writes `snap:` keys + * unconditionally, so there is no other keyspace to point it at. + */ +const SNAPSHOT_KEYSPACE_PREFIX = "snap:"; +const DEFAULT_BATCH_SIZE = 1000; + +/** + * How long a rule 2 candidate must have been marked before it may be deleted. It has to exceed the + * interval between passes, or a candidate is never sighted twice and never converts. + */ +const DEFAULT_ORPHAN_CONFIRM_MS = 60 * 60 * 1000; + +export type SweepResult = { + /** Keyspaces examined. */ + scanned: number; + /** Rule 1: terminal runs whose keyspace was given the completion expiry. */ + expired: number; + /** Rule 2: keyspaces with no run row, deleted. */ + deleted: number; + /** Left alone: a live run, a young orphan, or a batch whose Postgres lookup failed. */ + skipped: number; + /** + * Rule 2 candidates that were marked but not deleted, because deletion needs a second sighting + * in a later pass. A number that never converts to `deleted` means the confirm window is longer + * than the interval between passes, or the marker TTL is shorter than it. + */ + pendingDeletion: number; + /** + * Connections the pass iterated: every master of a cluster, or 1 standalone. Reported because the + * failure this component cannot tolerate is a false green, and a pass that covered one node of + * six is indistinguishable from a complete one by any other field here. TRI-13453 gates the dial + * on an observed sweep pass, so the observation has to carry its own coverage. + */ + nodes: number; + /** True when the pass stopped early on its deadline or abort signal, so coverage is incomplete. */ + partial: boolean; +}; + +/** + * Exactly one of `redisOptions` or `client`, enforced by the type rather than a runtime check. + * With `redisOptions` the sweep opens its OWN connection, which is the preferred shape: a long + * scan can then never stall a hot-path client. `client` exists for a caller that has already built + * a client and wants the sweep to use it; a borrowed client is left open by `quit()`. + * + * What the sweep needs is a connection of its OWN, not one it built itself. A caller pointing at a + * cluster should build a SECOND, sweep-dedicated cluster client and pass it here: that keeps a long + * scan off the hot path just as well as `redisOptions` does. Handing over the client the snapshot + * store is using is the case to avoid. + */ +export type SnapshotOrphanSweeperConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type SnapshotOrphanSweeperOptions = SnapshotOrphanSweeperConnection & { + /** + * Resolved through the run store, not a raw client. Under the run-ops split a run row can live on + * either database, and only the store knows which — a raw lookup would report a live run as an + * orphan and delete its keyspace. + */ + runStore: RunStore; + completedTtlMs: number; + orphanAgeMs?: number; + /** + * How long a rule 2 candidate must stay marked before the sweep will delete it. Defaults to one + * hour. + * + * Set it at or below the interval between passes, or the second sighting arrives too early to + * count and every candidate needs three passes instead of two. It does NOT need to exceed the + * interval; the constraint people reach for ("longer than the interval") is the wrong one and + * only costs latency. + * + * There is no marker-lifetime constraint to satisfy alongside it. The marker is a field on the + * run's `seq` hash, so it lives exactly as long as the keyspace it describes: it cannot expire + * out from under a candidate that is still waiting for its second sighting, and it cannot outlive + * a keyspace that was deleted. + */ + confirmOrphanAfterMs?: number; + logger?: Logger; +}; + +export class SnapshotOrphanSweeper { + readonly #redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + readonly #ownsClient: boolean; + readonly #runStore: RunStore; + readonly #completedTtlMs: number; + readonly #orphanAgeMs: number; + readonly #confirmOrphanAfterMs: number; + /** + * The ioredis client-level prefix, which is NOT the keyspace prefix. ioredis prepends it to keys + * for ordinary commands, but it does not prepend it to a SCAN MATCH pattern, and it does return + * matched keys with it still attached. Unhandled, a prefixed client makes the sweep match nothing + * and report a clean pass, which is the worst outcome for a safety net. + */ + readonly #clientPrefix: string; + readonly #logger: Logger; + #quit?: Promise; + + constructor(options: SnapshotOrphanSweeperOptions) { + this.#logger = options.logger ?? new Logger("SnapshotOrphanSweeper", "debug"); + this.#runStore = options.runStore; + this.#completedTtlMs = options.completedTtlMs; + this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; + this.#confirmOrphanAfterMs = options.confirmOrphanAfterMs ?? DEFAULT_ORPHAN_CONFIRM_MS; + this.#ownsClient = options.client === undefined; + this.#redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => + this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), + }); + this.#clientPrefix = clientPrefixOf(this.#redis); + } + + async quit(): Promise { + // A borrowed client belongs to the caller; closing it here would take down a connection the + // snapshot store may still be using. + if (!this.#ownsClient) return; + if (!this.#quit) { + this.#quit = this.#redis.quit().then( + () => undefined, + () => undefined + ); + } + await this.#quit; + } + + /** + * One full pass over the keyspace. `dryRun` reports what it would do and changes nothing. + * + * `deadline` and `signal` let the caller stop a pass cleanly instead of having it killed + * mid-cursor. The scheduler needs this: redis-worker moves a dequeued item's score to + * `now + visibilityTimeoutMs` and nothing extends it, so a pass that outlives its timeout is + * redelivered and runs concurrently with itself. A pass that stops inside its budget cannot. + * + * Whichever way it stops, `partial` comes back true. Reporting a truncated pass as a full one is + * the same false green as under-scanning a cluster: TRI-13453 gates the dial on an OBSERVED + * sweep pass, so the observation has to say how much of the keyspace it actually reached. + */ + async sweep(opts?: { + batchSize?: number; + dryRun?: boolean; + /** Epoch ms. The pass stops at the next batch boundary once passed. */ + deadline?: number; + signal?: AbortSignal; + }): Promise { + const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE; + const dryRun = opts?.dryRun ?? false; + const result: SweepResult = { + scanned: 0, + expired: 0, + deleted: 0, + skipped: 0, + pendingDeletion: 0, + nodes: 0, + partial: false, + }; + + // Checked at batch boundaries only. Stopping mid-batch would leave a run half-acted-on, and a + // batch is bounded work, so the boundary is both the safe and the timely place. + const outOfBudget = () => + opts?.signal?.aborted === true || + (opts?.deadline !== undefined && Date.now() >= opts.deadline); + + // SCAN carries no key, so a cluster cannot route it: one connection iterates ONE node's + // keyspace and then reports a completed cursor. A single-client sweep against a cluster would + // therefore return a clean-looking result having examined roughly 1/N of the keyspace, and the + // rest would leak with nothing to revisit it. Both rules are unbounded leaks when missed, so + // the pass fans out over every master and only reports done when all of them are done. + const nodes = this.#scanTargets(); + result.nodes = nodes.length; + + for (const node of nodes) { + if (outOfBudget()) { + result.partial = true; + break; + } + + let cursor = "0"; + do { + // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry + // is valid, so a keyspace whose entries are all invalid would never be discovered and would + // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is + // written by every append. + const [next, keys] = await node.scan( + cursor, + "MATCH", + `${this.#clientPrefix}${SNAPSHOT_KEYSPACE_PREFIX}{*}:e`, + "COUNT", + batchSize + ); + cursor = next; + + const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; + if (runIds.length === 0) continue; + + await this.#sweepBatch(runIds, dryRun, result); + + if (outOfBudget()) { + // A cursor mid-iteration means this node is not finished, so the pass is not either. + result.partial = true; + break; + } + } while (cursor !== "0"); + + if (result.partial) break; + } + + this.#logger.log("SnapshotOrphanSweeper pass complete", { ...result, dryRun }); + return result; + } + + #scanTargets(): Redis[] { + return scanTargetsOf(this.#redis); + } + + async #sweepBatch(runIds: string[], dryRun: boolean, result: SweepResult): Promise { + result.scanned += runIds.length; + + let rows: Map; + try { + rows = (await this.#runStore.findRunsByIds(runIds, { + select: { id: true, status: true }, + })) as unknown as Map; + } catch (error) { + // Never reap on an unknown answer. A lookup that failed says nothing about whether the run + // exists, and rule 2 deletes a whole keyspace. + this.#logger.error("SnapshotOrphanSweeper skipped a batch after a failed run lookup", { + count: runIds.length, + error, + }); + result.skipped += runIds.length; + return; + } + + // Every run that EXISTS clears its rule 2 marker, live ones included. It has to be every one, + // not just the terminal ones: a keyspace marked by an incomplete lookup, then seen alive, then + // missed again would otherwise present a mature marker on what is really a first sighting, and + // the two-sighting rule would be gone exactly when it was needed. This costs one DEL per + // existing run per pass, which is the price of the guard being sound rather than nearly sound. + const present = runIds.filter((runId) => rows.has(runId)); + if (!dryRun && present.length > 0) { + // Individual commands, never one pipeline: these keys span runs, so they span cluster slots. + await Promise.all(present.map((runId) => this.#clearOrphanMarker(runId))); + } + + for (const runId of runIds) { + const run = rows.get(runId); + + if (!run) { + await this.#applyRuleTwo(runId, dryRun, result); + continue; + } + + if (!FINAL.has(run.status)) { + // A live run. A SUSPENDED run can legitimately wait for weeks, so this is never touched. + result.skipped += 1; + continue; + } + + await this.#applyRuleOne(runId, dryRun, result); + } + } + + /** Rule 1: a terminal run whose keyspace never received the completion expiry. */ + async #applyRuleOne(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const ttls = await Promise.all(keys.map((key) => this.#redis.pttl(key))); + // -1 is "exists, no expiry". Anything already counting down was set by a healthy append. + if (!ttls.some((ttl) => ttl === -1)) { + result.skipped += 1; + return; + } + + if (!dryRun) { + const pipeline = this.#redis.pipeline(); + for (const key of keys) { + pipeline.pexpire(key, this.#completedTtlMs); + } + await pipeline.exec(); + } + + result.expired += 1; + } + + /** + * Rule 2: a keyspace with no run row at all, past the age threshold. + * + * TWO SIGHTINGS ARE REQUIRED. The `catch` in #sweepBatch covers a lookup that THROWS, but it + * cannot see a lookup that succeeds and is incomplete: a row that exists but did not come back + * reads exactly like a run that never existed, and the response to that is deleting a live run's + * execution state. `findRunsByIds` routes through RoutingRunStore.#findRunsByIdSet, which + * partitions ids by residency and asks each store only for its own — and with no client passed it + * reads each store's REPLICA. Both are sound today (id classification is authoritative for runs, + * and replica lag is nowhere near the 24h age gate), but each is an assumption held somewhere + * else in the codebase, not something this delete path can check. + * + * The asymmetry decides it: a false negative leaks keys, which is bounded and recoverable, while + * a false positive destroys live state. So an absent row marks the keyspace and returns; only a + * candidate still absent in a LATER pass is deleted. Any transient incomplete answer, whatever + * its cause, has to occur twice across the confirm window to do damage. + */ + async #applyRuleTwo(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const age = await this.#newestEntryAgeMs(runId); + if (age === undefined || age < this.#orphanAgeMs) { + // Either the keyspace carries no readable timestamp, or a birth may still be in flight. + result.skipped += 1; + return; + } + + const seqKey = snapshotKeys(runId).seq; + const markedAtRaw = await this.#redis.hget(seqKey, ORPHAN_MARKER_FIELD); + const markedAt = markedAtRaw === null ? undefined : Number(markedAtRaw); + + if (markedAt === undefined || Number.isNaN(markedAt)) { + if (!dryRun) { + // The field carries no TTL of its own; it lives and dies with the seq hash, which the + // keyspace's own completion expiry already governs. That removes the marker-lifetime knob + // whose derivation was wrong in the first place. + await this.#redis.hset(seqKey, ORPHAN_MARKER_FIELD, String(Date.now())); + } + result.pendingDeletion += 1; + return; + } + + if (Date.now() - markedAt < this.#confirmOrphanAfterMs) { + result.pendingDeletion += 1; + return; + } + + if (!dryRun) { + // One slot: every key here carries the same `{runId}` hash tag. The marker is a field on + // `seq`, which is in `keys`, so it goes with the keyspace rather than needing its own entry. + await this.#redis.del(...keys); + } + + result.deleted += 1; + } + + /** + * Clears a rule 2 marker for a keyspace whose run turned out to exist after all, so a later + * genuine absence still needs its own two sightings rather than inheriting a stale one. + * + * Called for EVERY run row the lookup returned, live ones included, and it has to be: a keyspace + * marked by an earlier incomplete lookup can belong to a run that is perfectly alive, and a + * SUSPENDED run can sit that way for weeks. Leaving the marker in place would let a later genuine + * absence delete on what is really a first sighting, which is the hole the two-sighting rule + * exists to close. It costs one DEL per existing run per pass; that is the price of the guard + * being sound rather than nearly sound. + */ + async #clearOrphanMarker(runId: string): Promise { + try { + await this.#redis.hdel(snapshotKeys(runId).seq, ORPHAN_MARKER_FIELD); + } catch { + // Best effort. A marker that outlives its usefulness expires on its own TTL. + } + } + + /** + * Every key for one run: the four core keys plus each wait-cycle key. + * + * The cycle keys are enumerated from the `c` high-water field on the seq hash, which the append + * script mints densely with HINCRBY, so 1..high covers every wp key that was ever written. This + * is the same source the store's own terminal-expiry loop uses. + * + * It deliberately does NOT use `KEYS`. That command iterates the whole database and blocks while + * it does, and a hash tag routes a key without scoping the scan, so one sweep pass over a batch + * would issue a full keyspace scan per run. + * + * The trade-off: if the seq hash is evicted while a wp key survives, `high` reads 0 and that + * orphaned cycle key is left behind. That is the right way to be wrong here. Leaving one small + * key costs bytes, where scanning the keyspace to find it costs every hot-path client latency on + * every pass. + */ + async #allKeys(runId: string): Promise { + const core = snapshotKeys(runId); + + const high = Number((await this.#redis.hget(core.seq, "c")) ?? "0"); + const cycles: string[] = []; + for (let n = 1; n <= high; n++) { + cycles.push(`${SNAPSHOT_KEYSPACE_PREFIX}{${runId}}:wp:${n}`); + } + + const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; + + // One round trip for the whole set, rather than one per candidate. Cluster-safe: every key here + // carries the same `{runId}` hash tag, so the whole pipeline lands in one slot on one node. + // The same holds for the pexpire pipeline and the multi-key DEL above. + const pipeline = this.#redis.pipeline(); + for (const key of candidates) { + pipeline.exists(key); + } + const replies = await pipeline.exec(); + + return candidates.filter((_key, index) => replies?.[index]?.[1] === 1); + } + + /** + * Age of the newest entry, so a keyspace still being written to is never treated as an orphan. + * The newest is the right end: an old first entry says nothing about whether the run is dead. + */ + async #newestEntryAgeMs(runId: string): Promise { + const core = snapshotKeys(runId); + + const newest = await this.#redis.zrevrange(core.idx, 0, 0); + const id = newest[0]; + + const raw = id + ? await this.#redis.hget(core.e, id) + : // The index holds valid entries only, so an all-invalid keyspace has an empty index. Fall + // back to the newest instant in the entry hash, or that keyspace is never old enough to + // reap and the leak survives the scan fix above. + await this.#newestRawFromEntries(core.e); + + if (!raw) return undefined; + + try { + const createdAt = (JSON.parse(raw) as { createdAt?: string }).createdAt; + if (!createdAt) return undefined; + const parsed = Date.parse(createdAt); + return Number.isNaN(parsed) ? undefined : Date.now() - parsed; + } catch { + return undefined; + } + } + + /** + * The newest entry document in the hash, by its own createdAt. Only reached for a keyspace with + * no index, which is rare, so the whole-hash read is acceptable where it would not be on the + * indexed path. + */ + async #newestRawFromEntries(eKey: string): Promise { + const all = await this.#redis.hgetall(eKey); + let newestRaw: string | undefined; + let newestAt = -Infinity; + + for (const [field, raw] of Object.entries(all)) { + // Sidecar fields hang off the entry ids as `#s` and `#c`; skip them. + if (field.includes("#")) continue; + try { + const at = Date.parse((JSON.parse(raw) as { createdAt?: string }).createdAt ?? ""); + if (!Number.isNaN(at) && at > newestAt) { + newestAt = at; + newestRaw = raw; + } + } catch { + continue; + } + } + + return newestRaw; + } + + /** + * The run id is whatever sits inside the hash tag, so a client prefix on the returned key does not + * need stripping: `engine:snap:{run_x}:e` and `snap:{run_x}:e` both yield `run_x`. + */ + #runIdFrom(key: string): string | undefined { + const open = key.indexOf("{"); + const close = key.indexOf("}", open + 1); + if (open === -1 || close === -1 || close === open + 1) return undefined; + return key.slice(open + 1, close); + } +} + +/** + * Every connection a pass must iterate to cover the whole keyspace: each master of a cluster, or + * the one standalone connection. Replicas are excluded — they hold the same keys as their master, + * so scanning them would double-count and act on one keyspace twice. + * + * Module-level and exported so the fan-out decision can be pinned on its own. It is the whole of + * the defect this guards against: everything the sweep does AFTER the scan is key-addressed and a + * cluster client routes it correctly without help, so the node list is the only place a cluster + * can silently cost the pass coverage. + * + * Resolved per pass, never cached: cluster topology changes under failover and resharding, and a + * stale node list is the same silent under-scan this exists to prevent. + */ + +/** + * Rule 2's "seen absent once" marker is a FIELD on the run's `seq` hash, not a key of its own. + * + * As a separate key its removal depended on the deleting call site remembering to append it to the + * DEL, which is the kind of contract a later edit breaks with no test noticing: a marker outliving + * its keyspace would let a recreated keyspace be deleted on what is really a first sighting. `seq` + * is already in `#allKeys`, so as a field the marker cannot outlive the keyspace at all. + */ +const ORPHAN_MARKER_FIELD = "orph"; + +export function scanTargetsOf(client: RedisClient): Redis[] { + return client instanceof Cluster ? client.nodes("master") : [client]; +} + +/** + * The ioredis client-level prefix for either endpoint shape. On a Cluster it lives on the nested + * `redisOptions`, not on the top-level options, and reading the wrong one yields "" — which makes + * every SCAN MATCH miss and the pass report a clean sweep of nothing. + */ +export function clientPrefixOf(client: RedisClient): string { + if (client instanceof Cluster) { + return (client.options.redisOptions?.keyPrefix as string | undefined) ?? ""; + } + return (client.options.keyPrefix as string | undefined) ?? ""; +} + +function isString(value: string | undefined): value is string { + return typeof value === "string"; +} diff --git a/internal-packages/run-store/src/snapshotReadShapes.test.ts b/internal-packages/run-store/src/snapshotReadShapes.test.ts new file mode 100644 index 00000000000..258731a60b5 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.test.ts @@ -0,0 +1,151 @@ +// A matcher that is too loose is the dangerous failure: it answers a query Redis cannot actually +// serve, and the caller gets a wrong answer rather than a slow one. So most of these tests are +// about what must NOT match. +import { describe, expect, it } from "vitest"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; + +const cursorArgs = { + where: { id: "snap_1", runId: "run_1" }, + select: { createdAt: true }, +}; + +const windowArgs = { + where: { runId: "run_1", isValid: true, createdAt: { gt: new Date("2026-08-24T00:00:00Z") } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, +}; + +describe("matchSinceCursorLookup", () => { + it("matches the engine's since-cursor lookup", () => { + expect(matchSinceCursorLookup(cursorArgs)).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: "env_1" }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1", environmentId: "env_1" }); + }); + + it("ignores keys explicitly set to undefined", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: undefined }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("refuses a selection of anything but createdAt", () => { + expect( + matchSinceCursorLookup({ where: cursorArgs.where, select: { description: true } }) + ).toBeUndefined(); + expect( + matchSinceCursorLookup({ + where: cursorArgs.where, + select: { createdAt: true, description: true }, + }) + ).toBeUndefined(); + }); + + it("refuses a where with no run id, because there is no keyspace to look in", () => { + expect( + matchSinceCursorLookup({ where: { id: "snap_1" }, select: { createdAt: true } }) + ).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, isValid: true }, + select: { createdAt: true }, + }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect( + matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } }) + ).toBeUndefined(); + }); + + it("refuses anything that is not an argument object", () => { + expect(matchSinceCursorLookup(undefined)).toBeUndefined(); + expect(matchSinceCursorLookup(null)).toBeUndefined(); + expect(matchSinceCursorLookup("where")).toBeUndefined(); + expect(matchSinceCursorLookup([cursorArgs])).toBeUndefined(); + }); +}); + +describe("matchSinceWindow", () => { + it("matches the engine's window query", () => { + expect(matchSinceWindow(windowArgs)).toEqual({ + runId: "run_1", + createdAt: new Date("2026-08-24T00:00:00Z"), + take: 50, + }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, environmentId: "env_1" }, + }) + ).toMatchObject({ environmentId: "env_1" }); + }); + + it("refuses a query that also wants the completed waitpoints", () => { + // The engine omits them on purpose to avoid an N x M read. An include that asks for them is a + // different query, and answering it from this path would return them empty. + expect( + matchSinceWindow({ + ...windowArgs, + include: { checkpoint: true, completedWaitpoints: true }, + }) + ).toBeUndefined(); + }); + + it("refuses ascending order", () => { + expect(matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } })).toBeUndefined(); + }); + + it("refuses a window that does not filter to valid entries", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, isValid: false } }) + ).toBeUndefined(); + }); + + it("refuses a cursor that is not a strict greater-than on a Date", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gte: new Date() } }, + }) + ).toBeUndefined(); + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gt: "2026-08-24T00:00:00Z" } }, + }) + ).toBeUndefined(); + }); + + it("refuses a missing take", () => { + const { take: _dropped, ...withoutTake } = windowArgs; + expect(matchSinceWindow(withoutTake)).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, batchId: "batch_1" } }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect(matchSinceWindow({ ...windowArgs, skip: 10 })).toBeUndefined(); + }); +}); diff --git a/internal-packages/run-store/src/snapshotReadShapes.ts b/internal-packages/run-store/src/snapshotReadShapes.ts new file mode 100644 index 00000000000..31dda17f7f0 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.ts @@ -0,0 +1,95 @@ +// Shape matchers for the two generic Prisma-args snapshot reads. +// +// `findExecutionSnapshot` and `findManyExecutionSnapshots` take arbitrary Prisma arguments, and a +// key-value store cannot answer an arbitrary query. Only three production call sites exist, all in +// the engine's executionSnapshotSystem, and both generic ones send a single fixed shape. So these +// matchers recognise exactly those shapes and return undefined for anything else, which sends the +// call to Postgres. +// +// Each matcher rejects an argument object carrying any key it does not know about. A query that has +// drifted must fall through and be answered correctly by Postgres, never answered approximately +// from Redis. + +type Unknown = Record; + +function isPlainObject(value: unknown): value is Unknown { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** True when `value` has exactly `allowed` keys, ignoring keys explicitly set to undefined. */ +function hasOnlyKeys(value: Unknown, allowed: string[]): boolean { + const present = Object.keys(value).filter((k) => value[k] !== undefined); + return present.every((k) => allowed.includes(k)); +} + +function isString(value: unknown): value is string { + return typeof value === "string"; +} + +export type SinceCursorLookup = { id: string; runId: string; environmentId?: string }; + +/** + * Step 1 of `getExecutionSnapshotsSince`: resolve a known snapshot id to its createdAt. + * + * { where: { id, runId, environmentId? }, select: { createdAt: true } } + */ +export function matchSinceCursorLookup(args: unknown): SinceCursorLookup | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "select"])) return undefined; + + const { where, select } = args; + if (!isPlainObject(where) || !isPlainObject(select)) return undefined; + if (!hasOnlyKeys(where, ["id", "runId", "environmentId"])) return undefined; + if (!hasOnlyKeys(select, ["createdAt"]) || select.createdAt !== true) return undefined; + if (!isString(where.id) || !isString(where.runId)) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + return { + id: where.id, + runId: where.runId, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} + +export type SinceWindow = { + runId: string; + createdAt: Date; + take: number; + environmentId?: string; +}; + +/** + * Step 2 of `getExecutionSnapshotsSince`: the capped window after a createdAt cursor. + * + * { where: { runId, isValid: true, createdAt: { gt }, environmentId? }, + * include: { checkpoint: true }, orderBy: { createdAt: "desc" }, take: N } + * + * The engine deliberately omits completedWaitpoints from the include to avoid an N x M read, so an + * include asking for them is a different query and is not matched. + */ +export function matchSinceWindow(args: unknown): SinceWindow | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "include", "orderBy", "take"])) { + return undefined; + } + + const { where, include, orderBy, take } = args; + if (!isPlainObject(where) || !isPlainObject(include) || !isPlainObject(orderBy)) return undefined; + if (typeof take !== "number") return undefined; + + if (!hasOnlyKeys(where, ["runId", "isValid", "createdAt", "environmentId"])) return undefined; + if (!isString(where.runId) || where.isValid !== true) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + if (!hasOnlyKeys(include, ["checkpoint"]) || include.checkpoint !== true) return undefined; + if (!hasOnlyKeys(orderBy, ["createdAt"]) || orderBy.createdAt !== "desc") return undefined; + + const cursor = where.createdAt; + if (!isPlainObject(cursor) || !hasOnlyKeys(cursor, ["gt"])) return undefined; + if (!(cursor.gt instanceof Date)) return undefined; + + return { + runId: where.runId, + createdAt: cursor.gt, + take, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts new file mode 100644 index 00000000000..12ab76a0f26 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts @@ -0,0 +1,285 @@ +// A birth writes Redis FIRST. The order is proved by crashing between the two writes and observing +// which side survived: an orphaned key with no run row is the harmless state, and a run with no +// snapshot at all is the one the order exists to prevent. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + unreachableRedis?: boolean; + } +) { + // An unreachable port makes every append throw for real, which is the failure the retry loop and + // the mode-dependent refusal are about. A fault injector cannot stand in: an injected fault means + // "the process died", and the two are handled differently on purpose. + const redis = new RedisSnapshotStore({ + redisOptions: opts?.unreachableRedis + ? ({ ...(redisOptions as object), port: 1, retryStrategy: () => null } as never) + : redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + } + ); + + return { decorated, redis }; +} + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +function cancelledData(runId: string, env: SnapshotFixtureEnv) { + return { + ...buildCreateRunData(runId, env), + status: "CANCELED" as const, + error: { type: "STRING_ERROR", raw: "cancelled" } as never, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0 as const, + }; +} + +describe("birth write ordering", () => { + containerTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("RUN_CREATED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const { id: _omitted, ...withoutId } = birthSnapshot(generateInternalId(), env); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot: withoutId }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + // The same minted id must reach both stores, or the comparator chases a difference that is + // not real. + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { runId } }); + expect(read!.entry.id).toBe(row.id); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a crash after the Redis append leaves an orphan key and no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterRedisBirthBeforePg") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + + // The harmless state: a keyspace nothing can reach, and no run that lacks a snapshot. + expect(await redis.getLatest(runId)).not.toBeNull(); + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "creates the run anyway when the birth append fails before redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + // Postgres is authoritative in every position before redis-only, so a Redis outage must not + // stop runs being created. + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "refuses to create the run when the birth append fails at redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "redis-only", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + // At redis-only Postgres writes no snapshot, so a run created without its Redis birth would + // have no snapshot anywhere. Failing before the run row exists lets the caller retry clean. + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toThrow(); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(snapshotId, env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + const read = await redis.getLatest(runId); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a born-terminal run gets the completion expiry immediately", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(generateInternalId(), env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + // A born-terminal run never transitions again, so the completion TTL has to be applied by + // the birth itself or the keyspace never expires. + const nonTerminal = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(nonTerminal, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + const terminal = await redis.getLatest(runId); + const alive = await redis.getLatest(nonTerminal); + expect(terminal).not.toBeNull(); + expect(alive).not.toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { mode: "off" }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts new file mode 100644 index 00000000000..437872106e7 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -0,0 +1,100 @@ +// Mode off is the merge-test position: the decorator must be indistinguishable from its delegate and +// must not touch Redis at all. A Redis store whose every member throws proves the second half, and +// enumerating the generated name list proves the first for every method rather than a chosen few. +import { describe, expect, it } from "vitest"; +import { RUN_STORE_METHOD_NAMES } from "./runStoreMethodNames.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +function explodingRedisStore(): RedisSnapshotStore { + return new Proxy({} as RedisSnapshotStore, { + get(_target, prop) { + return () => { + throw new Error(`the Redis store must not be called at mode off, but ${String(prop)} was`); + }; + }, + }); +} + +/** + * Records what the decorator forwarded, and answers with a per-member sentinel. No database is + * involved in whether mode off is a pass-through, so none is started; the behavioural suites for + * every other mode run against a real Postgres and a real Redis. + */ +function forwardingProbe(): { store: RunStore; calls: string[] } { + const calls: string[] = []; + + const store = new Proxy({} as Record, { + get(_target, prop: string) { + return (...args: unknown[]) => { + calls.push(prop); + return `result:${prop}`; + }; + }, + }); + + return { store: store as unknown as RunStore, calls }; +} + +describe("TaskRunExecutionSnapshotStore at mode off", () => { + it("defaults to mode off", () => { + const { store } = forwardingProbe(); + + const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore() }); + + expect(decorated.mode).toBe("off"); + }); + + it("forwards every method to the delegate and never calls Redis", async () => { + const { store, calls } = forwardingProbe(); + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode: "off", + }) as unknown as Record unknown>; + + for (const name of RUN_STORE_METHOD_NAMES) { + if (name === "runInTransaction") continue; + expect(await decorated[name]("arg-one", "arg-two")).toBe(`result:${name}`); + } + + expect(calls).toEqual(RUN_STORE_METHOD_NAMES.filter((n) => n !== "runInTransaction")); + }); + + it("hands the delegate's own store to a transaction callback", async () => { + const inner = forwardingProbe().store; + let seen: unknown; + const delegate = { + runInTransaction: async ( + _runId: string | undefined, + fn: (store: RunStore, tx: unknown) => Promise + ) => { + await fn(inner, "tx"); + }, + } as unknown as RunStore; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: explodingRedisStore(), + mode: "off", + }); + + await decorated.runInTransaction("run_1", async (store) => { + seen = store; + }); + + expect(seen).toBe(inner); + }); + + it("reports every other dial position as one that writes Redis", () => { + const { store } = forwardingProbe(); + const modes = ["dual-write", "redis-read", "redis-only"] as const; + + for (const mode of modes) { + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode, + }); + expect(decorated.mode).toBe(mode); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts new file mode 100644 index 00000000000..58c80b4df72 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -0,0 +1,104 @@ +// The read cohort is pure arithmetic on the run id, so it needs no containers. Keeping it out of the +// container-backed suite also keeps that suite small enough to run reliably. +import { describe, expect, it } from "vitest"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +type CohortProbe = { readsFromRedis(runId: string): boolean }; + +function probe(mode: SnapshotStoreMode, readPercent: number): CohortProbe { + // lazyConnect keeps the client from dialling anything: no read in this suite reaches the store. + const redis = new RedisSnapshotStore({ + redisOptions: { host: "127.0.0.1", port: 1, lazyConnect: true, retryStrategy: () => null }, + completedTtlMs: 1, + }); + + return new TaskRunExecutionSnapshotStore({} as RunStore, { + store: redis, + mode, + readPercent, + }) as unknown as CohortProbe; +} + +const ids = Array.from({ length: 500 }, (_, n) => `run_cohort_${n}_${n * 7919}`); + +describe("the read cohort", () => { + it("reads nothing from Redis before the read positions", () => { + for (const mode of ["off", "dual-write"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads everything from Redis at 100 percent", () => { + for (const mode of ["redis-read", "redis-only"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads nothing from Redis at 0 percent", () => { + const store = probe("redis-read", 0); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + }); + + it("ignores the dial at redis-only, whatever it is set to", () => { + // Postgres holds no snapshot rows at that position, so a run routed away from Redis reads + // nothing at all. The percentage is meaningful only while both stores hold the data. + for (const percent of [0, 1, 50, 99]) { + const store = probe("redis-only", percent); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + + it("gives one run the same answer every time", () => { + // A run that changed store between two reads of one poll could show the log going backwards. + const store = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + const first = store.readsFromRedis(id); + for (let i = 0; i < 5; i++) { + expect(store.readsFromRedis(id)).toBe(first); + } + } + }); + + it("gives two instances of the same dial the same answer", () => { + // The cohort must not depend on process state, or a redeploy reshuffles every in-flight run. + const first = probe("redis-read", 50); + const second = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + expect(second.readsFromRedis(id)).toBe(first.readsFromRedis(id)); + } + }); + + it("spreads a population across the dial", () => { + const store = probe("redis-read", 50); + const enabled = ids.filter((id) => store.readsFromRedis(id)).length; + + // A wide band: this asserts the hash spreads at all, not that it is uniform. + expect(enabled).toBeGreaterThan(150); + expect(enabled).toBeLessThan(350); + }); + + it("grows the cohort monotonically as the dial rises", () => { + const at = (percent: number) => { + const store = probe("redis-read", percent); + return new Set(ids.filter((id) => store.readsFromRedis(id))); + }; + + const ten = at(10); + const fifty = at(50); + const ninety = at(90); + + // Raising the dial must only ever add runs. A run that fell out on the way up would flip back to + // Postgres mid-flight, which is the thing the stable hash exists to prevent. + expect([...ten].every((id) => fifty.has(id))).toBe(true); + expect([...fifty].every((id) => ninety.has(id))).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts new file mode 100644 index 00000000000..a458e9070ed --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -0,0 +1,420 @@ +// Reads served from Redis must be indistinguishable from the Postgres reads they replace: the same +// payload shape, the same tenant boundary, the same fallback when Redis does not hold the answer. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { mode?: SnapshotStoreMode; readPercent?: number } +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "redis-read", + readPercent: opts?.readPercent ?? 100, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + return runId; +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("snapshot reads", () => { + containerTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.id).toBe(created.id); + expect(latest!.executionStatus).toBe("EXECUTING"); + expect(latest!.description).toBe("Run started"); + expect(latest!.runId).toBe(runId); + expect(latest!.checkpoint).toBeNull(); + expect(latest!.completedWaitpoints).toEqual([]); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest("returns the same payload Postgres would", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + expect(fromRedis!.id).toBe(fromPostgres!.id); + expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); + expect(fromRedis!.description).toBe(fromPostgres!.description); + expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); + expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); + expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); + expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); + expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); + } finally { + await redis.quit(); + } + }); + + containerTest( + "returns the same field set Postgres does, key for key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + // Not a value comparison: a column the hydrator forgets is absent rather than wrong, so it + // shows up as a missing KEY. lastHeartbeatAt was omitted this way and read back undefined + // where Postgres returns null, on every Redis-served read. + expect(Object.keys(fromRedis!).sort()).toEqual(Object.keys(fromPostgres!).sort()); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "reads a foreign environment as not found, so the caller's 404 still fires", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const foreign = await decorated.findLatestExecutionSnapshot(runId, undefined, "env_other"); + + expect(foreign).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "falls back to Postgres for a run with no keyspace", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A pre-cutover run: it exists in Postgres and Redis has never seen it. + await postgresOnly.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.executionStatus).toBe("RUN_CREATED"); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "postgres" }); + } finally { + await redis.quit(); + } + } + ); + + containerTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + readPercent: 0, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + containerTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + containerTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const cursor = await decorated.findExecutionSnapshot({ + where: { id: created.id, runId }, + select: { createdAt: true }, + }); + + expect(cursor).not.toBeNull(); + expect((cursor as { createdAt: Date }).createdAt.toISOString()).toBe( + created.createdAt.toISOString() + ); + expect(reads).toContainEqual({ method: "findExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "delegates a snapshot lookup it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // A different selection: Redis must not answer it approximately. + const row = await decorated.findExecutionSnapshot({ + where: { id: created.id }, + select: { description: true }, + }); + + expect(row).toEqual({ description: "Run started" }); + expect(reads.filter((r) => r.method === "findExecutionSnapshot")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const first = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const second = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Second")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const third = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Third")); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // Descending, exactly as the engine asked; it reverses app-side. + expect(window.map((s) => s.id)).toEqual([third.id, second.id]); + expect(reads).toContainEqual({ method: "findManyExecutionSnapshots", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "delegates a window query it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + + const rows = await decorated.findManyExecutionSnapshots({ + where: { runId }, + orderBy: { createdAt: "asc" }, + }); + + expect(rows.length).toBeGreaterThan(0); + expect(reads.filter((r) => r.method === "findManyExecutionSnapshots")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "serves the waitpoint id projections from Redis", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + + expect(ids).toEqual([]); + // present distinguishes "no waitpoints" from "this reader cannot see the snapshot", which is + // what the engine's read-repair keys off. + expect(withPresence).toEqual({ present: true, ids: [] }); + expect(reads).toContainEqual({ + method: "findSnapshotCompletedWaitpointIds", + source: "redis", + }); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "delegates a waitpoint id projection with no run id", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // Without a run id there is no keyspace to look in. + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id); + + expect(ids).toEqual([]); + expect(reads.filter((r) => r.method.startsWith("findSnapshot"))).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "off", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(await redis.getLatest(runId)).toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts new file mode 100644 index 00000000000..b308cb20988 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts @@ -0,0 +1,321 @@ +// `redis-only` is the terminal cutover, and it is the only dial position where Postgres stops being +// authoritative: it cannot be rolled back by turning the dial down, because the snapshots written +// while it was on exist nowhere else. It is also the only position that is a PAIR of settings, not +// one — the decorator's mode AND `snapshotWrites: false` on the store underneath it — and the two +// are set by different tickets. Every test here builds the pair, because testing the mode against a +// store that still writes snapshots would exercise a configuration that never ships. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +/** The shipping pair: decorator at `redis-only` over a store that writes no snapshot rows. */ +function build(prisma: never, redisOptions: never) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +function birth(env: SnapshotFixtureEnv, id: string) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +async function seedRun(decorated: TaskRunExecutionSnapshotStore, env: SnapshotFixtureEnv) { + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birth(env, snapshotId), + }); + return { runId, snapshotId }; +} + +function transition(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("redis-only: Postgres stops holding snapshots", () => { + containerTest("the run row lands but no snapshot row does", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + // The run itself is still Postgres-authoritative at this position. Only its snapshots move. + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + + // And the snapshot is genuinely in Redis under the id the caller minted. + const head = await redis.getLatest(runId); + expect(head?.id).toBe(snapshotId); + } finally { + await redis.quit(); + } + }); + + containerTest("transitions write no snapshot row either", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + await decorated.createExecutionSnapshot(transition(runId, env, "Run continued")); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const since = await redis.getSinceCreatedAt(runId, new Date(Date.now() - 60_000), { + limit: 50, + }); + expect(since.kind).toBe("hit"); + expect(since.kind === "hit" ? since.entries.length : 0).toBeGreaterThanOrEqual(2); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a completion still updates the run row while writing no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.completeAttemptSuccess( + runId, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id: generateInternalId(), + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + // The mutation half of a nested write must still land, or the run never finishes. + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(run.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "no completed-waitpoint join rows are written for a snapshot Postgres does not have", + async ({ prisma, redisOptions }) => { + // The join rows point at a snapshot row. With snapshot writes off there is no such row, so + // inserting them would leave links dangling at a snapshot only Redis holds. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ], + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const joins = await prisma.$queryRawUnsafe<{ n: bigint }[]>( + `SELECT count(*) AS n FROM "_completedWaitpoints" WHERE "B" = ANY($1::text[])`, + [wpA, wpB] + ); + expect(Number(joins[0]!.n)).toBe(0); + } finally { + await redis.quit(); + } + } + ); +}); + +describe("redis-only: every read is served from Redis", () => { + containerTest( + "the hot read, the since window and the waitpoint lookups all come from Redis", + async ({ prisma, redisOptions }) => { + // At every earlier position a Redis miss falls back to Postgres and the caller never notices. + // Here Postgres holds nothing, so a read that fell back would answer empty rather than wrong, + // and a run would silently lose its state. Each read is asserted to be Redis-sourced. + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const created = await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [{ id: wpA, index: 0 }], + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.id).toBe(created.id); + expect(latest!.completedWaitpointOrder).toEqual([wpA]); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(window.length).toBeGreaterThan(0); + + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + + expect(reads.length).toBeGreaterThan(0); + expect(reads.every((r) => r.source === "redis")).toBe(true); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an unrecognised read shape falls through to a Postgres that holds nothing", + async ({ prisma, redisOptions }) => { + // CHARACTERISATION, NOT AN ENDORSEMENT. `findManyExecutionSnapshots` serves from Redis only + // for the since-window shape `matchSinceWindow` recognises; anything else delegates. At every + // dial position before this one that is harmless, because Postgres holds the same rows. Here + // it holds none, so the caller gets an EMPTY result rather than an error, and empty is a + // valid answer to this query. The same is true of the `miss` and `danglingCycle` fallbacks in + // that method: all three are safe everywhere except the one position that cannot fall back. + // + // Only the engine's own call shapes reach this method today, and it issues the since-window + // one, so nothing is broken. It is pinned here so the terminal-cutover ticket decides + // deliberately whether a fall-through at `redis-only` should throw instead of answering + // empty, rather than discovering this shape in production. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + + // No `createdAt` cursor, so the shape does not match and the read is delegated. + const unmatched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(unmatched).toEqual([]); + + // The same run, asked the shape the engine actually issues, answers in full from Redis. + const matched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(matched.length).toBeGreaterThan(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the read cohort dial cannot route a run away from Redis", + async ({ prisma, redisOptions }) => { + // readPercent is a ramp control for `redis-read`. At `redis-only` a run routed to Postgres + // would read a database that holds no snapshots at all, so the dial must be ignored here + // whatever it is set to. + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: string[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + readPercent: 0, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (_m, source) => reads.push(source), + }, + } + ); + + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest!.id).toBe(snapshotId); + expect(reads).not.toContain("postgres"); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts new file mode 100644 index 00000000000..9952f569e2b --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -0,0 +1,286 @@ +// Inside a transaction the Redis append cannot run until the Postgres side commits, or a rollback +// leaves Redis holding a transition that never happened. These tests observe the buffer from inside +// the callback, so the deferral is proved rather than assumed. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build(prisma: never, redisOptions: never, mode: "off" | "dual-write" = "dual-write") { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode } + ); + return { decorated, redis }; +} + +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, id: string, description: string) { + return { + id, + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("the staging facade", () => { + containerTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + + // Still inside the transaction: nothing has reached Redis yet. + expect(await redis.getById(runId, id)).toBeNull(); + }); + + expect(await redis.getById(runId, id)).not.toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest( + "writes nothing to Redis when the transaction rolls back", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await expect( + decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + throw new Error("rolled back"); + }) + ).rejects.toThrow("rolled back"); + + // Both sides agree that the transition never happened. + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(0); + expect(await redis.getById(runId, id)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const first = generateInternalId(); + const second = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, first, "First"), tx); + await store.createExecutionSnapshot(snapshotInput(runId, env, second, "Second"), tx); + }); + + const firstRead = await redis.getById(runId, first); + const secondRead = await redis.getById(runId, second); + expect(firstRead).not.toBeNull(); + expect(secondRead).not.toBeNull(); + // Order matters: the log is append-only and its seq is what orders a read. + expect(firstRead!.seq).toBeLessThan(secondRead!.seq); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps the fork guard on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + // A stale expectation: this names a head that was never current. Outside a transaction the + // append is rejected as forked and never written. Staging must not weaken that, or a write + // the store would have refused becomes the head purely because it ran inside a transaction. + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "stale"), previousSnapshotId: generateInternalId() }, + tx + ); + }); + + expect(await redis.getById(runId, id)).toBeNull(); + + const head = await redis.getLatest(runId); + expect(head?.id).not.toBe(id); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "honours a correct expectation on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const head = await redis.getLatest(runId); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "expected"), previousSnapshotId: head!.id }, + tx + ); + }); + + expect((await redis.getById(runId, id))?.entry.description).toBe("expected"); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "hands the transaction callback a decorated store", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect((seen as TaskRunExecutionSnapshotStore).mode).toBe("dual-write"); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "hands the transaction callback the plain delegate at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "wraps the store handle from forWaitpointCompletion", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + // No snapshot write goes through this handle today. Wrapping it is what stops a future one + // from bypassing the decorator with no signal. + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "returns the plain handle from forWaitpointCompletion at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + expect(handle).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts new file mode 100644 index 00000000000..e6468c4f09f --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -0,0 +1,454 @@ +// A transition writes Postgres first and Redis second. The order is proved by observation, not by +// reading the code: with the Redis half made to fail, the Postgres row is still there and the caller +// sees no error, which is only possible if Postgres went first. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +type Harness = { + decorated: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + writes: { site: string; outcome: string }[]; +}; + +function harness( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + } +): Harness { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const repairs: Harness["repairs"] = []; + const writes: Harness["writes"] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + + return { decorated, redis, repairs, writes }; +} + +/** + * Creates the run and its keyspace, so a following transition is not skippedNoKeyspace. + * + * The birth is appended through the raw store rather than the decorator, because the decorator's + * own birth path is a separate concern with its own suite. Keeping it out here means a failure in + * this file is a failure of the transition path and nothing else. + */ +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function completionInput(env: SnapshotFixtureEnv) { + return { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +function expireInput(env: SnapshotFixtureEnv) { + return { + error: { type: "STRING_ERROR" as const, raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +describe("transition write ordering", () => { + containerTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.completeAttemptSuccess(runId, completionInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(row.id); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(writes).toContainEqual({ site: "completeAttemptSuccess", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps the Postgres write and enqueues one repair when the append fails", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs } = harness(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterPgBeforeRedis") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // The caller must NOT see an error: the Postgres mutation already committed, and the stall + // watchdog is the designed compensator. + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + + expect(await redis.getById(runId, row.id)).toBeNull(); + expect(repairs).toEqual([{ runId, snapshotId: row.id, executionStatus: "FINISHED" }]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "treats a transition on a run with no keyspace as skipped, not failed", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + // No birth: this is every pre-cutover run's first transition after the dial moves. + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.expireRun(run.id, expireInput(env), { select: { id: true } }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + expect(repairs).toEqual([]); + expect(writes).toEqual([{ site: "expireRun", outcome: "skippedNoKeyspace" }]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for expireRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.expireRun(runId, expireInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + expect(read?.entry.description).toBe("Run expired"); + } finally { + await redis.quit(); + } + }); + + containerTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + await prisma.taskRun.update({ where: { id: runId }, data: { status: "PENDING_VERSION" } }); + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { ...expireInput(env).snapshot, description: "Parked run expired" }, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe("Parked run expired"); + } finally { + await redis.quit(); + } + }); + + containerTest( + "appends nothing when expireParkedRun matches no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + // The run is PENDING, so the delegate's `status: PENDING_VERSION` guard matches nothing. + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + }); + + expect(result.count).toBe(0); + expect(writes.filter((w) => w.site === "expireParkedRun")).toEqual([]); + const latest = await redis.getLatest(runId); + expect(latest?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "DELAYED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe( + "Delayed run was rescheduled to a future date" + ); + } finally { + await redis.quit(); + } + }); + + containerTest( + "appends nothing when rescheduleRun carries no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { delayUntil: new Date(Date.now() + 60_000) }); + + expect(writes.filter((w) => w.site === "rescheduleRun")).toEqual([]); + expect((await redis.getLatest(runId))?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + const read = await redis.getById(runId, snapshotId); + expect(read?.entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(read?.entry.previousSnapshotId).toBe(head!.id); + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "reports a forked append without enqueuing a repair", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // A stale previousSnapshotId: another writer advanced the head. A repair cannot help, so the + // outcome is counted and dropped. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "forked" }); + expect(repairs).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "appends for the standalone createExecutionSnapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const created = await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const read = await redis.getById(runId, created.id); + expect(read).not.toBeNull(); + expect(read!.entry.executionStatus).toBe("EXECUTING"); + // The standalone path is the one whose delegate returns the row, so both stores agree exactly. + expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); + } finally { + await redis.quit(); + } + } + ); + + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); + try { + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.completeAttemptSuccess(run.id, completionInput(env), { + select: { id: true }, + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts new file mode 100644 index 00000000000..c6bc145a91d --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -0,0 +1,980 @@ +// Decorates any RunStore so execution snapshots also land in Redis. It overrides only the methods +// that touch a snapshot and inherits the rest from the generated pass-through base. +// +// Write ORDER is the correctness property, and the two orders are deliberately different: +// +// transition Postgres first, Redis second. A crash in the gap leaves a run whose latest snapshot +// is stale, which is exactly the state the heartbeat stall watchdog already heals. +// birth Redis first, Postgres second. A crash in the gap leaves an unreachable key for a run +// that does not exist. Postgres-first would leave a run with no snapshot at all, and +// getLatestExecutionSnapshot treats that as a hard error. +// +// Each order is chosen so the crash state is the harmless one. A lost cross-store write is never +// recovered by a transaction or an outbox: recovery is always the existing stall-and-repair job. +import { Logger } from "@trigger.dev/core/logger"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import type { + CompletedWaitpointRef, + RedisSnapshotStore, + SnapshotEntryInput, + SnapshotRead, +} from "./redisSnapshotStore.js"; +import { deriveDistinctIds, deriveOrder } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; +import { isInjectedFault, type SnapshotFaultInjector } from "./snapshotFaultInjection.js"; +import type { + ReadClient, + CompletionSnapshotInput, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateRunInput, + ExpireSnapshotInput, + LockRunData, + RescheduleSnapshotInput, + RunStore, + TaskRunWithWaitpoint, +} from "./types.js"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; +import { boundedIn } from "@trigger.dev/database"; +import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; + +/** One initial attempt plus three retries, per the write protocol. */ +const APPEND_ATTEMPTS = 4; + +/** + * Matches the engine's own chunked waitpoint fetch. A batch can complete a thousand waitpoints at + * once, and an unbounded `in:` makes each distinct list length its own prepared statement. + */ +const WAITPOINT_CHUNK_SIZE = 100; + +/** + * The rollout dial. Postgres stays fully written and authoritative in every position before + * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. + * + * A `compare` position was named here before its behaviour existed, and it read from this type as a + * real dial position while behaving in every respect exactly like `dual-write`. A dial value that + * silently does something other than its name is worse than a missing one: turning it on would have + * looked like enabling divergence reporting and delivered plain dual-write. It is added back by the + * ticket that implements the sampled dual-read and diff, at which point the name will be true. + */ +export type SnapshotStoreMode = "off" | "dual-write" | "redis-read" | "redis-only"; + +/** + * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in + * run-store and cannot reach the engine's worker, so the binding is injected. That binding must + * reuse the stall watchdog's job id for the run, or the watchdog and this path can start two + * concurrent repairs on one run. + */ +export type SnapshotRepairEnqueuer = (args: { + runId: string; + snapshotId: string; + executionStatus: string; +}) => Promise; + +export type DecoratorMetrics = { + recordWrite(site: string, outcome: string): void; + recordAppendFailed(site: string): void; + recordRead(method: string, source: "redis" | "postgres"): void; +}; + +export type TaskRunExecutionSnapshotStoreOptions = { + store: RedisSnapshotStore; + /** Defaults to `off`, which is a pure pass-through that never touches Redis. */ + mode?: SnapshotStoreMode; + /** Percentage of runs whose reads come from Redis at `redis-read` and `redis-only`. Defaults to 0. */ + readPercent?: number; + onAppendFailure?: SnapshotRepairEnqueuer; + faults?: SnapshotFaultInjector; + metrics?: DecoratorMetrics; + logger?: Logger; + /** + * Internal. Set only by the staging facade this class builds for `runInTransaction`. When present, + * an intercepted write does its Postgres half and pushes its entry here instead of appending, and + * the outer instance flushes the buffer after the transaction commits. + */ + staging?: StagedAppend[]; +}; + +/** One deferred append: the entry, plus the wait cycle it carries, if any. */ +export type StagedAppend = { + entry: SnapshotEntryInput; + /** + * The head this append expects, carried through staging so the compare-and-set survives the + * deferral. Dropping it would silently disable the fork guard for every snapshot written inside a + * transaction, and a stale append that should be rejected would instead become the head. + */ + expectedCur?: string; + completedWaitpoints?: CompletedWaitpointRef[]; +}; + +export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { + readonly mode: SnapshotStoreMode; + protected readonly redis: RedisSnapshotStore; + protected readonly readPercent: number; + protected readonly onAppendFailure?: SnapshotRepairEnqueuer; + protected readonly faults?: SnapshotFaultInjector; + protected readonly metrics?: DecoratorMetrics; + protected readonly logger: Logger; + protected readonly staging?: StagedAppend[]; + + constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { + super(delegate); + this.redis = options.store; + this.mode = options.mode ?? "off"; + this.readPercent = options.readPercent ?? 0; + this.onAppendFailure = options.onAppendFailure; + this.faults = options.faults; + this.metrics = options.metrics; + this.logger = options.logger ?? new Logger("TaskRunExecutionSnapshotStore", "debug"); + this.staging = options.staging; + } + + /** True in every position that appends to Redis. */ + protected get writesRedis(): boolean { + return this.mode !== "off"; + } + + /** + * The staging facade. Two writes share one Postgres transaction here, and the Redis half of each + * cannot run until that transaction commits: a rollback would otherwise leave Redis holding a + * transition that never happened. + * + * The callback gets a second decorator over the transaction-bound store, carrying a staging + * buffer. An intercepted write does its Postgres half through that store and pushes its entry + * onto the buffer. After the transaction resolves, this instance flushes the buffer in order + * through the same retry-and-repair path a lone transition uses. If the callback throws, the + * delegate rejects, the flush never runs, and the buffer goes away with the stack — so the + * Postgres rollback and the Redis silence agree. + */ + override async runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + if (!this.writesRedis) { + // At `off` the callback must receive the delegate's own store, untouched, so a transaction + // behaves exactly as it does without the decorator in the chain. + return this.delegate.runInTransaction(runId, fn); + } + + const staged: StagedAppend[] = []; + + const result = await this.delegate.runInTransaction(runId, (store, tx) => + fn(this.#wrap(store, staged), tx) + ); + + // The transaction committed. Only now can a snapshot claim its partner is durable. + for (const item of staged) { + await this.#appendTransition( + "runInTransaction", + item.entry, + item.expectedCur, + item.completedWaitpoints + ); + } + + return result; + } + + /** + * `forWaitpointCompletion` hands the caller a store to apply a completion on. No snapshot write + * goes through that handle today, so wrapping it changes nothing now; leaving it unwrapped is the + * one hole that would let a future snapshot write bypass the decorator with no signal at all. + */ + override async forWaitpointCompletion( + waitpointId: string, + context: Parameters[1] + ): Promise { + const store = await this.delegate.forWaitpointCompletion(waitpointId, context); + + if (!this.writesRedis) { + return store; + } + + // Carry the staging buffer through. Without it, a handle taken inside a transaction appends + // immediately, which is the exact ordering the facade exists to prevent. + return this.#wrap(store, this.staging); + } + + /** + * A second decorator over another store, sharing this one's options. One class in both roles keeps + * the write-ordering logic in exactly one place. Passing no buffer gives a plain decorator that + * appends immediately; passing one makes it stage instead. + */ + #wrap(store: RunStore, staging?: StagedAppend[]): TaskRunExecutionSnapshotStore { + return new TaskRunExecutionSnapshotStore(store, { + store: this.redis, + mode: this.mode, + readPercent: this.readPercent, + logger: this.logger, + ...(this.onAppendFailure && { onAppendFailure: this.onAppendFailure }), + ...(this.faults && { faults: this.faults }), + ...(this.metrics && { metrics: this.metrics }), + ...(staging && { staging }), + }); + } + + // --------------------------------------------------------------------------------------------- + // Births: Redis first, Postgres second. + // --------------------------------------------------------------------------------------------- + + override async createRun( + params: CreateRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; + + await this.#appendBirth("createRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createRun({ ...params, snapshot }, tx); + } + + override async createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createCancelledRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; + + await this.#appendBirth("createCancelledRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createCancelledRun({ ...params, snapshot }, tx); + } + + // --------------------------------------------------------------------------------------------- + // Transitions: Postgres first, Redis second. + // --------------------------------------------------------------------------------------------- + + override async completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.completeAttemptSuccess(runId, withId, args, tx); + + await this.#appendTransition( + "completeAttemptSuccess", + entryFromCompletion(ctx, withId.snapshot) + ); + return result; + } + + override async expireRun( + runId: string, + data: { error: unknown; completedAt: Date; expiredAt: Date; snapshot: ExpireSnapshotInput }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.expireRun(runId, data as never, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.expireRun(runId, withId as never, args, tx); + + await this.#appendTransition("expireRun", entryFromExpire(ctx, withId.snapshot)); + return result; + } + + override async expireParkedRun( + runId: string, + data: { + error: unknown; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + if (!this.writesRedis) { + return this.delegate.expireParkedRun(runId, data as never, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.expireParkedRun(runId, withId as never, tx); + + // The delegate writes nothing when the run is no longer PENDING_VERSION, so neither does Redis. + if (result.count > 0) { + await this.#appendTransition("expireParkedRun", entryFromExpire(ctx, withId.snapshot)); + } + return result; + } + + override async rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + // The delegate writes a snapshot only when one is supplied, so an absent snapshot is a plain run + // update with nothing for Redis to mirror. + if (!this.writesRedis || !data.snapshot) { + return this.delegate.rescheduleRun(runId, data, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.rescheduleRun(runId, withId, tx); + + await this.#appendTransition("rescheduleRun", entryFromReschedule(ctx, withId.snapshot)); + return result; + } + + override async lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise>> { + if (!this.writesRedis) { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + // This is the one transition whose input already carries both an id and the previous snapshot + // id, so it is also the one that can append under a compare-and-set on the current head. + const ctx = { id: data.snapshot.id, runId, createdAt: new Date() }; + const withStamp = { ...data, snapshot: { ...data.snapshot, createdAt: ctx.createdAt } }; + + const result = await this.delegate.lockRunToWorker(runId, withStamp, tx); + + await this.#appendTransition( + "lockRunToWorker", + entryFromLock(ctx, withStamp.snapshot), + withStamp.snapshot.previousSnapshotId, + // Built from the COMPLETE id set, which is what the delegate connects in Postgres, with the + // index taken from the ordered list where the id appears in it. Building from the ordered list + // instead would drop every id with no batch index, exactly the ids Postgres still records. + lockCycleRefs( + withStamp.snapshot.completedWaitpointIds, + withStamp.snapshot.completedWaitpointOrder + ) + ); + return result; + } + + override async createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.createExecutionSnapshot(input, tx); + } + + const ctx = this.#context(input.run.id, input.id); + const created = await this.delegate.createExecutionSnapshot( + { ...input, id: ctx.id, createdAt: ctx.createdAt }, + tx + ); + + // The standalone path is the only one whose delegate returns the row, so its entry can take the + // exact createdAt Postgres recorded rather than the decorator's own clock. + await this.#appendTransition( + "createExecutionSnapshot", + entryFromCreateExecutionSnapshot(ctx, input), + input.previousSnapshotId, + input.completedWaitpoints + ); + return created; + } + + // --------------------------------------------------------------------------------------------- + // The append protocol. + // --------------------------------------------------------------------------------------------- + + /** Mints the id when the caller did not, and stamps one clock for both stores. */ + #context(runId: string, suppliedId?: string) { + return { id: suppliedId ?? generateInternalId(), runId, createdAt: new Date() }; + } + + /** + * Births invert the order. Postgres-first would leave a run with no snapshot at all, and + * `getLatestExecutionSnapshot` treats that as a hard error, so the run would be stuck. Redis-first + * leaves an orphaned keyspace for a run that does not exist, which nothing can reach and the + * sweep's second rule reaps. + * + * Being first is also what lets this path refuse. Before `redis-only` a failed birth append is + * survivable, because Postgres is authoritative and holds the snapshot; at `redis-only` Postgres + * writes no snapshot, so a run created without its Redis birth would have no snapshot anywhere. + * Throwing here happens before the run row exists, so the caller retries a clean creation. + */ + async #appendBirth(site: string, entry: SnapshotEntryInput): Promise { + if (this.staging) { + // A birth inside a transaction cannot be staged: staging flushes after the commit, which is + // the opposite of what a birth needs. No caller does this today, so say so and append now. + this.logger.error("a run birth inside a transaction cannot be staged", { + runId: entry.runId, + site, + }); + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + const result = await this.redis.append({ + entry, + kind: "birth", + isTerminal: isTerminalEntry(entry), + }); + this.#recordOutcome(site, entry, result); + + // Modelled AFTER the successful append: the crash this boundary represents is a process that + // died between the two stores, not an append that failed. + this.faults?.("afterRedisBirthBeforePg", { runId: entry.runId, snapshotId: entry.id }); + return; + } catch (error) { + if (isInjectedFault(error)) { + throw error; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot birth append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + mode: this.mode, + error, + }); + + if (this.mode === "redis-only") { + throw error; + } + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Postgres has already committed by the time this runs. A throw here would turn a gap the stall + * watchdog heals into a caller-visible failure, so it never rethrows: it retries, then hands the + * run to the repair job and returns. + */ + async #appendTransition( + site: string, + entry: SnapshotEntryInput, + expectedCur?: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise { + if (this.staging) { + // Inside a transaction the append cannot run until the Postgres side commits, or a rollback + // leaves Redis holding a transition that never happened. + this.staging.push({ + entry, + ...(expectedCur !== undefined && { expectedCur }), + ...(completedWaitpoints && { completedWaitpoints }), + }); + return; + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + this.faults?.(attempt === 0 ? "afterPgBeforeRedis" : "midFlushRetry", { + runId: entry.runId, + snapshotId: entry.id, + }); + + const cycle = await this.#resolveCycle(entry.runId, completedWaitpoints); + + const result = await this.redis.append({ + entry, + kind: "transition", + isTerminal: isTerminalEntry(entry), + ...(expectedCur !== undefined && { expectedCur }), + ...(cycle && { cycle }), + }); + + this.#recordOutcome(site, entry, result); + return; + } catch (error) { + // An injected fault models a dead process, not a retryable append failure. + if (isInjectedFault(error)) { + this.metrics?.recordAppendFailed(site); + await this.#enqueueRepair(entry); + return; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + error, + }); + await this.#enqueueRepair(entry); + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Decides whether this append mints a new wait cycle or points at the one already there. + * + * A resume append carries a newly-differing id set, so it mints a cycle and the record set is + * written once. Every copy-forward append that follows re-passes the SAME list, and re-minting on + * each would rewrite the record set once per entry in the resume chain — the write amplification + * the pointer model exists to remove. So an unchanged id set carries the previous cycleSeq + * forward and writes no key. + * + * The extra read only happens for an append that actually carries waitpoints, which is the resume + * path rather than the hot path. + * + * `records` is deliberately left unset. The record envelope belongs to the waitpoint lane and + * ships empty in this build, so dual-write never re-versions the entry when it arrives. + */ + async #resolveCycle( + runId: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise< + | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[] } + | { kind: "carryForward"; cycleSeq: number; completedWaitpoints: CompletedWaitpointRef[] } + | undefined + > { + if (!completedWaitpoints || completedWaitpoints.length === 0) { + return undefined; + } + + const order = deriveOrder(completedWaitpoints); + const distinct = deriveDistinctIds(completedWaitpoints); + + try { + const head = await this.redis.getLatest(runId); + const previousIds = head?.completedWaitpointIds; + + // Both halves must match. Comparing the order alone is not enough: it holds only indexed ids, + // so two DIFFERENT single waits both present an empty order and would compare equal, and the + // second would inherit the first's waitpoint set instead of minting its own. + if ( + head?.cycle && + previousIds && + sameOrder(previousIds.order, order) && + sameSet(previousIds.distinctIds, distinct) + ) { + return { + kind: "carryForward", + cycleSeq: head.cycle.cycleSeq, + completedWaitpoints, + }; + } + } catch (error) { + // A failed probe must not lose the waitpoints. Minting a fresh cycle is the safe direction: + // it costs one duplicated record set, where a wrong carryForward would point at another + // cycle's ids. + this.logger.warn("snapshot cycle probe failed, minting a new cycle", { runId, error }); + } + + return { kind: "new", completedWaitpoints }; + } + + /** + * None of the four append outcomes is a failure, and none of them enqueues a repair. + * + * `skippedNoKeyspace` is every pre-cutover run's transitions. `forked` means another writer + * advanced the head, which a repair cannot help. `duplicate` is a retry that already landed. + * `cycleMismatch` means the store refused an untrustworthy waitpoint pointer on purpose. + */ + #recordOutcome( + site: string, + entry: SnapshotEntryInput, + result: Awaited> + ): void { + this.metrics?.recordWrite(site, result.outcome); + + if (result.outcome === "forked") { + this.logger.warn("snapshot append forked", { + runId: entry.runId, + snapshotId: entry.id, + site, + actualCur: result.actualCur, + }); + } + } + + // --------------------------------------------------------------------------------------------- + // Reads. + // + // Only three production call sites exist, all in the engine's executionSnapshotSystem, all with + // fixed argument shapes. The two generic Prisma-args methods therefore recognise exactly the + // shapes the engine sends and delegate everything else: an unrecognised shape must go to Postgres, + // never get an approximate answer from Redis. + // --------------------------------------------------------------------------------------------- + + /** + * Whether this run's reads come from Redis. Hashed on the run id so a run does not change store + * between two reads of the same poll, which would let a caller see the log go backwards. + */ + protected readsFromRedis(runId: string): boolean { + if (this.mode !== "redis-read" && this.mode !== "redis-only") return false; + + // At `redis-only` the cohort dial has no meaning. Postgres holds no snapshot rows at that + // position, so a run routed away from Redis reads nothing at all. Ignoring the percentage here + // makes that misconfiguration unreachable rather than merely documented. + if (this.mode === "redis-only") return true; + + if (this.readPercent >= 100) return true; + if (this.readPercent <= 0) return false; + + let hash = 0; + for (let i = 0; i < runId.length; i++) { + hash = (hash * 31 + runId.charCodeAt(i)) >>> 0; + } + return hash % 100 < this.readPercent; + } + + override async findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + environmentId?: string + ): Promise | null> { + if (!this.readsFromRedis(runId)) { + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + const read = await this.redis.getLatest(runId, { ...(environmentId && { environmentId }) }); + if (!read) { + // A miss is the coexistence path: a pre-cutover run, or expired history. It is not an error. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + if (read.danglingCycle) { + // The entry says it has waitpoints and the cycle key holding them is gone. Serving it would + // hand back an empty set that looks authoritative, and the run would resume with no waits. + // Postgres still has the join rows. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); + return this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); + } + + override async findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + const shape = matchSinceCursorLookup(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findExecutionSnapshot(args, client); + } + + const found = await this.redis.getById(shape.runId, shape.id, { + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (!found) { + this.metrics?.recordRead("findExecutionSnapshot", "postgres"); + return this.delegate.findExecutionSnapshot(args, client); + } + + this.metrics?.recordRead("findExecutionSnapshot", "redis"); + // The engine selects createdAt only, so the answer is the cursor and nothing else. + return { + createdAt: new Date(found.entry.createdAt as string), + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload; + } + + override async findManyExecutionSnapshots( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + const shape = matchSinceWindow(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + const result = await this.redis.getSinceCreatedAt(shape.runId, shape.createdAt, { + limit: shape.take, + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (result.kind === "miss") { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + + if (result.entries.some((entry) => entry.danglingCycle)) { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + + this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); + + // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. + const descending = [...result.entries].reverse(); + // Rows are hydrated for no entry here: the engine fetches the head's waitpoints itself, from + // the ids this call's head row reports. Each row still carries its own order. + const hydrated = await Promise.all( + descending.map((entry) => this.#hydrate(entry, shape.runId, client)) + ); + return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; + } + + override async findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + // Without a run id there is no keyspace to look in, so the router's fan-out is the only answer. + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "redis"); + return ids.distinctIds; + } + + override async findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + // present=false means this reader cannot see the snapshot, so its empty list is not + // authoritative and the engine's read-repair needs the Postgres answer. + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "redis"); + return { present: true, ids: ids.distinctIds }; + } + + /** + * Turns a store entry into the Prisma payload the interface promises. + * + * The entry supplies every scalar column. `checkpoint` and the full waitpoint rows still live in + * Postgres, so they are read back through the delegate — but only when the entry says they exist, + * which keeps the common read (a running run with neither) free of any Postgres call at all. + */ + async #hydrate( + read: SnapshotRead, + runId: string, + client?: ReadClient, + opts?: { hydrateWaitpointRows?: boolean } + ): Promise< + Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }> + > { + const entry = read.entry as Record; + + const checkpoint = entry.checkpointId + ? await this.#hydrateCheckpoint(runId, read.id, client) + : null; + + // `completedWaitpointOrder` is a scalar column, NOT the join. The engine reads it off the head + // row as the index oracle that gives each completed waitpoint its position in a batch, so it + // must be populated even when the waitpoint ROWS are not fetched. Returning an empty order here + // resumes every batched triggerAndWait with `index: undefined`. + // Three cases, and only the last needs a second Redis call. The read already carries the ids + // when the store decoded them. An entry with no wait cycle has no waitpoints by construction, + // which is the common case and used to cost a round trip to rediscover. Anything else asks. + const ids = + read.completedWaitpointIds ?? + (read.cycle === undefined + ? { present: true, distinctIds: [], order: [] } + : await this.redis.getSnapshotWaitpointIds(runId, read.id)); + const completedWaitpointOrder = ids.order; + + // The rows themselves are head-only, mirroring the engine's own N x M avoidance. + const completedWaitpoints = opts?.hydrateWaitpointRows + ? await this.#fetchWaitpointsInChunks(ids.distinctIds, runId, client) + : []; + + return { + id: read.id, + engine: entry.engine ?? "V2", + executionStatus: entry.executionStatus, + description: entry.description, + previousSnapshotId: entry.previousSnapshotId ?? null, + runId: entry.runId, + runStatus: entry.runStatus, + attemptNumber: entry.attemptNumber ?? null, + batchId: entry.batchId ?? null, + environmentId: entry.environmentId, + environmentType: entry.environmentType, + projectId: entry.projectId, + organizationId: entry.organizationId, + checkpointId: entry.checkpointId ?? null, + workerId: entry.workerId ?? null, + runnerId: entry.runnerId ?? null, + metadata: entry.metadata ?? null, + // A column no code writes, so Postgres returns null for it on every row. The entry does not + // carry it, and omitting it here would hand back undefined where Postgres hands back null, + // on every single read served from Redis. + lastHeartbeatAt: null, + completedWaitpointOrder, + isValid: read.isValid, + error: entry.error ?? null, + createdAt: new Date(entry.createdAt as string), + // A snapshot row is write-once, so both columns hold the one instant the decorator minted. + updatedAt: new Date(entry.createdAt as string), + checkpoint, + completedWaitpoints, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }>; + } + + /** + * Chunked, and bounded within each chunk, mirroring the engine's own waitpoint fetch. The run id + * routes each chunk to the owning store rather than fanning every one across both databases. + */ + async #fetchWaitpointsInChunks( + waitpointIds: string[], + runId: string, + client?: ReadClient + ): Promise { + if (waitpointIds.length === 0) return []; + + const all: unknown[] = []; + for (let i = 0; i < waitpointIds.length; i += WAITPOINT_CHUNK_SIZE) { + const chunk = waitpointIds.slice(i, i + WAITPOINT_CHUNK_SIZE); + const rows = await this.delegate.findManyWaitpoints( + { where: { id: { in: boundedIn(chunk) } } }, + client, + runId + ); + all.push(...rows); + } + return all; + } + + /** + * Reads the checkpoint row through the snapshot the delegate still holds, so the read stays + * residency-aware: the run id in the where is what routes it to the owning database, and the + * decorator sits above the router and has no client of its own. + * + * At `redis-only` the Postgres snapshot row is gone, so this returns null. The checkpoint row + * itself stays in Postgres, but the interface has no residency-aware way to read one directly. + * Closing that needs a narrow lookup on the interface, which the plan freezes for this ticket. + */ + async #hydrateCheckpoint( + runId: string, + snapshotId: string, + client?: ReadClient + ): Promise { + const row = await this.delegate.findExecutionSnapshot( + { where: { id: snapshotId, runId }, include: { checkpoint: true } }, + client + ); + return (row as { checkpoint?: unknown } | null)?.checkpoint ?? null; + } + + async #enqueueRepair(entry: SnapshotEntryInput): Promise { + if (!this.onAppendFailure) { + return; + } + + try { + await this.onAppendFailure({ + runId: entry.runId, + snapshotId: entry.id, + executionStatus: entry.executionStatus, + }); + } catch (error) { + // The repair enqueue is itself best-effort. Failing it must not fail the caller's write. + this.logger.error("snapshot repair enqueue failed", { runId: entry.runId, error }); + } + } +} + +/** Position-sensitive: the same ids in a different order are a different wait cycle. */ +function sameOrder(a: string[], b: string[]): boolean { + return a.length === b.length && a.every((id, index) => id === b[index]); +} + +/** + * Turns the lock site's two lists into cycle refs. `completedWaitpointIds` is the complete set the + * delegate connects; `completedWaitpointOrder` gives a position only to the ids that have one, and a + * repeated id keeps each of its positions. + */ +function lockCycleRefs(ids: string[], order: string[]): { id: string; index?: number }[] { + const refs: { id: string; index?: number }[] = []; + const indexed = new Set(); + + order.forEach((id, index) => { + refs.push({ id, index }); + indexed.add(id); + }); + + for (const id of ids) { + if (!indexed.has(id)) refs.push({ id }); + } + + return refs; +} + +/** Membership only, for the id set, which has no meaningful order. */ +function sameSet(a: string[], b: string[]): boolean { + if (a.length !== b.length) return false; + const seen = new Set(a); + return b.every((id) => seen.has(id)); +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts new file mode 100644 index 00000000000..4bc2d0dc271 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -0,0 +1,706 @@ +// The completed-waitpoint path, which every other suite here was blind to. +// +// Two defects hid behind that blindness. The decorator passed no cycle to `append`, so no +// wp: key was ever written and the Redis waitpoint side was permanently empty. And the +// since-window hydration returned an empty `completedWaitpointOrder`, which is the index oracle the +// engine uses to give each completed waitpoint its position in a batch — an empty order resumes +// every batched triggerAndWait with `index: undefined`. +// +// So these tests all use a snapshot that ACTUALLY carries waitpoints. A test that does not cannot +// tell a working cycle from a missing one. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + mode: "dual-write" | "redis-read" = "redis-read" +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const writes: { site: string; outcome: string }[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode, + readPercent: 100, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + return { decorated, redis, writes }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); + return runId; +} + +function resumeInput( + runId: string, + env: SnapshotFixtureEnv, + completedWaitpoints: { id: string; index?: number }[], + description = "Run resumed" +) { + return { + id: generateInternalId(), + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + completedWaitpoints, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("completed-waitpoint cycles", () => { + containerTest("a resume append mints a cycle key", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The key exists at all — before the fix, none was ever written. + const cycleKeys = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cycleKeys.length).toBe(1); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.order).toEqual([wpA, wpB]); + expect(ids.distinctIds).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "a copy-forward reuses the cycle and writes no second key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const waitpoints = [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]; + + await decorated.createExecutionSnapshot(resumeInput(runId, env, waitpoints, "resume")); + // The same id set again: this is the copy-forward every dequeue and checkpoint site does. + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry one") + ); + const third = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry two") + ); + + // Still ONE key. Re-minting per entry is the write amplification the pointer model removes. + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + + // And every entry still resolves the same order. + for (const id of [second.id, third.id]) { + expect((await redis.getSnapshotWaitpointIds(runId, id)).order).toEqual([wpA, wpB]); + } + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a newly-differing id set mints a second cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "first wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB, index: 0 }], "second wait") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).order).toEqual([wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same ids in a different order are a new cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + // Order IS the index oracle, so a reordering is a different cycle, not a carry-forward. + const reordered = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpB, index: 0 }, + { id: wpA, index: 1 }, + ]) + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, reordered.id)).order).toEqual([ + wpB, + wpA, + ]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a repeated id keeps both of its positions", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpX] = await seedSnapshotWaitpoints(prisma, env, 1); + + // One run batched twice under a single idempotency key: the id repeats, and each position + // must survive, because the runner matches results to positions. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpX, index: 0 }, + { id: wpX, index: 1 }, + ]) + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.order).toEqual([wpX, wpX]); + expect(ids.distinctIds).toEqual([wpX]); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps a completed waitpoint that has no batch index", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // Every wait.for, every single triggerAndWait and every token resumes with no batch index: + // the engine passes `index: b.batchIndex ?? undefined`. Postgres records the id in the + // completed-waitpoint join regardless. The ordered list cannot hold it, because its + // positions ARE the indexes, so the complete set has to be stored separately or the wait's + // result vanishes on a Redis read. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "single wait") + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + // No position, so it is absent from the oracle. That part is correct. + expect(ids.order).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "matches the Postgres join for a mix of indexed and index-less waits", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB, wpC] = await seedSnapshotWaitpoints(prisma, env, 3); + + const created = await decorated.createExecutionSnapshot( + resumeInput( + runId, + env, + [{ id: wpA, index: 0 }, { id: wpB }, { id: wpC, index: 1 }], + "mixed wait" + ) + ); + + // Parity with what Postgres holds is the actual requirement: the engine iterates the rows + // this set fetches, and uses the order only to assign each one its index. + const fromRedis = await redis.getSnapshotWaitpointIds(runId, created.id); + const fromPostgres = await new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + }).findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + + expect([...fromRedis.distinctIds].sort()).toEqual([...fromPostgres].sort()); + expect(fromRedis.order).toEqual([wpA, wpC]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "two consecutive index-less waits do not share a cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + // Neither wait has a batch index, so both present an EMPTY order. Deciding carry-forward on + // the order alone makes them compare equal, and the second silently inherits the first's + // waitpoint set: its own result is never stored and a read returns the wrong id. + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "first single wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB }], "second single wait") + ); + + expect((await redis.getSnapshotWaitpointIds(runId, first.id)).distinctIds).toEqual([wpA]); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).distinctIds).toEqual([wpB]); + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same index-less wait repeated does still carry forward", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // The copy-forward case must survive the stricter comparison: the same id set, still one key. + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "wait")); + const carried = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "carry") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + expect((await redis.getSnapshotWaitpointIds(runId, carried.id)).distinctIds).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the dequeue snapshot keeps an index-less waitpoint", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + // Postgres connects completedWaitpointIds, the COMPLETE set. Building the Redis refs from + // completedWaitpointOrder instead drops every id that has no position in it. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect([...ids.distinctIds].sort()).toEqual([wpA, wpB].sort()); + expect(ids.order).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "findLatestExecutionSnapshot hydrates an index-less waitpoint row", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "single")); + + // The hot read hydrates the rows from the id set, so an incomplete set means the resume + // gets no waitpoint at all. + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "a refused carry mints a fresh cycle rather than writing a pointerless head", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const snapshotId = generateInternalId(); + + // Driven at the store, because the decorator cannot reach this state deliberately: its + // probe reads the head first, sees the id set no longer matches, and mints a new cycle. + // The refusal is only reachable when the key vanishes BETWEEN that probe and the append, + // which is a race. Naming a cycle this incarnation never minted reproduces the same + // refusal deterministically. + const result = await redis.append({ + entry: { + id: snapshotId, + engine: "V2", + executionStatus: "EXECUTING", + description: "carry a cycle that was never minted", + runId, + runStatus: "EXECUTING", + createdAt: new Date().toISOString(), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + kind: "transition", + isTerminal: false, + cycle: { + kind: "carryForward", + cycleSeq: 9999, + completedWaitpoints: [{ id: wpA, index: 0 }], + }, + }); + + expect(result.outcome).toBe("written"); + if (result.outcome !== "written") return; + + // Refusing the pointer is right. Writing the entry with NO pointer is not: it becomes the + // head, and a read of it answers present-with-nothing, which is the one answer that stops + // the engine's read-repair from looking. + expect(result.cycleMismatch).toBe(true); + expect(result.cycleSeq).toBeGreaterThan(0); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "a dangling pointer reads as not present, not as an empty set", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + + // The entry keeps its pointer and the cycle key goes. Reachable by eviction, and by the + // completion TTL, which is set on every key for a run at once but expires them separately. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + // present:false is what sends the caller to Postgres, which still holds the join rows. + // present:true with an empty set would suppress the engine's read-repair. + expect(ids.present).toBe(false); + expect(ids.distinctIds).toEqual([]); + + // And the projections the engine actually calls fall back rather than answering empty. + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the hot read falls back to Postgres when the cycle key is gone", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // Served from Postgres, so the waitpoint is still there and the resume is not silently + // stripped of it. + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "findLatestExecutionSnapshot returns the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // completedWaitpointOrder is a scalar column, not the join. Empty here means every batched + // waitpoint resumes with index undefined. + expect(latest!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the since-window head carries the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // The head is first in a descending window. This is the row the engine reads the oracle off. + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the since-window falls back to Postgres when the head cycle key is gone", + async ({ prisma, redisOptions }) => { + // The hot read has always handled this. The since-window did not: its Lua returned the head's + // order and distinct set but never its dangling flag, so the decorator's fallback guard was + // dead code and an expired cycle key came back as an EMPTY order. Empty means "no indexed + // waitpoints" to the engine, which is how a batched triggerAndWait resumes with every + // position lost rather than falling back to the store that still knows them. + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The entry hash survives; only the cycle key goes. The completion TTL is applied per key, + // so this is a state the keyspace reaches on its own. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "lockRunToWorker carries its resolved order into the cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA, wpB], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.order).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an append with no waitpoints writes no cycle key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [], "no waitpoints")); + + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); diff --git a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts new file mode 100644 index 00000000000..e0b8a1a2107 --- /dev/null +++ b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts @@ -0,0 +1,167 @@ +// Shared setup for the snapshot-id, snapshot-writes and entry-parity suites. Modelled on the +// seedEnvironment/buildCreateRunInput pair in PostgresRunStore.test.ts; the slugs are suffixed so +// several fixtures can coexist in one database. +import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import type { CreateRunData } from "../types.js"; + +export type SnapshotFixtureEnv = { + id: string; + type: "DEVELOPMENT"; + projectId: string; + organizationId: string; +}; + +export type SnapshotIdFixture = { + run: { id: string }; + env: SnapshotFixtureEnv; +}; + +export async function seedSnapshotEnvironment(prisma: PrismaClient): Promise { + const suffix = generateInternalId().slice(-12); + + const organization = await prisma.organization.create({ + data: { title: `Snapshot Org ${suffix}`, slug: `snapshot-org-${suffix}` }, + }); + + const project = await prisma.project.create({ + data: { + name: `Snapshot Project ${suffix}`, + slug: `snapshot-project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + + return { + id: environment.id, + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + }; +} + +export function buildCreateRunData(runId: string, env: SnapshotFixtureEnv): CreateRunData { + return { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${runId.slice(-16)}`, + runtimeEnvironmentId: env.id, + environmentType: env.type, + organizationId: env.organizationId, + projectId: env.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${runId.slice(-8)}`, + spanId: `span_${runId.slice(-8)}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +export type SnapshotWorkerFixture = { workerId: string; taskId: string }; + +/** + * Seeds a BackgroundWorker and one of its tasks. The snapshot's `workerId` and the run's + * `lockedById` are both foreign keys, so a made-up id fails the constraint rather than the + * assertion, and the test reports a fixture fault as if it were a parity fault. + */ +export async function seedSnapshotWorker( + prisma: PrismaClient, + env: SnapshotFixtureEnv +): Promise { + const suffix = generateInternalId().slice(-12); + + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + engine: "V2", + contentHash: `hash_${suffix}`, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + version: "20260824.1", + metadata: {}, + }, + }); + + const task = await prisma.backgroundWorkerTask.create({ + data: { + slug: "my-task", + friendlyId: `task_${suffix}`, + filePath: "src/trigger/my-task.ts", + exportName: "myTask", + workerId: worker.id, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + }, + }); + + return { workerId: worker.id, taskId: task.id }; +} + +/** + * Seeds real Waitpoint rows and returns their ids. The legacy completed-waitpoint join carries a + * real foreign key, so an invented id fails the constraint rather than the assertion — the test + * then reports a fixture fault as if it were a defect in the code under test. + */ +export async function seedSnapshotWaitpoints( + prisma: PrismaClient, + env: SnapshotFixtureEnv, + count: number +): Promise { + const ids: string[] = []; + + for (let i = 0; i < count; i++) { + const suffix = generateInternalId().slice(-12); + const waitpoint = await prisma.waitpoint.create({ + data: { + friendlyId: `waitpoint_${suffix}`, + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date(), + idempotencyKey: `idem_${suffix}`, + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + ids.push(waitpoint.id); + } + + return ids; +} + +/** + * Seeds an environment plus one run in `status`, with no execution snapshot. The suites that use it + * assert on the snapshot rows a store method writes, so the run must start with none. + */ +export async function setupSnapshotIdFixture( + prisma: PrismaClient, + opts?: { status?: TaskRunStatus } +): Promise { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: opts?.status ?? "PENDING" }, + }); + + return { run: { id: runId }, env }; +} diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 7c7f9566893..9ea39473e5b 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -23,12 +23,19 @@ import type { Residency } from "@trigger.dev/core/v3/isomorphic"; export type ReadClient = PrismaClientOrTransaction | PrismaReplicaClient; export type IdempotencyKeyRunMatch = { + id: string; + createdAt: Date; friendlyId: string; idempotencyKey: string | null; idempotencyKeyExpiresAt: Date | null; }; export type CreateRunSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id?: string; engine: "V2"; executionStatus: TaskRunExecutionStatus; @@ -43,6 +50,14 @@ export type CreateRunSnapshotInput = { }; export type CompletionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; executionStatus: "FINISHED"; description: string; runStatus: TaskRunStatus; @@ -64,6 +79,14 @@ export type PromotePendingVersionArgs = { }; export type ExpireSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; engine: "V2"; executionStatus: "FINISHED"; description: string; @@ -75,6 +98,14 @@ export type ExpireSnapshotInput = { }; export type RescheduleSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; environmentId: string; environmentType: RuntimeEnvironmentType; projectId: string; @@ -85,6 +116,11 @@ export type RescheduleSnapshotInput = { }; export type LockSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id: string; previousSnapshotId: string; attemptNumber?: number; @@ -292,6 +328,14 @@ export type TaskRunWithWaitpoint = TaskRun & { associatedWaitpoint: Waitpoint | * input — callers pass the high-level shape, not a raw Prisma `data`/`include`. */ export type CreateExecutionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; run: { id: string; status: TaskRunStatus; attemptNumber?: number | null }; snapshot: { executionStatus: TaskRunExecutionStatus; diff --git a/internal-packages/testcontainers/src/clusterSlot.test.ts b/internal-packages/testcontainers/src/clusterSlot.test.ts new file mode 100644 index 00000000000..a7bec4313fa --- /dev/null +++ b/internal-packages/testcontainers/src/clusterSlot.test.ts @@ -0,0 +1,42 @@ +import { expect, it, describe } from "vitest"; +import { slotOf, expectOneSlot } from "./clusterSlot"; + +describe("slotOf", () => { + it("matches the published CRC16/XMODEM check value and known slots", () => { + expect(slotOf("123456789")).toBe(0x31c3); + expect(slotOf("engine:snap:{run_1}:e")).toBe(8108); + expect(slotOf("engine:snap:{run_2}:e")).toBe(12239); + }); + + it("groups keys that share a non-empty tag into one slot", () => { + expect(slotOf("a{tag}b")).toBe(slotOf("c{tag}d")); + }); + + it("hashes the whole key when the tag is empty (not the empty tag)", () => { + // If the empty `{}` were used as the tag, these would collide; hashing the whole key keeps them apart. + expect(slotOf("a{}b")).not.toBe(slotOf("c{}d")); + }); + + it("hashes the whole key when a brace is unclosed (malformed tag)", () => { + // `b` is not a tag here (no closing brace), so these must not share a slot the way `{b}` would. + expect(slotOf("a{b")).not.toBe(slotOf("x{b")); + }); + + it("hashes UTF-8 bytes, matching Redis for a non-ASCII tag", () => { + // Redis (cluster-key-slot) hashes the UTF-8 bytes of `é` to slot 10180. + expect(slotOf("{é}")).toBe(10180); + }); +}); + +describe("expectOneSlot", () => { + it("passes when every key shares one slot", () => { + expect(() => expectOneSlot(["snap:{r}:e", "snap:{r}:idx", "snap:{r}:cur"])).not.toThrow(); + }); + it("passes for zero or one key", () => { + expect(() => expectOneSlot([])).not.toThrow(); + expect(() => expectOneSlot(["snap:{r}:e"])).not.toThrow(); + }); + it("throws when two keys fall in different slots", () => { + expect(() => expectOneSlot(["snap:{run_1}:e", "snap:{run_2}:e"])).toThrow(/slot/i); + }); +}); diff --git a/internal-packages/testcontainers/src/clusterSlot.ts b/internal-packages/testcontainers/src/clusterSlot.ts new file mode 100644 index 00000000000..ccf74942176 --- /dev/null +++ b/internal-packages/testcontainers/src/clusterSlot.ts @@ -0,0 +1,34 @@ +// CRC16/XMODEM over a key's hash tag, computed here because CLUSTER KEYSLOT is unavailable on a +// standalone test container. Pinned against the cluster-key-slot package for our key shapes. Hashes +// UTF-8 BYTES (as Redis does), not UTF-16 code units, so a non-ASCII key still matches Redis's slot. + +function crc16(str: string): number { + let crc = 0; + for (const byte of Buffer.from(str, "utf8")) { + crc ^= byte << 8; + for (let j = 0; j < 8; j++) { + crc = crc & 0x8000 ? (crc << 1) ^ 0x1021 : crc << 1; + crc &= 0xffff; + } + } + return crc; +} + +/** The Redis cluster slot (0–16383) for a key, honouring `{…}` hash-tag extraction. */ +export function slotOf(key: string): number { + const start = key.indexOf("{"); + const end = start === -1 ? -1 : key.indexOf("}", start + 1); + const tag = start !== -1 && end !== -1 && end > start + 1 ? key.slice(start + 1, end) : key; + return crc16(tag) % 16384; +} + +/** Throws unless every key maps to one slot. A `[]` or single-key input passes. */ +export function expectOneSlot(keys: string[]): void { + if (keys.length <= 1) return; + const slots = new Set(keys.map(slotOf)); + if (slots.size !== 1) { + throw new Error( + `expected all keys in one cluster slot, got ${slots.size}: ${JSON.stringify(keys)}` + ); + } +} diff --git a/internal-packages/testcontainers/src/faultInjection.test.ts b/internal-packages/testcontainers/src/faultInjection.test.ts new file mode 100644 index 00000000000..fbcce7dc955 --- /dev/null +++ b/internal-packages/testcontainers/src/faultInjection.test.ts @@ -0,0 +1,58 @@ +import { expect, it, describe } from "vitest"; +import { createFaultInjector } from "./faultInjection"; + +type B = "afterPgBeforeRedis" | "midFlushRetry"; +class TestFault extends Error { + constructor(readonly boundary: B) { + super(`injected at ${boundary}`); + this.name = "TestFault"; + } +} +const make = () => createFaultInjector({ error: (b) => new TestFault(b) }); + +describe("createFaultInjector", () => { + it("does not throw when nothing is armed", () => { + const f = make(); + expect(() => f.hook("afterPgBeforeRedis", { runId: "r1" })).not.toThrow(); + expect(f.fired("afterPgBeforeRedis")).toBe(0); + }); + + it("throws the injected error while armed, and counts each throw", () => { + const f = make(); + f.arm("afterPgBeforeRedis"); + expect(() => f.hook("afterPgBeforeRedis")).toThrow(TestFault); + expect(f.fired("afterPgBeforeRedis")).toBe(1); + }); + + it("times limits the number of throws", () => { + const f = make(); + f.arm("midFlushRetry", { times: 2 }); + expect(() => f.hook("midFlushRetry")).toThrow(); + expect(() => f.hook("midFlushRetry")).toThrow(); + expect(() => f.hook("midFlushRetry")).not.toThrow(); + expect(f.fired("midFlushRetry")).toBe(2); + }); + + it("runId scopes throws to the matching run only", () => { + const f = make(); + f.arm("afterPgBeforeRedis", { runId: "r1" }); + expect(() => f.hook("afterPgBeforeRedis", { runId: "r2" })).not.toThrow(); + expect(() => f.hook("afterPgBeforeRedis", { runId: "r1" })).toThrow(); + expect(f.fired("afterPgBeforeRedis")).toBe(1); + }); + + it("rejects a non-integer or negative times, but allows the default (Infinity)", () => { + const f = make(); + expect(() => f.arm("midFlushRetry", { times: Number.NaN })).toThrow(RangeError); + expect(() => f.arm("midFlushRetry", { times: 1.5 })).toThrow(RangeError); + expect(() => f.arm("midFlushRetry", { times: -1 })).toThrow(RangeError); + expect(() => f.arm("midFlushRetry")).not.toThrow(); // unlimited + }); + + it("disarm clears a boundary", () => { + const f = make(); + f.arm("afterPgBeforeRedis"); + f.disarm("afterPgBeforeRedis"); + expect(() => f.hook("afterPgBeforeRedis")).not.toThrow(); + }); +}); diff --git a/internal-packages/testcontainers/src/faultInjection.ts b/internal-packages/testcontainers/src/faultInjection.ts new file mode 100644 index 00000000000..826cbbb3c36 --- /dev/null +++ b/internal-packages/testcontainers/src/faultInjection.ts @@ -0,0 +1,46 @@ +// Test-only fault-injection harness, shared by the snapshot decorator (crash-gap) and the waitpoint +// lane. Generic over the boundary union; the caller passes the error constructor, so this package +// takes no dependency on @internal/run-store (which would close a dependency cycle). The armed hook +// is SYNCHRONOUS: a crash at a write boundary must interrupt before the next write. + +export type FaultInjector = { + arm(boundary: TBoundary, opts?: { times?: number; runId?: string }): void; + disarm(boundary?: TBoundary): void; + hook: (boundary: TBoundary, context?: { runId?: string }) => void; + fired(boundary: TBoundary): number; +}; + +type Armed = { remaining: number; runId?: string }; + +export function createFaultInjector(opts: { + error: (boundary: TBoundary) => Error; +}): FaultInjector { + const armed = new Map(); + const counts = new Map(); + + return { + arm(boundary, o) { + const times = o?.times ?? Infinity; + if (times !== Infinity && (!Number.isInteger(times) || times < 0)) { + throw new RangeError("times must be a non-negative integer or Infinity"); + } + armed.set(boundary, { remaining: times, runId: o?.runId }); + }, + disarm(boundary) { + if (boundary === undefined) armed.clear(); + else armed.delete(boundary); + }, + hook: (boundary, context) => { + const a = armed.get(boundary); + if (!a || a.remaining <= 0) return; + if (a.runId !== undefined && a.runId !== context?.runId) return; + a.remaining -= 1; + if (a.remaining <= 0) armed.delete(boundary); + counts.set(boundary, (counts.get(boundary) ?? 0) + 1); + throw opts.error(boundary); + }, + fired(boundary) { + return counts.get(boundary) ?? 0; + }, + }; +} diff --git a/internal-packages/testcontainers/src/heteroRunOpsWithRedis.test.ts b/internal-packages/testcontainers/src/heteroRunOpsWithRedis.test.ts new file mode 100644 index 00000000000..192fd877a17 --- /dev/null +++ b/internal-packages/testcontainers/src/heteroRunOpsWithRedis.test.ts @@ -0,0 +1,23 @@ +import { expect } from "vitest"; +import Redis from "ioredis"; +import { heteroRunOpsWithRedisTest } from "./index"; + +heteroRunOpsWithRedisTest( + "provides both Postgres clients and a live Redis", + async ({ prisma14, prisma17, redisOptions }) => { + const a = await prisma14.$queryRaw`SELECT 1 as ok`; + const b = await prisma17.$queryRaw`SELECT 1 as ok`; + expect(a).toEqual([{ ok: 1 }]); + expect(b).toEqual([{ ok: 1 }]); + + const redis = new Redis(redisOptions); + try { + expect(await redis.dbsize()).toBe(0); + await redis.set("k", "v"); + expect(await redis.get("k")).toBe("v"); + } finally { + await redis.quit(); + } + }, + 120_000 +); diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index 820ca827aec..e72a63766e9 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -431,14 +431,19 @@ type HeteroRunOpsPostgresTestContext = { // control-plane schema on PG14 (legacy), prisma17 is a RunOpsPrismaClient over the dedicated SUBSET // schema on a SEPARATE PG17 container. Lets a test prove the two sides carry different schemas // without disturbing the existing heteroPostgresTest (which keeps the full schema on both sides). -export const heteroRunOpsPostgresTest = test.extend({ - postgresContainer14: async ({}, use) => { +// The six hetero run-ops fixtures, shared by heteroRunOpsPostgresTest and heteroRunOpsWithRedisTest +// so the two cannot drift. +const heteroRunOpsFixtures = { + postgresContainer14: async ({}, use: Use) => { await use(await getWorkerPostgresContainer()); }, - postgresContainer17: async ({}, use) => { + postgresContainer17: async ({}, use: Use) => { await use(await getRunOpsWorkerPostgresContainer17()); }, - uri14: async ({ postgresContainer14 }, use) => { + uri14: async ( + { postgresContainer14 }: { postgresContainer14: StartedPostgreSqlContainer }, + use: Use + ) => { const baseUri = postgresContainer14.getConnectionUri(); const cloneDb = `heteroRunOps14_${pgCloneCounter++}`; await createDatabaseFromTemplate(baseUri, cloneDb); @@ -448,7 +453,10 @@ export const heteroRunOpsPostgresTest = test.extend { + uri17: async ( + { postgresContainer17 }: { postgresContainer17: StartedPostgreSqlContainer }, + use: Use + ) => { const baseUri = postgresContainer17.getConnectionUri(); const cloneDb = `heteroRunOps17_${pgCloneCounter++}`; await createDatabaseFromTemplate(baseUri, cloneDb); @@ -458,7 +466,7 @@ export const heteroRunOpsPostgresTest = test.extend { + prisma14: async ({ uri14 }: { uri14: string }, use: Use) => { const prisma = new PrismaClient({ datasources: { db: { url: uri14 } } }); try { await use(prisma); @@ -466,7 +474,7 @@ export const heteroRunOpsPostgresTest = test.extend { + prisma17: async ({ uri17 }: { uri17: string }, use: Use) => { const prisma = new RunOpsPrismaClient({ datasources: { db: { url: uri17 } } }); try { await use(prisma); @@ -474,6 +482,10 @@ export const heteroRunOpsPostgresTest = test.extend({ + ...heteroRunOpsFixtures, }); type ThreeDbRunOpsPostgresTestContext = { @@ -554,6 +566,93 @@ export const threeDbRunOpsPostgresTest = test.extend + test.extend({ + legacyUri: async ({}, use) => { + const container = await getWorkerPostgresContainer(); + const baseUri = container.getConnectionUri(); + const cloneDb = `nShardLegacy_${pgCloneCounter++}`; + await createDatabaseFromTemplate(baseUri, cloneDb); + try { + await use(postgresUriWithDatabase(baseUri, cloneDb)); + } finally { + await dropCloneDatabase(baseUri, cloneDb); + } + }, + newUri: async ({}, use) => { + const container = await getRunOpsWorkerPostgresContainer17(); + const baseUri = container.getConnectionUri(); + const cloneDb = `nShardNew_${pgCloneCounter++}`; + await createDatabaseFromTemplate(baseUri, cloneDb); + try { + await use(postgresUriWithDatabase(baseUri, cloneDb)); + } finally { + await dropCloneDatabase(baseUri, cloneDb); + } + }, + shardUris: async ({}, use) => { + const container = await getRunOpsWorkerPostgresContainer17(); + const baseUri = container.getConnectionUri(); + const clones: string[] = []; + try { + for (let i = 0; i < gen2ShardCount; i++) { + const cloneDb = `nShardGen2_${pgCloneCounter++}`; + await createDatabaseFromTemplate(baseUri, cloneDb); + clones.push(cloneDb); + } + await use(clones.map((db) => postgresUriWithDatabase(baseUri, db))); + } finally { + for (const db of clones) { + await dropCloneDatabase(baseUri, db); + } + } + }, + legacyPrisma: async ({ legacyUri }, use) => { + const prisma = new PrismaClient({ datasources: { db: { url: legacyUri } } }); + try { + await use(prisma); + } finally { + await prisma.$disconnect(); + } + }, + newPrisma: async ({ newUri }, use) => { + const prisma = new RunOpsPrismaClient({ datasources: { db: { url: newUri } } }); + try { + await use(prisma); + } finally { + await prisma.$disconnect(); + } + }, + shardPrismas: async ({ shardUris }, use) => { + const clients = shardUris.map( + (url) => new RunOpsPrismaClient({ datasources: { db: { url } } }) + ); + try { + await use(clients); + } finally { + for (const c of clients) { + await c.$disconnect(); + } + } + }, + }); + export const redisContainer = async ( { network, task }: { network: StartedNetwork } & TestContext, use: Use @@ -635,6 +734,22 @@ const flushRedis = async ( await use(); }; +type HeteroRunOpsWithRedisContext = HeteroRunOpsPostgresTestContext & { + redisContainer: StartedRedisContainer; + resetRedis: void; + redisOptions: RedisOptions; +}; + +// heteroRunOpsPostgresTest (PG14 + PG17, dedicated-schema run-ops) composed with the WORKER-SCOPED +// Redis container — boots once per worker, FLUSHALL between tests, matching containerTest. Not +// postgresAndRedisTest, which boots a container per test and times out under load. +export const heteroRunOpsWithRedisTest = test.extend({ + ...heteroRunOpsFixtures, + redisContainer: [bootWorkerRedis, { scope: "worker" }], + resetRedis: [flushRedis, { auto: true }], + redisOptions, +}); + type RedisTestContext = { redisContainer: StartedRedisContainer; resetRedis: void; @@ -980,3 +1095,6 @@ export const postgresAndMinioTest = withWarmup( await getWorkerPostgresContainer(); } ); + +export { slotOf, expectOneSlot } from "./clusterSlot"; +export { createFaultInjector, type FaultInjector } from "./faultInjection"; diff --git a/internal-packages/testcontainers/src/nShardFixture.test.ts b/internal-packages/testcontainers/src/nShardFixture.test.ts new file mode 100644 index 00000000000..1fd3449616f --- /dev/null +++ b/internal-packages/testcontainers/src/nShardFixture.test.ts @@ -0,0 +1,27 @@ +import { expect } from "vitest"; +import { makeNShardRunOpsPostgresTest } from "./index.js"; + +const nShardTest = makeNShardRunOpsPostgresTest(2); + +// Booting the PG14 + PG17 containers and cloning four databases on a cold runner far exceeds +// vitest's 5s default (this package sets no global testTimeout), so pass a generous per-test one. +nShardTest( + "builds 4 distinct databases (legacy + new + 2 gen-2 shards)", + async ({ legacyUri, newUri, shardUris }) => { + expect(shardUris).toHaveLength(2); + const all = [legacyUri, newUri, ...shardUris]; + expect(new Set(all).size).toBe(4); + }, + 120_000 +); + +nShardTest( + "each gen-2 clone carries the run-ops subset schema", + async ({ shardPrismas }) => { + expect(shardPrismas).toHaveLength(2); + for (const prisma of shardPrismas) { + await expect(prisma.taskRun.count()).resolves.toBe(0); + } + }, + 120_000 +); diff --git a/knip.json b/knip.json index 84456756ca1..c6e8aee8977 100644 --- a/knip.json +++ b/knip.json @@ -25,7 +25,8 @@ "vite/node-globals-shim.js", "app/v3/otlpTransformWorker.ts" ], - "ignoreDependencies": ["@sentry/cli", "assert", "util"] + "ignoreDependencies": ["@sentry/cli", "assert", "util"], + "ignore": ["app/v3/runOpsMigration/runOpsMintShard.server.ts"] }, "internal-packages/dashboard-agent": { "entry": ["trigger.config.ts", "src/investigation-sweep.ts", "src/maintenance.ts"], diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index fba2e52e1ee..a7a07cf40eb 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -23,6 +23,7 @@ import { DevDisconnectResponseBody, EnvironmentVariableResponseBody, FailDeploymentResponseBody, + GetDeploymentBuildEnvVarsResponseBody, GetDeploymentResponseBody, GetEnvironmentVariablesResponseBody, GetLatestDeploymentResponseBody, @@ -689,6 +690,20 @@ export class CliApiClient { ); } + async getDeploymentBuildEnvVars(deploymentId: string) { + if (!this.accessToken) { + throw new Error("getDeploymentBuildEnvVars: No access token"); + } + + return wrapZodFetch( + GetDeploymentBuildEnvVarsResponseBody, + `${this.apiURL}/api/v1/deployments/${deploymentId}/build-env-vars`, + { + headers: this.getHeaders(), + } + ); + } + async getCliPlatformNotification(projectRef?: string, signal?: AbortSignal) { if (!this.accessToken) { return { success: true as const, data: { notification: null } }; @@ -1001,6 +1016,7 @@ export class CliApiClient { Authorization: `Bearer ${this.accessToken}`, "Content-Type": "application/json", "x-trigger-source": this.source, + "x-trigger-cli-version": VERSION, ...this.getBranchHeader(), }; } diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index 7afa06982ae..74052486c13 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -12,7 +12,7 @@ import type { DeploymentFinalizedEvent, DeploymentTriggeredVia, } from "@trigger.dev/core/v3/schemas"; -import { DeploymentEventFromString } from "@trigger.dev/core/v3/schemas"; +import { BuildManifest, DeploymentEventFromString } from "@trigger.dev/core/v3/schemas"; import type { Command } from "commander"; import { Option as CommandOption } from "commander"; import { join, relative, resolve } from "node:path"; @@ -24,6 +24,7 @@ import type { CliApiClient } from "../apiClient.js"; import { buildWorker } from "../build/buildWorker.js"; import { resolveAlwaysExternal } from "../build/externals.js"; import { createContextArchive, getArchiveSize } from "../deploy/archiveContext.js"; +import { createBundleArchive } from "../deploy/bundleArchive.js"; import { S2 } from "@s2-dev/streamstore"; import { mkdir, readFile, unlink } from "node:fs/promises"; import { @@ -90,6 +91,8 @@ const DeployCommandOptions = CommonCommandOptions.extend({ push: z.boolean().optional(), builder: z.string().default("trigger"), nativeBuildServer: z.boolean().default(false), + localBundle: z.boolean().default(false), + fromBundle: z.string().optional(), detach: z.boolean().default(false), plain: z.boolean().default(false), compression: z.enum(["zstd", "gzip"]).default("zstd"), @@ -248,6 +251,23 @@ export function configureDeployCommand(program: Command) { "Use the native build server for building the image" ) ) + .addOption( + new CommandOption( + "--local-bundle", + "Experimental: install and bundle locally, upload only the build output, and build the image remotely. Implies using the native build server." + ) + .implies({ nativeBuildServer: true }) + .conflicts(["localBuild", "forceLocalBuild"]) + ) + .addOption( + new CommandOption( + "--from-bundle ", + "Internal: build the image from a pre-built bundle directory. Implies a local build." + ) + .implies({ localBuild: true }) + .conflicts(["nativeBuildServer", "localBundle"]) + .hideHelp() + ) .addOption( new CommandOption( "--detach", @@ -335,6 +355,18 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { logger.debug("Using project ref from env", { ref: envVars.TRIGGER_PROJECT_REF }); } + if (options.fromBundle) { + await handleFromBundleDeploy({ + bundleDir: options.fromBundle, + options, + dashboardUrl: authorization.dashboardUrl, + auth: authorization.auth, + existingDeploymentId: envVars.TRIGGER_EXISTING_DEPLOYMENT_ID, + projectRefOverride: options.projectRef ?? envVars.TRIGGER_PROJECT_REF, + }); + return; + } + let resolvedConfig = await loadConfig({ cwd: projectPath, overrides: { project: options.projectRef ?? envVars.TRIGGER_PROJECT_REF }, @@ -413,6 +445,19 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { resolvedConfig.runtime = projectClient.defaultRuntime; } + if (options.localBundle) { + await handleLocalBundleDeploy({ + apiClient: projectClient.client, + config: resolvedConfig, + dashboardUrl: authorization.dashboardUrl, + options, + userId: userIdForDeploy(authorization), + gitMeta, + branch, + }); + return; + } + if (options.nativeBuildServer) { await handleNativeBuildServerDeploy({ apiClient: projectClient.client, @@ -1635,3 +1680,993 @@ export function verifyDirectory(dir: string, projectPath: string) { throw new Error(`Directory "${dir}" not found at ${projectPath}`); } } + +// --local-bundle: install + bundling happen locally, only the build output is +// uploaded, and the build server runs just the container build from it. +async function handleLocalBundleDeploy({ + apiClient, + options, + config, + dashboardUrl, + userId, + gitMeta, + branch, +}: { + apiClient: CliApiClient; + config: Awaited>; + dashboardUrl: string; + options: DeployCommandOptions; + userId?: string; + gitMeta?: GitMeta; + branch?: string; +}) { + const tmpDir = join(config.workingDir, ".trigger", "tmp"); + await mkdir(tmpDir, { recursive: true }); + + const archivePath = join(tmpDir, `deploy-${Date.now()}.tar.gz`); + + const ignoredBuildFlags = [ + options.compression !== "zstd" && "--compression", + options.cacheCompression !== "zstd" && "--cache-compression", + options.compressionLevel !== undefined && "--compression-level", + !options.forceCompression && "--no-force-compression", + !options.cache && "--no-cache", + options.builder !== "trigger" && "--builder", + options.network !== undefined && "--network", + options.push !== undefined && "--push/--no-push", + options.load !== undefined && "--load/--no-load", + ].filter((flag): flag is string => Boolean(flag)); + + if (ignoredBuildFlags.length > 0) { + log.warn( + `The following flags are ignored with --local-bundle (the image is built remotely): ${ignoredBuildFlags.join(", ")}` + ); + } + + const serverEnvVars = await apiClient.getEnvironmentVariables(config.project); + loadDotEnvVars(config.workingDir, options.envFile); + + // Keep the bundle dir around on dry runs so the printed path is inspectable + const destination = getTmpDir(config.workingDir, "build", options.dryRun); + const forcedExternals = await resolveAlwaysExternal(apiClient); + + const $buildSpinner = spinner({ plain: options.plain }); + + const [buildError, buildManifest] = await tryCatch( + buildWorker({ + target: "deploy", + environment: options.env, + branch, + destination: destination.path, + resolvedConfig: config, + rewritePaths: true, + envVars: serverEnvVars.success ? serverEnvVars.data.variables : {}, + forcedExternals, + plain: options.plain, + listener: { + onBundleStart() { + $buildSpinner.start("Building trigger code"); + }, + onBundleComplete(result) { + $buildSpinner.stop("Successfully built code"); + logger.debug("Bundle result", result); + }, + }, + }) + ); + + if (buildError) { + $buildSpinner.stop("Failed to build code"); + throw buildError; + } + + const bundleManifest = buildManifest; + const bundleOutputPath = destination.path; + + // Extensions can set undefined values at runtime despite the manifest type + const bundleBuildEnvVars = Object.fromEntries( + Object.entries(buildManifest.build.env ?? {}).filter( + (entry): entry is [string, string] => typeof entry[1] === "string" + ) + ); + + if (options.dryRun) { + logger.info(`Dry run complete. View the built bundle at ${destination.path}`); + return; + } + + // Sync BEFORE init: init enqueues the build synchronously, so a post-init sync races a fast build + const childVars = buildManifest.deploy.sync?.env ?? {}; + const parentVars = buildManifest.deploy.sync?.parentEnv ?? {}; + const secretChildVars = buildManifest.deploy.sync?.secretEnv ?? {}; + const secretParentVars = buildManifest.deploy.sync?.secretParentEnv ?? {}; + + const hasVarsToSync = + Object.keys(childVars).length > 0 || + Object.keys(secretChildVars).length > 0 || + // Only sync parent variables if this is a branch environment + (branch && (Object.keys(parentVars).length > 0 || Object.keys(secretParentVars).length > 0)); + + if (!options.skipSyncEnvVars) { + if (hasVarsToSync) { + const uploadResult = await syncEnvVarsWithServer( + apiClient, + config.project, + options.env, + childVars, + parentVars, + secretChildVars, + secretParentVars + ); + + if (!uploadResult.success) { + throw new Error(`Failed to sync env vars with the server: ${uploadResult.error}`); + } + + logger.debug("Synced env vars with the server"); + } + } else if (hasVarsToSync) { + logger.log( + "Skipping syncing env vars. The environment variables in your project have changed, but the --skip-sync-env-vars flag was provided." + ); + } + + const $deploymentSpinner = spinner(); + $deploymentSpinner.start("Preparing deployment files"); + + await createBundleArchive(bundleOutputPath, archivePath); + + const archiveSize = await getArchiveSize(archivePath); + const sizeMB = (archiveSize / 1024 / 1024).toFixed(2); + $deploymentSpinner.message(`Deployment files ready (${sizeMB} MB)`); + + const artifactResult = await apiClient.createArtifact({ + type: "deployment_bundle", + contentType: "application/gzip", + contentLength: archiveSize, + }); + + if (!artifactResult.success) { + $deploymentSpinner.stop("Failed creating deployment artifact"); + log.error(chalk.bold(chalkError(artifactResult.error))); + throw new OutroCommandError(`Deployment failed`); + } + + const { artifactKey, uploadUrl, uploadFields } = artifactResult.data; + + logger.debug("Artifact created", { artifactKey }); + + // Defense in depth: current older servers already reject the deployment_bundle + // type at createArtifact; this catches a server that accepts it but returns a + // non-bundle key, which would make the remote build treat the bundle as source. + if (!artifactKey.startsWith("bundles/")) { + $deploymentSpinner.stop("Failed creating deployment artifact"); + log.error( + chalk.bold( + chalkError( + "This server does not support --local-bundle deploys yet. Deploy without --local-bundle instead." + ) + ) + ); + throw new OutroCommandError(`Deployment failed`); + } + + $deploymentSpinner.message("Uploading deployment files"); + + const [readError, fileBuffer] = await tryCatch(readFile(archivePath)); + + if (readError) { + $deploymentSpinner.stop("Failed reading deployment archive"); + log.error(chalk.bold(chalkError(readError.message))); + throw new OutroCommandError(`Deployment failed`); + } + + const formData = new FormData(); + + for (const [key, value] of Object.entries(uploadFields)) { + formData.append(key, value); + } + + const blob = new Blob([new Uint8Array(fileBuffer)], { type: "application/gzip" }); + formData.append("file", blob, "deployment.tar.gz"); + + const [uploadError, uploadResponse] = await tryCatch( + fetch(uploadUrl, { + method: "POST", + body: formData, + }) + ); + + if (uploadError || !uploadResponse?.ok) { + $deploymentSpinner.stop("Failed to upload deployment files"); + log.error( + chalk.bold( + chalkError( + `${uploadError?.message} (${uploadResponse?.statusText} ${uploadResponse?.status})` + ) + ) + ); + throw new OutroCommandError(`Deployment failed`); + } + + const [unlinkError] = await tryCatch(unlink(archivePath)); + if (unlinkError) { + logger.debug("Failed to delete deployment artifact file", { archivePath, error: unlinkError }); + } + + $deploymentSpinner.message("Deployment files uploaded"); + + const configFilePath = + config.configFile !== undefined + ? relative(config.workspaceDir, config.configFile).replace(/\\/g, "/") + : undefined; + + const initializeDeploymentResult = await apiClient.initializeDeployment({ + contentHash: bundleManifest.contentHash, + userId, + gitMeta, + type: config.features.run_engine_v2 ? "MANAGED" : "V1", + // config.runtime (not the manifest runtime) to match classic native deploys + runtime: config.runtime, + isNativeBuild: true, + artifactKey, + skipPromotion: options.skipPromotion, + configFilePath, + triggeredVia: getTriggeredVia(), + externalId: options.externalId, + force: options.force, + fromBundle: true, + buildEnvVars: Object.keys(bundleBuildEnvVars).length > 0 ? bundleBuildEnvVars : undefined, + }); + + if (!initializeDeploymentResult.success) { + $deploymentSpinner.stop("Failed to initialize deployment"); + log.error(chalk.bold(chalkError(initializeDeploymentResult.error))); + throw new OutroCommandError(`Deployment failed`); + } + + const deployment = initializeDeploymentResult.data; + + const rawDeploymentLink = `${dashboardUrl}/projects/v3/${config.project}/deployments/${deployment.shortCode}`; + const rawTestLink = `${dashboardUrl}/projects/v3/${config.project}/test?environment=${ + options.env === "prod" ? "prod" : "stg" + }`; + + if (deployment.outcome === "existing") { + $deploymentSpinner.stop(`Version ${deployment.version} was already deployed`); + + setDeploymentGithubActionsOutput({ + version: deployment.version, + shortCode: deployment.shortCode, + rawDeploymentLink, + rawTestLink, + needsPromotion: !deployment.isPromoted, + }); + + warnAboutSkippedBuild(options.externalId, deployment.isPromoted); + + outro( + `Version ${deployment.version} was already deployed for --external-id ${options.externalId} — nothing to build ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : rawDeploymentLink + }` + ); + + return; + } + + const exposedDeploymentLink = isLinksSupported + ? cliLink(chalk.bold(rawDeploymentLink), rawDeploymentLink) + : chalk.bold(rawDeploymentLink); + $deploymentSpinner.stop("Deployment initialized"); + log.info(`View deployment: ${exposedDeploymentLink}`); + + warnAboutCanceledDeployments(deployment.canceledDeployments, options.externalId); + + setDeploymentGithubActionsOutput({ + version: deployment.version, + shortCode: deployment.shortCode, + rawDeploymentLink, + rawTestLink, + needsPromotion: options.skipPromotion, + }); + + if (options.detach) { + outro(`Version ${deployment.version} is being deployed`); + return; + } + + const { eventStream } = deployment; + + if (!eventStream) { + log.warn(`Failed streaming build logs, open the deployment in the dashboard to view the logs`); + + outro(`Version ${deployment.version} is being deployed`); + + return process.exit(0); + } + + const $queuedSpinner = spinner(); + $queuedSpinner.start("Build queued"); + + const abortController = new AbortController(); + + const s2 = new S2({ accessToken: eventStream.s2.accessToken }); + const basin = s2.basin(eventStream.s2.basin); + const stream = basin.stream(eventStream.s2.stream); + + const [readSessionError, readSession] = await tryCatch( + stream.readSession( + { + start: { from: { seqNum: 0 }, clamp: true }, + stop: { waitSecs: 60 * 20 }, // 20 minutes + }, + { signal: abortController.signal } + ) + ); + + if (readSessionError) { + $queuedSpinner.stop("Failed to query build progress"); + log.warn(`Failed streaming build logs, open the deployment in the dashboard to view the logs`); + + outro( + `Version ${deployment.version} is being deployed ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + + return process.exit(0); + } + + let finalDeploymentEvent: DeploymentFinalizedEvent["data"] | undefined; + let queuedSpinnerStopped = false; + + for await (const record of readSession) { + const decoded = record.body; + const result = DeploymentEventFromString.safeParse(decoded); + if (!result.success) { + logger.debug("Failed to parse deployment event, skipping", { + error: result.error, + record: decoded, + }); + continue; + } + + const event = result.data; + + switch (event.type) { + case "log": { + if (record.seqNum === 0) { + $queuedSpinner.stop("Build started"); + console.log("│"); + queuedSpinnerStopped = true; + } + + const formattedTimestamp = chalkGrey( + new Date(record.timestamp).toLocaleTimeString("en-US", { + hour12: false, + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + fractionalSecondDigits: 3, + }) + ); + + const { level, message } = event.data; + const formattedMessage = + level === "error" + ? chalk.bold(chalkError(message)) + : level === "warn" + ? chalkWarning(message) + : level === "debug" + ? chalkGrey(message) + : message; + + // We use console.log here instead of clack's logger as the current version does not support changing the line spacing. + // And the logs look verbose with the default spacing. + // We cannot upgrade because the newer versions introduced some weird issues with the spinner. + // Ideally, we'd use clack's `taskLog` to only show the recent n lines of logs as they are streamed, but that also seems brittle + // and has some issues with cursor movements/clearing lines that it shouldn't clear. + // We can revisit this on future versions of `@clack/prompts`. + console.log(`│ ${formattedTimestamp} ${formattedMessage}`); + break; + } + case "finalized": { + finalDeploymentEvent = event.data; + abortController.abort(); // stop the stream + break; + } + default: { + event satisfies never; + logger.debug("Unknown deployment event, skipping", { event }); + continue; + } + } + } + + if (!queuedSpinnerStopped && !finalDeploymentEvent) { + // unlikely that it happens in practice, only in rare corner cases + // the timeout would kick in earlier if the build server fails to dequeue the build + + $queuedSpinner.stop("Log stream stopped"); + + log.error("Failed dequeueing build, please try again shortly"); + + throw new OutroCommandError( + `Version ${deployment.version} ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + + if (!finalDeploymentEvent) { + log.error( + "Stopped receiving updates from the build server, please check the deployment status in the dashboard" + ); + + if (!isLinksSupported) { + log.info(`View deployment: ${rawDeploymentLink}`); + } + + throw new OutroCommandError( + `Version ${deployment.version} ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + + switch (finalDeploymentEvent.result) { + case "succeeded": { + queuedSpinnerStopped + ? log.success("Deployment completed successfully") + : $queuedSpinner.stop("Deployment completed successfully"); + + if (finalDeploymentEvent.message) { + log.success(finalDeploymentEvent.message); + } + + if (options.skipPromotion) { + log.info( + `This deployment was not automatically promoted. You can promote in the dashboard or via the promote command, e.g, \`npx trigger.dev promote ${deployment.version}\`.` + ); + } + + if (!isLinksSupported) { + log.info(`Test tasks: ${rawTestLink}`); + } + + outro( + `Version ${deployment.version} was deployed ${ + isLinksSupported + ? `| ${cliLink("Test tasks", rawTestLink)} | ${cliLink( + "View deployment", + rawDeploymentLink + )}` + : "" + }` + ); + return process.exit(0); + } + case "failed": { + if (!queuedSpinnerStopped) { + $queuedSpinner.stop("Deployment failed"); + } + + log.error( + chalk.bold( + chalkError( + "Deployment failed" + + (finalDeploymentEvent.message ? `: ${finalDeploymentEvent.message}` : "") + ) + ) + ); + + throw new OutroCommandError( + `Version ${deployment.version} deployment failed ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + case "timed_out": { + if (!queuedSpinnerStopped) { + $queuedSpinner.stop("Deployment timed out"); + } + + log.error( + chalk.bold( + chalkError( + "Deployment timed out" + + (finalDeploymentEvent.message ? `: ${finalDeploymentEvent.message}` : "") + ) + ) + ); + + throw new OutroCommandError( + `Version ${deployment.version} deployment timed out ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + case "canceled": { + if (!queuedSpinnerStopped) { + $queuedSpinner.stop("Deployment was canceled"); + } + + log.error( + chalk.bold( + chalkError( + "Deployment was canceled" + + (finalDeploymentEvent.message ? `: ${finalDeploymentEvent.message}` : "") + ) + ) + ); + + throw new OutroCommandError( + `Version ${deployment.version} deployment canceled ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + default: { + // This case is only relevant in case we extend the enum in the future. + // New enum values will not be treated as errors in older cli versions. + queuedSpinnerStopped + ? log.success("Log stream finished") + : $queuedSpinner.stop("Log stream finished"); + if (finalDeploymentEvent.message) { + log.message(finalDeploymentEvent.message); + } + + if (!isLinksSupported) { + log.info(`Test tasks: ${rawTestLink}`); + } + + outro( + `Version ${deployment.version} ${ + isLinksSupported + ? `| ${cliLink("Test tasks", rawTestLink)} | ${cliLink( + "View deployment", + rawDeploymentLink + )}` + : "" + }` + ); + return process.exit(0); + } + } +} + +// Builds the image locally from the bundle and finalizes the deployment. +async function buildAndFinalizeFromBundle({ + apiClient, + projectId, + projectRef, + deployment, + options, + dashboardUrl, + authAccessToken, + compilationPath, + buildEnvVars, + branch, + isLocalBuild, +}: { + apiClient: CliApiClient; + projectId: string; + projectRef: string; + deployment: Deployment; + options: DeployCommandOptions; + dashboardUrl: string; + authAccessToken: string; + compilationPath: string; + buildEnvVars: Record | undefined; + branch: string | undefined; + isLocalBuild: boolean; +}) { + const authenticateToTriggerRegistry = options.localBuild; + const skipServerSideRegistryPush = options.localBuild; + + const version = deployment.version; + + const { rawDeploymentLink, rawTestLink } = buildDeploymentLinks({ + dashboardUrl, + projectRef, + env: options.env, + shortCode: deployment.shortCode, + }); + + const deploymentLink = cliLink("View deployment", rawDeploymentLink); + const testLink = cliLink("Test tasks", rawTestLink); + + const $spinner = spinner({ plain: options.plain }); + + const buildSuffix = + isLocalBuild && process.env.TRIGGER_LOCAL_BUILD_LABEL_DISABLED !== "1" ? " (local)" : ""; + const deploySuffix = + isLocalBuild && process.env.TRIGGER_LOCAL_BUILD_LABEL_DISABLED !== "1" ? " (local build)" : ""; + + if (options.plain) { + $spinner.start(`Building version ${version}${buildSuffix}`); + } else if (isCI) { + log.step(`Building version ${version}\n`); + } else { + if (isLinksSupported) { + $spinner.start(`Building version ${version}${buildSuffix} ${deploymentLink}`); + } else { + $spinner.start(`Building version ${version}${buildSuffix}`); + } + } + + const buildResult = await buildImage({ + isLocalBuild, + useRegistryCache: options.useRegistryCache, + noCache: !options.cache, + deploymentId: deployment.id, + deploymentVersion: deployment.version, + imageTag: deployment.imageTag, + imagePlatform: deployment.imagePlatform, + load: options.load, + contentHash: deployment.contentHash, + externalBuildId: deployment.externalBuildData?.buildId, + externalBuildToken: deployment.externalBuildData?.buildToken, + externalBuildProjectId: deployment.externalBuildData?.projectId, + projectId, + projectRef, + apiUrl: apiClient.apiURL, + apiKey: apiClient.accessToken!, + apiClient, + branchName: branch, + authAccessToken, + compilationPath, + buildEnvVars, + compression: options.compression, + cacheCompression: options.cacheCompression, + compressionLevel: options.compressionLevel, + forceCompression: options.forceCompression, + onLog: (logMessage) => { + if (options.plain || isCI) { + console.log(logMessage); + return; + } + + if (isLinksSupported) { + $spinner.message( + `Building version ${version}${buildSuffix} ${deploymentLink}: ${logMessage}` + ); + } else { + $spinner.message(`Building version ${version}${buildSuffix}: ${logMessage}`); + } + }, + // Local build options + network: options.network, + builder: options.builder, + push: options.push, + authenticateToRegistry: authenticateToTriggerRegistry, + }); + + logger.debug("Build result", buildResult); + + const warnings = checkLogsForWarnings(buildResult.logs); + + const canShowLocalBuildHint = + !isLocalBuild && process.env.TRIGGER_LOCAL_BUILD_HINT_DISABLED !== "1"; + const buildFailed = !warnings.ok || !buildResult.ok; + + if (buildFailed && canShowLocalBuildHint) { + const providerStatus = await apiClient.getRemoteBuildProviderStatus(); + + if (providerStatus.success && providerStatus.data.status === "degraded") { + prettyWarning(providerStatus.data.message + "\n"); + } + } + + if (!warnings.ok) { + await failDeploy( + apiClient, + deployment, + { name: "BuildError", message: warnings.summary }, + buildResult.logs, + $spinner, + warnings.warnings, + warnings.errors + ); + + throw new SkipLoggingError("Failed to build image"); + } + + if (!buildResult.ok) { + await failDeploy( + apiClient, + deployment, + { name: "BuildError", message: buildResult.error }, + buildResult.logs, + $spinner, + warnings.warnings + ); + + throw new SkipLoggingError("Failed to build image"); + } + + const getDeploymentResponse = await apiClient.getDeployment(deployment.id); + + if (!getDeploymentResponse.success) { + await failDeploy( + apiClient, + deployment, + { name: "DeploymentError", message: getDeploymentResponse.error }, + buildResult.logs, + $spinner + ); + + throw new SkipLoggingError(getDeploymentResponse.error); + } + + const deploymentWithWorker = getDeploymentResponse.data; + + if (!deploymentWithWorker.worker) { + const errorData = deploymentWithWorker.errorData + ? prepareDeploymentError(deploymentWithWorker.errorData) + : undefined; + + await failDeploy( + apiClient, + deployment, + { + name: "DeploymentError", + message: errorData?.message ?? "Failed to get deployment with worker", + }, + buildResult.logs, + $spinner + ); + + throw new SkipLoggingError(errorData?.message ?? "Failed to get deployment with worker"); + } + + if (options.plain) { + $spinner.message(`Deploying version ${version}${deploySuffix}`); + } else if (isCI) { + log.step(`Deploying version ${version}${deploySuffix}\n`); + } else { + if (isLinksSupported) { + $spinner.message(`Deploying version ${version}${deploySuffix} ${deploymentLink}`); + } else { + $spinner.message(`Deploying version ${version}${deploySuffix}`); + } + } + + const finalizeResponse = await apiClient.finalizeDeployment( + deployment.id, + { + imageDigest: buildResult.digest, + skipPromotion: options.skipPromotion, + skipPushToRegistry: skipServerSideRegistryPush, + }, + (logMessage) => { + if (options.plain || isCI) { + console.log(logMessage); + return; + } + + if (isLinksSupported) { + $spinner.message( + `Deploying version ${version}${deploySuffix} ${deploymentLink}: ${logMessage}` + ); + } else { + $spinner.message(`Deploying version ${version}${deploySuffix}: ${logMessage}`); + } + } + ); + + if (!finalizeResponse.success) { + await failDeploy( + apiClient, + deployment, + { name: "FinalizeError", message: finalizeResponse.error }, + buildResult.logs, + $spinner + ); + + throw new SkipLoggingError("Failed to finalize deployment"); + } + + if (options.plain) { + console.log(`Successfully deployed version ${version}${deploySuffix}`); + } else if (isCI) { + log.step(`Successfully deployed version ${version}${deploySuffix}`); + } else { + $spinner.stop(`Successfully deployed version ${version}${deploySuffix}`); + } + + const taskCount = deploymentWithWorker.worker?.tasks.length ?? 0; + + if (options.plain) { + console.log( + `Version ${version} deployed with ${taskCount} detected task${taskCount === 1 ? "" : "s"}` + ); + + if (process.env.TRIGGER_DEPLOYMENT_LINK_OUTPUT_DISABLED !== "1") { + console.log(`Deployment: ${rawDeploymentLink}`); + console.log(`Test: ${rawTestLink}`); + } + } else { + outro( + `Version ${version} deployed with ${taskCount} detected task${taskCount === 1 ? "" : "s"} ${ + isLinksSupported ? `| ${deploymentLink} | ${testLink}` : "" + }` + ); + + if (!isLinksSupported) { + console.log("View deployment"); + console.log(rawDeploymentLink); + console.log(); // new line + console.log("Test tasks"); + console.log(rawTestLink); + } + } + + if (options.saveLogs) { + const logPath = await saveLogs(deployment.shortCode, buildResult.logs); + console.log(`Full build logs have been saved to ${logPath}`); + } + + setDeploymentGithubActionsOutput({ + version, + shortCode: deployment.shortCode, + rawDeploymentLink, + rawTestLink, + needsPromotion: options.skipPromotion, + }); +} + +// Runs only the container build from a pre-built bundle dir, skipping config loading +// entirely. Attach mode is the supported flow (build server); fresh-init is for testing. +async function handleFromBundleDeploy({ + bundleDir, + options, + dashboardUrl, + auth, + existingDeploymentId, + projectRefOverride, +}: { + bundleDir: string; + options: DeployCommandOptions; + dashboardUrl: string; + auth: { accessToken: string; apiUrl: string }; + existingDeploymentId?: string; + projectRefOverride?: string; +}) { + const bundlePath = resolve(process.cwd(), bundleDir); + + if (!isDirectory(bundlePath)) { + throw new Error(`Bundle directory not found at ${bundlePath}`); + } + + const [manifestReadError, manifestRaw] = await tryCatch( + readFile(join(bundlePath, "build.json"), "utf-8") + ); + + if (manifestReadError) { + throw new Error( + `Failed to read build.json in the bundle directory: ${manifestReadError.message}` + ); + } + + let manifestJson: unknown; + try { + manifestJson = JSON.parse(manifestRaw); + } catch { + throw new Error(`Invalid build.json in the bundle directory: not valid JSON`); + } + + const manifestResult = BuildManifest.safeParse(manifestJson); + + if (!manifestResult.success) { + throw new Error(`Invalid build.json in the bundle directory: ${manifestResult.error.message}`); + } + + const bundleManifest = manifestResult.data; + + // --dry-run must never touch the server + if (options.dryRun) { + logger.info(`Dry run complete. Validated bundle at ${bundlePath}`); + return; + } + + const projectRef = projectRefOverride ?? bundleManifest.config.project; + + const branch = options.env === "preview" ? getBranch({ specified: options.branch }) : undefined; + + if (options.env === "preview" && !branch) { + throw new Error( + "Preview deploys from a bundle require an explicit branch. Pass --branch ." + ); + } + + // In attach mode the branch env already exists + if (options.env === "preview" && branch && !existingDeploymentId) { + await upsertBranch({ + accessToken: auth.accessToken, + apiUrl: auth.apiUrl, + projectRef, + branch, + gitMeta: undefined, + }); + } + + const projectClient = await getProjectClient({ + accessToken: auth.accessToken, + apiUrl: auth.apiUrl, + projectRef, + env: options.env, + branch, + profile: options.profile, + }); + + if (!projectClient) { + throw new Error("Failed to get project client"); + } + + // In attach mode the build-arg values are stored encrypted on the deployment + let buildEnvVars: Record | undefined; + + if (existingDeploymentId) { + const buildEnvVarsResult = + await projectClient.client.getDeploymentBuildEnvVars(existingDeploymentId); + + if (!buildEnvVarsResult.success) { + throw new Error( + `Failed to fetch the build environment variables for deployment ${existingDeploymentId}: ${buildEnvVarsResult.error}` + ); + } + + buildEnvVars = buildEnvVarsResult.data.variables; + } else if (bundleManifest.build.env && Object.keys(bundleManifest.build.env).length > 0) { + // Extensions can set undefined values at runtime despite the manifest type + buildEnvVars = Object.fromEntries( + Object.entries(bundleManifest.build.env).filter( + (entry): entry is [string, string] => typeof entry[1] === "string" + ) + ); + } + + if (!existingDeploymentId) { + logger.warn( + "No existing deployment to attach to — initializing a fresh local-build deployment from the bundle. This path is intended for testing." + ); + } + + const deployment = await initializeOrAttachDeployment( + projectClient.client, + { + contentHash: bundleManifest.contentHash, + type: "MANAGED", + runtime: bundleManifest.runtime, + isLocalBuild: true, + isNativeBuild: false, + triggeredVia: getTriggeredVia(), + }, + existingDeploymentId + ); + + // Fail fast if we know local builds will fail + const buildxResult = await x("docker", ["buildx", "version"]); + + if (buildxResult.exitCode !== 0) { + logger.debug(`"docker buildx version" failed (${buildxResult.exitCode}):`, buildxResult); + throw new Error( + "Failed to find docker buildx. Please install it: https://github.com/docker/buildx#installing." + ); + } + + await buildAndFinalizeFromBundle({ + apiClient: projectClient.client, + projectId: projectClient.id, + projectRef, + deployment, + options, + dashboardUrl, + authAccessToken: auth.accessToken, + compilationPath: bundlePath, + buildEnvVars, + branch, + isLocalBuild: true, + }); +} diff --git a/packages/cli-v3/src/deploy/bundleArchive.test.ts b/packages/cli-v3/src/deploy/bundleArchive.test.ts new file mode 100644 index 00000000000..efd79b5aefd --- /dev/null +++ b/packages/cli-v3/src/deploy/bundleArchive.test.ts @@ -0,0 +1,106 @@ +import { mkdtemp, mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as tar from "tar"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { createBundleArchive } from "./bundleArchive.js"; + +describe("createBundleArchive", () => { + let bundleDir: string; + let outDir: string; + + beforeEach(async () => { + bundleDir = await mkdtemp(join(tmpdir(), "bundle-src-")); + outDir = await mkdtemp(join(tmpdir(), "bundle-out-")); + }); + + afterEach(async () => { + await rm(bundleDir, { recursive: true, force: true }); + await rm(outDir, { recursive: true, force: true }); + }); + + it("archives bundle contents at the root, including dotfiles and nested dirs", async () => { + await writeFile(join(bundleDir, "build.json"), JSON.stringify({ contentHash: "abc" })); + await writeFile(join(bundleDir, "Containerfile"), "FROM scratch"); + await writeFile(join(bundleDir, "package.json"), "{}"); + await writeFile(join(bundleDir, "index.mjs"), "export {}"); + await writeFile(join(bundleDir, ".dockerignore"), "*.log\n"); + await mkdir(join(bundleDir, ".trigger", "skills", "my-skill"), { recursive: true }); + await writeFile(join(bundleDir, ".trigger", "skills", "my-skill", "SKILL.md"), "# skill"); + + const archivePath = join(outDir, "bundle.tar.gz"); + await createBundleArchive(bundleDir, archivePath); + + const extractDir = join(outDir, "extracted"); + await mkdir(extractDir); + await tar.extract({ file: archivePath, cwd: extractDir }); + + const rootEntries = (await readdir(extractDir)).sort(); + expect(rootEntries).toEqual( + [ + ".dockerignore", + ".trigger", + "Containerfile", + "build.json", + "index.mjs", + "package.json", + ].sort() + ); + + const skill = await readFile( + join(extractDir, ".trigger", "skills", "my-skill", "SKILL.md"), + "utf-8" + ); + expect(skill).toBe("# skill"); + }); + + it("excludes only .DS_Store — node_modules paths must survive", async () => { + await writeFile(join(bundleDir, "build.json"), "{}"); + await writeFile(join(bundleDir, ".DS_Store"), "junk"); + // Under npx the controller entry points live beneath a node_modules segment + const controllerDir = join( + bundleDir, + ".npm", + "_npx", + "abc123", + "node_modules", + "trigger.dev", + "dist" + ); + await mkdir(controllerDir, { recursive: true }); + await writeFile(join(controllerDir, "managed-index-controller.mjs"), "x"); + await mkdir(join(bundleDir, "dist"), { recursive: true }); + await writeFile(join(bundleDir, "dist", "chunk.mjs"), "x"); + + const archivePath = join(outDir, "bundle.tar.gz"); + await createBundleArchive(bundleDir, archivePath); + + const extractDir = join(outDir, "extracted"); + await mkdir(extractDir); + await tar.extract({ file: archivePath, cwd: extractDir }); + + const rootEntries = (await readdir(extractDir)).sort(); + expect(rootEntries).toEqual(["build.json", "dist", ".npm"].sort()); + + const controller = await readFile( + join( + extractDir, + ".npm", + "_npx", + "abc123", + "node_modules", + "trigger.dev", + "dist", + "managed-index-controller.mjs" + ), + "utf-8" + ); + expect(controller).toBe("x"); + }); + + it("throws when the bundle dir is empty", async () => { + await expect(createBundleArchive(bundleDir, join(outDir, "bundle.tar.gz"))).rejects.toThrow( + /No files found/ + ); + }); +}); diff --git a/packages/cli-v3/src/deploy/bundleArchive.ts b/packages/cli-v3/src/deploy/bundleArchive.ts new file mode 100644 index 00000000000..f53c820df89 --- /dev/null +++ b/packages/cli-v3/src/deploy/bundleArchive.ts @@ -0,0 +1,40 @@ +import { glob } from "tinyglobby"; +import * as tar from "tar"; +import { logger } from "../utilities/logger.js"; + +// The bundle dir is generated build output, so the usual source ignores (dist, +// node_modules, ...) would strip load-bearing files: under npx the controller +// entry points live beneath a node_modules path segment. +const BUNDLE_IGNORES = ["**/.DS_Store"]; + +// Bundle contents land at the archive root; the build server extracts without stripping +export async function createBundleArchive(bundleDir: string, outputPath: string) { + logger.debug("Creating bundle archive", { bundleDir, outputPath }); + + const files = await glob(["**/*"], { + cwd: bundleDir, + ignore: BUNDLE_IGNORES, + dot: true, // .trigger/skills and .dockerignore must be included + absolute: false, + onlyFiles: true, + followSymbolicLinks: false, + }); + + if (files.length === 0) { + throw new Error("No files found in the bundle output. This is likely a bug."); + } + + await tar.create( + { + gzip: true, + file: outputPath, + cwd: bundleDir, + portable: true, + preservePaths: false, + mtime: new Date(0), + }, + files + ); + + logger.debug("Bundle archive created", { outputPath, fileCount: files.length }); +} diff --git a/packages/core/src/v3/isomorphic/friendlyId.test.ts b/packages/core/src/v3/isomorphic/friendlyId.test.ts index 2e3ba4d83a5..8d818ae60bc 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.test.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.test.ts @@ -11,13 +11,21 @@ import { RUN_OPS_ID_VERSION, RUN_OPS_ID_VERSION_2, RUN_OPS_ID_VERSION_INDEX, + WAITPOINT_ID_TYPE_INDEX, + WAITPOINT_ID_VERSION, base32hexDecode, base32hexEncode, + deriveWaitpointIdFromAnchor, + generateFriendlyId, generateRunOpsId, generateRunOpsIdV2, + generateWaitpointId, + isValidShardChar, parseRunId, parseRunOpsIdBody, parseRunOpsIdV2Body, + parseWaitpointId, + type WaitpointIdType, } from "./friendlyId.js"; /** Every legal gen-2 shard char: the full DNS-safe lowercase range. */ @@ -410,3 +418,173 @@ describe("parseRunId — v2 arm", () => { expect(parseRunId(`waitpoint_${generateRunOpsIdV2("a")}`).format).toBe("legacy"); }); }); + +describe("isValidShardChar", () => { + it("accepts a single [a-z0-9] char", () => { + expect(isValidShardChar("a")).toBe(true); + expect(isValidShardChar("0")).toBe(true); + expect(isValidShardChar("w")).toBe(true); + }); + it("rejects multi-char, empty, uppercase, and punctuation", () => { + expect(isValidShardChar("")).toBe(false); + expect(isValidShardChar("ab")).toBe(false); + expect(isValidShardChar("A")).toBe(false); + expect(isValidShardChar("-")).toBe(false); + expect(isValidShardChar("legacy")).toBe(false); + }); +}); + +describe("waitpoint ids: run-ops format with version char w", () => { + it("mints a 26-char body per type, with the type char at index 24 and version w at 25", () => { + const cases: Array<[WaitpointIdType, string]> = [ + ["RUN", "r"], + ["BATCH", "b"], + ["DATETIME", "d"], + ["MANUAL", "m"], + ]; + + for (const [type, typeChar] of cases) { + const body = generateWaitpointId(type); + expect(body.length).toBe(RUN_OPS_ID_LENGTH); + expect(body[WAITPOINT_ID_TYPE_INDEX]).toBe(typeChar); + expect(body[RUN_OPS_ID_VERSION_INDEX]).toBe(WAITPOINT_ID_VERSION); + } + }); + + it("round-trips every type char through parseWaitpointId", () => { + for (const type of ["RUN", "BATCH", "DATETIME", "MANUAL"] as WaitpointIdType[]) { + const parsed = parseWaitpointId(generateWaitpointId(type)); + expect(parsed.format).toBe("b32hexW"); + if (parsed.format !== "b32hexW") throw new Error("unreachable"); + expect(parsed.type).toBe(type); + } + }); + + it("classifies both the prefixed and the bare form identically", () => { + const body = generateWaitpointId("MANUAL"); + const bare = parseWaitpointId(body); + const prefixed = parseWaitpointId(`waitpoint_${body}`); + expect(bare).toEqual(prefixed); + expect(bare).toEqual({ format: "b32hexW", type: "MANUAL", timestamp: expect.any(Date) }); + }); + + it("recovers the mint timestamp from the core", () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date("2026-08-21T12:00:00.000Z")); + const parsed = parseWaitpointId(generateWaitpointId("DATETIME")); + if (parsed.format !== "b32hexW") throw new Error("unreachable"); + expect(parsed.timestamp.toISOString()).toBe("2026-08-21T12:00:00.000Z"); + } finally { + vi.useRealTimers(); + } + }); + + it("classifies every legacy shape as legacy", () => { + const legacy = [ + WaitpointId.generate().id, + WaitpointId.generate().friendlyId, + generateFriendlyId("waitpoint"), + "", + "waitpoint_", + "a".repeat(27), + "a".repeat(26), + ]; + + for (const id of legacy) { + expect(parseWaitpointId(id).format).toBe("legacy"); + } + }); + + it("rejects a 26-char body whose version is w but whose type char is not r/b/d/m", () => { + const body = generateWaitpointId("RUN"); + const bad = `${body.slice(0, WAITPOINT_ID_TYPE_INDEX)}x${WAITPOINT_ID_VERSION}`; + expect(parseWaitpointId(bad).format).toBe("legacy"); + }); + + it("rejects a body whose core is outside the base32hex alphabet", () => { + const body = generateWaitpointId("RUN"); + // "w" is outside [0-9a-v], so the core no longer decodes. + expect(parseWaitpointId(`w${body.slice(1)}`).format).toBe("legacy"); + }); + + it("never parses a run id as a waitpoint id, or the reverse", () => { + expect(parseWaitpointId(generateRunOpsId()).format).toBe("legacy"); + expect(parseWaitpointId(generateRunOpsIdV2("7")).format).toBe("legacy"); + expect(parseRunId(`run_${generateWaitpointId("RUN")}`).format).toBe("legacy"); + }); + + it("rejects a well-formed waitpoint body wearing a foreign prefix", () => { + const body = `${"0".repeat(24)}rw`; // valid core + RUN type char + version w + expect(parseWaitpointId(`run_${body}`).format).toBe("legacy"); + expect(parseWaitpointId(`batch_${body}`).format).toBe("legacy"); + expect(parseWaitpointId(`waitpoint_${body}`)).toEqual({ + format: "b32hexW", + type: "RUN", + timestamp: expect.any(Date), + }); + expect(parseWaitpointId(body).format).toBe("b32hexW"); + }); + + it("handles a bare body that happens to contain an underscore sanely (never throws, never misclassifies)", () => { + const body = generateWaitpointId("BATCH"); + const withUnderscore = `_${body.slice(1)}`; + expect(() => parseWaitpointId(withUnderscore)).not.toThrow(); + // "_" is outside the base32hex alphabet, so this can never be a real waitpoint id. + expect(parseWaitpointId(withUnderscore).format).toBe("legacy"); + }); +}); + +describe("deriveWaitpointIdFromAnchor", () => { + it("is deterministic: the same anchor and type always give the same id", () => { + const anchor = `run_${generateRunOpsId("us-east-1")}`; + const first = deriveWaitpointIdFromAnchor(anchor, "RUN"); + expect(first).toBeDefined(); + expect(first).toBe(deriveWaitpointIdFromAnchor(anchor, "RUN")); + }); + + it("shares the anchor's 24-char core and replaces the region and version chars", () => { + const anchorBody = generateRunOpsId("us-east-1"); + const derived = deriveWaitpointIdFromAnchor(`run_${anchorBody}`, "RUN"); + expect(derived).toBeDefined(); + expect(derived!.slice(0, WAITPOINT_ID_TYPE_INDEX)).toBe( + anchorBody.slice(0, WAITPOINT_ID_TYPE_INDEX) + ); + expect(derived![WAITPOINT_ID_TYPE_INDEX]).toBe("r"); + expect(derived![RUN_OPS_ID_VERSION_INDEX]).toBe(WAITPOINT_ID_VERSION); + }); + + it("accepts a bare anchor body as well as a prefixed one", () => { + const anchorBody = generateRunOpsId(); + expect(deriveWaitpointIdFromAnchor(anchorBody, "RUN")).toBe( + deriveWaitpointIdFromAnchor(`run_${anchorBody}`, "RUN") + ); + }); + + it("accepts a gen-2 anchor", () => { + const derived = deriveWaitpointIdFromAnchor(`run_${generateRunOpsIdV2("7")}`, "RUN"); + expect(derived).toBeDefined(); + expect(parseWaitpointId(derived!).format).toBe("b32hexW"); + }); + + it("derives a BATCH id from a run-ops format batch anchor", () => { + const derived = deriveWaitpointIdFromAnchor(`batch_${generateRunOpsId()}`, "BATCH"); + expect(derived).toBeDefined(); + const parsed = parseWaitpointId(derived!); + if (parsed.format !== "b32hexW") throw new Error("unreachable"); + expect(parsed.type).toBe("BATCH"); + }); + + it("returns undefined for a legacy anchor, so the caller falls back to a legacy mint", () => { + expect(deriveWaitpointIdFromAnchor(RunId.generate().friendlyId, "RUN")).toBeUndefined(); + expect(deriveWaitpointIdFromAnchor("run_", "RUN")).toBeUndefined(); + expect(deriveWaitpointIdFromAnchor("", "RUN")).toBeUndefined(); + }); + + it("gives a different id per type from one anchor", () => { + const anchor = `run_${generateRunOpsId()}`; + expect(deriveWaitpointIdFromAnchor(anchor, "RUN")).not.toBe( + deriveWaitpointIdFromAnchor(anchor, "BATCH") + ); + }); +}); diff --git a/packages/core/src/v3/isomorphic/friendlyId.ts b/packages/core/src/v3/isomorphic/friendlyId.ts index c468de65319..416660ce446 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.ts @@ -40,6 +40,11 @@ export const DEFAULT_REGION_CHAR = "0"; const REGION_CHAR_PATTERN = /^[a-z0-9]$/; // Same slot, same range: the gen-2 shard key is a region char's positional twin. const SHARD_CHAR_PATTERN = REGION_CHAR_PATTERN; +/** True iff `value` is a single valid gen-2 shard char. The descriptor validator and + * `resolveShard` share this so a configured key and a decoded key cannot drift. */ +export function isValidShardChar(value: string): boolean { + return SHARD_CHAR_PATTERN.test(value); +} /** One lowercase [a-z0-9] char per supported region, at RUN_OPS_ID_REGION_INDEX. */ export const REGION_CODES: Readonly> = { "us-east-1": "e", @@ -238,6 +243,105 @@ export function parseRunId(id: string): ParsedRunId { return LEGACY_RUN_ID; } +// Waitpoint ids reuse the run-ops body layout — 24-char base32hex core, then a +// positional char, then a version char — so the body parses positionally instead of +// splitting on "_". Index 24 carries the TYPE (the slot a run uses for its region or +// shard char), which leaves room to move to a shard char under a later version. +export const WAITPOINT_ID_VERSION = "w"; +export const WAITPOINT_ID_TYPE_INDEX = RUN_OPS_ID_REGION_INDEX; + +export type WaitpointIdType = "RUN" | "BATCH" | "DATETIME" | "MANUAL"; + +// "w" sits OUTSIDE the base32hex alphabet [0-9a-v], so the version char can never be +// mistaken for a core char, and it can never collide with a numeric run generation. +const WAITPOINT_TYPE_CHARS: Readonly> = { + RUN: "r", + BATCH: "b", + DATETIME: "d", + MANUAL: "m", +}; + +const WAITPOINT_TYPES_BY_CHAR: Readonly> = { + r: "RUN", + b: "BATCH", + d: "DATETIME", + m: "MANUAL", +}; + +export type ParsedWaitpointId = + | { format: "b32hexW"; type: WaitpointIdType; timestamp: Date } + | { format: "legacy" }; + +const LEGACY_WAITPOINT_ID: ParsedWaitpointId = { format: "legacy" }; + +/** + * Mint a standalone waitpoint id body (26 chars, no prefix) for DATETIME and MANUAL: a + * fresh core, the type char, then the waitpoint version char. + */ +export function generateWaitpointId(type: WaitpointIdType): string { + return `${mintRunOpsIdCore()}${WAITPOINT_TYPE_CHARS[type]}${WAITPOINT_ID_VERSION}`; +} + +/** + * Derive the 1:1 waitpoint id body for a RUN or BATCH anchor by reusing the anchor's + * 24-char core. Pure, so create-if-absent is idempotent without a lock. Returns + * undefined when the anchor is not a run-ops id, which is the caller's signal to mint a + * legacy waitpoint instead. + * + * Only the core survives: the anchor's region or shard char and its version char are + * both replaced. So the anchor id is NOT recoverable from the waitpoint id — the reverse + * direction uses the completedBy* back-pointer. + */ +export function deriveWaitpointIdFromAnchor( + anchorId: string, + type: WaitpointIdType +): string | undefined { + const body = stripAnchorPrefix(anchorId); + if (!parseRunOpsIdBody(body) && !parseRunOpsIdV2Body(body)) { + return undefined; + } + + return `${body.slice(0, RUN_OPS_ID_CORE_LENGTH)}${WAITPOINT_TYPE_CHARS[type]}${WAITPOINT_ID_VERSION}`; +} + +/** + * Classify a waitpoint id. Accepts the prefixed (`waitpoint_`) and bare forms, but + * NOT another entity's prefix (`run_`, `batch_`, ...) — this is the discriminator a + * later ticket uses to route a possibly customer-supplied id, so a foreign prefix must + * classify legacy rather than have its body reinterpreted as a waitpoint id. Total: + * never throws. + */ +export function parseWaitpointId(id: string): ParsedWaitpointId { + const body = stripWaitpointIdPrefix(id); + if (body.length !== RUN_OPS_ID_LENGTH) return LEGACY_WAITPOINT_ID; + if (body[RUN_OPS_ID_VERSION_INDEX] !== WAITPOINT_ID_VERSION) return LEGACY_WAITPOINT_ID; + + const type = WAITPOINT_TYPES_BY_CHAR[body[WAITPOINT_ID_TYPE_INDEX] ?? ""]; + if (!type) return LEGACY_WAITPOINT_ID; + + const timestamp = parseRunOpsIdCoreTimestamp(body); + if (timestamp === undefined) return LEGACY_WAITPOINT_ID; + + return { format: "b32hexW", type, timestamp }; +} + +// Strip any `_` if present. Prefix-agnostic is correct ONLY here: the caller +// already knows anchorId names a run or batch anchor, so there is no foreign prefix to +// guard against. Do not reuse for parseWaitpointId — see stripWaitpointIdPrefix. +function stripAnchorPrefix(id: string): string { + const underscore = id.indexOf("_"); + return underscore === -1 ? id : id.slice(underscore + 1); +} + +const WAITPOINT_ID_PREFIX = "waitpoint_"; + +// Strip the `waitpoint_` prefix if present; any other prefix, or a bare body, is left +// as-is. Unlike stripAnchorPrefix, this must never strip a foreign prefix down to a body +// that then happens to pass the run-ops shape check. +function stripWaitpointIdPrefix(id: string): string { + return id.startsWith(WAITPOINT_ID_PREFIX) ? id.slice(WAITPOINT_ID_PREFIX.length) : id; +} + export function generateInternalId(): string { return cuid(); } diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 08dc4d9ca0a..14b6f54ea79 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -654,6 +654,7 @@ export const BuildServerMetadata = z.object({ skipPromotion: z.boolean().optional(), configFilePath: z.string().optional(), skipEnqueue: z.boolean().optional(), + fromBundle: z.boolean().optional(), }); export type BuildServerMetadata = z.infer; @@ -720,7 +721,7 @@ export const UpsertBranchResponseBody = z.object({ export type UpsertBranchResponseBody = z.infer; export const CreateArtifactRequestBody = z.object({ - type: z.enum(["deployment_context"]).default("deployment_context"), + type: z.enum(["deployment_context", "deployment_bundle"]).default("deployment_context"), contentType: z.string().default("application/gzip"), contentLength: z.number().optional(), }); @@ -784,6 +785,8 @@ type NativeBuildOutput = BaseOutput & { artifactKey?: string; configFilePath?: string; skipEnqueue?: boolean; + fromBundle?: boolean; + buildEnvVars?: Record; }; type NonNativeBuildOutput = BaseOutput & { @@ -792,6 +795,8 @@ type NonNativeBuildOutput = BaseOutput & { artifactKey?: never; configFilePath?: never; skipEnqueue?: never; + fromBundle?: never; + buildEnvVars?: never; }; const InitializeDeploymentRequestBodyFull = InitializeDeploymentRequestBodyBase.extend({ @@ -800,6 +805,10 @@ const InitializeDeploymentRequestBodyFull = InitializeDeploymentRequestBodyBase. artifactKey: z.string().optional(), configFilePath: z.string().optional(), skipEnqueue: z.boolean().optional().default(false), + // The artifact is a pre-built bundle; the build server only runs the container build + fromBundle: z.boolean().optional(), + // Build-time env var values for fromBundle deploys, stored encrypted on the deployment + buildEnvVars: z.record(z.string()).optional(), }).superRefine((data, ctx) => { if (data.force && !data.externalId) { ctx.addIssue({ @@ -815,7 +824,15 @@ export const InitializeDeploymentRequestBody = InitializeDeploymentRequestBodyFu if (data.isNativeBuild) { return { ...data, isNativeBuild: true as const }; } - const { skipPromotion, artifactKey, configFilePath, skipEnqueue, ...rest } = data; + const { + skipPromotion, + artifactKey, + configFilePath, + skipEnqueue, + fromBundle, + buildEnvVars, + ...rest + } = data; return { ...rest, isNativeBuild: false as const }; } ); @@ -921,6 +938,15 @@ export const GetDeploymentResponseBody = z.object({ export type GetDeploymentResponseBody = z.infer; +// Secret material, deliberately kept off GetDeploymentResponseBody +export const GetDeploymentBuildEnvVarsResponseBody = z.object({ + variables: z.record(z.string()), +}); + +export type GetDeploymentBuildEnvVarsResponseBody = z.infer< + typeof GetDeploymentBuildEnvVarsResponseBody +>; + export const GetLatestDeploymentResponseBody = GetDeploymentResponseBody.omit({ worker: true, }); @@ -1824,6 +1850,11 @@ export const SessionTriggerConfig = z.object({ lockToVersion: z.string().optional(), /** Region to schedule runs in. Forwarded to `TaskRunOptions.region`. */ region: z.string().optional(), + /** + * How long a run may sit undequeued before it expires (duration string + * like `"2m"`, or seconds). Forwarded to `TaskRunOptions.ttl`. + */ + ttl: z.string().or(z.number().nonnegative().int()).optional(), /** Convenience field surfaced to chat.agent via the wire payload. */ idleTimeoutInSeconds: z.number().int().positive().max(3600).optional(), }); diff --git a/packages/trigger-sdk/src/v3/ai.ts b/packages/trigger-sdk/src/v3/ai.ts index c241930323b..dfee30f039a 100644 --- a/packages/trigger-sdk/src/v3/ai.ts +++ b/packages/trigger-sdk/src/v3/ai.ts @@ -10448,6 +10448,7 @@ function createChatStartSessionAction( const maxDuration = params.triggerConfig?.maxDuration ?? options?.triggerConfig?.maxDuration; const idleTimeoutInSeconds = params.triggerConfig?.idleTimeoutInSeconds ?? options?.triggerConfig?.idleTimeoutInSeconds; + const ttl = params.triggerConfig?.ttl ?? options?.triggerConfig?.ttl; const triggerConfig: SessionTriggerConfig = { basePayload: { @@ -10470,6 +10471,7 @@ function createChatStartSessionAction( ...(options?.triggerConfig?.region || params.triggerConfig?.region ? { region: params.triggerConfig?.region ?? options?.triggerConfig?.region } : {}), + ...(ttl !== undefined ? { ttl } : {}), ...(options?.triggerConfig?.lockToVersion || params.triggerConfig?.lockToVersion ? { lockToVersion: diff --git a/packages/trigger-sdk/src/v3/chat-server.test.ts b/packages/trigger-sdk/src/v3/chat-server.test.ts index 539fe0247ad..92ffd77502c 100644 --- a/packages/trigger-sdk/src/v3/chat-server.test.ts +++ b/packages/trigger-sdk/src/v3/chat-server.test.ts @@ -216,7 +216,7 @@ describe("chat.headStart (route handler)", () => { expect(body.triggerConfig.basePayload.idleTimeoutInSeconds).toBe(60); }); - it("merges triggerConfig tags and queue into createSession", async () => { + it("merges triggerConfig tags, queue and ttl into createSession", async () => { const requests: CapturedRequest[] = []; global.fetch = vi.fn().mockImplementation(async (url: string | URL, init?: RequestInit) => { const urlStr = typeof url === "string" ? url : url.toString(); @@ -248,6 +248,7 @@ describe("chat.headStart (route handler)", () => { triggerConfig: { tags: ["org:acme", "agentic-run:xyz"], queue: "my-queue", + ttl: "2m", }, run: async ({ chat: chatHelper }) => { return streamText({ @@ -276,6 +277,7 @@ describe("chat.headStart (route handler)", () => { const body = JSON.parse(sessionCreate!.init!.body as string); expect(body.triggerConfig.tags).toEqual(["chat:chat-1", "org:acme", "agentic-run:xyz"]); expect(body.triggerConfig.queue).toBe("my-queue"); + expect(body.triggerConfig.ttl).toBe("2m"); expect(body.triggerConfig.basePayload.trigger).toBe("handover-prepare"); expect(body.triggerConfig.basePayload.chatId).toBe("chat-1"); }); diff --git a/packages/trigger-sdk/src/v3/chat-server.ts b/packages/trigger-sdk/src/v3/chat-server.ts index 5e48e3b24b6..dc850d118ed 100644 --- a/packages/trigger-sdk/src/v3/chat-server.ts +++ b/packages/trigger-sdk/src/v3/chat-server.ts @@ -550,6 +550,7 @@ async function openHandoverSession(opts: { ? { maxDuration: opts.triggerConfig.maxDuration } : {}), ...(opts.triggerConfig?.region ? { region: opts.triggerConfig.region } : {}), + ...(opts.triggerConfig?.ttl !== undefined ? { ttl: opts.triggerConfig.ttl } : {}), ...(opts.triggerConfig?.lockToVersion ? { lockToVersion: opts.triggerConfig.lockToVersion } : {}), diff --git a/packages/trigger-sdk/src/v3/chat.ts b/packages/trigger-sdk/src/v3/chat.ts index a7c7125575e..55889bd58e5 100644 --- a/packages/trigger-sdk/src/v3/chat.ts +++ b/packages/trigger-sdk/src/v3/chat.ts @@ -870,6 +870,10 @@ export class TriggerChatTransport implements ChatTransport { this.activeStreams.delete(chatId); } + // A stop that never saw its TURN_COMPLETE leaves the flag set, and the new + // turn would be skipped record by record. + state.skipToTurnComplete = false; + state.isStreaming = true; this.notifySessionChange(chatId, state); @@ -1281,6 +1285,10 @@ export class TriggerChatTransport implements ChatTransport { this.activeStreams.delete(chatId); } + // A stop that never saw its TURN_COMPLETE leaves the flag set, and the new + // turn would be skipped record by record. + state.skipToTurnComplete = false; + // Mark streaming + persist so a reload mid-action resumes (reconnectToStream // no-ops when the persisted session says isStreaming: false). state.isStreaming = true; diff --git a/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts b/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts index ca18ce59985..ca51282e614 100644 --- a/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts +++ b/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts @@ -115,7 +115,7 @@ describe("chat.createStartSessionAction — runtime", () => { ]); }); - it("forwards maxDuration, region, and lockToVersion from triggerConfig", async () => { + it("forwards maxDuration, region, lockToVersion, and ttl from triggerConfig", async () => { installStartFixture(); const start = chat.createStartSessionAction("fake-chat", { @@ -123,6 +123,7 @@ describe("chat.createStartSessionAction — runtime", () => { maxDuration: 120, region: "us-east-1", lockToVersion: "20260101.1", + ttl: "2m", }, }); await start({ chatId: "chat-parity" }); @@ -130,6 +131,16 @@ describe("chat.createStartSessionAction — runtime", () => { expect(lastStartBody?.triggerConfig.maxDuration).toBe(120); expect(lastStartBody?.triggerConfig.region).toBe("us-east-1"); expect(lastStartBody?.triggerConfig.lockToVersion).toBe("20260101.1"); + expect(lastStartBody?.triggerConfig.ttl).toBe("2m"); + }); + + it("omits ttl when triggerConfig does not set it", async () => { + installStartFixture(); + + const start = chat.createStartSessionAction("fake-chat"); + await start({ chatId: "chat-no-ttl" }); + + expect(lastStartBody?.triggerConfig).not.toHaveProperty("ttl"); }); it("server-mints override tokens for additional API keys", async () => { diff --git a/packages/trigger-sdk/test/chat-transport-events.test.ts b/packages/trigger-sdk/test/chat-transport-events.test.ts index 39f4e53d722..a35e78ad519 100644 --- a/packages/trigger-sdk/test/chat-transport-events.test.ts +++ b/packages/trigger-sdk/test/chat-transport-events.test.ts @@ -174,6 +174,70 @@ describe("transport send events", () => { }); }); +describe("stopped turn followed by a new turn", () => { + /** + * `.out` stub that honours the `Last-Event-ID` cursor like the server does, so + * a resubscribe cannot replay records the reader already consumed. A stop that + * never saw its turn-complete is therefore unrecoverable unless the new send + * clears the skip state. + */ + function cursoredOneTurnTransport() { + const frames = [ + { id: "1", data: `{"type":"text-delta","id":"t1","delta":"hello"}` }, + { id: "2", data: `{"type":"trigger:turn-complete"}` }, + ]; + + return makeTransport({ + fetch: async (_url, init, ctx) => { + if (ctx.endpoint === "in") return jsonOk(); + + const cursor = new Headers(init.headers).get("Last-Event-ID"); + const from = cursor ? frames.findIndex((f) => f.id === cursor) + 1 : 0; + const remaining = frames.slice(from); + const response = sseResponse( + remaining.map((f) => `id: ${f.id}\ndata: ${f.data}\n\n`).join("") + ); + // Nothing left to send: the session is settled, so the reader stops + // instead of resubscribing. + if (remaining.length === 0) response.headers.set("X-Session-Settled", "true"); + return response; + }, + }); + } + + it("streams a sendMessages turn after a stop that never saw turn-complete", async () => { + const { transport, events } = cursoredOneTurnTransport(); + + expect(await transport.stopGeneration("c1")).toBe(true); + events.length = 0; + + const stream = await transport.sendMessages({ + trigger: "submit-message", + chatId: "c1", + messageId: undefined, + messages: [user("after stop", "u-2")], + abortSignal: undefined, + }); + const chunks = await readAll(stream); + + expect(chunks).toEqual([{ type: "text-delta", id: "t1", delta: "hello" }]); + expect(events.some((e) => e.type === "turn-completed")).toBe(true); + }); + + it("streams a sendAction turn after a stop that never saw turn-complete", async () => { + const { transport, events } = cursoredOneTurnTransport(); + + expect(await transport.stopGeneration("c1")).toBe(true); + events.length = 0; + + const stream = await transport.sendAction("c1", { type: "undo" }); + const chunks = await readAll(stream); + + expect(chunks).toEqual([{ type: "text-delta", id: "t1", delta: "hello" }]); + expect(events.some((e) => e.type === "turn-completed")).toBe(true); + }); +}); + describe("transport stream events", () => { it("marks reconnectToStream subscriptions as resumed", async () => { const { transport, events } = makeTransport({