From f9d674e06127450318ad04756ae77ebab32468b0 Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Mon, 10 Aug 2026 12:37:52 +0000 Subject: [PATCH 1/6] feat(webapp): enforce watch plan limits Refuse a watch whose window exceeds the plan's agentWatchMaxHours, or that would push the org past its agentWatchers count, with a new watch_limit_reached result carrying an upgrade hint. Plan limits are a floor below the existing code ceilings (min(plan, WATCH_MAX_HOURS=24) and the per-chat cap of 3, which still apply independently). Fails open: an absent limit resolves to unlimited, so self-hosted is unaffected and the upgrade nudge is gated on billing presence. TRI-12863 --- .server-changes/agent-watch-plan-limits.md | 6 + .../routes/api.v1.dashboard-agent.watches.ts | 4 +- .../dashboardAgentWatchLimits.server.ts | 53 +++ .../services/dashboardAgentWatches.server.ts | 42 +++ .../test/dashboardAgentWatchLimits.test.ts | 327 ++++++++++++++++++ .../dashboard-agent-db/src/watch-queries.ts | 16 + 6 files changed, 447 insertions(+), 1 deletion(-) create mode 100644 .server-changes/agent-watch-plan-limits.md create mode 100644 apps/webapp/app/services/dashboardAgentWatchLimits.server.ts create mode 100644 apps/webapp/test/dashboardAgentWatchLimits.test.ts diff --git a/.server-changes/agent-watch-plan-limits.md b/.server-changes/agent-watch-plan-limits.md new file mode 100644 index 00000000000..d323e7194f0 --- /dev/null +++ b/.server-changes/agent-watch-plan-limits.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Watches now respect your plan's limits: free plans can run a limited number of watches at once and for a shorter window, with a prompt to upgrade for more. diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts index 0fbe83e3463..5833ea016df 100644 --- a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts @@ -111,7 +111,9 @@ export async function action({ request }: ActionFunctionArgs) { if (!result.ok) { const status = - result.code === "limit_reached" || result.code === "duplicate" + result.code === "limit_reached" || + result.code === "watch_limit_reached" || + result.code === "duplicate" ? 409 : result.code === "invalid_target" ? 404 diff --git a/apps/webapp/app/services/dashboardAgentWatchLimits.server.ts b/apps/webapp/app/services/dashboardAgentWatchLimits.server.ts new file mode 100644 index 00000000000..d7f3454760a --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchLimits.server.ts @@ -0,0 +1,53 @@ +import type { Limits } from "@trigger.dev/platform"; +import { WATCH_MAX_HOURS } from "@internal/dashboard-agent-contracts"; +import { getCachedLimit, isBillingConfigured } from "./platform.v3.server"; + +// The unlimited sentinel, matching the message quota (TRI-12863 P1). Never Infinity: it +// serializes to null in the limit cache. +export const UNLIMITED_WATCH_LIMIT = 100_000_000; + +// Filled by cloud billing (TRI-12863 P0). Absent until then, and always on self-hosted, so +// the fallback applies and the plan floor is off. +const WATCH_MAX_HOURS_LIMIT_KEY = "agentWatchMaxHours" as keyof Limits; +const WATCH_COUNT_LIMIT_KEY = "agentWatchers" as keyof Limits; + +export type WatchPlanLimits = { + /** Longest window one watch may run for, in hours. */ + maxHours: number; + /** How many active watches the org may run at once. */ + watchers: number; +}; + +async function readLimit(organizationId: string, key: keyof Limits): Promise { + const cached = await getCachedLimit(organizationId, key, UNLIMITED_WATCH_LIMIT); + // A cache error leaves `val` empty; fall open to unlimited. + return cached.val ?? UNLIMITED_WATCH_LIMIT; +} + +/** + * The org's plan floors for watches. Fails open: an absent limit (self-hosted, or before the + * cloud side ships) resolves to the unlimited sentinel, so neither floor bites. + */ +export async function resolveWatchPlanLimits(organizationId: string): Promise { + const [maxHours, watchers] = await Promise.all([ + readLimit(organizationId, WATCH_MAX_HOURS_LIMIT_KEY), + readLimit(organizationId, WATCH_COUNT_LIMIT_KEY), + ]); + return { maxHours, watchers }; +} + +/** + * The window ceiling actually in force: the plan floor under the code ceiling. A plan that + * allows 100 hours still caps at {@link WATCH_MAX_HOURS}. + */ +export function effectiveWatchMaxHours(planMaxHours: number): number { + return Math.min(planMaxHours, WATCH_MAX_HOURS); +} + +/** + * A watch-limit refusal, plus an upgrade nudge when billing is present. Self-hosted never + * hits this (fails open above), and the nudge is gated so a stray refusal stays quiet there. + */ +export function watchLimitHint(base: string, billingConfigured = isBillingConfigured()): string { + return billingConfigured ? `${base} Upgrade your plan for more.` : base; +} diff --git a/apps/webapp/app/services/dashboardAgentWatches.server.ts b/apps/webapp/app/services/dashboardAgentWatches.server.ts index d6167627fee..595be8f3f26 100644 --- a/apps/webapp/app/services/dashboardAgentWatches.server.ts +++ b/apps/webapp/app/services/dashboardAgentWatches.server.ts @@ -10,6 +10,7 @@ import { cancelWatch, chatExists, claimWatchSubmission, + countActiveWatchesForOrg, createChat, createWatch, generateWatchId, @@ -68,6 +69,12 @@ import { import { watchCreationCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; import { normalizeErrorFingerprint } from "~/services/dashboardAgentWatchErrorChecks"; import { subscribeUserToWatchAlerts } from "~/services/dashboardAgentWatchAlerts.server"; +import { + effectiveWatchMaxHours, + resolveWatchPlanLimits, + watchLimitHint, + type WatchPlanLimits, +} from "~/services/dashboardAgentWatchLimits.server"; import { mintDashboardAgentWatchBatchToken, mintDashboardAgentWatchToken, @@ -165,6 +172,7 @@ export async function authorizeWatchEnvironmentById(params: { export type CreateWatchErrorCode = | "limit_reached" + | "watch_limit_reached" | "duplicate" | "invalid_target" | "chat_not_found" @@ -273,6 +281,12 @@ export async function createDashboardAgentWatch(params: { scheduleTick?: typeof scheduleWatchTick; /** Skip the real trigger-config gate when a tick scheduler is injected. */ configured?: () => boolean; + /** Plan floors on window and count. Fails open to unlimited when absent. */ + resolveLimits?: (organizationId: string) => Promise; + /** Org-wide active-watch count, for the watcher-count floor. */ + countActiveWatches?: (organizationId: string) => Promise; + /** Gates the upgrade nudge, so self-hosted stays quiet. */ + billingConfigured?: () => boolean; }; }): Promise { const { environment, userId, chatId } = params; @@ -284,6 +298,11 @@ export async function createDashboardAgentWatch(params: { const buildCheckDeps = params.deps?.checkDeps ?? watchCreationCheckDeps; const scheduleTick = params.deps?.scheduleTick ?? scheduleWatchTick; const isDashboardAgentConfigured = params.deps?.configured ?? isDashboardAgentConfiguredDefault; + const resolveLimits = params.deps?.resolveLimits ?? resolveWatchPlanLimits; + const countActiveWatches = + params.deps?.countActiveWatches ?? + ((organizationId: string) => countActiveWatchesForOrg(dashboardAgentDb, { organizationId })); + const hint = (base: string) => watchLimitHint(base, params.deps?.billingConfigured?.()); const checkDeps = buildCheckDeps(environment, now); if (!isDashboardAgentConfigured()) { @@ -317,6 +336,17 @@ export async function createDashboardAgentWatch(params: { }); if (!precheck.ok) return creationGuardrailError(precheck); + // Plan floors sit below the code ceilings (min(plan, ceiling)). Fails open: an absent + // limit resolves to unlimited, so neither floor bites on self-hosted. + const planLimits = await resolveLimits(environment.organizationId); + if (spec.maxHours > effectiveWatchMaxHours(planLimits.maxHours)) { + return { + ok: false, + code: "watch_limit_reached", + error: hint("That watch window is longer than your plan allows."), + }; + } + // `since` is server-set so the model can't backdate a recurrence window. const persistedSpec: PersistedWatchSpec = spec.kind === "error_recurrence" ? { ...spec, since: now.toISOString() } : spec; @@ -331,6 +361,18 @@ export async function createDashboardAgentWatch(params: { return { ok: true, watching: false, identity, immediate }; } + // Counted only now the immediate check didn't answer: a one-shot creates no row and so + // consumes no watcher slot. The per-chat cap of 3 still applies independently, in + // `createWatch`. + const activeCount = await countActiveWatches(environment.organizationId); + if (activeCount >= planLimits.watchers) { + return { + ok: false, + code: "watch_limit_reached", + error: hint("You've reached the number of active watches your plan allows."), + }; + } + const expiresAt = new Date(now.getTime() + spec.maxHours * 60 * 60 * 1000); const created = await createWatch(dashboardAgentDb, { diff --git a/apps/webapp/test/dashboardAgentWatchLimits.test.ts b/apps/webapp/test/dashboardAgentWatchLimits.test.ts new file mode 100644 index 00000000000..95d499558e7 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchLimits.test.ts @@ -0,0 +1,327 @@ +import { + countActiveWatchesForOrg, + createChat, + createDashboardAgentDb, + listActiveWatchesForChat, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { WatchCheckDeps, WatchRunRow } from "~/services/dashboardAgentWatchChecks"; +import type { WatchPlanLimits } from "~/services/dashboardAgentWatchLimits.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("@trigger.dev/sdk", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + TriggerClient: class { + tasks = { trigger: async () => ({ id: "run_test" }) }; + }, + }; +}); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-limits"; + +const { createDashboardAgentWatch } = await import("~/services/dashboardAgentWatches.server"); +const { effectiveWatchMaxHours, watchLimitHint, UNLIMITED_WATCH_LIMIT } = + await import("~/services/dashboardAgentWatchLimits.server"); + +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + const migrations = readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort(); + for (const name of migrations) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 4 }); + ctx.agentDb = agentDbClient.db; +} + +async function seed(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const user = await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `p${slug.slice(0, 6)}`, + }, + }); + return { user, organization, project, environment }; +} + +type Seeded = Awaited>; + +function authenticated(seeded: Seeded) { + return { + id: seeded.environment.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + slug: "prod", + type: "PRODUCTION", + project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, + organization: { id: seeded.organization.id, slug: seeded.organization.slug }, + } as any; +} + +async function seedChat(seeded: Seeded, chatId: string) { + await createChat(ctx.agentDb, { + id: chatId, + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + return chatId; +} + +function runRow(overrides: Partial = {}): WatchRunRow { + return { + friendlyId: "run_1", + status: "PENDING", + queue: "task/my-task", + createdAt: new Date(), + queuedAt: null, + startedAt: null, + completedAt: null, + delayUntil: null, + ...overrides, + }; +} + +function fakeCheckDeps(overrides: Partial = {}): WatchCheckDeps { + return { + readRun: async () => runRow(), + queueExists: async () => true, + readQueueDepth: async () => ({ depth: 7, source: "live_queue", current: true }), + readQueueOldestAge: async () => ({ ageMs: 30_000, source: "live_queue", current: true }), + readErrorRecurrence: async () => null, + readHealth: async () => ({ trustworthy: true, severity: "warn" }), + ...overrides, + }; +} + +const UNLIMITED: WatchPlanLimits = { + maxHours: UNLIMITED_WATCH_LIMIT, + watchers: UNLIMITED_WATCH_LIMIT, +}; + +function runStart(runId: string, maxHours = 2): WatchSpec { + return { kind: "run_start", runId, checkEveryMinutes: 1, maxHours, note: "tell me" }; +} + +function create(args: { + seeded: Seeded; + spec: WatchSpec; + chatId: string; + limits?: WatchPlanLimits; + billingConfigured?: boolean; + countActiveWatches?: (organizationId: string) => Promise; +}) { + return createDashboardAgentWatch({ + environment: authenticated(args.seeded), + userId: args.seeded.user.id, + chatId: args.chatId, + spec: args.spec, + deps: { + configured: () => true, + checkDeps: () => fakeCheckDeps(), + scheduleTick: async () => {}, + resolveLimits: async () => args.limits ?? UNLIMITED, + ...(args.countActiveWatches ? { countActiveWatches: args.countActiveWatches } : {}), + ...(args.billingConfigured === undefined + ? {} + : { billingConfigured: () => args.billingConfigured! }), + }, + }); +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +describe("watch plan limits (pure)", () => { + it("caps the window ceiling at the code ceiling of 24 hours", () => { + expect(effectiveWatchMaxHours(100)).toBe(24); + expect(effectiveWatchMaxHours(1)).toBe(1); + expect(effectiveWatchMaxHours(0.5)).toBe(0.5); + }); + + it("adds an upgrade nudge only when billing is configured", () => { + expect(watchLimitHint("too long.", true)).toBe("too long. Upgrade your plan for more."); + expect(watchLimitHint("too long.", false)).toBe("too long."); + }); +}); + +describe("createDashboardAgentWatch plan enforcement", () => { + postgresTest( + "refuses a window longer than the plan allows", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "window"); + await seedChat(seeded, "chat_1"); + + const result = await create({ + seeded, + chatId: "chat_1", + spec: runStart("run_1", 2), + limits: { maxHours: 1, watchers: UNLIMITED_WATCH_LIMIT }, + billingConfigured: true, + }); + + expect(result).toMatchObject({ ok: false, code: "watch_limit_reached" }); + if (result.ok) return; + expect(result.error).toContain("Upgrade your plan"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "creates a watch whose window is within the plan", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "within"); + await seedChat(seeded, "chat_1"); + + const result = await create({ + seeded, + chatId: "chat_1", + spec: runStart("run_1", 1), + limits: { maxHours: 1, watchers: UNLIMITED_WATCH_LIMIT }, + }); + + expect(result.ok).toBe(true); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(1); + } + ); + + postgresTest( + "refuses once the org is at its watcher count, counting active watches for real", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "count"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + + const limits: WatchPlanLimits = { maxHours: UNLIMITED_WATCH_LIMIT, watchers: 1 }; + + const first = await create({ seeded, chatId: "chat_1", spec: runStart("run_1"), limits }); + expect(first.ok).toBe(true); + expect( + await countActiveWatchesForOrg(ctx.agentDb, { organizationId: seeded.organization.id }) + ).toBe(1); + + const second = await create({ seeded, chatId: "chat_2", spec: runStart("run_2"), limits }); + expect(second).toMatchObject({ ok: false, code: "watch_limit_reached" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_2" })).toHaveLength(0); + } + ); + + postgresTest( + "fails open: an absent limit resolves to unlimited and a 2h watch is created", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "failopen"); + await seedChat(seeded, "chat_1"); + + const result = await create({ + seeded, + chatId: "chat_1", + spec: runStart("run_1", 2), + limits: UNLIMITED, + }); + + expect(result.ok).toBe(true); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(1); + } + ); + + postgresTest( + "leaves no upgrade nudge on a refusal when billing is unconfigured", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "selfhosted"); + await seedChat(seeded, "chat_1"); + + const result = await create({ + seeded, + chatId: "chat_1", + spec: runStart("run_1", 2), + limits: { maxHours: 1, watchers: UNLIMITED_WATCH_LIMIT }, + billingConfigured: false, + }); + + expect(result).toMatchObject({ ok: false, code: "watch_limit_reached" }); + if (result.ok) return; + expect(result.error).not.toContain("Upgrade"); + } + ); + + postgresTest( + "min semantics: a plan of 100 hours still permits only up to the 24h ceiling", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "minsem"); + await seedChat(seeded, "chat_1"); + + const created = await create({ + seeded, + chatId: "chat_1", + spec: runStart("run_1", 24), + limits: { maxHours: 100, watchers: UNLIMITED_WATCH_LIMIT }, + }); + expect(created.ok).toBe(true); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(1); + } + ); +}); diff --git a/internal-packages/dashboard-agent-db/src/watch-queries.ts b/internal-packages/dashboard-agent-db/src/watch-queries.ts index 39364511d16..ff777303d5a 100644 --- a/internal-packages/dashboard-agent-db/src/watch-queries.ts +++ b/internal-packages/dashboard-agent-db/src/watch-queries.ts @@ -494,6 +494,22 @@ export async function countUnreadWatchWakes( return rows[0]?.count ?? 0; } +/** + * How many active watches an org has, across all its chats and users. The plan-limit floor + * is org-wide, so this is org-scoped only; a chat deletion cancels its watches, so `active` + * is the whole count. + */ +export async function countActiveWatchesForOrg( + db: DashboardAgentDb, + params: { organizationId: string } +): Promise { + const rows = await db + .select({ count: sql`count(*)::int` }) + .from(watches) + .where(and(eq(watches.status, "active"), eq(watches.organizationId, params.organizationId))); + return rows[0]?.count ?? 0; +} + /** * Whether this user has a watch that can still wake them here. Covered by * `watches_org_user_active_idx`; a chat deletion cancels its watches, so `active` is enough. From 2dd410d99d85b762d50e571342d7d71ac12648d6 Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Mon, 10 Aug 2026 15:50:51 +0000 Subject: [PATCH 2/6] fix(webapp,dashboard-agent-db): stop a stuck investigation pinning the sweep --- ...board-agent-investigation-sweep-backoff.md | 6 + ...dashboardAgentInvestigationSweep.server.ts | 60 + .../dashboardAgentInvestigationPoison.test.ts | 166 ++ .../drizzle/0005_ambitious_mordo.sql | 2 + .../drizzle/meta/0005_snapshot.json | 1375 +++++++++++++++++ .../drizzle/meta/_journal.json | 9 +- .../dashboard-agent-db/src/queries.ts | 33 +- .../dashboard-agent-db/src/schema.ts | 4 + 8 files changed, 1653 insertions(+), 2 deletions(-) create mode 100644 .server-changes/dashboard-agent-investigation-sweep-backoff.md create mode 100644 apps/webapp/test/dashboardAgentInvestigationPoison.test.ts create mode 100644 internal-packages/dashboard-agent-db/drizzle/0005_ambitious_mordo.sql create mode 100644 internal-packages/dashboard-agent-db/drizzle/meta/0005_snapshot.json diff --git a/.server-changes/dashboard-agent-investigation-sweep-backoff.md b/.server-changes/dashboard-agent-investigation-sweep-backoff.md new file mode 100644 index 00000000000..b49c7881cb4 --- /dev/null +++ b/.server-changes/dashboard-agent-investigation-sweep-backoff.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +A single stuck assistant investigation can no longer hold up others from being tidied away. diff --git a/apps/webapp/app/services/dashboardAgentInvestigationSweep.server.ts b/apps/webapp/app/services/dashboardAgentInvestigationSweep.server.ts index b10ee6b06af..38853ea0ddd 100644 --- a/apps/webapp/app/services/dashboardAgentInvestigationSweep.server.ts +++ b/apps/webapp/app/services/dashboardAgentInvestigationSweep.server.ts @@ -5,8 +5,11 @@ import { listStaleOpenInvestigations, + recordInvestigationSweepAttempt, settleInvestigationAndCloseCard, + settleInvestigationAsInconclusive, type Investigation, + type SettledInvestigation, type SettledInvestigationCard, } from "@internal/dashboard-agent-db"; import { UNSETTLED_INVESTIGATION_NOTE } from "@internal/dashboard-agent-contracts"; @@ -22,6 +25,13 @@ export const INVESTIGATION_STALE_MS = 30 * 60 * 1000; /** Per-run cap. Oldest first, so the rest land next run. */ const SWEEP_BATCH_LIMIT = 100; +/** + * After this many failed settle attempts a row is force-abandoned: settled `inconclusive` + * WITHOUT the closing card, so a card that never renders leaves the queue instead of + * looping forever. The rare stuck spinner is the price of not starving every other row. + */ +export const MAX_SWEEP_ATTEMPTS = 5; + export type InvestigationSweepResult = { /** Stale `in_progress` rows seen. */ stale: number; @@ -30,6 +40,8 @@ export type InvestigationSweepResult = { closed: number; /** A turn (or another sweep) settled it first. */ alreadySettled: number; + /** Rows past the attempt cap, force-settled without a card so they leave the queue. */ + abandoned: number; failed: number; }; @@ -46,6 +58,10 @@ export type InvestigationSweepDeps = { chatId: string; note: string; }) => Promise; + /** Record a failed settle out-of-band; returns the new attempt count, or null if gone. */ + recordAttempt?: (params: { id: string }) => Promise; + /** Force a poison row terminal without the failing render path. */ + forceAbandon?: (params: { id: string; note: string }) => Promise; }; /** @@ -61,12 +77,17 @@ export async function sweepDashboardAgentInvestigations( deps.listStale ?? ((params) => listStaleOpenInvestigations(dashboardAgentDb, params)); const settleAndClose = deps.settleAndClose ?? ((params) => settleInvestigationAndCloseCard(dashboardAgentDb, params)); + const recordAttempt = + deps.recordAttempt ?? ((params) => recordInvestigationSweepAttempt(dashboardAgentDb, params)); + const forceAbandon = + deps.forceAbandon ?? ((params) => settleInvestigationAsInconclusive(dashboardAgentDb, params)); const result: InvestigationSweepResult = { stale: 0, settled: 0, closed: 0, alreadySettled: 0, + abandoned: 0, failed: 0, }; @@ -93,10 +114,49 @@ export async function sweepDashboardAgentInvestigations( result.settled++; if (outcome.closed) result.closed++; } catch (error) { + // The settle rolled back, so the row is still `in_progress`. Record the attempt in + // its own write — this rotates the row to the back of the sweep order (see + // `listStaleOpenInvestigations`) so it can't pin the head and starve newer rows. + let attempts: number | null = null; + try { + attempts = await recordAttempt({ id: investigation.id }); + } catch (recordError) { + logger.error("Dashboard agent investigation sweep: failed to record a sweep attempt", { + investigationId: investigation.id, + chatId: investigation.chatId, + error: recordError, + }); + } + + // Past the cap the card will never render; force it terminal without the render + // path so it leaves the queue instead of looping forever. + if (attempts !== null && attempts >= MAX_SWEEP_ATTEMPTS) { + try { + await forceAbandon({ id: investigation.id, note: UNSETTLED_INVESTIGATION_NOTE }); + result.abandoned++; + logger.warn( + "Dashboard agent investigation sweep: abandoned a card past the attempt cap", + { + investigationId: investigation.id, + chatId: investigation.chatId, + attempts, + } + ); + continue; + } catch (abandonError) { + logger.error("Dashboard agent investigation sweep: failed to abandon a poison card", { + investigationId: investigation.id, + chatId: investigation.chatId, + error: abandonError, + }); + } + } + result.failed++; logger.error("Dashboard agent investigation sweep: failed to settle an investigation", { investigationId: investigation.id, chatId: investigation.chatId, + attempts, error, }); } diff --git a/apps/webapp/test/dashboardAgentInvestigationPoison.test.ts b/apps/webapp/test/dashboardAgentInvestigationPoison.test.ts new file mode 100644 index 00000000000..52b87f35c17 --- /dev/null +++ b/apps/webapp/test/dashboardAgentInvestigationPoison.test.ts @@ -0,0 +1,166 @@ +import { + createChat, + createDashboardAgentDb, + getInvestigation, + settleInvestigationAndCloseCard, + upsertInvestigationRevision, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { + investigationStateSchema, + type InvestigationState, +} from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, vi } from "vitest"; + +const ctx = vi.hoisted(() => ({ + agentDb: undefined as unknown as DashboardAgentDb, +})); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +const { sweepDashboardAgentInvestigations, INVESTIGATION_STALE_MS, MAX_SWEEP_ATTEMPTS } = + await import("~/services/dashboardAgentInvestigationSweep.server"); + +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + const migrations = readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort(); + for (const name of migrations) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; +let prismaForRaw: PrismaClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 2 }); + ctx.agentDb = agentDbClient.db; + prismaForRaw = prisma; +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +const ORG = "org_poison"; +const USER = "user_poison"; + +function openState(): InvestigationState { + return investigationStateSchema.parse({ + outcome: "in_progress", + severity: "warn", + confidence: "medium", + title: "a stuck card", + headline: "Still checking.", + progress: "Reading spans", + checkNext: [], + hypotheses: [], + evidence: [], + }); +} + +async function seedInvestigation(chatId: string, ageMs: number): Promise { + await createChat(ctx.agentDb, { id: chatId, organizationId: ORG, userId: USER }); + const created = await upsertInvestigationRevision(ctx.agentDb, { + chatId, + projectRef: "proj", + environmentRef: "env", + state: openState(), + }); + if (!created.ok) throw new Error("fixture investigation not created"); + await prismaForRaw!.$executeRawUnsafe( + `update trigger_dashboard_agent.investigations + set updated_at = now() - ($2 || ' milliseconds')::interval where id = $1`, + created.id, + String(ageMs) + ); + return created.id; +} + +async function outcomeOf(id: string): Promise { + const row = await getInvestigation(ctx.agentDb, { id }); + return row ? (row.state as { outcome?: string }).outcome : undefined; +} + +const STALE_AGE_MS = INVESTIGATION_STALE_MS + 60_000; +const OLDER_AGE_MS = STALE_AGE_MS + 60_000; + +describe("the investigation sweep with a poison row", () => { + postgresTest( + "a row that always fails to settle cannot pin the head and starve a newer row", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + + // Poison sorts first (older `updated_at`); renderable is newer. + const poisonId = await seedInvestigation("chat_poison", OLDER_AGE_MS); + const renderableId = await seedInvestigation("chat_ok", STALE_AGE_MS); + + // Only the poison row's settle throws; the renderable one goes through the real path. + const settleAndClose = (params: { id: string; chatId: string; note: string }) => { + if (params.id === poisonId) throw new Error("state isn't renderable"); + return settleInvestigationAndCloseCard(ctx.agentDb, params); + }; + + // limit 1 forces head contention: without backoff the poison row would win every run. + // A failed run throws so the job retries, but the attempt is recorded before it does. + await expect( + sweepDashboardAgentInvestigations({ limit: 1, settleAndClose }) + ).rejects.toThrow(); + expect(await outcomeOf(poisonId)).toBe("in_progress"); + expect(await outcomeOf(renderableId)).toBe("in_progress"); + + // Next run: the poison row now sorts behind the never-attempted renderable one, + // so the newer row is picked and settled despite the poison row still being stale. + const second = await sweepDashboardAgentInvestigations({ limit: 1, settleAndClose }); + expect(second).toMatchObject({ stale: 1, settled: 1, failed: 0 }); + expect(await outcomeOf(renderableId)).toBe("inconclusive"); + expect(await outcomeOf(poisonId)).toBe("in_progress"); + }, + 30_000 + ); + + postgresTest( + "after the attempt cap the poison row is abandoned and leaves the queue", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const poisonId = await seedInvestigation("chat_poison", STALE_AGE_MS); + + const settleAndClose = () => { + throw new Error("state isn't renderable"); + }; + + // The first MAX_SWEEP_ATTEMPTS-1 runs record a failed attempt and throw; the row stays stale. + for (let i = 1; i < MAX_SWEEP_ATTEMPTS; i++) { + await expect(sweepDashboardAgentInvestigations({ settleAndClose })).rejects.toThrow(); + expect(await outcomeOf(poisonId)).toBe("in_progress"); + } + + // The capped run force-settles the row without the render path, so it leaves the queue. + const capped = await sweepDashboardAgentInvestigations({ settleAndClose }); + expect(capped).toMatchObject({ stale: 1, abandoned: 1, failed: 0 }); + expect(await outcomeOf(poisonId)).toBe("inconclusive"); + + // Nothing stale remains, so the poison row is no longer swept. + const after = await sweepDashboardAgentInvestigations({ settleAndClose }); + expect(after).toMatchObject({ stale: 0 }); + }, + 30_000 + ); +}); diff --git a/internal-packages/dashboard-agent-db/drizzle/0005_ambitious_mordo.sql b/internal-packages/dashboard-agent-db/drizzle/0005_ambitious_mordo.sql new file mode 100644 index 00000000000..3d56343452d --- /dev/null +++ b/internal-packages/dashboard-agent-db/drizzle/0005_ambitious_mordo.sql @@ -0,0 +1,2 @@ +ALTER TABLE "trigger_dashboard_agent"."investigations" ADD COLUMN "sweep_attempts" integer DEFAULT 0 NOT NULL;--> statement-breakpoint +ALTER TABLE "trigger_dashboard_agent"."investigations" ADD COLUMN "last_sweep_attempt_at" timestamp with time zone; \ No newline at end of file diff --git a/internal-packages/dashboard-agent-db/drizzle/meta/0005_snapshot.json b/internal-packages/dashboard-agent-db/drizzle/meta/0005_snapshot.json new file mode 100644 index 00000000000..e3a9a8082a4 --- /dev/null +++ b/internal-packages/dashboard-agent-db/drizzle/meta/0005_snapshot.json @@ -0,0 +1,1375 @@ +{ + "id": "9f0a4739-19ca-4a15-82dd-25598116feb9", + "prevId": "f7cbfef4-7fc8-4deb-8da2-59248b242a60", + "version": "7", + "dialect": "postgresql", + "tables": { + "trigger_dashboard_agent.agent_message_usage": { + "name": "agent_message_usage", + "schema": "trigger_dashboard_agent", + "columns": { + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "period": { + "name": "period", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_message_usage_organization_id_period_pk": { + "name": "agent_message_usage_organization_id_period_pk", + "columns": [ + "organization_id", + "period" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "trigger_dashboard_agent.chat_messages": { + "name": "chat_messages", + "schema": "trigger_dashboard_agent", + "columns": { + "chat_id": { + "name": "chat_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_chat_user_role_idx": { + "name": "chat_messages_chat_user_role_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"chat_messages\".\"role\" = 'user'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "chat_messages_chat_id_message_id_pk": { + "name": "chat_messages_chat_id_message_id_pk", + "columns": [ + "chat_id", + "message_id" + ] + } + }, + "uniqueConstraints": { + "chat_messages_chat_position_key": { + "name": "chat_messages_chat_position_key", + "nullsNotDistinct": false, + "columns": [ + "chat_id", + "position" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "trigger_dashboard_agent.chat_sessions": { + "name": "chat_sessions", + "schema": "trigger_dashboard_agent", + "columns": { + "chat_id": { + "name": "chat_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_access_token": { + "name": "public_access_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_event_id": { + "name": "last_event_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "trigger_dashboard_agent.chat_turn_evals": { + "name": "chat_turn_evals", + "schema": "trigger_dashboard_agent", + "columns": { + "chat_id": { + "name": "chat_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "turn": { + "name": "turn", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_run_id": { + "name": "agent_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "eval_run_id": { + "name": "eval_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_ref": { + "name": "project_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "environment": { + "name": "environment", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "current_page": { + "name": "current_page", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt_slug": { + "name": "prompt_slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt_version": { + "name": "prompt_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "tools_used": { + "name": "tools_used", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "tool_error": { + "name": "tool_error", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "judge_model": { + "name": "judge_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "score_grounded": { + "name": "score_grounded", + "type": "smallint", + "primaryKey": false, + "notNull": false + }, + "score_answered": { + "name": "score_answered", + "type": "smallint", + "primaryKey": false, + "notNull": false + }, + "score_concise": { + "name": "score_concise", + "type": "smallint", + "primaryKey": false, + "notNull": false + }, + "passed": { + "name": "passed", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "intent_category": { + "name": "intent_category", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sentiment": { + "name": "sentiment", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "capability_gap": { + "name": "capability_gap", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "docs_gap": { + "name": "docs_gap", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "support_opportunity": { + "name": "support_opportunity", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "feature_request": { + "name": "feature_request", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "topics": { + "name": "topics", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "signals": { + "name": "signals", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_text": { + "name": "user_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "judge": { + "name": "judge", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_turn_evals_org_created_idx": { + "name": "chat_turn_evals_org_created_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_turn_evals_created_idx": { + "name": "chat_turn_evals_created_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_turn_evals_org_opps_idx": { + "name": "chat_turn_evals_org_opps_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"chat_turn_evals\".\"capability_gap\" or \"trigger_dashboard_agent\".\"chat_turn_evals\".\"docs_gap\" or \"trigger_dashboard_agent\".\"chat_turn_evals\".\"support_opportunity\" or \"trigger_dashboard_agent\".\"chat_turn_evals\".\"feature_request\"", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "chat_turn_evals_chat_id_turn_pk": { + "name": "chat_turn_evals_chat_id_turn_pk", + "columns": [ + "chat_id", + "turn" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "trigger_dashboard_agent.chats": { + "name": "chats", + "schema": "trigger_dashboard_agent", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'New chat'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "messages": { + "name": "messages", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_message_position": { + "name": "next_message_position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chats_org_user_last_msg_idx": { + "name": "chats_org_user_last_msg_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_message_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"chats\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "trigger_dashboard_agent.investigations": { + "name": "investigations", + "schema": "trigger_dashboard_agent", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_ref": { + "name": "project_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "environment_ref": { + "name": "environment_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "state": { + "name": "state", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "sweep_attempts": { + "name": "sweep_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_sweep_attempt_at": { + "name": "last_sweep_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "investigations_chat_idx": { + "name": "investigations_chat_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_open_updated_idx": { + "name": "investigations_open_updated_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"investigations\".\"state\"->>'outcome' = 'in_progress'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "trigger_dashboard_agent.watch_batches": { + "name": "watch_batches", + "schema": "trigger_dashboard_agent", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cadence_minutes": { + "name": "cadence_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "armed_at": { + "name": "armed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_tick_at": { + "name": "last_tick_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "watch_batches_environment_id_cadence_minutes_pk": { + "name": "watch_batches_environment_id_cadence_minutes_pk", + "columns": [ + "environment_id", + "cadence_minutes" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "trigger_dashboard_agent.watch_submissions": { + "name": "watch_submissions", + "schema": "trigger_dashboard_agent", + "columns": { + "chat_id": { + "name": "chat_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_request_id": { + "name": "client_request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_hash": { + "name": "draft_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft": { + "name": "draft", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "watch_id": { + "name": "watch_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "unavailable": { + "name": "unavailable", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "external_notification_status": { + "name": "external_notification_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_requested'" + }, + "external_notification_reason": { + "name": "external_notification_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "immediate_result": { + "name": "immediate_result", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_error": { + "name": "refusal_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_existing_id": { + "name": "refusal_existing_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "watch_submissions_created_idx": { + "name": "watch_submissions_created_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "watch_submissions_chat_id_client_request_id_pk": { + "name": "watch_submissions_chat_id_client_request_id_pk", + "columns": [ + "chat_id", + "client_request_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "trigger_dashboard_agent.watches": { + "name": "watches", + "schema": "trigger_dashboard_agent", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "spec": { + "name": "spec", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "delivery_status": { + "name": "delivery_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_required'" + }, + "cancel_reason": { + "name": "cancel_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolution": { + "name": "resolution", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "observed_outcome": { + "name": "observed_outcome", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "investigate_on_attention": { + "name": "investigate_on_attention", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_ref": { + "name": "project_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_attempted_at": { + "name": "last_attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "fired_at": { + "name": "fired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "delivery_claimed_at": { + "name": "delivery_claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "delivery_claim_id": { + "name": "delivery_claim_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_result": { + "name": "last_result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "tick_count": { + "name": "tick_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "alert_dispatch_key": { + "name": "alert_dispatch_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "retention_at": { + "name": "retention_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "greatest(delivered_at, cancelled_at, fired_at, last_checked_at, created_at)", + "type": "stored" + } + }, + "cadence_minutes": { + "name": "cadence_minutes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "((spec ->> 'checkEveryMinutes')::int)", + "type": "stored" + } + } + }, + "indexes": { + "watches_chat_idx": { + "name": "watches_chat_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "watches_chat_active_identity_key": { + "name": "watches_chat_active_identity_key", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "identity", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"trigger_dashboard_agent\".\"watches\".\"status\" = 'active'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "watches_status_expires_idx": { + "name": "watches_status_expires_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "watches_pending_delivery_idx": { + "name": "watches_pending_delivery_idx", + "columns": [ + { + "expression": "fired_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"watches\".\"delivery_status\" in ('pending', 'delivering')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "watches_org_user_wake_idx": { + "name": "watches_org_user_wake_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"fired_at\", \"last_checked_at\") desc", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"watches\".\"delivery_status\" = 'delivered' and \"trigger_dashboard_agent\".\"watches\".\"status\" in ('fired', 'expired')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "watches_org_user_active_idx": { + "name": "watches_org_user_active_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"watches\".\"status\" = 'active'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "watches_active_env_cadence_idx": { + "name": "watches_active_env_cadence_idx", + "columns": [ + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cadence_minutes", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"last_attempted_at\", \"last_checked_at\", \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"watches\".\"status\" = 'active'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "watches_env_cadence_delivery_idx": { + "name": "watches_env_cadence_delivery_idx", + "columns": [ + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cadence_minutes", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivery_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"fired_at\", \"last_checked_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"watches\".\"status\" in ('fired', 'expired') and \"trigger_dashboard_agent\".\"watches\".\"delivery_status\" in ('pending', 'delivering')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "watches_retention_idx": { + "name": "watches_retention_idx", + "columns": [ + { + "expression": "retention_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"trigger_dashboard_agent\".\"watches\".\"status\" in ('fired', 'expired', 'cancelled') and \"trigger_dashboard_agent\".\"watches\".\"delivery_status\" in ('not_required', 'delivered')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "trigger_dashboard_agent": "trigger_dashboard_agent" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/internal-packages/dashboard-agent-db/drizzle/meta/_journal.json b/internal-packages/dashboard-agent-db/drizzle/meta/_journal.json index 1f33e4ddf8f..64c3fadddaa 100644 --- a/internal-packages/dashboard-agent-db/drizzle/meta/_journal.json +++ b/internal-packages/dashboard-agent-db/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1786359241538, "tag": "0004_stale_corsair", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1786376934874, + "tag": "0005_ambitious_mordo", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/internal-packages/dashboard-agent-db/src/queries.ts b/internal-packages/dashboard-agent-db/src/queries.ts index cd964f053f4..37e90a9a6c8 100644 --- a/internal-packages/dashboard-agent-db/src/queries.ts +++ b/internal-packages/dashboard-agent-db/src/queries.ts @@ -1057,6 +1057,10 @@ export async function listChatIdsWithOpenInvestigations( /** * Sweep for investigations nothing else settles. `olderThan` is on `updated_at`, * which every revision bumps, so a card a live turn is writing to stays out. + * + * Order is `last_sweep_attempt_at` nulls first, then `updated_at`: a never-attempted + * row is always seen before one a prior sweep already failed on, so a row that can't + * settle rotates to the back instead of pinning the head and starving newer rows. */ export async function listStaleOpenInvestigations( db: DashboardAgentDb, @@ -1074,12 +1078,39 @@ export async function listStaleOpenInvestigations( sql`${investigations.updatedAt} <= ${params.olderThan.toISOString()}::timestamptz` ) ) - .orderBy(investigations.updatedAt) + .orderBy(sql`${investigations.lastSweepAttemptAt} asc nulls first`, investigations.updatedAt) .limit(params.limit ?? 100); return rows.map((row) => row.investigation); } +/** + * Record a failed stale-sweep settle on its own, committed outside the settle tx that + * rolled back. Bumps the attempt count and stamps `last_sweep_attempt_at` — which does + * NOT touch `updated_at`, so the row still reads as stale, only later in the order. + * Returns the new count, or null when the row is no longer `in_progress`. + */ +export async function recordInvestigationSweepAttempt( + db: DashboardAgentDbOrTx, + params: { id: string } +): Promise { + const rows = await db + .update(investigations) + .set({ + sweepAttempts: sql`${investigations.sweepAttempts} + 1`, + lastSweepAttemptAt: sql`now()`, + }) + .where( + and( + eq(investigations.id, params.id), + sql`${investigations.state}->>'outcome' = 'in_progress'` + ) + ) + .returning({ sweepAttempts: investigations.sweepAttempts }); + + return rows[0]?.sweepAttempts ?? null; +} + /** What the settle wrote, which is what the closing card has to render. */ export type SettledInvestigation = { id: string; revision: number; state: unknown }; diff --git a/internal-packages/dashboard-agent-db/src/schema.ts b/internal-packages/dashboard-agent-db/src/schema.ts index 73dafc64b0c..d080d759915 100644 --- a/internal-packages/dashboard-agent-db/src/schema.ts +++ b/internal-packages/dashboard-agent-db/src/schema.ts @@ -170,6 +170,10 @@ export const investigations = dashboardAgentSchema.table( // Monotonic; bumped by a single atomic UPDATE. revision: integer("revision").notNull().default(0), state: jsonb("state").$type().notNull(), + // Failed stale-sweep settle attempts. Bumped outside the rolled-back settle tx so a + // row that can't render rotates to the back of the sweep order instead of pinning it. + sweepAttempts: integer("sweep_attempts").notNull().default(0), + lastSweepAttemptAt: timestamp("last_sweep_attempt_at", { withTimezone: true }), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }, From bce03cb2c7247442720d83b6fd266dc9fd61311d Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Mon, 10 Aug 2026 19:01:07 +0000 Subject: [PATCH 3/6] fix(webapp): map a watch plan-limit refusal to 409, not 500 The card-submit route's status ladder didn't handle watch_limit_reached, so a plan-limit refusal fell through to HTTP 500. Match the MCP route and return 409. --- ...jectParam.env.$envParam.dashboard-agent.ts | 1 + .../dashboardAgentWatchLimitStatus.test.ts | 161 ++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 apps/webapp/test/dashboardAgentWatchLimitStatus.test.ts diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts index c01d9cc2d42..d69133f398e 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts @@ -541,6 +541,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.ok) { const status = result.code === "limit_reached" || + result.code === "watch_limit_reached" || result.code === "duplicate" || result.code === "request_conflict" ? 409 diff --git a/apps/webapp/test/dashboardAgentWatchLimitStatus.test.ts b/apps/webapp/test/dashboardAgentWatchLimitStatus.test.ts new file mode 100644 index 00000000000..3f34737c7cd --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchLimitStatus.test.ts @@ -0,0 +1,161 @@ +import { + createDashboardAgentDb, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, vi } from "vitest"; +import type * as WatchLimitsModule from "~/services/dashboardAgentWatchLimits.server"; + +// A plan-limit refusal (`watch_limit_reached`) is a 409, not a 500. The card submit's status +// ladder must map it the same way the MCP route does, or a full org sees an "unexpected error". + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + userId: "", +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/session.server", () => ({ + requireUser: async () => ({ id: ctx.userId, admin: false, isImpersonating: false }), +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +// The only stub: the plan floor billing would resolve. A 1-hour window makes a 2-hour watch +// exceed the plan, so the real submit path returns `watch_limit_reached`. Everything else runs. +vi.mock("~/services/dashboardAgentWatchLimits.server", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveWatchPlanLimits: async () => ({ + maxHours: 1, + watchers: actual.UNLIMITED_WATCH_LIMIT, + }), + }; +}); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-limit-status"; + +const { action } = + await import("~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent"); + +/** Replays every migration in order, so a new migration can't leave the suite on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function seed(prisma: PrismaClient) { + const slug = `limit_status_${Math.random().toString(36).slice(2, 10)}`; + const user = await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `p${slug.slice(0, 6)}`, + }, + }); + ctx.userId = user.id; + return { user, organization, project }; +} + +// error_recurrence resolves its target with no run/queue read, so the plan floor is the only +// thing standing between a valid submit and a created watch. +const DRAFT = JSON.stringify({ + spec: { + kind: "error_recurrence", + fingerprint: "a1b2c3", + checkEveryMinutes: 5, + maxHours: 2, + note: "ping me if it happens again", + }, + followUp: { investigateOnAttention: false, notifyExternally: false }, +}); + +function submitRequest(slug: string, body: Record) { + const form = new URLSearchParams(body); + return action({ + request: new Request( + `https://app.trigger.dev/resources/orgs/${slug}/projects/${slug}/env/prod/dashboard-agent`, + { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: form.toString(), + } + ), + params: { organizationSlug: slug, projectParam: slug, envParam: "prod" }, + context: {}, + } as never) as Promise; +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +describe("the watch card submit's status for a plan-limit refusal", () => { + postgresTest( + "answers 409, not 500, when the window is longer than the plan allows", + async ({ prisma, postgresContainer }) => { + ctx.prisma = prisma; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(postgresContainer.getConnectionUri(), { max: 4 }); + ctx.agentDb = agentDbClient.db; + + const seeded = await seed(prisma); + + const response = await submitRequest(seeded.organization.slug, { + intent: "watch-create", + draft: DRAFT, + clientRequestId: "wreq_limit_1", + }); + + expect(response.status).toBe(409); + expect(await response.json()).toMatchObject({ code: "watch_limit_reached" }); + }, + 30_000 + ); +}); From 2db7c39567f39ce5565638989d6d79d15afa07b0 Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Mon, 10 Aug 2026 19:01:16 +0000 Subject: [PATCH 4/6] style(dashboard-agent-db): oxfmt the drizzle meta files drizzle-kit generated them unformatted, failing the oxfmt --check code-quality gate. --- .../drizzle/meta/0005_snapshot.json | 32 ++++--------------- .../drizzle/meta/_journal.json | 2 +- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/internal-packages/dashboard-agent-db/drizzle/meta/0005_snapshot.json b/internal-packages/dashboard-agent-db/drizzle/meta/0005_snapshot.json index e3a9a8082a4..b00ae150c57 100644 --- a/internal-packages/dashboard-agent-db/drizzle/meta/0005_snapshot.json +++ b/internal-packages/dashboard-agent-db/drizzle/meta/0005_snapshot.json @@ -47,10 +47,7 @@ "compositePrimaryKeys": { "agent_message_usage_organization_id_period_pk": { "name": "agent_message_usage_organization_id_period_pk", - "columns": [ - "organization_id", - "period" - ] + "columns": ["organization_id", "period"] } }, "uniqueConstraints": {}, @@ -128,20 +125,14 @@ "compositePrimaryKeys": { "chat_messages_chat_id_message_id_pk": { "name": "chat_messages_chat_id_message_id_pk", - "columns": [ - "chat_id", - "message_id" - ] + "columns": ["chat_id", "message_id"] } }, "uniqueConstraints": { "chat_messages_chat_position_key": { "name": "chat_messages_chat_position_key", "nullsNotDistinct": false, - "columns": [ - "chat_id", - "position" - ] + "columns": ["chat_id", "position"] } }, "policies": {}, @@ -462,10 +453,7 @@ "compositePrimaryKeys": { "chat_turn_evals_chat_id_turn_pk": { "name": "chat_turn_evals_chat_id_turn_pk", - "columns": [ - "chat_id", - "turn" - ] + "columns": ["chat_id", "turn"] } }, "uniqueConstraints": {}, @@ -764,10 +752,7 @@ "compositePrimaryKeys": { "watch_batches_environment_id_cadence_minutes_pk": { "name": "watch_batches_environment_id_cadence_minutes_pk", - "columns": [ - "environment_id", - "cadence_minutes" - ] + "columns": ["environment_id", "cadence_minutes"] } }, "uniqueConstraints": {}, @@ -920,10 +905,7 @@ "compositePrimaryKeys": { "watch_submissions_chat_id_client_request_id_pk": { "name": "watch_submissions_chat_id_client_request_id_pk", - "columns": [ - "chat_id", - "client_request_id" - ] + "columns": ["chat_id", "client_request_id"] } }, "uniqueConstraints": {}, @@ -1372,4 +1354,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/internal-packages/dashboard-agent-db/drizzle/meta/_journal.json b/internal-packages/dashboard-agent-db/drizzle/meta/_journal.json index 64c3fadddaa..213320fa640 100644 --- a/internal-packages/dashboard-agent-db/drizzle/meta/_journal.json +++ b/internal-packages/dashboard-agent-db/drizzle/meta/_journal.json @@ -45,4 +45,4 @@ "breakpoints": true } ] -} \ No newline at end of file +} From f695267560ed9f72e07480d3ab3ce2d5fdd8ba1a Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Tue, 11 Aug 2026 09:25:22 +0000 Subject: [PATCH 5/6] fix(webapp): hoist a type-only import so oxlint stops failing --- apps/webapp/test/dashboardAgentWatchLimits.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/webapp/test/dashboardAgentWatchLimits.test.ts b/apps/webapp/test/dashboardAgentWatchLimits.test.ts index 95d499558e7..a00a3207eed 100644 --- a/apps/webapp/test/dashboardAgentWatchLimits.test.ts +++ b/apps/webapp/test/dashboardAgentWatchLimits.test.ts @@ -7,6 +7,7 @@ import { type DashboardAgentDbClient, } from "@internal/dashboard-agent-db"; import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import type * as TriggerSdk from "@trigger.dev/sdk"; import { postgresTest } from "@internal/testcontainers"; import type { PrismaClient } from "@trigger.dev/database"; import { readdirSync, readFileSync } from "node:fs"; @@ -37,7 +38,7 @@ vi.mock("~/services/dashboardAgentDb.server", () => ({ })); vi.mock("@trigger.dev/sdk", async (importOriginal) => { - const actual = await importOriginal(); + const actual = await importOriginal(); return { ...actual, TriggerClient: class { From 3fd5cf42449e65cb38cf903fb45e7caa1dcef680 Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Tue, 11 Aug 2026 10:55:25 +0000 Subject: [PATCH 6/6] chore(server-changes): consolidate the watch-limits notes into one --- .../dashboard-agent-investigation-sweep-backoff.md | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .server-changes/dashboard-agent-investigation-sweep-backoff.md diff --git a/.server-changes/dashboard-agent-investigation-sweep-backoff.md b/.server-changes/dashboard-agent-investigation-sweep-backoff.md deleted file mode 100644 index b49c7881cb4..00000000000 --- a/.server-changes/dashboard-agent-investigation-sweep-backoff.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -A single stuck assistant investigation can no longer hold up others from being tidied away.