diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index b80dda73fab..f2853288477 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -9997,7 +9997,7 @@ "description": "Whether a paused execution was cancelled." }, "reason": { - "description": "Machine-readable cancellation outcome, present on every cancellation including full successes. `recorded` is the success value. `already_cancelled`, `already_completed`, and `already_failed` mean the run had already reached that terminal state, so nothing was cancelled and `durablyRecorded` is false. `redis_unavailable` and `redis_write_failed` mean the distributed cancellation signal was not written, so an already-running execution may not observe the cancellation. `paused_event_publish_failed` and `paused_database_cancel_failed` name the failing step for a paused run.", + "description": "Machine-readable cancellation outcome, present on every cancellation including full successes. `recorded` and `queue_cancelled` are successful cancellation values. `already_cancelled`, `already_completed`, and `already_failed` mean the run had already reached that terminal state, so nothing was cancelled and `durablyRecorded` is false. The remaining values identify a degraded or incomplete cancellation step.", "type": "string", "enum": [ "recorded", @@ -10007,7 +10007,10 @@ "redis_unavailable", "redis_write_failed", "paused_event_publish_failed", - "paused_database_cancel_failed" + "paused_database_cancel_failed", + "queue_cancelled", + "active_resume_signal_failed", + "cancellation_not_finalized" ] } }, diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.test.ts b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.test.ts index 982c69d0ee2..8de3f2cb24e 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.test.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.test.ts @@ -11,15 +11,14 @@ import { } from '@sim/testing' import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { WorkflowRunAlreadyTerminalError } from '@/lib/execution/workflow-run-already-terminal-error' const mocks = vi.hoisted(() => ({ cancel: vi.fn(), - capture: vi.fn(), })) vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) -vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.capture })) vi.mock('@/lib/workflows/application/cancel-run', () => ({ cancelWorkflowRun: { operation: { id: 'workflows.runs.cancel' }, execute: mocks.cancel }, })) @@ -91,18 +90,10 @@ describe('POST /api/v2/workflows/[workflowId]/runs/[runId]/cancel', () => { }) }) - /** - * The published outcome of a cancel against a run that had already finished. - * `durablyRecorded: true` here is the defect this suite pins: nothing was - * written, so a caller reconciling on that flag would trust a write that never - * happened. - */ - it.each([ - ['cancelled', 'already_cancelled'], - ['completed', 'already_completed'], - ['failed', 'already_failed'], - ])('reports a terminal %s run as a no-op the caller can tell apart', async (_status, reason) => { - mocks.cancel.mockResolvedValue(serviceResult({ success: true, durablyRecorded: false, reason })) + it('reports an already-cancelled run as an idempotent no-op', async () => { + mocks.cancel.mockResolvedValue( + serviceResult({ success: true, durablyRecorded: false, reason: 'already_cancelled' }) + ) const response = await POST(request(), context) @@ -111,7 +102,39 @@ describe('POST /api/v2/workflows/[workflowId]/runs/[runId]/cancel', () => { success: true, runId: RUN_ID, durablyRecorded: false, - reason, + reason: 'already_cancelled', }) }) + + it.each([ + ['completed', 'already_completed'], + ['failed', 'already_failed'], + ] as const)( + 'preserves the v2 terminal no-op response when a standalone run is already %s', + async (executionStatus, reason) => { + mocks.cancel.mockRejectedValue( + new WorkflowRunAlreadyTerminalError({ + executionId: RUN_ID, + executionStatus, + redisAvailable: true, + locallyAborted: false, + }) + ) + + const response = await POST(request(), context) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + data: { + success: true, + runId: RUN_ID, + redisAvailable: true, + durablyRecorded: false, + locallyAborted: false, + pausedCancelled: false, + reason, + }, + }) + } + ) }) diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.ts b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.ts index ba04e01740b..c65e0c0d693 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.ts @@ -1,6 +1,5 @@ import { v2CancelWorkflowRunContract } from '@/lib/api/contracts/v2/workflows' import { defineV2JsonRoute, v2ApiKeyAuth, v2RateLimits } from '@/lib/api/server/routes' -import { captureServerEvent } from '@/lib/posthog/server' import { v2WorkflowErrorPolicies } from '@/lib/workflows/api' import { cancelWorkflowRun } from '@/lib/workflows/application/cancel-run' import { workflowOperations } from '@/lib/workflows/application/operations' @@ -13,7 +12,7 @@ export const POST = defineV2JsonRoute({ auth: v2ApiKeyAuth, operation: workflowOperations.cancelRun, rateLimit: v2RateLimits.publicApi, - errorPolicy: v2WorkflowErrorPolicies.concealRunAuthorization, + errorPolicy: v2WorkflowErrorPolicies.cancelRun, mapInput: ({ params }) => ({ workflowId: params.workflowId, runId: params.runId }), useCase: cancelWorkflowRun, present: (result) => ({ @@ -27,21 +26,4 @@ export const POST = defineV2JsonRoute({ reason: result.reason, }, }), - /** - * Reports a cancellation, so it needs the run to have actually been - * cancelled. `success` alone no longer implies that: a cancel against an - * already-terminal run satisfies the request without writing anything, and - * reports `success: true` with `durablyRecorded: false`. Requiring both also - * keeps the event off a cancellation that reached the row but failed its - * paused reconciliation, which reports the inverse pair. - */ - onSuccess: ({ principal, result }) => { - if (!result.success || !result.durablyRecorded || principal.kind !== 'personal_api_key') return - captureServerEvent( - principal.userId, - 'workflow_execution_cancelled', - { workflow_id: result.workflowId, workspace_id: result.workspaceId }, - { groups: { workspace: result.workspaceId } } - ) - }, }) diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/route.test.ts b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/route.test.ts index e289efc3283..60a093868c9 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/route.test.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/route.test.ts @@ -14,7 +14,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ cancel: vi.fn(), - capture: vi.fn(), readRun: vi.fn(), authorizeReadRun: vi.fn(), })) @@ -22,8 +21,6 @@ const mocks = vi.hoisted(() => ({ vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) -vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.capture })) - vi.mock('@/lib/workflows/application/read-workflow-run', () => ({ readWorkflowRun: { operation: { id: 'workflows.runs.read' }, @@ -370,7 +367,6 @@ describe('v2 run detail and cancel adapters', () => { 'v2:workflows.runs.cancel:api-key:key-1', expect.anything() ) - expect(mocks.capture).not.toHaveBeenCalled() }) it('keeps cancellation request-rate admission separate from run control', async () => { @@ -399,13 +395,17 @@ describe('v2 run detail and cancel adapters', () => { code: 'FORBIDDEN', message: 'Insufficient workspace permissions', }) - expect(mocks.capture).not.toHaveBeenCalled() }) - it('projects cancellation analytics only after a successful personal-key result', async () => { + it('passes a personal-key principal to the cancellation use case', async () => { + const personalPrincipal = { + kind: 'personal_api_key' as const, + userId: 'key-user', + keyId: 'personal-key', + } v2RouteMocks.authenticate.mockResolvedValueOnce({ ...auth, - principal: { kind: 'personal_api_key', userId: 'key-user', keyId: 'personal-key' }, + principal: personalPrincipal, rateLimitSubjectIds: ['api-key:personal-key', 'user:key-user'], keyType: 'personal', }) @@ -415,12 +415,10 @@ describe('v2 run detail and cancel adapters', () => { }) expect(response.status).toBe(200) - expect(mocks.capture).toHaveBeenCalledOnce() - expect(mocks.capture).toHaveBeenCalledWith( - 'key-user', - 'workflow_execution_cancelled', - { workflow_id: 'workflow-1', workspace_id: 'workspace-1' }, - { groups: { workspace: 'workspace-1' } } - ) + expect(mocks.cancel).toHaveBeenCalledWith({ + principal: personalPrincipal, + input: { workflowId: 'workflow-1', runId: 'run-1' }, + request: expect.anything(), + }) }) }) diff --git a/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.test.ts b/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.test.ts index 21c2fa13e9b..7208c0129bc 100644 --- a/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.test.ts +++ b/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.test.ts @@ -1,1381 +1,101 @@ /** * @vitest-environment node */ - -import { - databaseMock, - dbChainMockFns, - hybridAuthMockFns, - posthogServerMock, - resetDbChainMock, - workflowAuthzMockFns, - workflowsUtilsMock, -} from '@sim/testing' import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { WorkflowRunAlreadyTerminalError } from '@/lib/execution/workflow-run-already-terminal-error' -const { - mockMarkExecutionCancelled, - mockClearExecutionCancellation, - mockAbortManualExecution, - mockBeginPausedCancellation, - mockStagePausedCancellation, - mockBlockQueuedResumesForCancellation, - mockClearPausedCancellationIntent, - mockCompletePausedCancellation, - mockGetPausedCancellationStatus, - mockGetActiveResumeCancellationTarget, - mockRollbackActiveResumeCancellation, - mockFinalizeExecutionStream, - mockReadExecutionMetaState, - mockWriteEvent, - mockWriteTerminalEvent, - mockCancelByExecution, - mockGetJobQueue, - mockReleaseExecutionSlot, - mockCancelWorkflowGroupExecution, - mockPublishWorkflowGroupCancellationEvent, -} = vi.hoisted(() => ({ - mockMarkExecutionCancelled: vi.fn(), - mockClearExecutionCancellation: vi.fn(), - mockAbortManualExecution: vi.fn(), - mockBeginPausedCancellation: vi.fn(), - mockStagePausedCancellation: vi.fn(), - mockBlockQueuedResumesForCancellation: vi.fn(), - mockClearPausedCancellationIntent: vi.fn(), - mockCompletePausedCancellation: vi.fn(), - mockGetPausedCancellationStatus: vi.fn(), - mockGetActiveResumeCancellationTarget: vi.fn(), - mockRollbackActiveResumeCancellation: vi.fn(), - mockFinalizeExecutionStream: vi.fn(), - mockReadExecutionMetaState: vi.fn(), - mockWriteEvent: vi.fn(), - mockWriteTerminalEvent: vi.fn(), - mockCancelByExecution: vi.fn(), - mockGetJobQueue: vi.fn(), - mockReleaseExecutionSlot: vi.fn(), - mockCancelWorkflowGroupExecution: vi.fn(), - mockPublishWorkflowGroupCancellationEvent: vi.fn(), -})) - -vi.mock('@/lib/core/async-jobs', () => ({ - getJobQueue: mockGetJobQueue, -})) - -vi.mock('@/lib/billing/calculations/usage-reservation', () => ({ - releaseExecutionSlot: mockReleaseExecutionSlot, -})) - -vi.mock('@/lib/execution/cancellation', () => ({ - markExecutionCancelled: (...args: unknown[]) => mockMarkExecutionCancelled(...args), - clearExecutionCancellation: (...args: unknown[]) => mockClearExecutionCancellation(...args), +const mocks = vi.hoisted(() => ({ + cancel: vi.fn(), + getSession: vi.fn(), })) -vi.mock('@/lib/execution/manual-cancellation', () => ({ - abortManualExecution: (...args: unknown[]) => mockAbortManualExecution(...args), -})) - -vi.mock('@/lib/workflows/executor/human-in-the-loop-manager', () => ({ - PauseResumeManager: { - beginPausedCancellation: (...args: unknown[]) => mockBeginPausedCancellation(...args), - stagePausedCancellation: (...args: unknown[]) => mockStagePausedCancellation(...args), - blockQueuedResumesForCancellation: (...args: unknown[]) => - mockBlockQueuedResumesForCancellation(...args), - clearPausedCancellationIntent: (...args: unknown[]) => - mockClearPausedCancellationIntent(...args), - completePausedCancellation: (...args: unknown[]) => mockCompletePausedCancellation(...args), - getPausedCancellationStatus: (...args: unknown[]) => mockGetPausedCancellationStatus(...args), - getActiveResumeCancellationTarget: (...args: unknown[]) => - mockGetActiveResumeCancellationTarget(...args), - rollbackActiveResumeCancellation: (...args: unknown[]) => - mockRollbackActiveResumeCancellation(...args), - }, -})) - -vi.mock('@/lib/workflows/utils', () => workflowsUtilsMock) - -vi.mock('@/lib/posthog/server', () => posthogServerMock) - -vi.mock('@/lib/table/workflow-group-cancellation', () => ({ - cancelWorkflowGroupExecution: (...args: unknown[]) => mockCancelWorkflowGroupExecution(...args), - publishWorkflowGroupCancellationEvent: (...args: unknown[]) => - mockPublishWorkflowGroupCancellationEvent(...args), -})) +vi.mock('@/lib/auth', () => ({ getSession: mocks.getSession })) -vi.mock('@/lib/execution/event-buffer', () => ({ - finalizeExecutionStream: (...args: unknown[]) => mockFinalizeExecutionStream(...args), - readExecutionMetaState: (...args: unknown[]) => mockReadExecutionMetaState(...args), - createExecutionEventWriter: () => ({ - write: (...args: unknown[]) => mockWriteEvent(...args), - writeTerminal: (...args: unknown[]) => mockWriteTerminalEvent(...args), - close: vi.fn().mockResolvedValue(undefined), - }), +vi.mock('@/lib/workflows/application/cancel-run', () => ({ + cancelWorkflowRun: { operation: { id: 'workflows.runs.cancel' }, execute: mocks.cancel }, })) -import { cancelWorkflowExecutionContract } from '@/lib/api/contracts/workflows' -import { POST as cancelExecution } from './route' +import { POST } from '@/app/api/workflows/[id]/executions/[executionId]/cancel/route' -/** - * Drives the route and validates every success body against the contract that - * `requestJson` enforces on the client. The route builds its responses by hand - * rather than through a declarative builder, so nothing else checks that the - * two agree — and a `reason` the contract omits makes the client throw on a - * cancellation that actually applied. Routing every case in this suite through - * here covers each `NextResponse.json` shape the route can return. - */ -const POST = async (...args: Parameters) => { - const response = await cancelExecution(...args) - if (response.status < 400) { - cancelWorkflowExecutionContract.response.schema.parse(await response.clone().json()) - } - return response -} +const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } +const context = { params: Promise.resolve({ id: 'workflow-1', executionId: 'execution-1' }) } -const makeRequest = () => - new NextRequest('http://localhost/api/workflows/wf-1/executions/ex-1/cancel', { - method: 'POST', - }) - -const makeParams = () => ({ params: Promise.resolve({ id: 'wf-1', executionId: 'ex-1' }) }) - -const ACTIVE_RESUME_TARGET = { - resumeEntryId: 'resume-entry-1', - pausedExecutionId: 'paused-1', - parentExecutionId: 'ex-1', - resumeExecutionId: 'resume-ex-1', +function request() { + return new NextRequest( + 'http://localhost/api/workflows/workflow-1/executions/execution-1/cancel', + { method: 'POST' } + ) } describe('POST /api/workflows/[id]/executions/[executionId]/cancel', () => { beforeEach(() => { vi.clearAllMocks() - resetDbChainMock() - hybridAuthMockFns.mockCheckHybridAuth.mockResolvedValue({ success: true, userId: 'user-1' }) - workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValue({ - allowed: true, - workflow: { workspaceId: 'workspace-1' }, - }) - dbChainMockFns.limit.mockResolvedValue([ - { - executionDeadlineAt: null, - executionOrigin: null, - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - dbChainMockFns.returning.mockResolvedValue([{ status: 'cancelled' }]) - mockCancelByExecution.mockReset().mockResolvedValue(0) - mockGetJobQueue.mockReset().mockResolvedValue({ cancelByExecution: mockCancelByExecution }) - mockReleaseExecutionSlot.mockReset().mockResolvedValue(undefined) - mockCancelWorkflowGroupExecution.mockReset().mockResolvedValue({ kind: 'not_workflow_group' }) - mockPublishWorkflowGroupCancellationEvent.mockReset().mockResolvedValue(undefined) - mockClearExecutionCancellation.mockReset().mockResolvedValue(undefined) - mockMarkExecutionCancelled - .mockReset() - .mockResolvedValue({ durablyRecorded: false, reason: 'redis_unavailable' }) - mockAbortManualExecution.mockReset().mockReturnValue(false) - mockBeginPausedCancellation.mockReset().mockResolvedValue(false) - mockStagePausedCancellation.mockReset().mockResolvedValue({ kind: 'not_paused' }) - mockBlockQueuedResumesForCancellation.mockReset().mockResolvedValue(false) - mockClearPausedCancellationIntent.mockReset().mockResolvedValue(undefined) - mockCompletePausedCancellation.mockReset().mockResolvedValue(false) - mockGetPausedCancellationStatus.mockReset().mockResolvedValue(null) - mockGetActiveResumeCancellationTarget.mockReset().mockResolvedValue(null) - mockRollbackActiveResumeCancellation.mockReset().mockResolvedValue(true) - mockFinalizeExecutionStream.mockReset().mockResolvedValue(true) - mockReadExecutionMetaState.mockReset().mockResolvedValue({ status: 'missing' }) - mockWriteEvent.mockReset().mockResolvedValue({ eventId: 1 }) - mockWriteTerminalEvent.mockReset().mockResolvedValue({ eventId: 1 }) - }) - - it('returns success when cancellation was durably recorded', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: true, - reason: 'recorded', + mocks.getSession.mockResolvedValue({ + user: { id: principal.userId }, + session: { id: principal.sessionId }, }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ + mocks.cancel.mockResolvedValue({ success: true, - executionId: 'ex-1', + executionId: 'execution-1', redisAvailable: true, durablyRecorded: true, locallyAborted: false, pausedCancelled: false, reason: 'recorded', - }) - expect(mockCancelByExecution).toHaveBeenCalledWith( - { - workflowId: 'wf-1', - executionId: 'ex-1', - }, - 'standalone' - ) - expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1', { - executionDeadlineAt: null, - }) - expect(mockClearExecutionCancellation).not.toHaveBeenCalled() - }) - - it('atomically claims one workflow-group attempt before signalling it', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: true, - reason: 'recorded', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - executionId: 'ex-1', - redisAvailable: true, - durablyRecorded: true, - reason: 'recorded', - }) - expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledWith({ + workflowId: 'workflow-1', workspaceId: 'workspace-1', - workflowId: 'wf-1', - executionId: 'ex-1', }) - expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( - mockMarkExecutionCancelled.mock.invocationCallOrder[0] - ) - expect(mockMarkExecutionCancelled.mock.invocationCallOrder[0]).toBeLessThan( - mockPublishWorkflowGroupCancellationEvent.mock.invocationCallOrder[0] - ) - expect(mockCancelByExecution).not.toHaveBeenCalled() - expect(mockStagePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') - expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() }) - it('keeps a claimed group cancellation retryable when signalling fails', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - - const response = await POST(makeRequest(), makeParams()) + it('passes the authenticated principal and canonical run input to the application use case', async () => { + const response = await POST(request(), context) expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: false, - durablyRecorded: false, - reason: 'redis_unavailable', - }) - expect(mockMarkExecutionCancelled).toHaveBeenCalledOnce() - expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() - expect(mockCancelByExecution).not.toHaveBeenCalled() - }) - - it('accepts an exact in-process group abort without cancelling its carrier', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - mockAbortManualExecution.mockReturnValue(true) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - durablyRecorded: false, - locallyAborted: true, - reason: 'redis_unavailable', - }) - expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() - expect(mockCancelByExecution).not.toHaveBeenCalled() - }) - - it('does not cancel a shared carrier when an idle workflow-group pause appears', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockStagePausedCancellation - .mockResolvedValueOnce({ kind: 'not_paused' }) - .mockResolvedValue({ kind: 'idle' }) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - pausedCancelled: true, - reason: 'recorded', - }) - expect(mockCancelByExecution).not.toHaveBeenCalled() - expect(mockStagePausedCancellation.mock.invocationCallOrder[1]).toBeLessThan( - mockWriteTerminalEvent.mock.invocationCallOrder[0] - ) - expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') - }) - - it('leaves a late workflow-group pause retryable when event publication fails', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockStagePausedCancellation - .mockResolvedValueOnce({ kind: 'not_paused' }) - .mockResolvedValue({ kind: 'idle' }) - mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: false, - pausedCancelled: false, - reason: 'paused_event_publish_failed', - }) - expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() - expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( - mockWriteTerminalEvent.mock.invocationCallOrder[0] - ) - expect(mockCompletePausedCancellation).not.toHaveBeenCalled() - expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() - }) - - it('cancels an active workflow-group resume without cancelling its shared carrier', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockStagePausedCancellation.mockResolvedValue({ - kind: 'active_resume', - target: ACTIVE_RESUME_TARGET, - }) - mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - mockCancelByExecution.mockResolvedValue(1) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - pausedCancelled: true, - reason: 'recorded', - }) - expect(mockCancelByExecution).toHaveBeenCalledOnce() - expect(mockCancelByExecution).toHaveBeenCalledWith( - { workflowId: 'wf-1', executionId: 'ex-1' }, - 'resume' - ) - expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('resume-ex-1', { - executionDeadlineAt: null, - }) - expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() - expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( - mockMarkExecutionCancelled.mock.invocationCallOrder[0] - ) - expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') - expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( - mockCompletePausedCancellation.mock.invocationCallOrder[0] - ) - }) - - it('keeps a claimed active group resume retryable when no stop backend accepts it', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockStagePausedCancellation.mockResolvedValue({ - kind: 'active_resume', - target: ACTIVE_RESUME_TARGET, - }) - mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: false, - durablyRecorded: true, - locallyAborted: false, - pausedCancelled: false, - reason: 'active_resume_signal_failed', - }) - expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('resume-ex-1', { - executionDeadlineAt: null, - }) - expect(mockCancelByExecution).toHaveBeenCalledWith( - { workflowId: 'wf-1', executionId: 'ex-1' }, - 'resume' - ) - expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( - mockMarkExecutionCancelled.mock.invocationCallOrder[0] - ) - expect(mockRollbackActiveResumeCancellation).not.toHaveBeenCalled() - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - }) - - it('rechecks for a pause after the group terminal claim waits on persistence', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockStagePausedCancellation - .mockResolvedValueOnce({ kind: 'not_paused' }) - .mockResolvedValue({ kind: 'idle' }) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - pausedCancelled: true, - reason: 'recorded', - }) - expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( - mockStagePausedCancellation.mock.invocationCallOrder[1] - ) - expect(mockStagePausedCancellation.mock.invocationCallOrder[1]).toBeLessThan( - mockWriteTerminalEvent.mock.invocationCallOrder[0] - ) - expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( - mockCompletePausedCancellation.mock.invocationCallOrder[0] - ) - expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() - }) - - it('rechecks for a regular pause after the terminal claim waits on persistence', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - mockStagePausedCancellation - .mockResolvedValueOnce({ kind: 'not_paused' }) - .mockResolvedValue({ kind: 'idle' }) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - pausedCancelled: true, - reason: 'recorded', - }) - expect(databaseMock.db.update.mock.invocationCallOrder[0]).toBeLessThan( - mockStagePausedCancellation.mock.invocationCallOrder[1] - ) - expect(mockStagePausedCancellation.mock.invocationCallOrder[1]).toBeLessThan( - mockWriteTerminalEvent.mock.invocationCallOrder[0] - ) - expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( - mockCompletePausedCancellation.mock.invocationCallOrder[0] - ) - }) - - it('uses generic cancellation for a regular Table-trigger-block execution', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: null, - status: 'running', - trigger: 'table', - workspaceId: 'workspace-1', - }, - ]) - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: true, - reason: 'recorded', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - expect(mockCancelWorkflowGroupExecution).not.toHaveBeenCalled() - expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1', { - executionDeadlineAt: null, - }) - expect(mockCancelByExecution).toHaveBeenCalledOnce() - }) - - it('returns 409 without generic cancellation when the exact group attempt is stale', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'conflict', - status: 'no_longer_active', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(409) await expect(response.json()).resolves.toEqual({ - error: 'Workflow group execution cannot be cancelled while no_longer_active', - }) - expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - expect(mockCancelByExecution).not.toHaveBeenCalled() - expect(mockStagePausedCancellation).toHaveBeenCalledOnce() - expect(mockClearExecutionCancellation).not.toHaveBeenCalled() - expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() - }) - - it('terminalizes a durable workflow-group log after its table sidecar was deleted', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ kind: 'cancelled_without_sidecar' }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ success: true, - executionId: 'ex-1', - reason: 'recorded', - }) - expect(mockMarkExecutionCancelled).toHaveBeenCalledOnce() - expect(mockCancelByExecution).not.toHaveBeenCalled() - expect(mockClearExecutionCancellation).not.toHaveBeenCalled() - expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() - }) - - it('returns unsuccessful response when Redis is unavailable', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - success: false, - executionId: 'ex-1', - redisAvailable: false, - durablyRecorded: false, - locallyAborted: false, - pausedCancelled: false, - reason: 'redis_unavailable', - }) - }) - - it('returns unsuccessful response when Redis persistence fails', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_write_failed', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - success: false, - executionId: 'ex-1', - redisAvailable: true, - durablyRecorded: false, - locallyAborted: false, - pausedCancelled: false, - reason: 'redis_write_failed', - }) - }) - - it('returns success when local fallback aborts execution without Redis durability', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - mockAbortManualExecution.mockReturnValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - success: true, - executionId: 'ex-1', - redisAvailable: false, - durablyRecorded: false, - locallyAborted: true, - pausedCancelled: false, - reason: 'redis_unavailable', - }) - }) - - it('returns success when the queue backend cancels the active job', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - mockCancelByExecution.mockResolvedValue(1) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - executionId: 'ex-1', - durablyRecorded: false, - locallyAborted: false, - reason: 'queue_cancelled', - }) - expect(mockClearExecutionCancellation).not.toHaveBeenCalled() - }) - - it('cancels a queued execution before its workflow log exists', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([]) - mockCancelByExecution.mockResolvedValueOnce(1) - mockMarkExecutionCancelled.mockResolvedValueOnce({ - durablyRecorded: true, - reason: 'recorded', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - success: true, - executionId: 'ex-1', + executionId: 'execution-1', redisAvailable: true, durablyRecorded: true, locallyAborted: false, pausedCancelled: false, - reason: 'queue_cancelled', - }) - expect(mockCancelByExecution).toHaveBeenCalledWith( - { - workflowId: 'wf-1', - executionId: 'ex-1', - }, - 'standalone' - ) - expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1') - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('ex-1') - expect(mockClearExecutionCancellation).not.toHaveBeenCalled() - }) - - it('does not use an unscoped local abort before its workflow log exists', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([]) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(404) - await expect(response.json()).resolves.toEqual({ error: 'Execution not found' }) - expect(mockAbortManualExecution).not.toHaveBeenCalled() - expect(mockCancelByExecution).toHaveBeenCalledWith( - { - workflowId: 'wf-1', - executionId: 'ex-1', - }, - 'standalone' - ) - expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() - expect(mockClearExecutionCancellation).not.toHaveBeenCalled() - }) - - it('returns success when a paused HITL execution is cancelled directly in the database', async () => { - mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - success: true, - executionId: 'ex-1', - redisAvailable: true, - durablyRecorded: true, - locallyAborted: false, - pausedCancelled: true, reason: 'recorded', }) - expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() - expect(mockWriteTerminalEvent).toHaveBeenCalledWith( - expect.objectContaining({ - type: 'execution:cancelled', - executionId: 'ex-1', - workflowId: 'wf-1', - }), - 'cancelled' - ) - expect(mockFinalizeExecutionStream).not.toHaveBeenCalled() - }) - - it('claims the paused workflow-group sidecar before publishing and finalizing', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', + expect(mocks.cancel).toHaveBeenCalledWith({ + principal, + input: { + workflowId: 'workflow-1', + runId: 'execution-1', + abortSignal: expect.any(AbortSignal), }, - ]) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - durablyRecorded: true, - pausedCancelled: true, - reason: 'recorded', - }) - expect(mockStagePausedCancellation.mock.invocationCallOrder[0]).toBeLessThan( - mockWriteTerminalEvent.mock.invocationCallOrder[0] - ) - expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( - mockWriteTerminalEvent.mock.invocationCallOrder[0] - ) - expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( - mockCompletePausedCancellation.mock.invocationCallOrder[0] - ) - expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() - expect(mockCancelByExecution).not.toHaveBeenCalled() - expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() - expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') - }) - - it('keeps a paused workflow-group cancellation reserved when event publication fails', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'running', - workspaceId: 'workspace-1', - }, - ]) - mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: false, - pausedCancelled: false, - reason: 'paused_event_publish_failed', - }) - expect(mockCompletePausedCancellation).not.toHaveBeenCalled() - expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() - expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() - expect(mockCancelByExecution).not.toHaveBeenCalled() - }) - - it('publishes paused cancellation event even when Redis cancellation is recorded', async () => { - mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - executionId: 'ex-1', - durablyRecorded: true, - pausedCancelled: true, - }) - expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() - expect(mockWriteTerminalEvent).toHaveBeenCalledWith( - expect.objectContaining({ - type: 'execution:cancelled', - executionId: 'ex-1', - workflowId: 'wf-1', - }), - 'cancelled' - ) - expect(mockFinalizeExecutionStream).not.toHaveBeenCalled() - }) - - it('does not confirm paused cancellation when terminal event publication fails', async () => { - mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - mockCompletePausedCancellation.mockResolvedValue(true) - mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - success: false, - executionId: 'ex-1', - redisAvailable: false, - durablyRecorded: true, - locallyAborted: false, - pausedCancelled: false, - reason: 'paused_event_publish_failed', - }) - expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() - expect(mockCompletePausedCancellation).not.toHaveBeenCalled() - expect(mockWriteTerminalEvent).toHaveBeenCalledWith( - expect.objectContaining({ - type: 'execution:cancelled', - executionId: 'ex-1', - workflowId: 'wf-1', - }), - 'cancelled' - ) - expect(mockFinalizeExecutionStream).not.toHaveBeenCalled() - }) - - it('finishes reconciliation when the pause row is already cancelled', async () => { - mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - pausedCancelled: true, - reason: 'recorded', + request: expect.any(NextRequest), }) - expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() - expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( - mockCompletePausedCancellation.mock.invocationCallOrder[0] - ) }) - it('returns 401 when auth fails', async () => { - hybridAuthMockFns.mockCheckHybridAuth.mockResolvedValue({ - success: false, - error: 'Unauthorized', - }) + it('rejects unauthenticated requests before invoking cancellation', async () => { + mocks.getSession.mockResolvedValue(null) - const response = await POST(makeRequest(), makeParams()) + const response = await POST(request(), context) expect(response.status).toBe(401) + expect(mocks.cancel).not.toHaveBeenCalled() }) - it('returns 403 when workflow access is denied', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValue({ - allowed: false, - message: 'Access denied', - status: 403, - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(403) - }) - - it('returns 404 when the execution does not belong to the workflow', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([]) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(404) - expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() - expect(mockCancelByExecution).toHaveBeenCalledWith( - { - workflowId: 'wf-1', - executionId: 'ex-1', - }, - 'standalone' - ) - }) - - it('treats an already-cancelled execution as an idempotent success', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { executionDeadlineAt: null, status: 'cancelled', workspaceId: 'workspace-1' }, - ]) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - durablyRecorded: false, - reason: 'already_cancelled', - }) - expect(mockCancelByExecution).not.toHaveBeenCalled() - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('ex-1') - }) - - it('reconciles the exact sidecar when a workflow-group log is already cancelled', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'cancelled', - workspaceId: 'workspace-1', - }, - ]) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'already_cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - reason: 'already_cancelled', - }) - expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledWith({ - workspaceId: 'workspace-1', - workflowId: 'wf-1', - executionId: 'ex-1', - }) - expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1', { - executionDeadlineAt: null, - }) - expect(mockPublishWorkflowGroupCancellationEvent).toHaveBeenCalledOnce() - expect(mockCancelByExecution).not.toHaveBeenCalled() - }) - - it('does not finalize an already-cancelled group retry until exact stop is accepted', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'cancelled', - workspaceId: 'workspace-1', - }, - ]) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'already_cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: false, - redisAvailable: false, - reason: 'redis_unavailable', - }) - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - expect(mockWriteTerminalEvent).not.toHaveBeenCalled() - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() - }) - - it('repairs a stranded active resume when the group log is already cancelled', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'cancelled', - workspaceId: 'workspace-1', - }, - ]) - mockStagePausedCancellation.mockResolvedValue({ - kind: 'active_resume', - target: ACTIVE_RESUME_TARGET, - }) - mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) - mockAbortManualExecution.mockReturnValue(true) - mockCancelByExecution.mockResolvedValue(1) - mockCompletePausedCancellation.mockResolvedValue(true) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'already_cancelled', - tableId: 'table-1', - rowId: 'row-1', - groupId: 'group-1', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - locallyAborted: true, - pausedCancelled: true, - reason: 'already_cancelled', - }) - expect(mockCancelByExecution).toHaveBeenCalledWith( - { workflowId: 'wf-1', executionId: 'ex-1' }, - 'resume' + it('projects application conflicts without reimplementing cancellation errors', async () => { + mocks.cancel.mockRejectedValue( + new WorkflowRunAlreadyTerminalError({ + executionId: 'execution-1', + executionStatus: 'completed', + redisAvailable: true, + locallyAborted: false, + }) ) - expect(mockAbortManualExecution).toHaveBeenCalledWith('resume-ex-1') - expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') - expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() - expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( - mockCompletePausedCancellation.mock.invocationCallOrder[0] - ) - }) - - it('repairs a stranded active resume when a regular workflow log is already cancelled', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: null, - status: 'cancelled', - workspaceId: 'workspace-1', - }, - ]) - mockStagePausedCancellation.mockResolvedValue({ - kind: 'active_resume', - target: ACTIVE_RESUME_TARGET, - }) - mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) - mockAbortManualExecution.mockReturnValue(true) - mockCancelByExecution.mockResolvedValue(1) - mockCompletePausedCancellation.mockResolvedValue(true) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - locallyAborted: true, - pausedCancelled: true, - reason: 'already_cancelled', - }) - expect(mockCancelByExecution).toHaveBeenCalledWith( - { workflowId: 'wf-1', executionId: 'ex-1' }, - 'resume' - ) - expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( - mockCompletePausedCancellation.mock.invocationCallOrder[0] - ) - expect(mockCancelWorkflowGroupExecution).not.toHaveBeenCalled() - }) - - it('keeps an already-cancelled active resume retryable when publication fails', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: null, - status: 'cancelled', - workspaceId: 'workspace-1', - }, - ]) - mockStagePausedCancellation.mockResolvedValue({ - kind: 'active_resume', - target: ACTIVE_RESUME_TARGET, - }) - mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: false, - pausedCancelled: false, - reason: 'paused_event_publish_failed', - }) - expect(mockCompletePausedCancellation).not.toHaveBeenCalled() - }) - - it('fails closed when an already-cancelled group sidecar cannot be reconciled', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'cancelled', - workspaceId: 'workspace-1', - }, - ]) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'conflict', - status: 'completed', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(409) - await expect(response.json()).resolves.toEqual({ - error: 'Workflow group execution cannot be reconciled while completed', - }) - }) - - it('reports a reconciliation failure for an already-cancelled group sidecar', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { - executionDeadlineAt: null, - executionOrigin: 'workflow_group', - status: 'cancelled', - workspaceId: 'workspace-1', - }, - ]) - mockCancelWorkflowGroupExecution.mockRejectedValue(new Error('database unavailable')) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(500) - await expect(response.json()).resolves.toEqual({ error: 'database unavailable' }) - }) - - it('returns 409 when the execution is already terminal', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([ - { status: 'completed', workspaceId: 'workspace-1' }, - ]) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(409) - expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() - expect(mockCancelByExecution).not.toHaveBeenCalled() - }) - - it('returns 409 when completion wins the terminal database race', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - const returning = vi.fn().mockResolvedValue([]) - const where = vi.fn(() => ({ returning })) - databaseMock.db.update.mockReturnValueOnce({ set: vi.fn(() => ({ where })) }) - dbChainMockFns.limit - .mockResolvedValueOnce([ - { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, - ]) - .mockResolvedValueOnce([{ status: 'completed' }]) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(409) - await expect(response.json()).resolves.toEqual({ - error: 'Execution cannot be cancelled while completed', - }) - expect(returning).toHaveBeenCalledOnce() - expect(mockClearExecutionCancellation).toHaveBeenCalledWith('ex-1') - expect(mockWriteTerminalEvent).not.toHaveBeenCalled() - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() - }) - - it('rolls back paused cancellation intent when resume completion wins the log claim', async () => { - mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - const returning = vi.fn().mockResolvedValue([]) - const where = vi.fn(() => ({ returning })) - databaseMock.db.update.mockReturnValueOnce({ set: vi.fn(() => ({ where })) }) - dbChainMockFns.limit - .mockResolvedValueOnce([ - { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, - ]) - .mockResolvedValueOnce([{ status: 'completed' }]) - const response = await POST(makeRequest(), makeParams()) + const response = await POST(request(), context) expect(response.status).toBe(409) - await expect(response.json()).resolves.toEqual({ + await expect(response.json()).resolves.toMatchObject({ error: 'Execution cannot be cancelled while completed', }) - expect(mockWriteTerminalEvent).not.toHaveBeenCalled() - expect(mockCompletePausedCancellation).not.toHaveBeenCalled() - expect(mockClearPausedCancellationIntent).toHaveBeenCalledWith('ex-1', 'wf-1') - }) - - it('treats a concurrent cancellation as an idempotent success', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - dbChainMockFns.limit - .mockResolvedValueOnce([ - { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, - ]) - .mockResolvedValueOnce([{ status: 'cancelled' }]) - dbChainMockFns.returning.mockResolvedValueOnce([]) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ success: true, reason: 'recorded' }) - expect(mockClearExecutionCancellation).not.toHaveBeenCalled() - expect(mockWriteTerminalEvent).toHaveBeenCalledTimes(1) - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('ex-1') - }) - - it('updates execution log status in DB when durably recorded', async () => { - const mockWhere = vi.fn().mockResolvedValue(undefined) - const mockSet = vi.fn(() => ({ where: mockWhere })) - databaseMock.db.update.mockReturnValueOnce({ set: mockSet }) - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: true, - reason: 'recorded', - }) - - await POST(makeRequest(), makeParams()) - - expect(databaseMock.db.update).toHaveBeenCalled() - expect(mockSet).toHaveBeenCalledWith({ - status: 'cancelled', - endedAt: expect.any(Date), - totalDurationMs: expect.anything(), - executionDeadlineAt: null, - }) - }) - - it('updates execution log status in DB when locally aborted', async () => { - const mockWhere = vi.fn().mockResolvedValue(undefined) - const mockSet = vi.fn(() => ({ where: mockWhere })) - databaseMock.db.update.mockReturnValueOnce({ set: mockSet }) - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - mockAbortManualExecution.mockReturnValue(true) - - await POST(makeRequest(), makeParams()) - - expect(databaseMock.db.update).toHaveBeenCalled() - expect(mockSet).toHaveBeenCalledWith({ - status: 'cancelled', - endedAt: expect.any(Date), - totalDurationMs: expect.anything(), - executionDeadlineAt: null, - }) - }) - - it('claims the execution log before finalizing a paused cancellation', async () => { - mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - - await POST(makeRequest(), makeParams()) - - expect(databaseMock.db.update).toHaveBeenCalled() - }) - - it('does not confirm cancellation until the terminal database update succeeds', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: true, - reason: 'recorded', - }) - databaseMock.db.update.mockReturnValueOnce({ - set: vi.fn(() => ({ - where: vi.fn(() => { - throw new Error('DB connection failed') - }), - })), - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - const data = await response.json() - expect(data).toMatchObject({ - success: false, - reason: 'cancellation_not_finalized', - }) - expect(mockClearExecutionCancellation).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.ts b/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.ts index d696a5e6f90..d516f33c489 100644 --- a/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.ts +++ b/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.ts @@ -1,1006 +1,36 @@ -import { db } from '@sim/db' -import { workflowExecutionLogs } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' -import { toError } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' -import { and, eq, inArray } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' +import { cancelWorkflowExecutionContract } from '@/lib/api/contracts/workflows' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { - type CancelWorkflowExecutionResponse, - cancelWorkflowExecutionContract, -} from '@/lib/api/contracts/workflows' -import { parseRequest } from '@/lib/api/server' -import { WORKSPACE_KEY_SCOPE_DENIED } from '@/lib/api-key/policy-messages' -import { checkHybridAuth } from '@/lib/auth/hybrid' -import { releaseExecutionSlot } from '@/lib/billing/calculations/usage-reservation' -import { getJobQueue } from '@/lib/core/async-jobs' -import type { ExecutionJobCancellationScope } from '@/lib/core/async-jobs/types' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - clearExecutionCancellation, - type ExecutionCancellationRecordResult, - markExecutionCancelled, -} from '@/lib/execution/cancellation' -import { createExecutionEventWriter, readExecutionMetaState } from '@/lib/execution/event-buffer' -import { abortManualExecution } from '@/lib/execution/manual-cancellation' -import { cancelledExecutionLogFields } from '@/lib/logs/execution/cancellation' -import { workflowExecutionOriginSql } from '@/lib/logs/execution-origin' -import { captureServerEvent } from '@/lib/posthog/server' -import { - cancelWorkflowGroupExecution, - type PublishableWorkflowGroupCancellation, - publishWorkflowGroupCancellationEvent, -} from '@/lib/table/workflow-group-cancellation' -import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager' - -const logger = createLogger('CancelExecutionAPI') -const PAUSED_CANCELLATION_DB_ATTEMPTS = 3 -const PAUSED_CANCELLATION_DB_RETRY_MS = 200 - -/** - * Builds the single success shape this route returns. The route hand-builds its - * responses rather than going through a declarative builder, so nothing else - * checks that they satisfy the contract the client validates against — and - * several outcomes the route resolves itself (a still-queued run, an - * already-cancelled run) ride on `success: true`, where a rejected body turns a - * cancellation that worked into a client-side failure. - */ -function cancellationOutcome(body: CancelWorkflowExecutionResponse) { - return NextResponse.json(body) -} - -async function cancelQueuedExecutionJobs( - workflowId: string, - executionId: string, - scope: ExecutionJobCancellationScope -): Promise { - try { - const queue = await getJobQueue() - return await queue.cancelByExecution({ workflowId, executionId }, scope) - } catch (error) { - logger.warn('Failed to cancel queued execution jobs', { - workflowId, - executionId, - error: toError(error).message, - }) - return 0 - } -} - -function abortLocalExecution(executionId: string): boolean { - try { - return abortManualExecution(executionId) - } catch (error) { - logger.warn('Failed to abort local execution', { - executionId, - error: toError(error).message, - }) - return false - } -} - -interface ExecutionStopSignalResult { - cancellation: ExecutionCancellationRecordResult - locallyAborted: boolean - queueJobsCancelled: number - accepted: boolean -} - -interface ExecutionStopSummary extends ExecutionStopSignalResult { - signalledExecutionIds: Set -} - -function createExecutionStopSummary(): ExecutionStopSummary { - return { - cancellation: { durablyRecorded: false, reason: 'redis_unavailable' }, - locallyAborted: false, - queueJobsCancelled: 0, - accepted: false, - signalledExecutionIds: new Set(), - } -} - -function mergeExecutionStopSignal( - summary: ExecutionStopSummary, - signalExecutionId: string, - result: ExecutionStopSignalResult -): void { - if (result.cancellation.durablyRecorded || !summary.cancellation.durablyRecorded) { - summary.cancellation = result.cancellation - } - summary.locallyAborted = summary.locallyAborted || result.locallyAborted - summary.queueJobsCancelled += result.queueJobsCancelled - summary.accepted = summary.accepted || result.accepted - summary.signalledExecutionIds.add(signalExecutionId) -} - -async function signalExecutionStop(args: { - workflowId: string - signalExecutionId: string - queueBindingExecutionId?: string - executionDeadlineAt: Date | null - queueScope?: ExecutionJobCancellationScope -}): Promise { - const cancellation = await markExecutionCancelled(args.signalExecutionId, { - executionDeadlineAt: args.executionDeadlineAt, - }) - const locallyAborted = abortLocalExecution(args.signalExecutionId) - const queueJobsCancelled = args.queueScope - ? await cancelQueuedExecutionJobs( - args.workflowId, - args.queueBindingExecutionId ?? args.signalExecutionId, - args.queueScope - ) - : 0 - return { - cancellation, - locallyAborted, - queueJobsCancelled, - accepted: cancellation.durablyRecorded || locallyAborted || queueJobsCancelled > 0, - } -} - -async function signalActiveResumeStop(args: { - workflowId: string - executionId: string - executionDeadlineAt: Date | null - target: NonNullable< - Awaited> - > -}): Promise<{ - target: NonNullable< - Awaited> - > - signal: ExecutionStopSignalResult -}> { - const signal = await signalExecutionStop({ - workflowId: args.workflowId, - signalExecutionId: args.target.resumeExecutionId, - queueBindingExecutionId: args.executionId, - executionDeadlineAt: args.executionDeadlineAt, - queueScope: 'resume', - }) - return { target: args.target, signal } -} - -async function didActiveResumeStop( - executionId: string, - workflowId: string, - target: NonNullable< - Awaited> - >, - signal: ExecutionStopSignalResult -): Promise { - if (signal.accepted) return true - const currentTarget = await PauseResumeManager.getActiveResumeCancellationTarget( - executionId, - workflowId - ) - if (currentTarget && currentTarget.resumeEntryId !== target.resumeEntryId) { - logger.warn('A replacement resume became active while cancellation was staged', { - executionId, - previousResumeEntryId: target.resumeEntryId, - currentResumeEntryId: currentTarget.resumeEntryId, - }) - } - return currentTarget === null -} - -type PausedCancellationStage = Awaited< - ReturnType -> - -function isPausedCancellationStage( - stage: PausedCancellationStage -): stage is Exclude { - return stage.kind !== 'not_paused' -} - -async function clearStopSignalMarkers(summary: ExecutionStopSummary): Promise { - await Promise.all( - [...summary.signalledExecutionIds].map((executionId) => clearExecutionCancellation(executionId)) - ) -} - -type ExecutionLogCancellationClaim = - | { kind: 'cancelled' } - | { kind: 'conflict'; status: string } - | { kind: 'not_found' } - -async function claimExecutionLogCancellation(args: { - executionId: string - workflowId: string - workspaceId: string -}): Promise { - const now = new Date() - const [cancelledExecution] = await db - .update(workflowExecutionLogs) - .set(cancelledExecutionLogFields(now)) - .where( - and( - eq(workflowExecutionLogs.executionId, args.executionId), - eq(workflowExecutionLogs.workflowId, args.workflowId), - eq(workflowExecutionLogs.workspaceId, args.workspaceId), - inArray(workflowExecutionLogs.status, ['running', 'pending']) - ) - ) - .returning({ status: workflowExecutionLogs.status }) - - if (cancelledExecution?.status === 'cancelled') return { kind: 'cancelled' } - - const currentExecution = await db - .select({ status: workflowExecutionLogs.status }) - .from(workflowExecutionLogs) - .where( - and( - eq(workflowExecutionLogs.executionId, args.executionId), - eq(workflowExecutionLogs.workflowId, args.workflowId), - eq(workflowExecutionLogs.workspaceId, args.workspaceId) - ) - ) - .limit(1) - .then((rows) => rows[0]) - - if (!currentExecution) return { kind: 'not_found' } - if (currentExecution.status === 'cancelled') return { kind: 'cancelled' } - return { kind: 'conflict', status: currentExecution.status } -} - -async function completePausedCancellationWithRetry( - executionId: string, - workflowId: string, - options: { logMissing?: boolean } = {} -): Promise { - for (let attempt = 1; attempt <= PAUSED_CANCELLATION_DB_ATTEMPTS; attempt++) { - try { - const cancelled = await PauseResumeManager.completePausedCancellation(executionId, workflowId) - if (cancelled) { - logger.info('Paused execution cancelled in database', { executionId, attempt }) - return true - } - if (options.logMissing !== false) { - logger.warn('Paused execution cancellation could not be completed in database', { - executionId, - attempt, - }) - } - return false - } catch (error) { - logger.warn('Failed to complete paused execution cancellation in database', { - executionId, - attempt, - error, - }) - if (attempt < PAUSED_CANCELLATION_DB_ATTEMPTS) { - await sleep(PAUSED_CANCELLATION_DB_RETRY_MS) - } - } - } - return false -} - -async function ensureCancellationEventPublished( - executionId: string, - workflowId: string, - context: { workspaceId?: string; userId?: string } = {} -): Promise { - try { - const metaState = await readExecutionMetaState(executionId) - if (metaState.status === 'found' && metaState.meta.status === 'cancelled') { - return true - } - } catch (error) { - logger.warn('Failed to read execution state before publishing cancellation', { - executionId, - error: toError(error).message, - }) - } - - const writer = createExecutionEventWriter(executionId, { - workspaceId: context.workspaceId, - workflowId, - userId: context.userId, - }) - try { - await writer.writeTerminal( - { - type: 'execution:cancelled', - timestamp: new Date().toISOString(), - executionId, - workflowId, - data: { duration: 0 }, - }, - 'cancelled' - ) - return true - } catch (error) { - logger.warn('Failed to publish execution cancellation event', { - executionId, - error, - }) - return false - } finally { - await writer.close().catch((error) => { - logger.warn('Failed to close cancellation event writer', { - executionId, - error, - }) - }) - } -} + internalWorkflowErrorPolicies, + internalWorkflowSessionOrApiKeyAuth, +} from '@/lib/workflows/api' +import { cancelWorkflowRun } from '@/lib/workflows/application/cancel-run' +import { workflowOperations } from '@/lib/workflows/application/operations' export const runtime = 'nodejs' export const dynamic = 'force-dynamic' -export const POST = withRouteHandler( - async (req: NextRequest, context: { params: Promise<{ id: string; executionId: string }> }) => { - const auth = await checkHybridAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(cancelWorkflowExecutionContract, req, context) - if (!parsed.success) return parsed.response - const { id: workflowId, executionId } = parsed.data.params - - try { - const workflowAuthorization = await authorizeWorkflowByWorkspacePermission({ - workflowId, - userId: auth.userId, - action: 'write', - }) - if (!workflowAuthorization.allowed) { - return NextResponse.json( - { error: workflowAuthorization.message || 'Access denied' }, - { status: workflowAuthorization.status } - ) - } - - if ( - auth.apiKeyType === 'workspace' && - workflowAuthorization.workflow?.workspaceId !== auth.workspaceId - ) { - return NextResponse.json({ error: WORKSPACE_KEY_SCOPE_DENIED }, { status: 403 }) - } - - const execution = await db - .select({ - executionDeadlineAt: workflowExecutionLogs.executionDeadlineAt, - executionOrigin: workflowExecutionOriginSql(), - status: workflowExecutionLogs.status, - workspaceId: workflowExecutionLogs.workspaceId, - }) - .from(workflowExecutionLogs) - .where( - and( - eq(workflowExecutionLogs.executionId, executionId), - eq(workflowExecutionLogs.workflowId, workflowId) - ) - ) - .limit(1) - .then((rows) => rows[0]) - - if (!execution) { - const queueJobsCancelled = await cancelQueuedExecutionJobs( - workflowId, - executionId, - 'standalone' - ) - if (queueJobsCancelled > 0) { - const locallyAborted = abortLocalExecution(executionId) - const cancellation = await markExecutionCancelled(executionId) - await PauseResumeManager.blockQueuedResumesForCancellation(executionId, workflowId).catch( - (error) => { - logger.warn('Failed to block queued resumes after queued-run cancellation', { - executionId, - error, - }) - } - ) - await releaseExecutionSlot(executionId).catch((error) => { - logger.warn('Failed to release reservation after queued-run cancellation', { - executionId, - error, - }) - }) - - const workspaceId = workflowAuthorization.workflow?.workspaceId - captureServerEvent( - auth.userId, - 'workflow_execution_cancelled', - { workflow_id: workflowId, workspace_id: workspaceId ?? '' }, - workspaceId ? { groups: { workspace: workspaceId } } : undefined - ) - - return cancellationOutcome({ - success: true, - executionId, - redisAvailable: cancellation.reason !== 'redis_unavailable', - durablyRecorded: cancellation.durablyRecorded, - locallyAborted, - pausedCancelled: false, - reason: 'queue_cancelled', - }) - } - - return NextResponse.json({ error: 'Execution not found' }, { status: 404 }) - } - - const authorizedWorkspaceId = workflowAuthorization.workflow?.workspaceId - if (authorizedWorkspaceId && execution.workspaceId !== authorizedWorkspaceId) { - return NextResponse.json({ error: 'Execution not found' }, { status: 404 }) - } - - const isWorkflowGroupExecution = execution.executionOrigin === 'workflow_group' - - if (execution.status === 'cancelled') { - let groupCancellationToPublish: PublishableWorkflowGroupCancellation | null = null - if (isWorkflowGroupExecution) { - const workflowGroupCancellation = await cancelWorkflowGroupExecution({ - workspaceId: execution.workspaceId, - workflowId, - executionId, - }) - if (workflowGroupCancellation.kind === 'conflict') { - return NextResponse.json( - { - error: `Workflow group execution cannot be reconciled while ${workflowGroupCancellation.status}`, - }, - { status: 409 } - ) - } - if (workflowGroupCancellation.kind === 'not_workflow_group') { - return NextResponse.json( - { error: 'Workflow group execution is no longer the active table execution' }, - { status: 409 } - ) - } - if ( - workflowGroupCancellation.kind === 'cancelled' || - workflowGroupCancellation.kind === 'already_cancelled' - ) { - groupCancellationToPublish = workflowGroupCancellation - } - } - - const stopSummary = createExecutionStopSummary() - let pausedCancelled = false - const pausedCancellationStage = await PauseResumeManager.stagePausedCancellation( - executionId, - workflowId - ) - const hasPausedCancellation = isPausedCancellationStage(pausedCancellationStage) - const requiresCancellationEvent = hasPausedCancellation || isWorkflowGroupExecution - let cancellationEventPublished = !requiresCancellationEvent - let activeResumeSignalFailed = false - let exactStopSatisfied = true - if (pausedCancellationStage.kind === 'active_resume') { - const activeResumeStop = await signalActiveResumeStop({ - workflowId, - executionId, - executionDeadlineAt: execution.executionDeadlineAt, - target: pausedCancellationStage.target, - }) - mergeExecutionStopSignal( - stopSummary, - activeResumeStop.target.resumeExecutionId, - activeResumeStop.signal - ) - exactStopSatisfied = await didActiveResumeStop( - executionId, - workflowId, - activeResumeStop.target, - activeResumeStop.signal - ) - activeResumeSignalFailed = !exactStopSatisfied - } else if (isWorkflowGroupExecution && !hasPausedCancellation) { - const retrySignal = await signalExecutionStop({ - workflowId, - signalExecutionId: executionId, - executionDeadlineAt: execution.executionDeadlineAt, - }) - mergeExecutionStopSignal(stopSummary, executionId, retrySignal) - exactStopSatisfied = retrySignal.accepted - } - - if (groupCancellationToPublish && exactStopSatisfied) { - await publishWorkflowGroupCancellationEvent(groupCancellationToPublish, executionId) - } - - if (requiresCancellationEvent && exactStopSatisfied) { - cancellationEventPublished = await ensureCancellationEventPublished( - executionId, - workflowId, - { - workspaceId: execution.workspaceId, - userId: auth.userId, - } - ) - } - if (hasPausedCancellation && cancellationEventPublished && exactStopSatisfied) { - pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId, { - logMissing: false, - }) - } - - if (exactStopSatisfied) { - await releaseExecutionSlot(executionId).catch((error) => { - logger.warn('Failed to release reservation while reconciling cancelled execution', { - executionId, - error: toError(error).message, - }) - }) - } - - if (pausedCancelled) { - await clearStopSignalMarkers(stopSummary) - } - - const pausedReconciliationSucceeded = - exactStopSatisfied && - (!hasPausedCancellation || (cancellationEventPublished && pausedCancelled)) - return cancellationOutcome({ - success: pausedReconciliationSucceeded, - executionId, - redisAvailable: requiresCancellationEvent ? cancellationEventPublished : true, - durablyRecorded: false, - locallyAborted: stopSummary.locallyAborted, - pausedCancelled, - reason: activeResumeSignalFailed - ? 'active_resume_signal_failed' - : !exactStopSatisfied - ? stopSummary.cancellation.reason - : hasPausedCancellation && !cancellationEventPublished - ? 'paused_event_publish_failed' - : hasPausedCancellation && !pausedCancelled - ? 'paused_database_cancel_failed' - : 'already_cancelled', - }) - } - - if (execution.status !== 'running' && execution.status !== 'pending') { - return NextResponse.json( - { error: `Execution cannot be cancelled while ${execution.status}` }, - { status: 409 } - ) - } - - logger.info('Cancel execution requested', { workflowId, executionId, userId: auth.userId }) - - const stopSummary = createExecutionStopSummary() - let pausedCancelled = false - let pausedCancellationStage = await PauseResumeManager.stagePausedCancellation( - executionId, - workflowId - ) - let effectivePausedCancellationPath = isPausedCancellationStage(pausedCancellationStage) - let activeResumeTarget = - pausedCancellationStage.kind === 'active_resume' ? pausedCancellationStage.target : null - let activeResumeEntryId = activeResumeTarget?.resumeEntryId ?? null - let activeResumeSignalAccepted = false - - if (activeResumeTarget && !isWorkflowGroupExecution) { - const activeResumeStop = await signalActiveResumeStop({ - workflowId, - executionId, - executionDeadlineAt: execution.executionDeadlineAt, - target: activeResumeTarget, - }) - mergeExecutionStopSignal( - stopSummary, - activeResumeStop.target.resumeExecutionId, - activeResumeStop.signal - ) - activeResumeSignalAccepted = await didActiveResumeStop( - executionId, - workflowId, - activeResumeStop.target, - activeResumeStop.signal - ) - - if (!activeResumeSignalAccepted) { - const failedResumeEntryId = activeResumeTarget.resumeEntryId - await PauseResumeManager.rollbackActiveResumeCancellation( - executionId, - workflowId, - failedResumeEntryId - ).catch((error) => { - logger.warn('Failed to roll back active resume cancellation intent', { - executionId, - activeResumeEntryId: failedResumeEntryId, - error: toError(error).message, - }) - }) - await clearStopSignalMarkers(stopSummary) - return cancellationOutcome({ - success: false, - executionId, - redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', - durablyRecorded: stopSummary.cancellation.durablyRecorded, - locallyAborted: stopSummary.locallyAborted, - pausedCancelled: false, - reason: 'active_resume_signal_failed', - }) - } - } else if (!effectivePausedCancellationPath && !isWorkflowGroupExecution) { - const signal = await signalExecutionStop({ - workflowId, - signalExecutionId: executionId, - executionDeadlineAt: execution.executionDeadlineAt, - queueScope: 'standalone', - }) - mergeExecutionStopSignal(stopSummary, executionId, signal) - - if (!signal.accepted) { - pausedCancellationStage = await PauseResumeManager.stagePausedCancellation( - executionId, - workflowId - ) - effectivePausedCancellationPath = isPausedCancellationStage(pausedCancellationStage) - activeResumeTarget = - pausedCancellationStage.kind === 'active_resume' ? pausedCancellationStage.target : null - activeResumeEntryId = activeResumeTarget?.resumeEntryId ?? null - - if (activeResumeTarget) { - const activeResumeStop = await signalActiveResumeStop({ - workflowId, - executionId, - executionDeadlineAt: execution.executionDeadlineAt, - target: activeResumeTarget, - }) - mergeExecutionStopSignal( - stopSummary, - activeResumeStop.target.resumeExecutionId, - activeResumeStop.signal - ) - activeResumeSignalAccepted = await didActiveResumeStop( - executionId, - workflowId, - activeResumeStop.target, - activeResumeStop.signal - ) - if (!activeResumeSignalAccepted) { - const failedResumeEntryId = activeResumeTarget.resumeEntryId - await PauseResumeManager.rollbackActiveResumeCancellation( - executionId, - workflowId, - failedResumeEntryId - ).catch((error) => { - logger.warn('Failed to roll back late active resume cancellation intent', { - executionId, - activeResumeEntryId: failedResumeEntryId, - error: toError(error).message, - }) - }) - await clearStopSignalMarkers(stopSummary) - return cancellationOutcome({ - success: false, - executionId, - redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', - durablyRecorded: stopSummary.cancellation.durablyRecorded, - locallyAborted: stopSummary.locallyAborted, - pausedCancelled: false, - reason: 'active_resume_signal_failed', - }) - } - } else if (!effectivePausedCancellationPath) { - return cancellationOutcome({ - success: false, - executionId, - redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', - durablyRecorded: stopSummary.cancellation.durablyRecorded, - locallyAborted: stopSummary.locallyAborted, - pausedCancelled: false, - reason: stopSummary.cancellation.reason, - }) - } - } - } - - let terminalCancellationClaimed = false - let competingTerminalStatus: string | null = null - let workflowGroupNoLongerActive = false - let groupCancellationToPublish: PublishableWorkflowGroupCancellation | null = null - try { - if (isWorkflowGroupExecution) { - const workflowGroupCancellation = await cancelWorkflowGroupExecution({ - workspaceId: execution.workspaceId, - workflowId, - executionId, - }) - if (workflowGroupCancellation.kind === 'conflict') { - competingTerminalStatus = workflowGroupCancellation.status - } else if (workflowGroupCancellation.kind === 'not_workflow_group') { - workflowGroupNoLongerActive = true - } else { - terminalCancellationClaimed = true - if ( - workflowGroupCancellation.kind === 'cancelled' || - workflowGroupCancellation.kind === 'already_cancelled' - ) { - groupCancellationToPublish = workflowGroupCancellation - } - } - } else { - const claim = await claimExecutionLogCancellation({ - executionId, - workflowId, - workspaceId: execution.workspaceId, - }) - if (claim.kind === 'cancelled') { - terminalCancellationClaimed = true - } else { - competingTerminalStatus = claim.kind === 'conflict' ? claim.status : 'no_longer_active' - } - } - } catch (dbError) { - logger.warn('Failed to finalize cancelled execution directly', { - executionId, - error: toError(dbError).message, - }) - } - - if (workflowGroupNoLongerActive) { - await clearStopSignalMarkers(stopSummary) - if (activeResumeEntryId) { - await PauseResumeManager.rollbackActiveResumeCancellation( - executionId, - workflowId, - activeResumeEntryId - ).catch((error) => { - logger.warn('Failed to roll back active resume after group target disappeared', { - executionId, - activeResumeEntryId, - error: toError(error).message, - }) - }) - } else if (effectivePausedCancellationPath) { - await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId).catch( - (error) => { - logger.warn('Failed to clear cancellation intent after group target disappeared', { - executionId, - error: toError(error).message, - }) - } - ) - } - return NextResponse.json( - { error: 'Workflow group execution is no longer the active table execution' }, - { status: 409 } - ) - } - - if (competingTerminalStatus) { - await clearStopSignalMarkers(stopSummary) - if (activeResumeEntryId) { - await PauseResumeManager.rollbackActiveResumeCancellation( - executionId, - workflowId, - activeResumeEntryId - ).catch((error) => { - logger.warn('Failed to roll back active resume after terminal race', { - executionId, - activeResumeEntryId, - error: toError(error).message, - }) - }) - } else if (effectivePausedCancellationPath) { - await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId).catch( - (error) => { - logger.warn('Failed to clear cancellation intent after terminal race', { - executionId, - error: toError(error).message, - }) - } - ) - } - return NextResponse.json( - { - error: isWorkflowGroupExecution - ? `Workflow group execution cannot be cancelled while ${competingTerminalStatus}` - : `Execution cannot be cancelled while ${competingTerminalStatus}`, - }, - { status: 409 } - ) - } - - if (!terminalCancellationClaimed) { - if (effectivePausedCancellationPath && !stopSummary.accepted) { - if (activeResumeEntryId) { - await PauseResumeManager.rollbackActiveResumeCancellation( - executionId, - workflowId, - activeResumeEntryId - ) - } else { - await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId) - } - } - return cancellationOutcome({ - success: false, - executionId, - redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', - durablyRecorded: stopSummary.cancellation.durablyRecorded, - locallyAborted: stopSummary.locallyAborted, - pausedCancelled: false, - reason: 'cancellation_not_finalized', - }) - } - - let pauseReconciliationFailed = false - let activeResumeSignalFailed = false - if (isWorkflowGroupExecution) { - if (activeResumeTarget) { - const activeResumeStop = await signalActiveResumeStop({ - workflowId, - executionId, - executionDeadlineAt: execution.executionDeadlineAt, - target: activeResumeTarget, - }) - mergeExecutionStopSignal( - stopSummary, - activeResumeStop.target.resumeExecutionId, - activeResumeStop.signal - ) - activeResumeSignalAccepted = await didActiveResumeStop( - executionId, - workflowId, - activeResumeStop.target, - activeResumeStop.signal - ) - activeResumeSignalFailed = !activeResumeSignalAccepted - } else if (!effectivePausedCancellationPath) { - const groupSignal = await signalExecutionStop({ - workflowId, - signalExecutionId: executionId, - executionDeadlineAt: execution.executionDeadlineAt, - }) - mergeExecutionStopSignal(stopSummary, executionId, groupSignal) - } - } - - try { - const postClaimPausedCancellationStage = await PauseResumeManager.stagePausedCancellation( - executionId, - workflowId - ) - if (isPausedCancellationStage(postClaimPausedCancellationStage)) { - effectivePausedCancellationPath = true - pausedCancellationStage = postClaimPausedCancellationStage - if (postClaimPausedCancellationStage.kind === 'active_resume') { - const currentActiveResume = postClaimPausedCancellationStage.target - const alreadyAttemptedCurrentResume = - currentActiveResume.resumeEntryId === activeResumeEntryId && - stopSummary.signalledExecutionIds.has(currentActiveResume.resumeExecutionId) - if (!alreadyAttemptedCurrentResume) { - const activeResumeStop = await signalActiveResumeStop({ - workflowId, - executionId, - executionDeadlineAt: execution.executionDeadlineAt, - target: currentActiveResume, - }) - activeResumeEntryId = activeResumeStop.target.resumeEntryId - activeResumeTarget = activeResumeStop.target - mergeExecutionStopSignal( - stopSummary, - activeResumeStop.target.resumeExecutionId, - activeResumeStop.signal - ) - activeResumeSignalAccepted = await didActiveResumeStop( - executionId, - workflowId, - activeResumeStop.target, - activeResumeStop.signal - ) - } - activeResumeSignalFailed = !activeResumeSignalAccepted - } else { - activeResumeSignalFailed = false - } - } - } catch (error) { - pauseReconciliationFailed = true - effectivePausedCancellationPath = true - logger.warn('Failed to recheck paused execution after terminal cancellation claim', { - executionId, - error: toError(error).message, - }) - } - - const executionStopSatisfied = effectivePausedCancellationPath - ? !activeResumeSignalFailed - : stopSummary.accepted - if (groupCancellationToPublish && executionStopSatisfied && !pauseReconciliationFailed) { - await publishWorkflowGroupCancellationEvent(groupCancellationToPublish, executionId) - } - let cancellationEventPublished = false - if (executionStopSatisfied && !pauseReconciliationFailed) { - cancellationEventPublished = await ensureCancellationEventPublished( - executionId, - workflowId, - { - workspaceId: execution.workspaceId, - userId: auth.userId, - } - ) - } - - if (effectivePausedCancellationPath) { - if (cancellationEventPublished && !pauseReconciliationFailed && !activeResumeSignalFailed) { - pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId) - } - } else if (executionStopSatisfied) { - await releaseExecutionSlot(executionId).catch((error) => { - logger.warn('Failed to release reservation after execution cancellation', { - executionId, - error: toError(error).message, - }) - }) - } - - const success = effectivePausedCancellationPath - ? pausedCancelled && - cancellationEventPublished && - !pauseReconciliationFailed && - !activeResumeSignalFailed - : executionStopSatisfied - - if (effectivePausedCancellationPath && pausedCancelled && cancellationEventPublished) { - await clearStopSignalMarkers(stopSummary) - } - - if (success) { - const workspaceId = execution.workspaceId - captureServerEvent( - auth.userId, - 'workflow_execution_cancelled', - { workflow_id: workflowId, workspace_id: workspaceId ?? '' }, - workspaceId ? { groups: { workspace: workspaceId } } : undefined - ) - } - - const durablyRecorded = effectivePausedCancellationPath - ? true - : stopSummary.cancellation.durablyRecorded - const reason = activeResumeSignalFailed - ? 'active_resume_signal_failed' - : pauseReconciliationFailed - ? 'paused_database_cancel_failed' - : effectivePausedCancellationPath && !cancellationEventPublished - ? 'paused_event_publish_failed' - : effectivePausedCancellationPath && !pausedCancelled - ? 'paused_database_cancel_failed' - : effectivePausedCancellationPath - ? 'recorded' - : stopSummary.queueJobsCancelled > 0 && - !stopSummary.cancellation.durablyRecorded && - !stopSummary.locallyAborted - ? 'queue_cancelled' - : stopSummary.cancellation.reason - - return cancellationOutcome({ - success, - executionId, - redisAvailable: - effectivePausedCancellationPath || pausedCancelled - ? cancellationEventPublished - : stopSummary.cancellation.reason !== 'redis_unavailable', - durablyRecorded, - locallyAborted: stopSummary.locallyAborted, - pausedCancelled, - reason, - }) - } catch (error) { - logger.error('Failed to cancel execution', { - workflowId, - executionId, - error: toError(error).message, - }) - return NextResponse.json( - { error: toError(error).message || 'Failed to cancel execution' }, - { status: 500 } - ) - } - } -) +export const POST = defineInternalJsonRoute({ + contract: cancelWorkflowExecutionContract, + auth: internalWorkflowSessionOrApiKeyAuth, + operation: workflowOperations.cancelRun, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal cancellation behavior', + }), + errorPolicy: internalWorkflowErrorPolicies.concealRunAuthorization, + mapInput: ({ params }, { request }) => ({ + workflowId: params.id, + runId: params.executionId, + abortSignal: request.signal, + }), + useCase: cancelWorkflowRun, + present: (result) => ({ + success: result.success, + executionId: result.executionId, + redisAvailable: result.redisAvailable, + durablyRecorded: result.durablyRecorded, + locallyAborted: result.locallyAborted, + pausedCancelled: result.pausedCancelled, + reason: result.reason, + }), +}) diff --git a/apps/sim/lib/api/contracts/v2/workflows.ts b/apps/sim/lib/api/contracts/v2/workflows.ts index 2ac11bf28c2..05fa6837a73 100644 --- a/apps/sim/lib/api/contracts/v2/workflows.ts +++ b/apps/sim/lib/api/contracts/v2/workflows.ts @@ -1869,15 +1869,10 @@ export const v2CancelWorkflowRunDataSchema = z ), locallyAborted: z.boolean().describe('Whether an in-process execution was aborted.'), pausedCancelled: z.boolean().describe('Whether a paused execution was cancelled.'), - /** - * Always emitted by the cancellation service — it is not a partial-failure - * marker. `recorded` is the full-success value; the `already_*` values name - * a terminal no-op; the rest name the step that degraded. - */ reason: cancelWorkflowExecutionReasonSchema .optional() .describe( - 'Machine-readable cancellation outcome, present on every cancellation including full successes. `recorded` is the success value. `already_cancelled`, `already_completed`, and `already_failed` mean the run had already reached that terminal state, so nothing was cancelled and `durablyRecorded` is false. `redis_unavailable` and `redis_write_failed` mean the distributed cancellation signal was not written, so an already-running execution may not observe the cancellation. `paused_event_publish_failed` and `paused_database_cancel_failed` name the failing step for a paused run.' + 'Machine-readable cancellation outcome, present on every cancellation including full successes. `recorded` and `queue_cancelled` are successful cancellation values. `already_cancelled`, `already_completed`, and `already_failed` mean the run had already reached that terminal state, so nothing was cancelled and `durablyRecorded` is false. The remaining values identify a degraded or incomplete cancellation step.' ), }) .meta({ diff --git a/apps/sim/lib/api/contracts/workflows.test.ts b/apps/sim/lib/api/contracts/workflows.test.ts index 5f026cfa9b6..26d19fb7ab6 100644 --- a/apps/sim/lib/api/contracts/workflows.test.ts +++ b/apps/sim/lib/api/contracts/workflows.test.ts @@ -3,7 +3,6 @@ import { cancelWorkflowExecutionReasonSchema, executeWorkflowBodySchema, getWorkflowResponseDataSchema, - internalCancelWorkflowExecutionReasonSchema, updateWorkflowBodySchema, workflowListItemSchema, workflowStateSchema, @@ -131,34 +130,16 @@ describe('workflow contracts', () => { expect(forkPolicySchema.parse({}).forkSyncExcluded).toBe(false) }) - /** - * The v2 cancel endpoint presents `cancelWorkflowRun`'s result unchanged, and - * that use case delegates wholly to the cancellation service — so it cannot - * emit the outcomes the internal route resolves for itself. Folding those into - * the service enum would publish reasons v2 never returns, because the v2 - * contract documents this enum value by value. - */ - it('keeps internal-only cancellation reasons out of the enum v2 publishes', () => { + it('exposes the cancellation service vocabulary to every cancel surface', () => { for (const reason of [ + 'already_cancelled', + 'already_completed', + 'already_failed', 'queue_cancelled', 'active_resume_signal_failed', 'cancellation_not_finalized', ]) { - expect(internalCancelWorkflowExecutionReasonSchema.options).toContain(reason) - expect(cancelWorkflowExecutionReasonSchema.options).not.toContain(reason) - } - }) - - /** - * Both surfaces answer a cancel against an already-terminal run, so both name - * it with the same member. The service observes the terminal status itself now - * — the internal route no longer owns `already_cancelled` privately — and - * without these the v2 contract rejects the very body v2 emits. - */ - it('shares the terminal no-op vocabulary between both cancel surfaces', () => { - for (const reason of ['already_cancelled', 'already_completed', 'already_failed']) { expect(cancelWorkflowExecutionReasonSchema.options).toContain(reason) - expect(internalCancelWorkflowExecutionReasonSchema.options).toContain(reason) } }) diff --git a/apps/sim/lib/api/contracts/workflows.ts b/apps/sim/lib/api/contracts/workflows.ts index e6580a60954..5bd017e428d 100644 --- a/apps/sim/lib/api/contracts/workflows.ts +++ b/apps/sim/lib/api/contracts/workflows.ts @@ -655,18 +655,7 @@ export const workflowExecutionStatusQuerySchema = z.object({ ), }) -/** - * Cancellation outcomes produced by the cancellation service, and so the whole - * vocabulary the public v2 endpoint can return — `cancelWorkflowRun` delegates - * its outcome to that service. Mirrors `CancelWorkflowExecutionReason` in - * `lib/execution/cancel-workflow-execution` (contracts stay import-clean of - * server modules). Keeping the internal route's extra outcomes out of here is - * what stops the published v2 schema advertising reasons v2 cannot emit. - * - * `already_cancelled`/`already_completed`/`already_failed` report a run that was - * already terminal when the request arrived: the request is satisfied, but no - * durable write happened, so they always pair with `durablyRecorded: false`. - */ +/** Mirrors the surface-neutral cancellation service's complete outcome vocabulary. */ export const cancelWorkflowExecutionReasonSchema = z.enum([ 'recorded', 'already_cancelled', @@ -676,22 +665,6 @@ export const cancelWorkflowExecutionReasonSchema = z.enum([ 'redis_write_failed', 'paused_event_publish_failed', 'paused_database_cancel_failed', -]) - -/** - * The internal route's vocabulary. It reimplements cancellation rather than - * calling the service, so it resolves three further outcomes of its own: - * `queue_cancelled` (the run was still queued, so no execution log row existed), - * `active_resume_signal_failed`, and `cancellation_not_finalized`. Several ride - * on `success: true` responses, so validating them against the service enum - * makes `requestJson` reject cancellations that genuinely applied. - * - * The `already_*` outcomes are no longer route-local: the service now observes - * the run's terminal status itself, so both surfaces name a terminal no-op with - * the same member. - */ -export const internalCancelWorkflowExecutionReasonSchema = z.enum([ - ...cancelWorkflowExecutionReasonSchema.options, 'queue_cancelled', 'active_resume_signal_failed', 'cancellation_not_finalized', @@ -704,7 +677,7 @@ const cancelWorkflowExecutionResponseSchema = z.object({ durablyRecorded: z.boolean(), locallyAborted: z.boolean(), pausedCancelled: z.boolean(), - reason: internalCancelWorkflowExecutionReasonSchema.optional(), + reason: cancelWorkflowExecutionReasonSchema.optional(), }) export type CancelWorkflowExecutionResponse = z.output diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index bbb44542619..77dce348bd8 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -34,6 +34,7 @@ export interface ToolCatalogEntry { | 'browser_type' | 'browser_wait_for' | 'call_integration_tool' + | 'cancel_workflow_run' | 'connect_slack_bot' | 'cp' | 'create_empty_file' @@ -71,6 +72,7 @@ export interface ToolCatalogEntry { | 'load_deployment' | 'load_integration_tool' | 'load_skill' + | 'load_slide_layout' | 'manage_credential' | 'manage_custom_tool' | 'manage_knowledge_base' @@ -162,6 +164,7 @@ export interface ToolCatalogEntry { | 'browser_type' | 'browser_wait_for' | 'call_integration_tool' + | 'cancel_workflow_run' | 'connect_slack_bot' | 'cp' | 'create_empty_file' @@ -199,6 +202,7 @@ export interface ToolCatalogEntry { | 'load_deployment' | 'load_integration_tool' | 'load_skill' + | 'load_slide_layout' | 'manage_credential' | 'manage_custom_tool' | 'manage_knowledge_base' @@ -1612,6 +1616,31 @@ export const CallIntegrationTool: ToolCatalogEntry = { requiresApproval: true, } +export const CancelWorkflowRun: ToolCatalogEntry = { + id: 'cancel_workflow_run', + name: 'cancel_workflow_run', + route: 'sim', + mode: 'async', + parameters: { + type: 'object', + properties: { + executionId: { + type: 'string', + description: + 'Required workflow execution ID returned by run_workflow with async:true or found with query_logs. This identifies a workflow run, not an agent invocation or chat request.', + }, + workflowId: { + type: 'string', + description: + 'Workflow ID that owns the execution. Optional; omit it to target the current workflow. Pass it when cancelling a run from another workflow.', + }, + }, + required: ['executionId'], + }, + requiredPermission: 'write', + requiresApproval: true, +} + export const ConnectSlackBot: ToolCatalogEntry = { id: 'connect_slack_bot', name: 'connect_slack_bot', @@ -3143,6 +3172,24 @@ export const LoadSkill: ToolCatalogEntry = { }, } +export const LoadSlideLayout: ToolCatalogEntry = { + id: 'load_slide_layout', + name: 'load_slide_layout', + route: 'go', + mode: 'sync', + parameters: { + type: 'object', + properties: { + name: { + type: 'string', + description: + "Layout name exactly as it appears in the Layout Library index (e.g. 'metric-cards').", + }, + }, + required: ['name'], + }, +} + export const ManageCredential: ToolCatalogEntry = { id: 'manage_credential', name: 'manage_credential', @@ -4406,7 +4453,11 @@ export const Run: ToolCatalogEntry = { description: 'Pre-gathered context: workflow state, block IDs, input requirements.', type: 'string', }, - request: { description: 'What to run or what logs to check.', type: 'string' }, + request: { + description: + 'What to run or cancel, or what logs to check. Include a known workflow executionId when cancelling.', + type: 'string', + }, }, required: ['request'], type: 'object', @@ -7004,6 +7055,7 @@ export const TOOL_CATALOG: Record = { [BrowserType.id]: BrowserType, [BrowserWaitFor.id]: BrowserWaitFor, [CallIntegrationTool.id]: CallIntegrationTool, + [CancelWorkflowRun.id]: CancelWorkflowRun, [ConnectSlackBot.id]: ConnectSlackBot, [Cp.id]: Cp, [CreateEmptyFile.id]: CreateEmptyFile, @@ -7041,6 +7093,7 @@ export const TOOL_CATALOG: Record = { [LoadDeployment.id]: LoadDeployment, [LoadIntegrationTool.id]: LoadIntegrationTool, [LoadSkill.id]: LoadSkill, + [LoadSlideLayout.id]: LoadSlideLayout, [ManageCredential.id]: ManageCredential, [ManageCustomTool.id]: ManageCustomTool, [ManageKnowledgeBase.id]: ManageKnowledgeBase, diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index c7f0cfcd3bf..9d6a39d5114 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -1560,6 +1560,25 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, + cancel_workflow_run: { + parameters: { + type: 'object', + properties: { + executionId: { + type: 'string', + description: + 'Required workflow execution ID returned by run_workflow with async:true or found with query_logs. This identifies a workflow run, not an agent invocation or chat request.', + }, + workflowId: { + type: 'string', + description: + 'Workflow ID that owns the execution. Optional; omit it to target the current workflow. Pass it when cancelling a run from another workflow.', + }, + }, + required: ['executionId'], + }, + resultSchema: undefined, + }, connect_slack_bot: { parameters: { type: 'object', @@ -3028,6 +3047,20 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, + load_slide_layout: { + parameters: { + type: 'object', + properties: { + name: { + type: 'string', + description: + "Layout name exactly as it appears in the Layout Library index (e.g. 'metric-cards').", + }, + }, + required: ['name'], + }, + resultSchema: undefined, + }, manage_credential: { parameters: { type: 'object', @@ -4314,7 +4347,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { type: 'string', }, request: { - description: 'What to run or what logs to check.', + description: + 'What to run or cancel, or what logs to check. Include a known workflow executionId when cancelling.', type: 'string', }, }, diff --git a/apps/sim/lib/copilot/request/tools/permission.test.ts b/apps/sim/lib/copilot/request/tools/permission.test.ts index 58a6e6c8113..35268410ab4 100644 --- a/apps/sim/lib/copilot/request/tools/permission.test.ts +++ b/apps/sim/lib/copilot/request/tools/permission.test.ts @@ -213,6 +213,7 @@ describe('gated tools are askable', () => { .sort() ).toEqual([ 'call_integration_tool', + 'cancel_workflow_run', 'delete_workspace_mcp_server', 'deploy_as_api', 'deploy_as_chat', diff --git a/apps/sim/lib/copilot/tool-executor/register-handlers.ts b/apps/sim/lib/copilot/tool-executor/register-handlers.ts index a598d4135e7..5655d7aa2ba 100644 --- a/apps/sim/lib/copilot/tool-executor/register-handlers.ts +++ b/apps/sim/lib/copilot/tool-executor/register-handlers.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { + CancelWorkflowRun, ConnectSlackBot, Cp as CpTool, CreateWorkflow, @@ -92,6 +93,7 @@ import { executeVfsRm, } from '../tools/handlers/vfs-mutate' import { + executeCancelWorkflowRun, executeCreateWorkflow, executeGenerateApiKey, executeMoveWorkflow, @@ -146,6 +148,7 @@ function buildHandlerMap(): Record { rename_workflow: h(executeRenameWorkflow), move_workflow: h(executeMoveWorkflow), [RunWorkflow.id]: h(executeRunWorkflow), + [CancelWorkflowRun.id]: h(executeCancelWorkflowRun), [RunWorkflowUntilBlock.id]: h(executeRunWorkflowUntilBlock), [RunFromBlock.id]: h(executeRunFromBlock), [RunBlock.id]: h(executeRunBlock), diff --git a/apps/sim/lib/copilot/tool-executor/router.test.ts b/apps/sim/lib/copilot/tool-executor/router.test.ts new file mode 100644 index 00000000000..f031cfc18b5 --- /dev/null +++ b/apps/sim/lib/copilot/tool-executor/router.test.ts @@ -0,0 +1,29 @@ +/** + * @vitest-environment node + */ + +import { describe, expect, it } from 'vitest' +import { hasHandler } from '@/lib/copilot/tool-executor/executor' +import { ensureHandlersRegistered } from '@/lib/copilot/tool-executor/register-handlers' +import { + getToolEntry, + isSimExecuted, + toolRequiresApproval, +} from '@/lib/copilot/tool-executor/router' + +describe('workflow-run cancellation tool routing', () => { + it('routes cancellation through Sim with write permission and explicit approval', () => { + expect(getToolEntry('cancel_workflow_run')).toMatchObject({ + requiredPermission: 'write', + route: 'sim', + }) + expect(isSimExecuted('cancel_workflow_run')).toBe(true) + expect(toolRequiresApproval('cancel_workflow_run')).toBe(true) + }) + + it('registers the Sim cancellation handler', () => { + ensureHandlersRegistered() + + expect(hasHandler('cancel_workflow_run')).toBe(true) + }) +}) diff --git a/apps/sim/lib/copilot/tools/handlers/param-types.ts b/apps/sim/lib/copilot/tools/handlers/param-types.ts index afe24b1b1ed..441a61e2d8c 100644 --- a/apps/sim/lib/copilot/tools/handlers/param-types.ts +++ b/apps/sim/lib/copilot/tools/handlers/param-types.ts @@ -60,6 +60,12 @@ export interface RunWorkflowParams { useDeployedState?: boolean } +export interface CancelWorkflowRunParams { + workflowId?: string + /** The workflow execution ID returned by run_workflow or query_logs. */ + executionId?: string +} + export interface RunWorkflowUntilBlockParams { workflowId?: string workflow_input?: unknown diff --git a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts index 37f6860601b..22448ced238 100644 --- a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts @@ -3,6 +3,7 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ExecutionContext } from '@/lib/copilot/request/types' +import { WorkflowRunAlreadyTerminalError } from '@/lib/execution/workflow-run-already-terminal-error' const { mocks } = vi.hoisted(() => ({ mocks: { @@ -14,8 +15,8 @@ const { mocks } = vi.hoisted(() => ({ vi.mock('@/lib/copilot/application/execute-workflow-use-case', () => ({ executeCopilotWorkflowUseCase: mocks.executeWorkflowUseCase, - messageForCopilotWorkflowError: (_error: unknown, fallback = 'Workflow operation failed') => - fallback, + messageForCopilotWorkflowError: (error: unknown, fallback = 'Workflow operation failed') => + error instanceof Error && 'code' in error ? error.message : fallback, })) vi.mock('@/lib/copilot/application/execute-api-key-use-case', () => ({ @@ -35,6 +36,7 @@ vi.mock('@/lib/core/telemetry', () => ({ })) import { + executeCancelWorkflowRun, executeCreateWorkflow, executeGenerateApiKey, executeMoveWorkflow, @@ -50,6 +52,7 @@ const context = { workspaceId: 'workspace-1', workflowId: 'workflow-1', toolCallId: 'tool-call-1', + copilotToolExecution: true, billingAttribution: { workspaceId: 'workspace-1' }, } as ExecutionContext @@ -158,6 +161,74 @@ describe('workflow mutation Copilot adapters', () => { ) }) + it('cancels a workflow run through the canonical application use case', async () => { + mocks.executeWorkflowUseCase.mockResolvedValue({ + success: true, + executionId: 'execution-1', + redisAvailable: true, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'recorded', + }) + + const result = await executeCancelWorkflowRun( + { workflowId: 'workflow-1', executionId: 'execution-1' }, + context + ) + + expect(result).toEqual({ + success: true, + output: { + workflowId: 'workflow-1', + executionId: 'execution-1', + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'recorded', + }, + }) + expect(mocks.executeWorkflowUseCase).toHaveBeenCalledWith( + context, + expect.objectContaining({ + operation: expect.objectContaining({ id: 'workflows.runs.cancel' }), + }), + { + workflowId: 'workflow-1', + runId: 'execution-1', + assertedWorkspaceId: 'workspace-1', + } + ) + }) + + it('returns a cancellation application error to the Run agent', async () => { + mocks.executeWorkflowUseCase.mockRejectedValue( + new WorkflowRunAlreadyTerminalError({ + executionId: 'execution-1', + executionStatus: 'completed', + redisAvailable: true, + locallyAborted: false, + }) + ) + + const result = await executeCancelWorkflowRun( + { workflowId: 'workflow-1', executionId: 'execution-1' }, + context + ) + + expect(result).toEqual({ + success: false, + error: 'Execution cannot be cancelled while completed', + }) + }) + + it('requires an execution ID before attempting workflow-run cancellation', async () => { + const result = await executeCancelWorkflowRun({ workflowId: 'workflow-1' }, context) + + expect(result).toEqual({ success: false, error: 'executionId is required' }) + expect(mocks.executeWorkflowUseCase).not.toHaveBeenCalled() + }) + it.each([ { label: 'until', diff --git a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts index 164574a84cb..9437c40e052 100644 --- a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts +++ b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts @@ -10,6 +10,7 @@ import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/typ import { requireCopilotWorkspace } from '@/lib/copilot/tools/server/workspace-scope' import { decodeVfsPathSegments, encodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils' import { PlatformEvents } from '@/lib/core/telemetry' +import { cancelWorkflowRun } from '@/lib/workflows/application/cancel-run' import { createWorkflow } from '@/lib/workflows/application/create-workflow' import { moveWorkflowsBulk } from '@/lib/workflows/application/move-workflows-bulk' import { @@ -116,6 +117,7 @@ function copilotRunLifecycle(context: ExecutionContext) { } import type { + CancelWorkflowRunParams, CreateWorkflowParams, GenerateApiKeyParams, MoveWorkflowParams, @@ -227,6 +229,51 @@ export async function executeRunWorkflow( } } +export async function executeCancelWorkflowRun( + params: CancelWorkflowRunParams, + context: ExecutionContext +): Promise { + try { + const workflowId = params.workflowId || context.workflowId + if (!workflowId) { + return { success: false, error: 'workflowId is required' } + } + const executionId = resolveInputFromExecutionId(params.executionId) + if (!executionId) { + return { success: false, error: 'executionId is required' } + } + + assertWorkflowMutationNotAborted( + context, + 'Request aborted before workflow run cancellation could be applied.' + ) + const result = await executeCopilotWorkflowUseCase(context, cancelWorkflowRun, { + workflowId, + runId: executionId, + assertedWorkspaceId: context.workspaceId, + ...(context.abortSignal ? { abortSignal: context.abortSignal } : {}), + }) + + return { + success: result.success, + output: { + workflowId, + executionId: result.executionId, + durablyRecorded: result.durablyRecorded, + locallyAborted: result.locallyAborted, + pausedCancelled: result.pausedCancelled, + reason: result.reason, + }, + error: result.success ? undefined : 'Workflow run cancellation could not be completed', + } + } catch (error) { + return { + success: false, + error: messageForCopilotWorkflowError(error, 'Failed to cancel workflow run'), + } + } +} + export async function executeSetGlobalWorkflowVariables( params: SetGlobalWorkflowVariablesParams, context: ExecutionContext diff --git a/apps/sim/lib/copilot/tools/tool-display.test.ts b/apps/sim/lib/copilot/tools/tool-display.test.ts index 253dd9d94d5..d3e6e212d4f 100644 --- a/apps/sim/lib/copilot/tools/tool-display.test.ts +++ b/apps/sim/lib/copilot/tools/tool-display.test.ts @@ -646,6 +646,9 @@ describe('opaque id suppression', () => { it('falls back to the generic label instead of printing a workflow id', () => { expect(getToolDisplayTitle('run_workflow', { workflowName: uuid })).toBe('Running workflow') expect(getToolDisplayTitle('run_workflow', { name: uuid })).toBe('Running workflow') + expect(getToolDisplayTitle('cancel_workflow_run', { executionId: uuid })).toBe( + 'Cancelling workflow run' + ) }) it('suppresses a bare-hex id too', () => { diff --git a/apps/sim/lib/copilot/tools/tool-display.ts b/apps/sim/lib/copilot/tools/tool-display.ts index 947f0845854..9cc094d81c5 100644 --- a/apps/sim/lib/copilot/tools/tool-display.ts +++ b/apps/sim/lib/copilot/tools/tool-display.ts @@ -535,6 +535,7 @@ const TOOL_TITLES: Record = { search_integration_tools: 'Finding the right integration', load_integration_tool: 'Loading integration tools', load_skill: 'Loading skill', + load_slide_layout: 'Loading slide layout', read: 'Reading file', search_library_docs: 'Searching library docs', user_table: 'Managing table', @@ -549,6 +550,7 @@ const TOOL_TITLES: Record = { prepare_file_edit: 'Editing file', apply_file_edit: 'Writing changes', create_workflow: 'Creating workflow', + cancel_workflow_run: 'Cancelling workflow run', edit_workflow: 'Editing workflow', manage_knowledge_base: 'Managing knowledge base', search_knowledge_base: 'Searching knowledge base', diff --git a/apps/sim/lib/execution/cancel-workflow-execution.test.ts b/apps/sim/lib/execution/cancel-workflow-execution.test.ts index 1f51da65463..bbfc773d679 100644 --- a/apps/sim/lib/execution/cancel-workflow-execution.test.ts +++ b/apps/sim/lib/execution/cancel-workflow-execution.test.ts @@ -1,637 +1,1687 @@ /** * @vitest-environment node */ + +import { databaseMock, dbChainMockFns, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const { + mockMarkExecutionCancelled, + mockClearExecutionCancellation, mockAbortManualExecution, mockBeginPausedCancellation, - mockBlockQueuedResumes, - mockCancelWorkflowGroupExecution, - mockCaptureServerEvent, + mockStagePausedCancellation, + mockBlockQueuedResumesForCancellation, mockClearPausedCancellationIntent, mockCompletePausedCancellation, - mockGetJobQueue, mockGetPausedCancellationStatus, - mockMarkExecutionCancelled, - mockPublishWorkflowGroupCancellationEvent, + mockGetActiveResumeCancellationTarget, + mockRollbackActiveResumeCancellation, + mockFinalizeExecutionStream, + mockReadExecutionMetaState, + mockWriteEvent, + mockWriteTerminalEvent, + mockCancelByExecution, + mockGetJobQueue, mockReleaseExecutionSlot, - mockUpdateSet, - mockUpdateReturning, - mockResolveWorkflowExecutionOwnership, + mockCancelWorkflowGroupExecution, + mockPublishWorkflowGroupCancellationEvent, } = vi.hoisted(() => ({ + mockMarkExecutionCancelled: vi.fn(), + mockClearExecutionCancellation: vi.fn(), mockAbortManualExecution: vi.fn(), mockBeginPausedCancellation: vi.fn(), - mockBlockQueuedResumes: vi.fn(), - mockCancelWorkflowGroupExecution: vi.fn(), - mockCaptureServerEvent: vi.fn(), + mockStagePausedCancellation: vi.fn(), + mockBlockQueuedResumesForCancellation: vi.fn(), mockClearPausedCancellationIntent: vi.fn(), mockCompletePausedCancellation: vi.fn(), - mockGetJobQueue: vi.fn(), mockGetPausedCancellationStatus: vi.fn(), - mockMarkExecutionCancelled: vi.fn(), - mockPublishWorkflowGroupCancellationEvent: vi.fn(), + mockGetActiveResumeCancellationTarget: vi.fn(), + mockRollbackActiveResumeCancellation: vi.fn(), + mockFinalizeExecutionStream: vi.fn(), + mockReadExecutionMetaState: vi.fn(), + mockWriteEvent: vi.fn(), + mockWriteTerminalEvent: vi.fn(), + mockCancelByExecution: vi.fn(), + mockGetJobQueue: vi.fn(), mockReleaseExecutionSlot: vi.fn(), - mockUpdateSet: vi.fn(), - mockUpdateReturning: vi.fn(), - mockResolveWorkflowExecutionOwnership: vi.fn(), + mockCancelWorkflowGroupExecution: vi.fn(), + mockPublishWorkflowGroupCancellationEvent: vi.fn(), })) -vi.mock('@sim/db', () => ({ - db: { - update: () => ({ - set: (values: unknown) => { - mockUpdateSet(values) - return { where: () => ({ returning: () => Promise.resolve(mockUpdateReturning()) }) } - }, - }), - }, +vi.mock('@/lib/core/async-jobs', () => ({ + getJobQueue: mockGetJobQueue, })) vi.mock('@/lib/billing/calculations/usage-reservation', () => ({ releaseExecutionSlot: mockReleaseExecutionSlot, })) -vi.mock('@/lib/table/workflow-group-cancellation', () => ({ - cancelWorkflowGroupExecution: mockCancelWorkflowGroupExecution, - publishWorkflowGroupCancellationEvent: mockPublishWorkflowGroupCancellationEvent, -})) - vi.mock('@/lib/execution/cancellation', () => ({ - markExecutionCancelled: mockMarkExecutionCancelled, + markExecutionCancelled: (...args: unknown[]) => mockMarkExecutionCancelled(...args), + clearExecutionCancellation: (...args: unknown[]) => mockClearExecutionCancellation(...args), })) vi.mock('@/lib/execution/manual-cancellation', () => ({ - abortManualExecution: mockAbortManualExecution, + abortManualExecution: (...args: unknown[]) => mockAbortManualExecution(...args), })) -vi.mock('@/lib/execution/event-buffer', () => ({ - createExecutionEventWriter: () => ({ - writeTerminal: vi.fn().mockResolvedValue(undefined), - close: vi.fn().mockResolvedValue(undefined), - }), - readExecutionMetaState: vi.fn().mockResolvedValue({ status: 'missing' }), +vi.mock('@/lib/workflows/executor/human-in-the-loop-manager', () => ({ + PauseResumeManager: { + beginPausedCancellation: (...args: unknown[]) => mockBeginPausedCancellation(...args), + stagePausedCancellation: (...args: unknown[]) => mockStagePausedCancellation(...args), + blockQueuedResumesForCancellation: (...args: unknown[]) => + mockBlockQueuedResumesForCancellation(...args), + clearPausedCancellationIntent: (...args: unknown[]) => + mockClearPausedCancellationIntent(...args), + completePausedCancellation: (...args: unknown[]) => mockCompletePausedCancellation(...args), + getPausedCancellationStatus: (...args: unknown[]) => mockGetPausedCancellationStatus(...args), + getActiveResumeCancellationTarget: (...args: unknown[]) => + mockGetActiveResumeCancellationTarget(...args), + rollbackActiveResumeCancellation: (...args: unknown[]) => + mockRollbackActiveResumeCancellation(...args), + }, })) -vi.mock('@/lib/core/async-jobs', () => ({ - getJobQueue: mockGetJobQueue, +vi.mock('@/lib/table/workflow-group-cancellation', () => ({ + cancelWorkflowGroupExecution: (...args: unknown[]) => mockCancelWorkflowGroupExecution(...args), + publishWorkflowGroupCancellationEvent: (...args: unknown[]) => + mockPublishWorkflowGroupCancellationEvent(...args), })) -vi.mock('@/lib/posthog/server', () => ({ - captureServerEvent: mockCaptureServerEvent, +vi.mock('@/lib/execution/event-buffer', () => ({ + finalizeExecutionStream: (...args: unknown[]) => mockFinalizeExecutionStream(...args), + readExecutionMetaState: (...args: unknown[]) => mockReadExecutionMetaState(...args), + createExecutionEventWriter: () => ({ + write: (...args: unknown[]) => mockWriteEvent(...args), + writeTerminal: (...args: unknown[]) => mockWriteTerminalEvent(...args), + close: vi.fn().mockResolvedValue(undefined), + }), })) -vi.mock('@/lib/workflows/executor/execution-job-ids', () => ({ - WORKFLOW_EXECUTION_JOB_ID_PREFIX: 'workflow-execution:', -})) +import { cancelWorkflowExecutionContract } from '@/lib/api/contracts/workflows' +import { OrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { + type CancelWorkflowExecutionInput, + cancelWorkflowExecution, + WorkflowExecutionNotFoundError, +} from '@/lib/execution/cancel-workflow-execution' +import { WorkflowRunAlreadyTerminalError } from '@/lib/execution/workflow-run-already-terminal-error' + +const INPUT: CancelWorkflowExecutionInput = { + workflowId: 'wf-1', + executionId: 'ex-1', + workspaceId: 'workspace-1', + attributedUserId: 'user-1', +} -vi.mock('@/lib/workflows/executor/execution-queries', () => ({ - resolveWorkflowExecutionOwnership: mockResolveWorkflowExecutionOwnership, -})) +async function cancelAsResponse( + overrides: Partial = {} +): Promise { + try { + const result = await cancelWorkflowExecution({ ...INPUT, ...overrides }) + const body = cancelWorkflowExecutionContract.response.schema.parse(result) + return Response.json(body) + } catch (error) { + if (error instanceof WorkflowExecutionNotFoundError) { + return Response.json({ error: error.message }, { status: 404 }) + } + if (error instanceof OrchestrationError) { + return Response.json( + { error: error.message }, + { status: statusForOrchestrationError(error.code) } + ) + } + if (error instanceof Error) { + return Response.json({ error: error.message }, { status: 500 }) + } + throw error + } +} -vi.mock('@/lib/workflows/executor/human-in-the-loop-manager', () => ({ - PauseResumeManager: { - beginPausedCancellation: mockBeginPausedCancellation, - getPausedCancellationStatus: mockGetPausedCancellationStatus, - blockQueuedResumesForCancellation: mockBlockQueuedResumes, - clearPausedCancellationIntent: mockClearPausedCancellationIntent, - completePausedCancellation: mockCompletePausedCancellation, - }, -})) +const POST = async (..._args: unknown[]) => cancelAsResponse() +const makeRequest = () => undefined +const makeParams = () => undefined -import { cancelWorkflowExecution } from '@/lib/execution/cancel-workflow-execution' +const ACTIVE_RESUME_TARGET = { + resumeEntryId: 'resume-entry-1', + pausedExecutionId: 'paused-1', + parentExecutionId: 'ex-1', + resumeExecutionId: 'resume-ex-1', +} -/** - * The durable writes a workflow-group transition reports back. The transaction - * updates the workflow log only, the cell sidecar only, or both, so a single - * `kind` cannot answer whether this request wrote anything. - */ -const NO_WRITES = { workflowLogTerminalized: false, sidecarCancelled: false } as const -const LOG_WRITE = { workflowLogTerminalized: true, sidecarCancelled: false } as const -const SIDECAR_WRITE = { workflowLogTerminalized: false, sidecarCancelled: true } as const -const BOTH_WRITES = { workflowLogTerminalized: true, sidecarCancelled: true } as const - -const INPUT = { - executionId: 'execution-1', - workflowId: 'workflow-1', - userId: 'user-1', - workspaceId: 'workspace-1', +const REPLACEMENT_ACTIVE_RESUME_TARGET = { + ...ACTIVE_RESUME_TARGET, + resumeEntryId: 'resume-entry-2', + resumeExecutionId: 'resume-ex-2', } describe('cancelWorkflowExecution', () => { beforeEach(() => { vi.clearAllMocks() - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'running', - }) - mockUpdateReturning.mockReturnValue([{ id: 'log-1' }]) - mockBeginPausedCancellation.mockResolvedValue(false) - mockGetPausedCancellationStatus.mockResolvedValue(null) + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValue([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + dbChainMockFns.returning.mockResolvedValue([{ status: 'cancelled' }]) + mockCancelByExecution.mockReset().mockResolvedValue(0) + mockGetJobQueue.mockReset().mockResolvedValue({ cancelByExecution: mockCancelByExecution }) + mockReleaseExecutionSlot.mockReset().mockResolvedValue(undefined) + mockCancelWorkflowGroupExecution.mockReset().mockResolvedValue({ kind: 'not_workflow_group' }) + mockPublishWorkflowGroupCancellationEvent.mockReset().mockResolvedValue(undefined) + mockClearExecutionCancellation.mockReset().mockResolvedValue(undefined) + mockMarkExecutionCancelled + .mockReset() + .mockResolvedValue({ durablyRecorded: false, reason: 'redis_unavailable' }) + mockAbortManualExecution.mockReset().mockReturnValue(false) + mockBeginPausedCancellation.mockReset().mockResolvedValue(false) + mockStagePausedCancellation.mockReset().mockResolvedValue({ kind: 'not_paused' }) + mockBlockQueuedResumesForCancellation.mockReset().mockResolvedValue(false) + mockClearPausedCancellationIntent.mockReset().mockResolvedValue(undefined) + mockCompletePausedCancellation.mockReset().mockResolvedValue(false) + mockGetPausedCancellationStatus.mockReset().mockResolvedValue(null) + mockGetActiveResumeCancellationTarget.mockReset().mockResolvedValue(null) + mockRollbackActiveResumeCancellation.mockReset().mockResolvedValue(true) + mockFinalizeExecutionStream.mockReset().mockResolvedValue(true) + mockReadExecutionMetaState.mockReset().mockResolvedValue({ status: 'missing' }) + mockWriteEvent.mockReset().mockResolvedValue({ eventId: 1 }) + mockWriteTerminalEvent.mockReset().mockResolvedValue({ eventId: 1 }) + }) + + it('returns success when cancellation was durably recorded', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'recorded', + }) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { + workflowId: 'wf-1', + executionId: 'ex-1', + }, + 'standalone' + ) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1', { + executionDeadlineAt: null, + }) + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + }) + + it('atomically claims one workflow-group attempt before signalling it', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: true, + reason: 'recorded', + }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + workflowId: 'wf-1', + executionId: 'ex-1', + }) + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockMarkExecutionCancelled.mock.invocationCallOrder[0] + ) + expect(mockMarkExecutionCancelled.mock.invocationCallOrder[0]).toBeLessThan( + mockPublishWorkflowGroupCancellationEvent.mock.invocationCallOrder[0] + ) + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockStagePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + }) + + it('keeps a claimed group cancellation retryable when signalling fails', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + durablyRecorded: false, + reason: 'redis_unavailable', + }) + expect(mockMarkExecutionCancelled).toHaveBeenCalledOnce() + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) + + it('accepts an exact in-process group abort without cancelling its carrier', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + mockAbortManualExecution.mockReturnValue(true) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: false, + locallyAborted: true, + reason: 'redis_unavailable', + }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) + + it('does not cancel a shared carrier when an idle workflow-group pause appears', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - mockAbortManualExecution.mockReturnValue(false) - mockBlockQueuedResumes.mockResolvedValue(undefined) - mockClearPausedCancellationIntent.mockResolvedValue(undefined) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValue({ kind: 'idle' }) mockCompletePausedCancellation.mockResolvedValue(true) - mockReleaseExecutionSlot.mockResolvedValue(undefined) - mockPublishWorkflowGroupCancellationEvent.mockResolvedValue(undefined) - mockGetJobQueue.mockResolvedValue({ - getJob: vi.fn().mockResolvedValue(null), - cancelJob: vi.fn(), + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', }) + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockStagePausedCancellation.mock.invocationCallOrder[1]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') }) - /** - * The row reads `cancelled` after a successful cancel just as it does after - * someone else's, so a status re-read alone would report this run's own work - * as `already_cancelled`. Nothing is re-read once the claim moved a row. - */ - it('reports a durable write when an active run is cancelled', async () => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'running', - }) - .mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'cancelled', - }) + it('leaves a late workflow-group pause retryable when event publication fails', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValue({ kind: 'idle' }) + mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + pausedCancelled: false, + reason: 'paused_event_publish_failed', + }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + }) - const result = await cancelWorkflowExecution(INPUT) - - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledTimes(1) - }) - - /** - * A cancel against a run that already reached a terminal state changes - * nothing: the log claim's `status = 'running'` predicate matches no row and - * no terminal metadata moves. Reporting `recorded`/`durablyRecorded: true` - * there tells a caller a durable write happened when none did, so the outcome - * names the state that was actually observed instead. - */ - it.each([ - ['cancelled', 'already_cancelled'], - ['completed', 'already_completed'], - ['failed', 'already_failed'], - ])('reports a run already %s as a no-op rather than a durable write', async (status, reason) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: status, - }) - mockUpdateReturning.mockReturnValue([]) - - const result = await cancelWorkflowExecution(INPUT) - - expect(result).toMatchObject({ success: true, durablyRecorded: false, reason }) - }) - - /** - * Reclassification only ever applies to an otherwise-clean outcome. A run that - * reached any terminal status can still carry paused-HITL state — a - * force-failed run keeps whatever pause rows it had — and when reconciling - * that genuinely fails, the caller is owed the step that failed rather than a - * no-op that also flips `success` to `true`. - */ - it.each([['cancelled'], ['completed'], ['failed']])( - 'still reports the failing step when a %s run has paused work left over', - async (status) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: status, - }) - mockBeginPausedCancellation.mockResolvedValue(true) - mockCompletePausedCancellation.mockResolvedValue(false) + it('cancels an active workflow-group resume without cancelling its shared carrier', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + mockCancelByExecution.mockResolvedValue(1) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockCompletePausedCancellation.mockResolvedValue(true) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ - success: false, - durablyRecorded: true, - reason: 'paused_database_cancel_failed', - }) - } - ) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockCancelByExecution).toHaveBeenCalledOnce() + expect(mockCancelByExecution).toHaveBeenCalledWith( + { workflowId: 'wf-1', executionId: 'ex-1' }, + 'resume' + ) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockMarkExecutionCancelled.mock.invocationCallOrder[0] + ) + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) + }) - /** - * The status read at entry can be stale: a run that finishes after it and - * before the claim leaves a `running` snapshot on a cancel whose claim matched - * no row. The claim's own row count is what separates that from a cancel this - * request really performed. - */ - it.each([ - ['completed', 'already_completed'], - ['failed', 'already_failed'], - ])('reports a run that reached %s after the entry read as a no-op', async (status, reason) => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'running', - }) - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: status, - }) - mockUpdateReturning.mockReturnValue([]) + it('keeps a claimed active group resume retryable when no stop backend accepts it', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'active_resume_signal_failed', + }) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { workflowId: 'wf-1', executionId: 'ex-1' }, + 'resume' + ) + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockMarkExecutionCancelled.mock.invocationCallOrder[0] + ) + expect(mockRollbackActiveResumeCancellation).not.toHaveBeenCalled() + expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() + }) + + it('rechecks for a pause after the group terminal claim waits on persistence', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockStagePausedCancellation.mock.invocationCallOrder[1] + ) + expect(mockStagePausedCancellation.mock.invocationCallOrder[1]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) + expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() + }) + + it('rechecks for a regular pause after the terminal claim waits on persistence', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: false, reason }) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(databaseMock.db.update.mock.invocationCallOrder[0]).toBeLessThan( + mockStagePausedCancellation.mock.invocationCallOrder[1] + ) + expect(mockStagePausedCancellation.mock.invocationCallOrder[1]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) }) - it('reports an undifferentiated outcome when the claim finds no durable log row', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: null, + it('uses generic cancellation for a regular Table-trigger-block execution', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'running', + trigger: 'table', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', }) - mockUpdateReturning.mockReturnValue([]) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) + expect(response.status).toBe(200) + expect(mockCancelWorkflowGroupExecution).not.toHaveBeenCalled() + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1', { + executionDeadlineAt: null, + }) + expect(mockCancelByExecution).toHaveBeenCalledOnce() }) - /** - * The re-read is purely observational — it only refines *which* no-op the - * caller is told about. A database that cannot answer it must not take the - * cancel down with it: the run has already been cancelled in Redis and its - * reservation still has to be released, so the failure degrades to the - * undifferentiated outcome rather than propagating. - */ - it('degrades to the undifferentiated outcome when the status re-read fails', async () => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'running', - }) - .mockRejectedValueOnce(new Error('connection terminated')) - mockUpdateReturning.mockReturnValue([]) + it('returns 409 without generic cancellation when the exact group attempt is stale', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'conflict', + status: 'no_longer_active', + }) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledTimes(2) - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Workflow group execution cannot be cancelled while no_longer_active', + }) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockStagePausedCancellation).toHaveBeenCalledOnce() + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() }) - it('releases the plan concurrency reservation after a successful cancellation', async () => { - const result = await cancelWorkflowExecution(INPUT) + it('terminalizes a durable workflow-group log after its table sidecar was deleted', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ kind: 'cancelled_without_sidecar' }) - expect(result.success).toBe(true) - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + executionId: 'ex-1', + reason: 'recorded', + }) + expect(mockMarkExecutionCancelled).toHaveBeenCalledOnce() + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + }) + + it('returns unsuccessful response when Redis is unavailable', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: false, + executionId: 'ex-1', + redisAvailable: false, + durablyRecorded: false, + locallyAborted: false, + pausedCancelled: false, + reason: 'redis_unavailable', + }) + }) + + it('returns unsuccessful response when Redis persistence fails', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_write_failed', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: false, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: false, + locallyAborted: false, + pausedCancelled: false, + reason: 'redis_write_failed', + }) }) - it('keeps the reservation held when nothing could be cancelled', async () => { + it('returns success when local fallback aborts execution without Redis durability', async () => { mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, reason: 'redis_unavailable', }) + mockAbortManualExecution.mockReturnValue(true) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result.success).toBe(false) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + executionId: 'ex-1', + redisAvailable: false, + durablyRecorded: false, + locallyAborted: true, + pausedCancelled: false, + reason: 'redis_unavailable', + }) + }) + + it('returns success when the queue backend cancels the active job', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + mockCancelByExecution.mockResolvedValue(1) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + executionId: 'ex-1', + durablyRecorded: false, + locallyAborted: false, + reason: 'queue_cancelled', + }) + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + }) + + it('cancels a queued execution before its workflow log exists', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + mockCancelByExecution.mockResolvedValueOnce(1) + mockMarkExecutionCancelled.mockResolvedValueOnce({ + durablyRecorded: true, + reason: 'recorded', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'queue_cancelled', + }) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { + workflowId: 'wf-1', + executionId: 'ex-1', + }, + 'standalone' + ) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1') + expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('ex-1') + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + }) + + it('does not use an unscoped local abort before its workflow log exists', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ error: 'Execution not found' }) + expect(mockAbortManualExecution).not.toHaveBeenCalled() + expect(mockCancelByExecution).toHaveBeenCalledWith( + { + workflowId: 'wf-1', + executionId: 'ex-1', + }, + 'standalone' + ) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() }) - it('cancels and publishes the table cell sidecar of a workflow-group execution', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('finishes cancellation when a failed active-resume rollback detects a replacement', async () => { + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'active_resume', target: ACTIVE_RESUME_TARGET }) + .mockResolvedValueOnce({ + kind: 'active_resume', + target: REPLACEMENT_ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) + mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) + mockMarkExecutionCancelled + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + mockCompletePausedCancellation.mockResolvedValueOnce(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockRollbackActiveResumeCancellation).toHaveBeenCalledWith( + 'ex-1', + 'wf-1', + 'resume-entry-1' + ) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(1, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(2, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(3, 'resume-ex-2', { + executionDeadlineAt: null, + }) + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + }) + + it('finishes late pause cancellation when rollback detects a replacement resume', async () => { + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValueOnce({ kind: 'active_resume', target: ACTIVE_RESUME_TARGET }) + .mockResolvedValueOnce({ + kind: 'active_resume', + target: REPLACEMENT_ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) + mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) + mockMarkExecutionCancelled + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + mockCompletePausedCancellation.mockResolvedValueOnce(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockRollbackActiveResumeCancellation).toHaveBeenCalledWith( + 'ex-1', + 'wf-1', + 'resume-entry-1' + ) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(1, 'ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(2, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(3, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(4, 'resume-ex-2', { + executionDeadlineAt: null, + }) + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + }) + + it('does not treat replacement queue cancellation as confirmation of the original stop', async () => { + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'active_resume', target: ACTIVE_RESUME_TARGET }) + .mockResolvedValueOnce({ + kind: 'active_resume', + target: REPLACEMENT_ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget + .mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) + .mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) + mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) + mockCancelByExecution.mockResolvedValue(1) + mockMarkExecutionCancelled + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + pausedCancelled: false, + reason: 'active_resume_signal_failed', + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(1, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(2, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(3, 'resume-ex-2', { + executionDeadlineAt: null, + }) + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + }) + + it('returns success when a paused HITL execution is cancelled directly in the database', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: true, + reason: 'recorded', }) - const cancelled = { - kind: 'cancelled' as const, + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'execution:cancelled', + executionId: 'ex-1', + workflowId: 'wf-1', + }), + 'cancelled' + ) + expect(mockFinalizeExecutionStream).not.toHaveBeenCalled() + }) + + it('claims the paused workflow-group sidecar before publishing and finalizing', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: BOTH_WRITES, - } - mockCancelWorkflowGroupExecution.mockResolvedValue(cancelled) + }) + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result.success).toBe(true) - expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledWith({ - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }) - expect(mockPublishWorkflowGroupCancellationEvent).toHaveBeenCalledWith(cancelled, 'execution-1') - expect(mockUpdateSet).not.toHaveBeenCalled() - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') - }) - - /** - * A workflow-group log that is already `cancelled` can still own a cell - * sidecar left in `error`, and reconciling it to `cancelled` is a durable - * write this request performed. The terminal entry snapshot cannot see that - * work, so it must not reinterpret the outcome as a no-op — the API would - * otherwise tell the caller nothing changed and drop the cancellation event. - */ - it('reports a durable write when a cancelled group run still had its sidecar reconciled', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'cancelled', - }) - const cancelled = { - kind: 'cancelled' as const, + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockStagePausedCancellation.mock.invocationCallOrder[0]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + }) + + it('keeps a paused workflow-group cancellation reserved when event publication fails', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: SIDECAR_WRITE, - } - mockCancelWorkflowGroupExecution.mockResolvedValue(cancelled) + }) + mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockPublishWorkflowGroupCancellationEvent).toHaveBeenCalledWith(cancelled, 'execution-1') + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + pausedCancelled: false, + reason: 'paused_event_publish_failed', + }) + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() }) - /** - * The group path terminalizes the workflow log itself when the cell sidecar - * is already gone, so that outcome is a durable write too. - */ - it('reports a durable write when the group path cancels a run whose sidecar is gone', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('publishes paused cancellation event even when Redis cancellation is recorded', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + executionId: 'ex-1', + durablyRecorded: true, + pausedCancelled: true, }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled_without_sidecar', - writes: LOG_WRITE, + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'execution:cancelled', + executionId: 'ex-1', + workflowId: 'wf-1', + }), + 'cancelled' + ) + expect(mockFinalizeExecutionStream).not.toHaveBeenCalled() + }) + + it('does not confirm paused cancellation when terminal event publication fails', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: false, + executionId: 'ex-1', + redisAvailable: false, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'paused_event_publish_failed', }) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'execution:cancelled', + executionId: 'ex-1', + workflowId: 'wf-1', + }), + 'cancelled' + ) + expect(mockFinalizeExecutionStream).not.toHaveBeenCalled() + }) - const result = await cancelWorkflowExecution(INPUT) + it('finishes reconciliation when the pause row is already cancelled', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockUpdateSet).not.toHaveBeenCalled() + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) + }) + + it('stops before lookup when cancellation is already aborted', async () => { + const controller = new AbortController() + controller.abort() + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Request aborted before workflow run cancellation could be applied.', + }) + expect(databaseMock.db.select).not.toHaveBeenCalled() + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() }) - /** - * The mirror case: a group run that was already terminal and whose sidecar was - * already `cancelled` leaves both records untouched, so it still reports the - * state it observed rather than a durable write. - */ - it('reports a group run that changed nothing as a no-op', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'cancelled', + it('stops before mutation when cancellation is aborted during execution lookup', async () => { + const controller = new AbortController() + dbChainMockFns.limit.mockImplementationOnce(async () => { + controller.abort() + return [ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'running', + workspaceId: 'workspace-1', + }, + ] + }) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(409) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockAbortManualExecution).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) + + it('rolls back pause staging when cancellation is aborted during staging', async () => { + const controller = new AbortController() + mockStagePausedCancellation.mockImplementationOnce(async () => { + controller.abort() + return { kind: 'idle' } + }) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, }) + + expect(response.status).toBe(409) + expect(mockClearPausedCancellationIntent).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockAbortManualExecution).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) + + it('rolls back an active resume staged while cancellation is aborted', async () => { + const controller = new AbortController() + mockStagePausedCancellation.mockImplementationOnce(async () => { + controller.abort() + return { kind: 'active_resume', target: ACTIVE_RESUME_TARGET } + }) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(409) + expect(mockRollbackActiveResumeCancellation).toHaveBeenCalledWith( + 'ex-1', + 'wf-1', + 'resume-entry-1' + ) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + }) + + it('rolls back pause staging for an already-cancelled execution when cancellation is aborted', async () => { + const controller = new AbortController() + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockImplementationOnce(async () => { + controller.abort() + return { kind: 'idle' } + }) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(409) + expect(mockClearPausedCancellationIntent).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + }) + + it('finishes cancellation when an aborted active-resume stage cannot be rolled back', async () => { + const controller = new AbortController() + mockStagePausedCancellation.mockImplementationOnce(async () => { + controller.abort() + return { kind: 'active_resume', target: ACTIVE_RESUME_TARGET } + }) + mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, pausedCancelled: true }) + expect(mockRollbackActiveResumeCancellation).toHaveBeenCalledWith( + 'ex-1', + 'wf-1', + 'resume-entry-1' + ) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('resume-ex-1', { + executionDeadlineAt: null, + }) + }) + + it('returns 404 when the execution does not belong to the workflow', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(404) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCancelByExecution).toHaveBeenCalledWith( + { + workflowId: 'wf-1', + executionId: 'ex-1', + }, + 'standalone' + ) + }) + + it('treats an already-cancelled execution as an idempotent success', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'cancelled', workspaceId: 'workspace-1' }, + ]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: false, + reason: 'already_cancelled', + }) + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('ex-1') + }) + + it('reconciles the exact sidecar when a workflow-group log is already cancelled', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) mockCancelWorkflowGroupExecution.mockResolvedValue({ kind: 'already_cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: NO_WRITES, }) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, - durablyRecorded: false, reason: 'already_cancelled', }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + workflowId: 'wf-1', + executionId: 'ex-1', + }) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1', { + executionDeadlineAt: null, + }) + expect(mockPublishWorkflowGroupCancellationEvent).toHaveBeenCalledOnce() + expect(mockCancelByExecution).not.toHaveBeenCalled() }) - /** - * The same `already_cancelled` kind covers a transition that left the sidecar - * alone but still terminalized an active workflow log. That log write is - * durable, so the outcome must stay `recorded` and must not re-read a state - * this request itself wrote. - */ - it('reports a durable write when a group run only repaired its workflow log', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('finishes workflow-group reconciliation when abort arrives during its durable commit', async () => { + const controller = new AbortController() + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockImplementationOnce(async () => { + controller.abort() + return { + kind: 'already_cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + } + }) + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, }) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, pausedCancelled: true }) + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + expect(mockPublishWorkflowGroupCancellationEvent).toHaveBeenCalledOnce() + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + }) + + it('does not finalize an already-cancelled group retry until exact stop is accepted', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) mockCancelWorkflowGroupExecution.mockResolvedValue({ kind: 'already_cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: LOG_WRITE, }) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledOnce() + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + redisAvailable: false, + reason: 'redis_unavailable', + }) + expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() }) - /** - * The lost race the sidecar-bearing kind used to hide: a concurrent cancel - * terminalized both records between the entry snapshot and this transaction, - * which then found the sidecar already `cancelled` and the log already - * `cancelled` and wrote nothing. A non-terminal entry snapshot cannot catch - * that, so the transition's own report of having written nothing is what - * forces the re-read — otherwise the request would claim a durable write and - * fire the v2 cancel analytics gate on a no-op. - */ - it('reports a group run that lost the race with its sidecar already cancelled as a no-op', async () => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', - }) - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'cancelled', - }) + it('repairs a stranded active resume when the group log is already cancelled', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockAbortManualExecution.mockReturnValue(true) + mockCancelByExecution.mockResolvedValue(1) + mockCompletePausedCancellation.mockResolvedValue(true) mockCancelWorkflowGroupExecution.mockResolvedValue({ kind: 'already_cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: NO_WRITES, }) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, - durablyRecorded: false, + locallyAborted: true, + pausedCancelled: true, reason: 'already_cancelled', }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledTimes(2) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { workflowId: 'wf-1', executionId: 'ex-1' }, + 'resume' + ) + expect(mockAbortManualExecution).toHaveBeenCalledWith('resume-ex-1') + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) }) - /** - * The lost-race re-read applies to the group path as well: an entry snapshot - * can still read `running` when the sidecar-less transition finds the log - * already `cancelled` and writes nothing. - */ - it('reports a group run that lost the race to another cancel as a no-op', async () => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', - }) - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'cancelled', - }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'already_cancelled_without_sidecar', - writes: NO_WRITES, + it('repairs a stranded active resume when a regular workflow log is already cancelled', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockAbortManualExecution.mockReturnValue(true) + mockCancelByExecution.mockResolvedValue(1) + mockCompletePausedCancellation.mockResolvedValue(true) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, - durablyRecorded: false, + locallyAborted: true, + pausedCancelled: true, reason: 'already_cancelled', }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledTimes(2) - }) - - it.each([ - [ - { kind: 'conflict' as const, status: 'completed', writes: NO_WRITES }, - 'cannot be cancelled while completed', - ], - [ - { kind: 'not_workflow_group' as const, writes: NO_WRITES }, - 'no longer the active table execution', - ], - ])( - 'releases the reservation before reporting a refused workflow-group cell claim as a conflict', - async (outcome, message) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - }) - mockCancelWorkflowGroupExecution.mockResolvedValue(outcome) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { workflowId: 'wf-1', executionId: 'ex-1' }, + 'resume' + ) + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) + expect(mockCancelWorkflowGroupExecution).not.toHaveBeenCalled() + }) - await expect(cancelWorkflowExecution(INPUT)).rejects.toMatchObject({ - code: 'conflict', - message: expect.stringContaining(message), - }) - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') - } - ) + it('keeps an already-cancelled active resume retryable when publication fails', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) - it.each([ - [{ kind: 'conflict' as const, status: 'completed', writes: NO_WRITES }], - [{ kind: 'not_workflow_group' as const, writes: NO_WRITES }], - ])( - 'keeps the reservation held when a refused claim follows a paused cancellation', - async (outcome) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - }) - mockBeginPausedCancellation.mockResolvedValue(true) - mockCancelWorkflowGroupExecution.mockResolvedValue(outcome) + const response = await POST(makeRequest(), makeParams()) - await expect(cancelWorkflowExecution(INPUT)).rejects.toMatchObject({ code: 'conflict' }) - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() - } - ) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + pausedCancelled: false, + reason: 'paused_event_publish_failed', + }) + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + }) - it.each([ - [{ kind: 'conflict' as const, status: 'completed', writes: NO_WRITES }], - [{ kind: 'not_workflow_group' as const, writes: NO_WRITES }], - ])( - 'keeps the reservation held when a refused claim follows a failed cancellation', - async (outcome) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - }) - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - mockCancelWorkflowGroupExecution.mockResolvedValue(outcome) + it('fails closed when an already-cancelled group sidecar cannot be reconciled', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'conflict', + status: 'completed', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Workflow group execution cannot be reconciled while completed', + }) + }) + + it('reports a reconciliation failure for an already-cancelled group sidecar', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockRejectedValue(new Error('database unavailable')) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ error: 'database unavailable' }) + }) - await expect(cancelWorkflowExecution(INPUT)).rejects.toMatchObject({ code: 'conflict' }) - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + it.each(['completed', 'failed'] as const)( + 'raises a typed conflict when a standalone execution is already %s', + async (executionStatus) => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { status: executionStatus, workspaceId: 'workspace-1' }, + ]) + + const error = await cancelWorkflowExecution(INPUT).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(WorkflowRunAlreadyTerminalError) + expect(error).toEqual( + expect.objectContaining({ + code: 'conflict', + executionId: 'ex-1', + executionStatus, + redisAvailable: true, + locallyAborted: false, + }) + ) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() } ) - it('releases the reservation and rethrows when the workflow-group cancel fails unexpectedly', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('keeps workflow-group terminal conflicts strict', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionOrigin: 'workflow_group', + status: 'completed', + workspaceId: 'workspace-1', + }, + ]) + + await expect(cancelWorkflowExecution(INPUT)).rejects.toMatchObject({ + name: 'OrchestrationError', + code: 'conflict', + message: 'Execution cannot be cancelled while completed', }) - const failure = new Error('Workflow-group cancellation lost its locked workflow-log claim') - mockCancelWorkflowGroupExecution.mockRejectedValue(failure) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) - await expect(cancelWorkflowExecution(INPUT)).rejects.toBe(failure) - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - expect(mockUpdateSet).not.toHaveBeenCalled() + it('returns 409 when completion wins the terminal database race', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + const returning = vi.fn().mockResolvedValue([]) + const where = vi.fn(() => ({ returning })) + databaseMock.db.update.mockReturnValueOnce({ set: vi.fn(() => ({ where })) }) + dbChainMockFns.limit + .mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, + ]) + .mockResolvedValueOnce([{ status: 'completed' }]) + + const error = await cancelWorkflowExecution(INPUT).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(WorkflowRunAlreadyTerminalError) + expect(error).toEqual( + expect.objectContaining({ + code: 'conflict', + executionId: 'ex-1', + executionStatus: 'completed', + redisAvailable: true, + locallyAborted: false, + }) + ) + expect(returning).toHaveBeenCalledOnce() + expect(mockClearExecutionCancellation).toHaveBeenCalledWith('ex-1') + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() }) - it('keeps the reservation held when an unexpected workflow-group failure follows a paused cancellation', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('rolls back paused cancellation intent when resume completion wins the log claim', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + const returning = vi.fn().mockResolvedValue([]) + const where = vi.fn(() => ({ returning })) + databaseMock.db.update.mockReturnValueOnce({ set: vi.fn(() => ({ where })) }) + dbChainMockFns.limit + .mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, + ]) + .mockResolvedValueOnce([{ status: 'completed' }]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Execution cannot be cancelled while completed', }) - mockBeginPausedCancellation.mockResolvedValue(true) - mockCancelWorkflowGroupExecution.mockRejectedValue(new Error('serialization conflict')) + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockClearPausedCancellationIntent).toHaveBeenCalledWith('ex-1', 'wf-1') + }) - await expect(cancelWorkflowExecution(INPUT)).rejects.toThrow('serialization conflict') - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + it('treats a concurrent cancellation as an idempotent success', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + dbChainMockFns.limit + .mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, + ]) + .mockResolvedValueOnce([{ status: 'cancelled' }]) + dbChainMockFns.returning.mockResolvedValueOnce([]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, reason: 'recorded' }) + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledTimes(1) + expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('ex-1') }) - it('keeps the reservation held when an unexpected workflow-group failure follows a failed cancellation', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('updates execution log status in DB when durably recorded', async () => { + const mockWhere = vi.fn().mockResolvedValue(undefined) + const mockSet = vi.fn(() => ({ where: mockWhere })) + databaseMock.db.update.mockReturnValueOnce({ set: mockSet }) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', }) + + await POST(makeRequest(), makeParams()) + + expect(databaseMock.db.update).toHaveBeenCalled() + expect(mockSet).toHaveBeenCalledWith({ + status: 'cancelled', + endedAt: expect.any(Date), + totalDurationMs: expect.anything(), + executionDeadlineAt: null, + }) + }) + + it('updates execution log status in DB when locally aborted', async () => { + const mockWhere = vi.fn().mockResolvedValue(undefined) + const mockSet = vi.fn(() => ({ where: mockWhere })) + databaseMock.db.update.mockReturnValueOnce({ set: mockSet }) mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, reason: 'redis_unavailable', }) - mockCancelWorkflowGroupExecution.mockRejectedValue(new Error('serialization conflict')) + mockAbortManualExecution.mockReturnValue(true) - await expect(cancelWorkflowExecution(INPUT)).rejects.toThrow('serialization conflict') - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + await POST(makeRequest(), makeParams()) + + expect(databaseMock.db.update).toHaveBeenCalled() + expect(mockSet).toHaveBeenCalledWith({ + status: 'cancelled', + endedAt: expect.any(Date), + totalDurationMs: expect.anything(), + executionDeadlineAt: null, + }) }) - it('leaves a standalone execution untouched by the workflow-group path', async () => { - await cancelWorkflowExecution(INPUT) + it('claims the execution log before finalizing a paused cancellation', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - expect(mockCancelWorkflowGroupExecution).not.toHaveBeenCalled() - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - expect(mockUpdateSet).toHaveBeenCalledWith(expect.objectContaining({ status: 'cancelled' })) + await POST(makeRequest(), makeParams()) + + expect(databaseMock.db.update).toHaveBeenCalled() }) - /** - * A cancelled run is terminal, so it owes the same two fields every other - * terminal write records. Without the duration it is invisible to the - * `minDurationMs`/`maxDurationMs` filters on `GET /api/v2/logs`. - */ - it('records how long the cancelled run had been going, not just when it stopped', async () => { - await cancelWorkflowExecution(INPUT) + it('does not confirm cancellation until the terminal database update succeeds', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', + }) + databaseMock.db.update.mockReturnValueOnce({ + set: vi.fn(() => ({ + where: vi.fn(() => { + throw new Error('DB connection failed') + }), + })), + }) + + const response = await POST(makeRequest(), makeParams()) - const [values] = mockUpdateSet.mock.calls.at(-1) as [Record] - expect(values.endedAt).toBeInstanceOf(Date) - expect(values.totalDurationMs).toBeDefined() + expect(response.status).toBe(200) + const data = await response.json() + expect(data).toMatchObject({ + success: false, + reason: 'cancellation_not_finalized', + }) + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/execution/cancel-workflow-execution.ts b/apps/sim/lib/execution/cancel-workflow-execution.ts index 10e92517854..4dc978c6134 100644 --- a/apps/sim/lib/execution/cancel-workflow-execution.ts +++ b/apps/sim/lib/execution/cancel-workflow-execution.ts @@ -1,57 +1,39 @@ import { db } from '@sim/db' import { workflowExecutionLogs } from '@sim/db/schema' import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' -import { and, eq } from 'drizzle-orm' +import { and, eq, inArray } from 'drizzle-orm' import { releaseExecutionSlot } from '@/lib/billing/calculations/usage-reservation' import { getJobQueue } from '@/lib/core/async-jobs' +import type { ExecutionJobCancellationScope } from '@/lib/core/async-jobs/types' import { OrchestrationError } from '@/lib/core/orchestration/types' import { + clearExecutionCancellation, type ExecutionCancellationRecordResult, markExecutionCancelled, } from '@/lib/execution/cancellation' import { createExecutionEventWriter, readExecutionMetaState } from '@/lib/execution/event-buffer' import { abortManualExecution } from '@/lib/execution/manual-cancellation' +import { + isWorkflowRunAlreadyTerminalStatus, + WorkflowRunAlreadyTerminalError, +} from '@/lib/execution/workflow-run-already-terminal-error' import { cancelledExecutionLogFields } from '@/lib/logs/execution/cancellation' -import { captureServerEvent } from '@/lib/posthog/server' +import { workflowExecutionOriginSql } from '@/lib/logs/execution-origin' import { cancelWorkflowGroupExecution, type PublishableWorkflowGroupCancellation, publishWorkflowGroupCancellationEvent, - type WorkflowGroupCancellationWrites, } from '@/lib/table/workflow-group-cancellation' -import { WORKFLOW_EXECUTION_JOB_ID_PREFIX } from '@/lib/workflows/executor/execution-job-ids' -import { resolveWorkflowExecutionOwnership } from '@/lib/workflows/executor/execution-queries' import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager' const logger = createLogger('CancelWorkflowExecution') const PAUSED_CANCELLATION_DB_ATTEMPTS = 3 const PAUSED_CANCELLATION_DB_RETRY_MS = 200 +const CANCELLATION_ABORTED_MESSAGE = + 'Request aborted before workflow run cancellation could be applied.' -async function cancelActiveWorkflowJob(executionId: string): Promise { - try { - const queue = await getJobQueue() - const job = await queue.getJob(`${WORKFLOW_EXECUTION_JOB_ID_PREFIX}${executionId}`) - if (!job || (job.status !== 'pending' && job.status !== 'processing')) return false - await queue.cancelJob(job.id) - logger.info('Cancelled active workflow queue job', { executionId, jobId: job.id }) - return true - } catch (error) { - logger.warn('Failed to cancel active workflow queue job', { executionId, error }) - return false - } -} - -/** - * Cancellation outcome vocabulary produced by this service, and so the whole - * vocabulary the public v2 endpoint can return. `recorded`/`redis_unavailable`/ - * `redis_write_failed` come from the Redis record step; the two `paused_*` - * values from the paused-HITL path; the three `already_*` values report a run - * that was already terminal when the request arrived, where the cancel claim - * matched no row and nothing durable was written. The internal cancel route - * resolves further outcomes on top of these — see - * `internalCancelWorkflowExecutionReasonSchema` in `lib/api/contracts/workflows`. - */ export type CancelWorkflowExecutionReason = | 'recorded' | 'already_cancelled' @@ -61,107 +43,226 @@ export type CancelWorkflowExecutionReason = | 'redis_write_failed' | 'paused_event_publish_failed' | 'paused_database_cancel_failed' + | 'queue_cancelled' + | 'active_resume_signal_failed' + | 'cancellation_not_finalized' -/** Maps each log status a cancel claim can never move to the outcome that reports it. */ -const TERMINAL_NO_OP_REASONS = { - cancelled: 'already_cancelled', - completed: 'already_completed', - failed: 'already_failed', -} as const satisfies Record - -type TerminalExecutionStatus = keyof typeof TERMINAL_NO_OP_REASONS - -function toTerminalExecutionStatus( - status: string | null | undefined -): TerminalExecutionStatus | null { - return typeof status === 'string' && status in TERMINAL_NO_OP_REASONS - ? (status as TerminalExecutionStatus) - : null +export interface CancelWorkflowExecutionResult { + success: boolean + executionId: string + redisAvailable: boolean + durablyRecorded: boolean + locallyAborted: boolean + pausedCancelled: boolean + reason?: CancelWorkflowExecutionReason } -/** - * What this request's own terminal claim did: moved the run to `cancelled` here - * and now, provably matched no row, or ran on a path that cannot tell. Every - * path that can terminalize the run — the direct log claim and the - * workflow-group transition — answers in this one vocabulary, so the report can - * ask a single question: did this request durably write? - * - * Only the direct claim ever answers `unknown`, and only when it could not run - * or its statement failed. The workflow-group transition always knows: it - * reports the writes it performed. - */ -type TerminalWriteOutcome = 'applied' | 'no_row' | 'unknown' +async function cancelQueuedExecutionJobs( + workflowId: string, + executionId: string, + scope: ExecutionJobCancellationScope +): Promise { + try { + const queue = await getJobQueue() + return await queue.cancelByExecution({ workflowId, executionId }, scope) + } catch (error) { + logger.warn('Failed to cancel queued execution jobs', { + workflowId, + executionId, + error: toError(error).message, + }) + return 0 + } +} -/** - * Reads a workflow-group transition's durability off the writes it reported - * rather than off its `kind`. Terminalizing the workflow log and cancelling the - * cell sidecar are each a durable write this request performed, and a single - * `kind` covers both a transition that did one of them and one that did - * neither: `already_cancelled` leaves a sidecar that was already `cancelled` - * alone, but may still have terminalized an active workflow log. - */ -function toTerminalWriteOutcome(writes: WorkflowGroupCancellationWrites): TerminalWriteOutcome { - return writes.workflowLogTerminalized || writes.sidecarCancelled ? 'applied' : 'no_row' +function abortLocalExecution(executionId: string): boolean { + try { + return abortManualExecution(executionId) + } catch (error) { + logger.warn('Failed to abort local execution', { + executionId, + error: toError(error).message, + }) + return false + } +} + +interface ExecutionStopSignalResult { + cancellation: ExecutionCancellationRecordResult + locallyAborted: boolean + queueJobsCancelled: number + accepted: boolean +} + +interface ExecutionStopSummary extends ExecutionStopSignalResult { + signalledExecutionIds: Set +} + +type ActiveResumeCancellationTarget = NonNullable< + Awaited> +> + +function createExecutionStopSummary(): ExecutionStopSummary { + return { + cancellation: { durablyRecorded: false, reason: 'redis_unavailable' }, + locallyAborted: false, + queueJobsCancelled: 0, + accepted: false, + signalledExecutionIds: new Set(), + } +} + +function mergeExecutionStopSignal( + summary: ExecutionStopSummary, + signalExecutionId: string, + result: ExecutionStopSignalResult +): void { + if (result.cancellation.durablyRecorded || !summary.cancellation.durablyRecorded) { + summary.cancellation = result.cancellation + } + summary.locallyAborted = summary.locallyAborted || result.locallyAborted + summary.queueJobsCancelled += result.queueJobsCancelled + summary.accepted = summary.accepted || result.accepted + summary.signalledExecutionIds.add(signalExecutionId) } /** - * Names the terminal state the cancel could not move, or `null` when it did - * real work or when this path cannot tell — in which case the caller keeps the - * undifferentiated report rather than guessing. - * - * A request that durably wrote is never a no-op, whatever the entry snapshot - * said. A run can be terminal at entry and still owe this request a real write: - * a workflow-group run whose log is already `cancelled` can carry a sidecar left - * in `error`, and reconciling it is a durable cancellation that the entry - * snapshot cannot see. - * - * The status read at entry is not enough on its own in the other direction - * either: a run that finishes between that read and the claim leaves a stale - * non-terminal snapshot behind a cancel that wrote nothing. The claim's own row - * count settles that, and a plain post-read cannot: after a successful cancel - * the row reads `cancelled` too, so the state has to be attributed to whoever - * wrote it. A claim that moved no row against a non-terminal snapshot re-reads - * the row it lost the race to, through the same ownership query the entry read - * came from. - * - * Purely observational — it gates no effect, and a read failure falls back to - * the undifferentiated report rather than failing the cancel. + * Commits cancellation after the caller's final abort check. Once signalling begins, the + * operation must finish reconciliation because workers may already have observed the durable, + * local, or queued signal; attempting to honor a later abort could revive only part of a run. */ -async function resolveTerminalNoOpReason( +async function signalExecutionStop(args: { + workflowId: string + signalExecutionId: string + queueBindingExecutionId?: string + executionDeadlineAt: Date | null + queueScope?: ExecutionJobCancellationScope +}): Promise { + const cancellation = await markExecutionCancelled(args.signalExecutionId, { + executionDeadlineAt: args.executionDeadlineAt, + }) + const locallyAborted = abortLocalExecution(args.signalExecutionId) + const queueJobsCancelled = args.queueScope + ? await cancelQueuedExecutionJobs( + args.workflowId, + args.queueBindingExecutionId ?? args.signalExecutionId, + args.queueScope + ) + : 0 + return { + cancellation, + locallyAborted, + queueJobsCancelled, + accepted: cancellation.durablyRecorded || locallyAborted || queueJobsCancelled > 0, + } +} + +async function signalAndRecordActiveResumeStop(args: { + workflowId: string + executionId: string + executionDeadlineAt: Date | null + target: ActiveResumeCancellationTarget + summary: ExecutionStopSummary +}): Promise { + const signal = await signalExecutionStop({ + workflowId: args.workflowId, + signalExecutionId: args.target.resumeExecutionId, + queueBindingExecutionId: args.executionId, + executionDeadlineAt: args.executionDeadlineAt, + queueScope: 'resume', + }) + mergeExecutionStopSignal(args.summary, args.target.resumeExecutionId, signal) + return didActiveResumeStop(args.executionId, args.workflowId, args.target, signal) +} + +async function didActiveResumeStop( executionId: string, workflowId: string, - priorTerminalStatus: TerminalExecutionStatus | null, - terminalWrite: TerminalWriteOutcome -): Promise { - if (terminalWrite === 'applied') return null - if (priorTerminalStatus !== null) return TERMINAL_NO_OP_REASONS[priorTerminalStatus] - if (terminalWrite !== 'no_row') return null - try { - const { priorStatus } = await resolveWorkflowExecutionOwnership(executionId, workflowId) - const terminalStatus = toTerminalExecutionStatus(priorStatus) - return terminalStatus !== null ? TERMINAL_NO_OP_REASONS[terminalStatus] : null - } catch (error) { - logger.warn('Failed to re-read execution status after an unmatched cancel claim', { + target: ActiveResumeCancellationTarget, + signal: ExecutionStopSignalResult +): Promise { + if (signal.cancellation.durablyRecorded || signal.locallyAborted) return true + const currentTarget = await PauseResumeManager.getActiveResumeCancellationTarget( + executionId, + workflowId + ) + if (currentTarget?.resumeEntryId === target.resumeEntryId && signal.queueJobsCancelled > 0) { + return true + } + if (currentTarget && currentTarget.resumeEntryId !== target.resumeEntryId) { + logger.warn('A replacement resume became active while cancellation was staged', { executionId, - error, + previousResumeEntryId: target.resumeEntryId, + currentResumeEntryId: currentTarget.resumeEntryId, }) - return null } + return currentTarget === null } -export interface CancelWorkflowExecutionResult { - success: boolean +type PausedCancellationStage = Awaited< + ReturnType +> + +function isPausedCancellationStage( + stage: PausedCancellationStage +): stage is Exclude { + return stage.kind !== 'not_paused' +} + +async function clearStopSignalMarkers(summary: ExecutionStopSummary): Promise { + await Promise.all( + [...summary.signalledExecutionIds].map((executionId) => clearExecutionCancellation(executionId)) + ) +} + +type ExecutionLogCancellationClaim = + | { kind: 'cancelled' } + | { kind: 'conflict'; status: string } + | { kind: 'not_found' } + +async function claimExecutionLogCancellation(args: { executionId: string - redisAvailable: boolean - durablyRecorded: boolean - locallyAborted: boolean - pausedCancelled: boolean - reason?: CancelWorkflowExecutionReason + workflowId: string + workspaceId: string +}): Promise { + const now = new Date() + const [cancelledExecution] = await db + .update(workflowExecutionLogs) + .set(cancelledExecutionLogFields(now)) + .where( + and( + eq(workflowExecutionLogs.executionId, args.executionId), + eq(workflowExecutionLogs.workflowId, args.workflowId), + eq(workflowExecutionLogs.workspaceId, args.workspaceId), + inArray(workflowExecutionLogs.status, ['running', 'pending']) + ) + ) + .returning({ status: workflowExecutionLogs.status }) + + if (cancelledExecution?.status === 'cancelled') return { kind: 'cancelled' } + + const currentExecution = await db + .select({ status: workflowExecutionLogs.status }) + .from(workflowExecutionLogs) + .where( + and( + eq(workflowExecutionLogs.executionId, args.executionId), + eq(workflowExecutionLogs.workflowId, args.workflowId), + eq(workflowExecutionLogs.workspaceId, args.workspaceId) + ) + ) + .limit(1) + .then((rows) => rows[0]) + + if (!currentExecution) return { kind: 'not_found' } + if (currentExecution.status === 'cancelled') return { kind: 'cancelled' } + return { kind: 'conflict', status: currentExecution.status } } async function completePausedCancellationWithRetry( executionId: string, - workflowId: string + workflowId: string, + options: { logMissing?: boolean } = {} ): Promise { for (let attempt = 1; attempt <= PAUSED_CANCELLATION_DB_ATTEMPTS; attempt++) { try { @@ -170,10 +271,12 @@ async function completePausedCancellationWithRetry( logger.info('Paused execution cancelled in database', { executionId, attempt }) return true } - logger.warn('Paused execution cancellation could not be completed in database', { - executionId, - attempt, - }) + if (options.logMissing !== false) { + logger.warn('Paused execution cancellation could not be completed in database', { + executionId, + attempt, + }) + } return false } catch (error) { logger.warn('Failed to complete paused execution cancellation in database', { @@ -189,14 +292,21 @@ async function completePausedCancellationWithRetry( return false } -async function ensurePausedCancellationEventPublished( +async function ensureCancellationEventPublished( executionId: string, workflowId: string, context: { workspaceId?: string; userId?: string } = {} ): Promise { - const metaState = await readExecutionMetaState(executionId) - if (metaState.status === 'found' && metaState.meta.status === 'cancelled') { - return true + try { + const metaState = await readExecutionMetaState(executionId) + if (metaState.status === 'found' && metaState.meta.status === 'cancelled') { + return true + } + } catch (error) { + logger.warn('Failed to read execution state before publishing cancellation', { + executionId, + error: toError(error).message, + }) } const writer = createExecutionEventWriter(executionId, { @@ -217,14 +327,14 @@ async function ensurePausedCancellationEventPublished( ) return true } catch (error) { - logger.warn('Failed to publish paused execution cancellation event', { + logger.warn('Failed to publish execution cancellation event', { executionId, error, }) return false } finally { await writer.close().catch((error) => { - logger.warn('Failed to close paused cancellation event writer', { + logger.warn('Failed to close cancellation event writer', { executionId, error, }) @@ -233,14 +343,13 @@ async function ensurePausedCancellationEventPublished( } export interface CancelWorkflowExecutionInput { - executionId: string workflowId: string - /** Actor for the analytics event. */ - userId: string - /** Workflow's workspace; feeds the event writer + analytics grouping. */ - workspaceId?: string - /** Legacy callers emit product analytics here; migrated adapters emit it after success. */ - captureAnalytics?: boolean + executionId: string + /** Human attribution resolved by the authorized application use case. */ + attributedUserId: string + /** Canonical workspace resolved with the workflow run. */ + workspaceId: string + abortSignal?: AbortSignal } export class WorkflowExecutionNotFoundError extends Error { @@ -250,312 +359,722 @@ export class WorkflowExecutionNotFoundError extends Error { } } -/** - * Cancels a workflow execution across the Redis abort record, the in-process - * aborter, the paused-HITL machinery, the workflow-group table cell sidecar, and - * the plan concurrency reservation. The interleaving is order-sensitive. Auth is - * the caller's responsibility; this throws on unexpected infrastructure errors. - * - * A workflow-group run whose cell sidecar refuses the claim throws - * `OrchestrationError('conflict')` rather than returning a success-shaped result: - * the cancel did not fully happen, and the internal route already answers 409 for - * exactly these two outcomes. - */ -export async function cancelWorkflowExecution( - input: CancelWorkflowExecutionInput -): Promise { - const { executionId, workflowId, userId, workspaceId } = input - - const { belongsToWorkflow, workflowGroupWorkspaceId, priorStatus } = - await resolveWorkflowExecutionOwnership(executionId, workflowId) - if (!belongsToWorkflow) throw new WorkflowExecutionNotFoundError() - const priorTerminalStatus = toTerminalExecutionStatus(priorStatus) - - let pausedCancellationStarted = false - let pausedCancelled = false +function throwCancellationAborted(): never { + throw new OrchestrationError('conflict', CANCELLATION_ABORTED_MESSAGE) +} + +function throwIfCancellationAborted(abortSignal?: AbortSignal): void { + if (abortSignal?.aborted) throwCancellationAborted() +} + +async function rollbackPausedCancellationAfterAbort(args: { + stage: PausedCancellationStage + workflowId: string + executionId: string + abortSignal?: AbortSignal +}): Promise { + if (!args.abortSignal?.aborted) return false + + if (args.stage.kind === 'active_resume') { + const rolledBack = await PauseResumeManager.rollbackActiveResumeCancellation( + args.executionId, + args.workflowId, + args.stage.target.resumeEntryId + ) + if (!rolledBack) { + logger.warn('Aborted cancellation could not be rolled back; completing cancellation', { + executionId: args.executionId, + activeResumeEntryId: args.stage.target.resumeEntryId, + }) + return false + } + } else if (args.stage.kind === 'idle') { + await PauseResumeManager.clearPausedCancellationIntent(args.executionId, args.workflowId) + } + + return true +} + +async function rollbackActiveResumeAfterFailedSignal(args: { + executionId: string + workflowId: string + resumeEntryId: string +}): Promise { try { - pausedCancellationStarted = await PauseResumeManager.beginPausedCancellation( - executionId, - workflowId + const rolledBack = await PauseResumeManager.rollbackActiveResumeCancellation( + args.executionId, + args.workflowId, + args.resumeEntryId ) + if (!rolledBack) { + logger.warn('Active resume cancellation could not be rolled back; completing cancellation', { + executionId: args.executionId, + activeResumeEntryId: args.resumeEntryId, + }) + } + return rolledBack } catch (error) { - logger.warn('Failed to begin paused execution cancellation in database', { - executionId, - error, + logger.warn('Failed to roll back active resume cancellation; completing cancellation', { + executionId: args.executionId, + activeResumeEntryId: args.resumeEntryId, + error: toError(error).message, }) + return false } - const pendingPausedCancellation = pausedCancellationStarted - ? null - : await PauseResumeManager.getPausedCancellationStatus(executionId, workflowId) - const isPausedCancellationPath = pausedCancellationStarted || pendingPausedCancellation !== null - - const cancellation: ExecutionCancellationRecordResult = isPausedCancellationPath - ? { durablyRecorded: false, reason: 'redis_unavailable' } - : await markExecutionCancelled(executionId) - const locallyAborted = isPausedCancellationPath ? false : abortManualExecution(executionId) - const queuedJobCancelled = isPausedCancellationPath - ? false - : await cancelActiveWorkflowJob(executionId) - - if (pausedCancellationStarted) { - logger.info('Paused execution cancellation reserved in database', { executionId }) - } else if (cancellation.durablyRecorded) { - logger.info('Execution marked as cancelled in Redis', { executionId }) - } else if (queuedJobCancelled) { - logger.info('Execution cancelled in workflow queue', { executionId }) - } else if (locallyAborted) { - logger.info('Execution cancelled via local in-process fallback', { executionId }) - } else if (!pausedCancellationStarted) { - logger.warn('Execution cancellation was not durably recorded', { - executionId, - reason: cancellation.reason, - }) +} + +function activeResumeSignalFailureResult( + executionId: string, + stopSummary: ExecutionStopSummary +): CancelWorkflowExecutionResult { + return { + success: false, + executionId, + redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + durablyRecorded: stopSummary.cancellation.durablyRecorded, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled: false, + reason: 'active_resume_signal_failed', } +} +function resolveCancellationReason(args: { + activeResumeSignalFailed: boolean + pauseReconciliationFailed: boolean + effectivePausedCancellationPath: boolean + cancellationEventPublished: boolean + pausedCancelled: boolean + stopSummary: ExecutionStopSummary +}): CancelWorkflowExecutionReason { + if (args.activeResumeSignalFailed) return 'active_resume_signal_failed' + if (args.pauseReconciliationFailed) return 'paused_database_cancel_failed' + if (args.effectivePausedCancellationPath && !args.cancellationEventPublished) { + return 'paused_event_publish_failed' + } + if (args.effectivePausedCancellationPath && !args.pausedCancelled) { + return 'paused_database_cancel_failed' + } + if (args.effectivePausedCancellationPath) return 'recorded' if ( - !isPausedCancellationPath && - (cancellation.durablyRecorded || queuedJobCancelled || locallyAborted) + args.stopSummary.queueJobsCancelled > 0 && + !args.stopSummary.cancellation.durablyRecorded && + !args.stopSummary.locallyAborted ) { - await PauseResumeManager.blockQueuedResumesForCancellation(executionId, workflowId).catch( - (error) => { - logger.warn('Failed to block queued paused resumes after cancellation', { + return 'queue_cancelled' + } + return args.stopSummary.cancellation.reason +} + +/** + * Applies the full queued, active, paused, resumed, and workflow-group + * cancellation lifecycle to an already-authorized canonical workflow run. + * Authorization and principal handling belong to `cancelWorkflowRun`; this + * service accepts only canonical identifiers and returns transport-neutral + * results or orchestration errors. + */ +export async function cancelWorkflowExecution({ + workflowId, + executionId, + attributedUserId, + workspaceId, + abortSignal, +}: CancelWorkflowExecutionInput): Promise { + try { + throwIfCancellationAborted(abortSignal) + + const execution = await db + .select({ + executionDeadlineAt: workflowExecutionLogs.executionDeadlineAt, + executionOrigin: workflowExecutionOriginSql(), + status: workflowExecutionLogs.status, + workspaceId: workflowExecutionLogs.workspaceId, + }) + .from(workflowExecutionLogs) + .where( + and( + eq(workflowExecutionLogs.executionId, executionId), + eq(workflowExecutionLogs.workflowId, workflowId), + eq(workflowExecutionLogs.workspaceId, workspaceId) + ) + ) + .limit(1) + .then((rows) => rows[0]) + + throwIfCancellationAborted(abortSignal) + + if (!execution) { + const queueJobsCancelled = await cancelQueuedExecutionJobs( + workflowId, + executionId, + 'standalone' + ) + if (queueJobsCancelled > 0) { + const locallyAborted = abortLocalExecution(executionId) + const cancellation = await markExecutionCancelled(executionId) + await PauseResumeManager.blockQueuedResumesForCancellation(executionId, workflowId).catch( + (error) => { + logger.warn('Failed to block queued resumes after queued-run cancellation', { + executionId, + error, + }) + } + ) + await releaseExecutionSlot(executionId).catch((error) => { + logger.warn('Failed to release reservation after queued-run cancellation', { + executionId, + error, + }) + }) + + return { + success: true, + executionId, + redisAvailable: cancellation.reason !== 'redis_unavailable', + durablyRecorded: cancellation.durablyRecorded, + locallyAborted, + pausedCancelled: false, + reason: 'queue_cancelled', + } + } + + throw new WorkflowExecutionNotFoundError() + } + + const isWorkflowGroupExecution = execution.executionOrigin === 'workflow_group' + + if (execution.status === 'cancelled') { + let groupCancellationToPublish: PublishableWorkflowGroupCancellation | null = null + let groupCancellationCommitted = false + if (isWorkflowGroupExecution) { + throwIfCancellationAborted(abortSignal) + + const workflowGroupCancellation = await cancelWorkflowGroupExecution({ + workspaceId: execution.workspaceId, + workflowId, executionId, - error, }) + if (workflowGroupCancellation.kind === 'conflict') { + throw new OrchestrationError( + 'conflict', + `Workflow group execution cannot be reconciled while ${workflowGroupCancellation.status}` + ) + } + if (workflowGroupCancellation.kind === 'not_workflow_group') { + throw new OrchestrationError( + 'conflict', + 'Workflow group execution is no longer the active table execution' + ) + } + if ( + workflowGroupCancellation.kind === 'cancelled' || + workflowGroupCancellation.kind === 'already_cancelled' + ) { + groupCancellationToPublish = workflowGroupCancellation + groupCancellationCommitted = true + } } - ) - } else if (!isPausedCancellationPath) { - await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId).catch( - (error) => { - logger.warn('Failed to clear paused cancellation intent after unsuccessful cancellation', { + + const stopSummary = createExecutionStopSummary() + let pausedCancelled = false + const pausedCancellationStage = await PauseResumeManager.stagePausedCancellation( + executionId, + workflowId + ) + if ( + !groupCancellationCommitted && + (await rollbackPausedCancellationAfterAbort({ + stage: pausedCancellationStage, + workflowId, + executionId, + abortSignal, + })) + ) { + throwCancellationAborted() + } + + const hasPausedCancellation = isPausedCancellationStage(pausedCancellationStage) + const requiresCancellationEvent = hasPausedCancellation || isWorkflowGroupExecution + let cancellationEventPublished = !requiresCancellationEvent + let activeResumeSignalFailed = false + let exactStopSatisfied = true + if (pausedCancellationStage.kind === 'active_resume') { + exactStopSatisfied = await signalAndRecordActiveResumeStop({ + workflowId, executionId, - error, + executionDeadlineAt: execution.executionDeadlineAt, + target: pausedCancellationStage.target, + summary: stopSummary, + }) + activeResumeSignalFailed = !exactStopSatisfied + } else if (isWorkflowGroupExecution && !hasPausedCancellation) { + const retrySignal = await signalExecutionStop({ + workflowId, + signalExecutionId: executionId, + executionDeadlineAt: execution.executionDeadlineAt, }) + mergeExecutionStopSignal(stopSummary, executionId, retrySignal) + exactStopSatisfied = retrySignal.accepted } - ) - } - let pausedCancellationPublished = false - let pausedCancellationPublishFailed = false - if (pausedCancellationStarted) { - pausedCancellationPublished = await ensurePausedCancellationEventPublished( + if (groupCancellationToPublish && exactStopSatisfied) { + await publishWorkflowGroupCancellationEvent(groupCancellationToPublish, executionId) + } + + if (requiresCancellationEvent && exactStopSatisfied) { + cancellationEventPublished = await ensureCancellationEventPublished( + executionId, + workflowId, + { + workspaceId: execution.workspaceId, + userId: attributedUserId, + } + ) + } + if (hasPausedCancellation && cancellationEventPublished && exactStopSatisfied) { + pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId, { + logMissing: false, + }) + } + + if (exactStopSatisfied) { + await releaseExecutionSlot(executionId).catch((error) => { + logger.warn('Failed to release reservation while reconciling cancelled execution', { + executionId, + error: toError(error).message, + }) + }) + } + + if (pausedCancelled) { + await clearStopSignalMarkers(stopSummary) + } + + const pausedReconciliationSucceeded = + exactStopSatisfied && + (!hasPausedCancellation || (cancellationEventPublished && pausedCancelled)) + return { + success: pausedReconciliationSucceeded, + executionId, + redisAvailable: requiresCancellationEvent ? cancellationEventPublished : true, + durablyRecorded: false, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled, + reason: activeResumeSignalFailed + ? 'active_resume_signal_failed' + : !exactStopSatisfied + ? stopSummary.cancellation.reason + : hasPausedCancellation && !cancellationEventPublished + ? 'paused_event_publish_failed' + : hasPausedCancellation && !pausedCancelled + ? 'paused_database_cancel_failed' + : 'already_cancelled', + } + } + + if (execution.status !== 'running' && execution.status !== 'pending') { + if (!isWorkflowGroupExecution && isWorkflowRunAlreadyTerminalStatus(execution.status)) { + throw new WorkflowRunAlreadyTerminalError({ + executionId, + executionStatus: execution.status, + redisAvailable: true, + locallyAborted: false, + }) + } + throw new OrchestrationError( + 'conflict', + `Execution cannot be cancelled while ${execution.status}` + ) + } + + logger.info('Cancel execution requested', { workflowId, executionId, attributedUserId }) + + const stopSummary = createExecutionStopSummary() + let pausedCancelled = false + let pausedCancellationStage = await PauseResumeManager.stagePausedCancellation( executionId, - workflowId, - { workspaceId, userId } + workflowId ) - pausedCancellationPublishFailed = !pausedCancellationPublished - if (pausedCancellationPublished) { - pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId) - } - } else { - if (pendingPausedCancellation === 'cancelled') { - pausedCancellationPublished = await ensurePausedCancellationEventPublished( + if ( + await rollbackPausedCancellationAfterAbort({ + stage: pausedCancellationStage, + workflowId, executionId, + abortSignal, + }) + ) { + throwCancellationAborted() + } + + let effectivePausedCancellationPath = isPausedCancellationStage(pausedCancellationStage) + let activeResumeTarget = + pausedCancellationStage.kind === 'active_resume' ? pausedCancellationStage.target : null + let activeResumeEntryId = activeResumeTarget?.resumeEntryId ?? null + let activeResumeSignalAccepted = false + const activeResumeTargetsNeedingStopConfirmation: ActiveResumeCancellationTarget[] = [] + + if (activeResumeTarget && !isWorkflowGroupExecution) { + activeResumeSignalAccepted = await signalAndRecordActiveResumeStop({ workflowId, - { workspaceId, userId } - ) - pausedCancellationPublishFailed = !pausedCancellationPublished - pausedCancelled = pausedCancellationPublished - } else if (pendingPausedCancellation === 'cancelling') { - pausedCancellationPublished = await ensurePausedCancellationEventPublished( executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target: activeResumeTarget, + summary: stopSummary, + }) + + if (!activeResumeSignalAccepted) { + const failedResumeEntryId = activeResumeTarget.resumeEntryId + const rolledBack = await rollbackActiveResumeAfterFailedSignal({ + executionId, + workflowId, + resumeEntryId: failedResumeEntryId, + }) + if (rolledBack) { + await clearStopSignalMarkers(stopSummary) + return activeResumeSignalFailureResult(executionId, stopSummary) + } + activeResumeTargetsNeedingStopConfirmation.push(activeResumeTarget) + } + } else if (!effectivePausedCancellationPath && !isWorkflowGroupExecution) { + const signal = await signalExecutionStop({ workflowId, - { workspaceId, userId } - ) - pausedCancellationPublishFailed = !pausedCancellationPublished - if (pausedCancellationPublished) { - pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId) + signalExecutionId: executionId, + executionDeadlineAt: execution.executionDeadlineAt, + queueScope: 'standalone', + }) + mergeExecutionStopSignal(stopSummary, executionId, signal) + + if (!signal.accepted) { + pausedCancellationStage = await PauseResumeManager.stagePausedCancellation( + executionId, + workflowId + ) + const postLateStageAbort = await rollbackPausedCancellationAfterAbort({ + stage: pausedCancellationStage, + workflowId, + executionId, + abortSignal, + }) + if (postLateStageAbort) { + await clearStopSignalMarkers(stopSummary) + throwCancellationAborted() + } + + effectivePausedCancellationPath = isPausedCancellationStage(pausedCancellationStage) + activeResumeTarget = + pausedCancellationStage.kind === 'active_resume' ? pausedCancellationStage.target : null + activeResumeEntryId = activeResumeTarget?.resumeEntryId ?? null + + if (activeResumeTarget) { + activeResumeSignalAccepted = await signalAndRecordActiveResumeStop({ + workflowId, + executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target: activeResumeTarget, + summary: stopSummary, + }) + if (!activeResumeSignalAccepted) { + const failedResumeEntryId = activeResumeTarget.resumeEntryId + const rolledBack = await rollbackActiveResumeAfterFailedSignal({ + executionId, + workflowId, + resumeEntryId: failedResumeEntryId, + }) + if (rolledBack) { + await clearStopSignalMarkers(stopSummary) + return activeResumeSignalFailureResult(executionId, stopSummary) + } + activeResumeTargetsNeedingStopConfirmation.push(activeResumeTarget) + } + } else if (!effectivePausedCancellationPath) { + return { + success: false, + executionId, + redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + durablyRecorded: stopSummary.cancellation.durablyRecorded, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled: false, + reason: stopSummary.cancellation.reason, + } + } } } - } - if ( - pausedCancellationPublishFailed && - (pausedCancellationStarted || pendingPausedCancellation === 'cancelling') - ) { - await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId).catch( - (error) => { - logger.warn('Failed to clear paused cancellation intent after publish failure', { + let terminalCancellationClaimed = false + let competingTerminalStatus: string | null = null + let workflowGroupNoLongerActive = false + let groupCancellationToPublish: PublishableWorkflowGroupCancellation | null = null + try { + if (isWorkflowGroupExecution) { + const workflowGroupCancellation = await cancelWorkflowGroupExecution({ + workspaceId: execution.workspaceId, + workflowId, + executionId, + }) + if (workflowGroupCancellation.kind === 'conflict') { + competingTerminalStatus = workflowGroupCancellation.status + } else if (workflowGroupCancellation.kind === 'not_workflow_group') { + workflowGroupNoLongerActive = true + } else { + terminalCancellationClaimed = true + if ( + workflowGroupCancellation.kind === 'cancelled' || + workflowGroupCancellation.kind === 'already_cancelled' + ) { + groupCancellationToPublish = workflowGroupCancellation + } + } + } else { + const claim = await claimExecutionLogCancellation({ executionId, - error, + workflowId, + workspaceId: execution.workspaceId, }) + if (claim.kind === 'cancelled') { + terminalCancellationClaimed = true + } else { + competingTerminalStatus = claim.kind === 'conflict' ? claim.status : 'no_longer_active' + } } - ) - } - - const success = - (isPausedCancellationPath - ? pausedCancelled && pausedCancellationPublished - : cancellation.durablyRecorded || queuedJobCancelled) || locallyAborted - - /** - * Frees the plan concurrency reservation once the stop-the-work effects above - * have actually taken. The paused path keeps its reservation because a paused - * run never held an in-flight slot to give back, and an unsuccessful ordinary - * cancel keeps it because the run may still be executing. - */ - const releaseSlotForStoppedExecution = async (): Promise => { - if (!success || isPausedCancellationPath) return - await releaseExecutionSlot(executionId).catch((error) => { - logger.warn('Failed to release reservation after execution cancellation', { + } catch (dbError) { + logger.warn('Failed to finalize cancelled execution directly', { executionId, - error, + error: toError(dbError).message, }) - }) - } + } - /** - * The sidecar transition can fail outright — a lost claim, a serialization - * conflict, a connection blip. The stop-the-work effects above have already - * fired, so the run is going down regardless and the reservation must not be - * stranded; but the cell is left in an unknown state, so the failure is - * re-thrown rather than swallowed into a success-shaped result. - */ - let groupCancellation: Awaited> | null = null - if (workflowGroupWorkspaceId) { - try { - groupCancellation = await cancelWorkflowGroupExecution({ - workspaceId: workflowGroupWorkspaceId, + if (workflowGroupNoLongerActive) { + await clearStopSignalMarkers(stopSummary) + if (activeResumeEntryId) { + await PauseResumeManager.rollbackActiveResumeCancellation( + executionId, + workflowId, + activeResumeEntryId + ).catch((error) => { + logger.warn('Failed to roll back active resume after group target disappeared', { + executionId, + activeResumeEntryId, + error: toError(error).message, + }) + }) + } else if (effectivePausedCancellationPath) { + await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId).catch( + (error) => { + logger.warn('Failed to clear cancellation intent after group target disappeared', { + executionId, + error: toError(error).message, + }) + } + ) + } + throw new OrchestrationError( + 'conflict', + 'Workflow group execution is no longer the active table execution' + ) + } + + if (competingTerminalStatus) { + await clearStopSignalMarkers(stopSummary) + if (activeResumeEntryId) { + await PauseResumeManager.rollbackActiveResumeCancellation( + executionId, + workflowId, + activeResumeEntryId + ).catch((error) => { + logger.warn('Failed to roll back active resume after terminal race', { + executionId, + activeResumeEntryId, + error: toError(error).message, + }) + }) + } else if (effectivePausedCancellationPath) { + await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId).catch( + (error) => { + logger.warn('Failed to clear cancellation intent after terminal race', { + executionId, + error: toError(error).message, + }) + } + ) + } + if ( + !isWorkflowGroupExecution && + isWorkflowRunAlreadyTerminalStatus(competingTerminalStatus) + ) { + throw new WorkflowRunAlreadyTerminalError({ + executionId, + executionStatus: competingTerminalStatus, + redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + locallyAborted: stopSummary.locallyAborted, + }) + } + throw new OrchestrationError( + 'conflict', + isWorkflowGroupExecution + ? `Workflow group execution cannot be cancelled while ${competingTerminalStatus}` + : `Execution cannot be cancelled while ${competingTerminalStatus}` + ) + } + + if (!terminalCancellationClaimed) { + if (effectivePausedCancellationPath && !stopSummary.accepted) { + if (activeResumeEntryId) { + await PauseResumeManager.rollbackActiveResumeCancellation( + executionId, + workflowId, + activeResumeEntryId + ) + } else { + await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId) + } + } + return { + success: false, + executionId, + redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + durablyRecorded: stopSummary.cancellation.durablyRecorded, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled: false, + reason: 'cancellation_not_finalized', + } + } + + let pauseReconciliationFailed = false + let activeResumeSignalFailed = false + for (const target of activeResumeTargetsNeedingStopConfirmation) { + const stopConfirmed = await signalAndRecordActiveResumeStop({ workflowId, executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target, + summary: stopSummary, }) + if (target.resumeEntryId === activeResumeEntryId) { + activeResumeSignalAccepted = stopConfirmed + } + activeResumeSignalFailed = activeResumeSignalFailed || !stopConfirmed + } + if (isWorkflowGroupExecution) { + if (activeResumeTarget) { + activeResumeSignalAccepted = await signalAndRecordActiveResumeStop({ + workflowId, + executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target: activeResumeTarget, + summary: stopSummary, + }) + activeResumeSignalFailed = !activeResumeSignalAccepted + } else if (!effectivePausedCancellationPath) { + const groupSignal = await signalExecutionStop({ + workflowId, + signalExecutionId: executionId, + executionDeadlineAt: execution.executionDeadlineAt, + }) + mergeExecutionStopSignal(stopSummary, executionId, groupSignal) + } + } + + try { + const postClaimPausedCancellationStage = await PauseResumeManager.stagePausedCancellation( + executionId, + workflowId + ) + if (isPausedCancellationStage(postClaimPausedCancellationStage)) { + effectivePausedCancellationPath = true + if (postClaimPausedCancellationStage.kind === 'active_resume') { + const currentActiveResume = postClaimPausedCancellationStage.target + const alreadyAttemptedCurrentResume = + currentActiveResume.resumeEntryId === activeResumeEntryId && + stopSummary.signalledExecutionIds.has(currentActiveResume.resumeExecutionId) + if (!alreadyAttemptedCurrentResume) { + activeResumeSignalAccepted = await signalAndRecordActiveResumeStop({ + workflowId, + executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target: currentActiveResume, + summary: stopSummary, + }) + } + activeResumeSignalFailed = activeResumeSignalFailed || !activeResumeSignalAccepted + } + } } catch (error) { - logger.error('Workflow group execution cancellation failed unexpectedly', { + pauseReconciliationFailed = true + effectivePausedCancellationPath = true + logger.warn('Failed to recheck paused execution after terminal cancellation claim', { executionId, - error, + error: toError(error).message, }) - await releaseSlotForStoppedExecution() - throw error } - } - - /** - * Both refusals mean the cell claim was lost, never that the run is still - * going: the sidecar conflicts only on a terminal workflow log or a terminal - * cell, and `not_workflow_group` means the log is not a group run at all. - * Every refusal is a terminal-or-absent state that carries no evidence of - * liveness, so nothing is left running to hold the reservation and it is - * released before the 409 rather than left to expire. (The Redis abort record - * is reversible — see `clearExecutionCancellation` — but a refusal gives no - * reason to reverse it.) - */ - if (groupCancellation?.kind === 'conflict') { - logger.warn('Workflow group execution could not be cancelled', { - executionId, - status: groupCancellation.status, - }) - await releaseSlotForStoppedExecution() - throw new OrchestrationError( - 'conflict', - `Workflow group execution cannot be cancelled while ${groupCancellation.status}` - ) - } - if (groupCancellation?.kind === 'not_workflow_group') { - logger.warn('Workflow group execution is no longer the active table execution', { executionId }) - await releaseSlotForStoppedExecution() - throw new OrchestrationError( - 'conflict', - 'Workflow group execution is no longer the active table execution' - ) - } - const groupCancellationToPublish: PublishableWorkflowGroupCancellation | null = - groupCancellation?.kind === 'cancelled' || groupCancellation?.kind === 'already_cancelled' - ? groupCancellation - : null - - /** - * The claim's row count is read back only to report it — `returning` changes - * what the statement returns, never the row it writes or the rows it matches. - */ - let terminalWrite: TerminalWriteOutcome = 'unknown' - if (groupCancellation !== null) { - terminalWrite = toTerminalWriteOutcome(groupCancellation.writes) - } else if ( - (cancellation.durablyRecorded || queuedJobCancelled || locallyAborted) && - !pausedCancelled - ) { - try { - const cancelledAt = new Date() - const claimedRows = await db - .update(workflowExecutionLogs) - .set(cancelledExecutionLogFields(cancelledAt)) - .where( - and( - eq(workflowExecutionLogs.executionId, executionId), - eq(workflowExecutionLogs.status, 'running') - ) - ) - .returning({ id: workflowExecutionLogs.id }) - terminalWrite = claimedRows.length > 0 ? 'applied' : 'no_row' - } catch (dbError) { - logger.warn('Failed to update execution log status directly', { - executionId, - error: dbError, + const executionStopSatisfied = effectivePausedCancellationPath + ? !activeResumeSignalFailed + : stopSummary.accepted + if (groupCancellationToPublish && executionStopSatisfied && !pauseReconciliationFailed) { + await publishWorkflowGroupCancellationEvent(groupCancellationToPublish, executionId) + } + let cancellationEventPublished = false + if (executionStopSatisfied && !pauseReconciliationFailed) { + cancellationEventPublished = await ensureCancellationEventPublished(executionId, workflowId, { + workspaceId: execution.workspaceId, + userId: attributedUserId, }) } - } - if (groupCancellationToPublish && success) { - await publishWorkflowGroupCancellationEvent(groupCancellationToPublish, executionId) - } + if (effectivePausedCancellationPath) { + if (cancellationEventPublished && !pauseReconciliationFailed && !activeResumeSignalFailed) { + pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId) + } + } else if (executionStopSatisfied) { + await releaseExecutionSlot(executionId).catch((error) => { + logger.warn('Failed to release reservation after execution cancellation', { + executionId, + error: toError(error).message, + }) + }) + } - await releaseSlotForStoppedExecution() + const success = effectivePausedCancellationPath + ? pausedCancelled && + cancellationEventPublished && + !pauseReconciliationFailed && + !activeResumeSignalFailed + : executionStopSatisfied - if (success && input.captureAnalytics !== false) { - captureServerEvent( - userId, - 'workflow_execution_cancelled', - { workflow_id: workflowId, workspace_id: workspaceId ?? '' }, - workspaceId ? { groups: { workspace: workspaceId } } : undefined - ) - } + if (effectivePausedCancellationPath && pausedCancelled && cancellationEventPublished) { + await clearStopSignalMarkers(stopSummary) + } - const durablyRecorded = isPausedCancellationPath - ? pausedCancellationPublished - : pausedCancelled || cancellation.durablyRecorded || queuedJobCancelled - const reason: CancelWorkflowExecutionReason = pausedCancellationPublishFailed - ? 'paused_event_publish_failed' - : !pausedCancelled && isPausedCancellationPath - ? 'paused_database_cancel_failed' - : pausedCancelled && !pausedCancellationPublished - ? 'paused_event_publish_failed' - : pausedCancelled || isPausedCancellationPath - ? 'recorded' - : queuedJobCancelled - ? 'recorded' - : cancellation.reason - - /** - * A run that was already terminal when the request arrived cannot be - * cancelled again: the claim's `status = 'running'` predicate matched no row - * and no terminal metadata moved, so `recorded`/`durablyRecorded: true` would - * claim a durable write that never happened. Every effect above still ran - * exactly as before — only the report changes. The request is still satisfied, - * because the run is not running, so `success` stays `true`. - * - * Reinterpreting is only ever right when nothing else went wrong. A terminal - * run — cancelled, or force-failed with paused state left behind — can still - * carry real paused-HITL reconciliation work, and a genuine failure there owes - * the caller the step that failed, not a no-op. So only an otherwise-clean - * `recorded` is a candidate, whatever the prior status was — and only when - * this request wrote nothing durable on any path. - */ - const terminalNoOpReason = - reason === 'recorded' && !pausedCancelled - ? await resolveTerminalNoOpReason(executionId, workflowId, priorTerminalStatus, terminalWrite) - : null + const durablyRecorded = effectivePausedCancellationPath + ? true + : stopSummary.cancellation.durablyRecorded + const reason = resolveCancellationReason({ + activeResumeSignalFailed, + pauseReconciliationFailed, + effectivePausedCancellationPath, + cancellationEventPublished, + pausedCancelled, + stopSummary, + }) - return { - success: terminalNoOpReason ? true : success, - executionId, - redisAvailable: - isPausedCancellationPath || pausedCancelled - ? pausedCancellationPublished - : cancellation.reason !== 'redis_unavailable', - durablyRecorded: terminalNoOpReason ? false : durablyRecorded, - locallyAborted, - pausedCancelled, - reason: terminalNoOpReason ?? reason, + return { + success, + executionId, + redisAvailable: + effectivePausedCancellationPath || pausedCancelled + ? cancellationEventPublished + : stopSummary.cancellation.reason !== 'redis_unavailable', + durablyRecorded, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled, + reason, + } + } catch (error) { + const normalizedError = toError(error) + logger.error('Failed to cancel execution', { + workflowId, + executionId, + error: normalizedError.message, + }) + throw error } } diff --git a/apps/sim/lib/execution/workflow-run-already-terminal-error.ts b/apps/sim/lib/execution/workflow-run-already-terminal-error.ts new file mode 100644 index 00000000000..0ff4966aec4 --- /dev/null +++ b/apps/sim/lib/execution/workflow-run-already-terminal-error.ts @@ -0,0 +1,33 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' + +export type WorkflowRunAlreadyTerminalStatus = 'completed' | 'failed' + +interface WorkflowRunAlreadyTerminalErrorOptions { + executionId: string + executionStatus: WorkflowRunAlreadyTerminalStatus + redisAvailable: boolean + locallyAborted: boolean +} + +/** A standalone run reached a non-cancellable terminal state before cancellation won. */ +export class WorkflowRunAlreadyTerminalError extends OrchestrationError { + readonly executionId: string + readonly executionStatus: WorkflowRunAlreadyTerminalStatus + readonly redisAvailable: boolean + readonly locallyAborted: boolean + + constructor(options: WorkflowRunAlreadyTerminalErrorOptions) { + super('conflict', `Execution cannot be cancelled while ${options.executionStatus}`) + this.name = 'WorkflowRunAlreadyTerminalError' + this.executionId = options.executionId + this.executionStatus = options.executionStatus + this.redisAvailable = options.redisAvailable + this.locallyAborted = options.locallyAborted + } +} + +export function isWorkflowRunAlreadyTerminalStatus( + status: string +): status is WorkflowRunAlreadyTerminalStatus { + return status === 'completed' || status === 'failed' +} diff --git a/apps/sim/lib/workflows/api/index.ts b/apps/sim/lib/workflows/api/index.ts index 26e4ce576dd..44fde23abf1 100644 --- a/apps/sim/lib/workflows/api/index.ts +++ b/apps/sim/lib/workflows/api/index.ts @@ -2,6 +2,7 @@ export { createInternalWorkflowErrorPolicy, internalWorkflowErrorPolicies, internalWorkflowReadAuth, + internalWorkflowSessionOrApiKeyAuth, internalWorkflowSessionOrExecutorAuth, v2WorkflowErrorPolicies, WORKFLOW_NOT_FOUND_MESSAGE, diff --git a/apps/sim/lib/workflows/api/route-policies.ts b/apps/sim/lib/workflows/api/route-policies.ts index 488d747b343..e445a2766ef 100644 --- a/apps/sim/lib/workflows/api/route-policies.ts +++ b/apps/sim/lib/workflows/api/route-policies.ts @@ -1,4 +1,10 @@ -import type { Principal } from '@sim/auth/principal' +import type { + PersonalApiKeyPrincipal, + Principal, + SessionPrincipal, + WorkspaceApiKeyPrincipal, +} from '@sim/auth/principal' +import { v2CancelWorkflowRunDataSchema } from '@/lib/api/contracts/v2/workflows' import { createInternalResourceConcealmentPolicy, createInternalSessionOrExecutorAuth, @@ -8,15 +14,38 @@ import { InternalUnauthenticatedError, internalErrorResponse, internalOrchestrationErrorPolicy, + internalSessionAuth, type V2ErrorPolicy, v2OrchestrationErrorPolicy, } from '@/lib/api/server/routes' import { authenticateApiKeyFromHeader, updateApiKeyLastUsed } from '@/lib/api-key/service' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { WorkflowRunAlreadyTerminalError } from '@/lib/execution/workflow-run-already-terminal-error' import { WORKFLOW_DELEGATION_AUDIENCE } from '@/lib/workflows/application/authorization' import { WorkflowImportError } from '@/lib/workflows/application/workflow-import-error' import { WorkflowOperationsNotAppliedError } from '@/lib/workflows/application/workflow-operations-error' -import { v2CaughtOrchestrationError, v2ErrorForOrchestration } from '@/app/api/v2/lib/response' +import { + v2CaughtOrchestrationError, + v2Data, + v2ErrorForOrchestration, +} from '@/app/api/v2/lib/response' + +function v2CancelRunErrorResponse(error: unknown) { + if (error instanceof WorkflowRunAlreadyTerminalError) { + return v2Data( + v2CancelWorkflowRunDataSchema.parse({ + success: true, + runId: error.executionId, + redisAvailable: error.redisAvailable, + durablyRecorded: false, + locallyAborted: error.locallyAborted, + pausedCancelled: false, + reason: error.executionStatus === 'completed' ? 'already_completed' : 'already_failed', + }) + ) + } + return v2CaughtOrchestrationError(error) +} export const v2WorkflowErrorPolicies = { default: v2OrchestrationErrorPolicy, @@ -55,31 +84,50 @@ export const v2WorkflowErrorPolicies = { concealRunAuthorization: createV2ResourceConcealmentPolicy({ notFoundMessage: 'Run not found', }), + cancelRun: createV2ResourceConcealmentPolicy({ + notFoundMessage: 'Run not found', + render: v2CancelRunErrorResponse, + }), } as const export const internalWorkflowSessionOrExecutorAuth = createInternalSessionOrExecutorAuth({ audience: WORKFLOW_DELEGATION_AUDIENCE, }) +type WorkflowApiKeyPrincipal = PersonalApiKeyPrincipal | WorkspaceApiKeyPrincipal + +async function authenticateWorkflowApiKey(rawApiKey: string): Promise { + const result = await authenticateApiKeyFromHeader(rawApiKey) + if (!result.success || !result.keyId || !result.keyType) { + throw new InternalUnauthenticatedError('Unauthorized') + } + await updateApiKeyLastUsed(result.keyId) + + if (result.keyType === 'workspace') { + if (!result.workspaceId) throw new Error('Workspace API key is missing its workspace scope') + return { kind: 'workspace_api_key', workspaceId: result.workspaceId, keyId: result.keyId } + } + if (!result.userId) throw new Error('Personal API key is missing its credential owner') + return { kind: 'personal_api_key', userId: result.userId, keyId: result.keyId } +} + +export const internalWorkflowSessionOrApiKeyAuth: InternalAuthPolicy< + SessionPrincipal | WorkflowApiKeyPrincipal +> = { + async authenticate(request) { + const rawApiKey = request.headers.get('x-api-key') + if (!rawApiKey) return internalSessionAuth.authenticate() + return authenticateWorkflowApiKey(rawApiKey) + }, +} + export const internalWorkflowReadAuth: InternalAuthPolicy = { async authenticate(request, params) { const rawApiKey = request.headers.get('x-api-key') if (!rawApiKey) { return internalWorkflowSessionOrExecutorAuth.authenticate(request, params) } - - const result = await authenticateApiKeyFromHeader(rawApiKey) - if (!result.success || !result.keyId || !result.keyType) { - throw new InternalUnauthenticatedError('Unauthorized') - } - await updateApiKeyLastUsed(result.keyId) - - if (result.keyType === 'workspace') { - if (!result.workspaceId) throw new Error('Workspace API key is missing its workspace scope') - return { kind: 'workspace_api_key', workspaceId: result.workspaceId, keyId: result.keyId } - } - if (!result.userId) throw new Error('Personal API key is missing its credential owner') - return { kind: 'personal_api_key', userId: result.userId, keyId: result.keyId } + return authenticateWorkflowApiKey(rawApiKey) }, } @@ -125,4 +173,8 @@ export const internalWorkflowErrorPolicies = { base: internalOrchestrationErrorPolicy, notFoundMessage: WORKFLOW_NOT_FOUND_MESSAGE, }), + concealRunAuthorization: createInternalResourceConcealmentPolicy({ + base: internalOrchestrationErrorPolicy, + notFoundMessage: 'Execution not found', + }), } as const diff --git a/apps/sim/lib/workflows/application/cancel-run.ts b/apps/sim/lib/workflows/application/cancel-run.ts index 848fb3c64cc..90f4a91ae5b 100644 --- a/apps/sim/lib/workflows/application/cancel-run.ts +++ b/apps/sim/lib/workflows/application/cancel-run.ts @@ -4,6 +4,7 @@ import { cancelWorkflowExecution, WorkflowExecutionNotFoundError, } from '@/lib/execution/cancel-workflow-execution' +import { captureServerEvent } from '@/lib/posthog/server' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { resolveActiveWorkflowRunApplicationContext } from '@/lib/workflows/application/context' import { workflowOperations } from '@/lib/workflows/application/operations' @@ -11,6 +12,8 @@ import { workflowOperations } from '@/lib/workflows/application/operations' export interface CancelWorkflowRunInput { workflowId: string runId: string + assertedWorkspaceId?: string + abortSignal?: AbortSignal } export const cancelWorkflowRun = defineAuthorizedWorkflowUseCase({ @@ -19,8 +22,9 @@ export const cancelWorkflowRun = defineAuthorizedWorkflowUseCase({ resolveActiveWorkflowRunApplicationContext({ runId: input.runId, assertedWorkflowId: input.workflowId, + assertedWorkspaceId: input.assertedWorkspaceId, }), - async execute({ principal, context }) { + async execute({ principal, context, input }) { const attribution = resolvePrincipalAttribution(principal, { workspaceBillingOwnerUserId: context.billedAccountUserId, }) @@ -28,9 +32,9 @@ export const cancelWorkflowRun = defineAuthorizedWorkflowUseCase({ const result = await cancelWorkflowExecution({ executionId: context.runId, workflowId: context.workflowId, - userId: attribution.attributedUserId, + attributedUserId: attribution.attributedUserId, workspaceId: context.workspaceId, - captureAnalytics: false, + abortSignal: input.abortSignal, }) return { ...result, workflowId: context.workflowId, workspaceId: context.workspaceId } } catch (error) { @@ -40,4 +44,16 @@ export const cancelWorkflowRun = defineAuthorizedWorkflowUseCase({ throw error } }, + afterSuccess({ principal, context, result }) { + if (!result.success || result.reason === 'already_cancelled') return + const attribution = resolvePrincipalAttribution(principal, { + workspaceBillingOwnerUserId: context.billedAccountUserId, + }) + captureServerEvent( + attribution.attributedUserId, + 'workflow_execution_cancelled', + { workflow_id: context.workflowId, workspace_id: context.workspaceId }, + { groups: { workspace: context.workspaceId } } + ) + }, }) diff --git a/apps/sim/lib/workflows/application/workflow-run-control.test.ts b/apps/sim/lib/workflows/application/workflow-run-control.test.ts index 7478d7d6f91..08f09d66eea 100644 --- a/apps/sim/lib/workflows/application/workflow-run-control.test.ts +++ b/apps/sim/lib/workflows/application/workflow-run-control.test.ts @@ -11,6 +11,7 @@ const { MockWorkflowExecutionNotFoundError, mocks } = vi.hoisted(() => { mocks: { audit: vi.fn(), cancel: vi.fn(), + capture: vi.fn(), resolvePermission: vi.fn(), resolveRunContext: vi.fn(), resume: vi.fn(), @@ -19,6 +20,7 @@ const { MockWorkflowExecutionNotFoundError, mocks } = vi.hoisted(() => { }) vi.mock('@sim/audit', () => ({ recordAudit: mocks.audit })) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.capture })) vi.mock('@sim/platform-authz/workspace', () => ({ permissionSatisfies: (actual: string | null, required: string) => { @@ -121,14 +123,21 @@ describe('workflow run-control application use cases', () => { expect(mocks.resolveRunContext).toHaveBeenCalledWith({ runId: 'parent-run-1', assertedWorkflowId: 'workflow-1', + assertedWorkspaceId: undefined, }) expect(mocks.cancel).toHaveBeenCalledWith({ executionId: 'parent-run-1', workflowId: 'workflow-1', - userId: actorUserId, + attributedUserId: actorUserId, workspaceId: 'workspace-1', - captureAnalytics: false, + abortSignal: undefined, }) + expect(mocks.capture).toHaveBeenCalledWith( + actorUserId, + 'workflow_execution_cancelled', + { workflow_id: 'workflow-1', workspace_id: 'workspace-1' }, + { groups: { workspace: 'workspace-1' } } + ) expect(mocks.audit).not.toHaveBeenCalled() } ) diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index d623c667e6f..8040820d60a 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -1023,6 +1023,9 @@ type CancelWorkflowRunResponseRef0 = { | 'redis_write_failed' | 'paused_event_publish_failed' | 'paused_database_cancel_failed' + | 'queue_cancelled' + | 'active_resume_signal_failed' + | 'cancellation_not_finalized' } export type CancelWorkflowRunResponse = {