From caa4dae159d612fb7f27522d88469f6090b48df9 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Fri, 28 Aug 2026 18:56:05 -0700 Subject: [PATCH 1/7] feat(billing): record function sandbox usage --- .../handlers/pi/local/sim-tools.test.ts | 48 +++++++++ .../executor/handlers/pi/local/sim-tools.ts | 27 ++++- .../executor/handlers/pi/pi-handler.test.ts | 30 +++++- apps/sim/executor/handlers/pi/pi-handler.ts | 33 ++++-- apps/sim/lib/billing/sandbox-pricing.test.ts | 29 +++++ apps/sim/lib/billing/sandbox-pricing.ts | 99 +++++++++++++++++ .../remote-sandbox/conformance.test.ts | 83 +++++++++++++- .../lib/execution/remote-sandbox/daytona.ts | 14 ++- apps/sim/lib/execution/remote-sandbox/e2b.ts | 11 +- .../remote-sandbox/function-resources.ts | 1 + .../sim/lib/execution/remote-sandbox/index.ts | 102 ++++++++++++++---- .../sim/lib/execution/remote-sandbox/types.ts | 9 ++ .../execute-request.test.ts | 81 +++++++++++++- .../lib/function-execution/execute-request.ts | 26 +++++ .../build-function-daytona-snapshot.ts | 3 +- apps/sim/tools/function/execute.test.ts | 16 +++ apps/sim/tools/function/execute.ts | 1 + apps/sim/tools/function/types.ts | 5 + apps/sim/tools/index.test.ts | 4 +- apps/sim/tools/index.ts | 12 ++- 20 files changed, 587 insertions(+), 47 deletions(-) create mode 100644 apps/sim/lib/billing/sandbox-pricing.test.ts create mode 100644 apps/sim/lib/billing/sandbox-pricing.ts diff --git a/apps/sim/executor/handlers/pi/local/sim-tools.test.ts b/apps/sim/executor/handlers/pi/local/sim-tools.test.ts index 60b99bb95ac..1ed6d359c2d 100644 --- a/apps/sim/executor/handlers/pi/local/sim-tools.test.ts +++ b/apps/sim/executor/handlers/pi/local/sim-tools.test.ts @@ -225,6 +225,54 @@ describe('buildSimToolSpecs', () => { }) }) + it('accumulates cost only from successful canonical Function tool results', async () => { + mockTransformBlockTool + .mockResolvedValueOnce({ + id: 'function_execute', + name: 'Function Execute', + description: 'Execute code', + params: {}, + parameters: { type: 'object', properties: {} }, + }) + .mockResolvedValueOnce({ + id: 'exa_search', + name: 'Exa Search', + description: 'Search the web', + params: {}, + parameters: { type: 'object', properties: {} }, + }) + const functionToolCost = { total: 0 } + const [functionSpec, searchSpec] = await buildSimToolSpecs( + executionContext(undefined), + [ + { type: 'function', operation: 'execute', usageControl: 'auto' }, + { type: 'exa', operation: 'exa_search', usageControl: 'auto' }, + ], + functionToolCost + ) + + mockExecuteTool + .mockResolvedValueOnce({ + success: true, + output: { result: 'ok', cost: { total: 0.125 } }, + }) + .mockResolvedValueOnce({ + success: true, + output: { result: 'search result', cost: { total: 4 } }, + }) + .mockResolvedValueOnce({ + success: false, + output: { cost: { total: 8 } }, + error: 'execution failed', + }) + + await functionSpec.execute({}) + await searchSpec.execute({}) + await functionSpec.execute({}) + + expect(functionToolCost.total).toBe(0.125) + }) + it('projects named provenance in successful Sim tool output', async () => { mockToolAdapter({ apiKey: 'secret-value' }) encryptionMockFns.mockDecryptSecret.mockResolvedValue({ decrypted: 'secret-value' }) diff --git a/apps/sim/executor/handlers/pi/local/sim-tools.ts b/apps/sim/executor/handlers/pi/local/sim-tools.ts index 483eb87f0a6..1ccf9b47318 100644 --- a/apps/sim/executor/handlers/pi/local/sim-tools.ts +++ b/apps/sim/executor/handlers/pi/local/sim-tools.ts @@ -47,6 +47,10 @@ type PiToolResultProjection = | { safe: true; result: PiToolResult } | { safe: false; result: PiToolResult } +export interface PiFunctionToolCostAccumulator { + total: number +} + function projectToolResult( result: ToolResponse, registry: ResolvedSecretTraceRegistry | undefined @@ -107,7 +111,8 @@ function buildSimToolSpec( ctx: ExecutionContext, inputTools: ToolInput[], provider: ProviderToolConfig, - toolIndex: number + toolIndex: number, + functionToolCost?: PiFunctionToolCostAccumulator ): PiToolSpec { const toolId = provider.canonicalId ?? provider.id const preseededParams = provider.params || {} @@ -170,6 +175,21 @@ function buildSimToolSpec( resolvedSecretTraceRegistry: toolCallRegistry, } ) + const resultCost = result.output?.cost + const resultCostTotal = + resultCost && typeof resultCost === 'object' + ? (resultCost as Record).total + : undefined + if ( + toolId === 'function_execute' && + result.success && + functionToolCost && + typeof resultCostTotal === 'number' && + Number.isFinite(resultCostTotal) && + resultCostTotal > 0 + ) { + functionToolCost.total += resultCostTotal + } const projection = projectToolResult(result, toolCallRegistry?.forkForPropagatedEntries()) if (projection.safe && registry && toolCallRegistry?.isComplete()) { registry.mergeToolCallRegistry(toolCallRegistry) @@ -199,7 +219,8 @@ function buildSimToolSpec( */ export async function buildSimToolSpecs( ctx: ExecutionContext, - inputTools: unknown + inputTools: unknown, + functionToolCost?: PiFunctionToolCostAccumulator ): Promise { if (!Array.isArray(inputTools)) return [] @@ -243,6 +264,6 @@ export async function buildSimToolSpecs( await annotateDuplicateToolBindings(ctx, providers) assignProviderToolIdentities(providers) return configuredTools.map(({ provider, toolIndex }) => - buildSimToolSpec(ctx, inputTools, provider, toolIndex) + buildSimToolSpec(ctx, inputTools, provider, toolIndex, functionToolCost) ) } diff --git a/apps/sim/executor/handlers/pi/pi-handler.test.ts b/apps/sim/executor/handlers/pi/pi-handler.test.ts index 063b319c6ff..aee9b0fd0f6 100644 --- a/apps/sim/executor/handlers/pi/pi-handler.test.ts +++ b/apps/sim/executor/handlers/pi/pi-handler.test.ts @@ -20,6 +20,7 @@ const { mockResolveSearchKey, mockBuildSearchTool, mockAssertPermissionsAllowed, + mockBuildSimToolSpecs, MockToolNotAllowedError, } = vi.hoisted(() => ({ mockRunLocal: vi.fn(), @@ -38,6 +39,7 @@ const { mockResolveSearchKey: vi.fn(), mockBuildSearchTool: vi.fn(), mockAssertPermissionsAllowed: vi.fn(), + mockBuildSimToolSpecs: vi.fn(), MockToolNotAllowedError: class ToolNotAllowedError extends Error {}, })) @@ -64,7 +66,7 @@ vi.mock('@/executor/handlers/pi/core/context', () => ({ appendPiMemory: mockAppendMemory, })) vi.mock('@/executor/handlers/pi/local/sim-tools', () => ({ - buildSimToolSpecs: vi.fn().mockResolvedValue([]), + buildSimToolSpecs: mockBuildSimToolSpecs, })) vi.mock('@/executor/handlers/pi/local/backend', () => ({ runLocalPi: mockRunLocal })) vi.mock('@/executor/handlers/pi/cloud/authoring/backend', () => ({ @@ -153,6 +155,7 @@ describe('PiBlockHandler', () => { mockResolveSearchKey.mockReturnValue('search-key') mockBuildSearchTool.mockReturnValue({ name: 'web_search' }) mockAssertPermissionsAllowed.mockResolvedValue(undefined) + mockBuildSimToolSpecs.mockResolvedValue([]) mockResolveSkills.mockResolvedValue([]) mockLoadMemory.mockResolvedValue([]) mockAppendMemory.mockResolvedValue(undefined) @@ -275,6 +278,19 @@ describe('PiBlockHandler', () => { expect((output as Record).content).toBe('hi') }) + it('adds successful Function tool cost once to a non-streaming Local Dev result', async () => { + mockBuildSimToolSpecs.mockImplementation( + async (_ctx: unknown, _tools: unknown, functionToolCost: { total: number }) => { + functionToolCost.total += 0.125 + return [] + } + ) + + const output = (await handler.execute(ctx(), block, localInputs())) as { cost: unknown } + + expect(output.cost).toEqual({ input: 0, output: 0, toolCost: 0.125, total: 0.125 }) + }) + it('routes Create PR to the cloud backend and surfaces PR output', async () => { const output = (await handler.execute(ctx(), block, { mode: 'cloud', @@ -942,6 +958,12 @@ describe('PiBlockHandler', () => { }) it('streams text when the block is selected for streaming output', async () => { + mockBuildSimToolSpecs.mockImplementation( + async (_ctx: unknown, _tools: unknown, functionToolCost: { total: number }) => { + functionToolCost.total += 0.25 + return [] + } + ) mockRunLocal.mockImplementation(async (_params, runCtx) => { runCtx.onEvent({ type: 'text', text: 'streamed' }) return { totals: { finalText: 'streamed', inputTokens: 0, outputTokens: 0, toolCalls: [] } } @@ -965,6 +987,12 @@ describe('PiBlockHandler', () => { } expect(text).toContain('streamed') expect(result.execution.output.content).toBe('streamed') + expect(result.execution.output.cost).toEqual({ + input: 0, + output: 0, + toolCost: 0.25, + total: 0.25, + }) }) it('streams only the canonical final document for Plan mode', async () => { diff --git a/apps/sim/executor/handlers/pi/pi-handler.ts b/apps/sim/executor/handlers/pi/pi-handler.ts index 2f42719b71c..d152fa52cf4 100644 --- a/apps/sim/executor/handlers/pi/pi-handler.ts +++ b/apps/sim/executor/handlers/pi/pi-handler.ts @@ -45,7 +45,10 @@ import { resolvePiSearchKey, } from '@/executor/handlers/pi/core/keys' import { runLocalPi } from '@/executor/handlers/pi/local/backend' -import { buildSimToolSpecs } from '@/executor/handlers/pi/local/sim-tools' +import { + buildSimToolSpecs, + type PiFunctionToolCostAccumulator, +} from '@/executor/handlers/pi/local/sim-tools' import { buildPiSearchToolSpec } from '@/executor/handlers/pi/search/tool' import type { BlockHandler, @@ -267,7 +270,8 @@ export class PiBlockHandler implements BlockHandler { } const usePrivateKey = inputs.authMethod === 'privateKey' const port = parseOptionalNumberInput(inputs.port, 'port', { integer: true, min: 1 }) ?? 22 - const tools = await buildSimToolSpecs(ctx, inputs.tools) + const functionToolCost: PiFunctionToolCostAccumulator = { total: 0 } + const tools = await buildSimToolSpecs(ctx, inputs.tools, functionToolCost) const params: PiLocalRunParams = { ...contextualBase, mode: 'local', @@ -282,7 +286,7 @@ export class PiBlockHandler implements BlockHandler { passphrase: usePrivateKey ? asRawString(inputs.passphrase) : undefined, }, } - return this.runPi(ctx, block, runLocalPi, params, memoryConfig) + return this.runPi(ctx, block, runLocalPi, params, memoryConfig, functionToolCost) } const owner = asOptString(inputs.owner) @@ -473,10 +477,20 @@ export class PiBlockHandler implements BlockHandler { model: string, isBYOK: boolean, startTime: number, - startTimeISO: string + startTimeISO: string, + functionToolCost = 0 ): NormalizedBlockOutput { const { totals } = result const endTime = Date.now() + const modelCost = computePiCost(model, totals.inputTokens, totals.outputTokens, isBYOK) + const cost = + functionToolCost > 0 + ? { + ...modelCost, + toolCost: functionToolCost, + total: modelCost.total + functionToolCost, + } + : modelCost return { content: totals.finalText, model, @@ -505,7 +519,7 @@ export class PiBlockHandler implements BlockHandler { output: totals.outputTokens, total: totals.inputTokens + totals.outputTokens, }, - cost: computePiCost(model, totals.inputTokens, totals.outputTokens, isBYOK), + cost, providerTiming: { startTime: startTimeISO, endTime: new Date(endTime).toISOString(), @@ -519,7 +533,8 @@ export class PiBlockHandler implements BlockHandler { block: SerializedBlock, backend: PiBackendRun

, params: P, - memoryConfig?: PiMemoryConfig + memoryConfig?: PiMemoryConfig, + functionToolCost?: PiFunctionToolCostAccumulator ): Promise { const startTime = Date.now() const startTimeISO = new Date(startTime).toISOString() @@ -561,7 +576,8 @@ export class PiBlockHandler implements BlockHandler { params.model, params.isBYOK, startTime, - startTimeISO + startTimeISO, + functionToolCost?.total ) ) if (memoryConfig) { @@ -610,7 +626,8 @@ export class PiBlockHandler implements BlockHandler { params.model, params.isBYOK, startTime, - startTimeISO + startTimeISO, + functionToolCost?.total ) } } diff --git a/apps/sim/lib/billing/sandbox-pricing.test.ts b/apps/sim/lib/billing/sandbox-pricing.test.ts new file mode 100644 index 00000000000..658cd7a1cbe --- /dev/null +++ b/apps/sim/lib/billing/sandbox-pricing.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest' +import { createSandboxPricing, priceSandboxUsage } from '@/lib/billing/sandbox-pricing' + +describe('sandbox pricing', () => { + it.each([ + ['e2b', 0.1656], + ['daytona', 0.16668], + ] as const)('prices one hour of the Function profile on %s', (provider, expected) => { + const pricing = createSandboxPricing(provider, 1) + + expect(priceSandboxUsage(pricing, 3_600_000, 3_600_000).rawCost).toBeCloseTo(expected, 8) + }) + + it('applies the multiplier once and rounds the final cost to eight decimals', () => { + const pricing = createSandboxPricing('e2b', 1.75) + + expect(priceSandboxUsage(pricing, 1234, 10_000).billedCost).toBe(0.00009934) + }) + + it('caps duration at the provider lifetime', () => { + const pricing = createSandboxPricing('daytona', 1) + + expect(priceSandboxUsage(pricing, 90_000, 60_000).durationMs).toBe(60_000) + }) + + it('rejects a non-positive multiplier', () => { + expect(() => createSandboxPricing('e2b', 0)).toThrow('finite positive') + }) +}) diff --git a/apps/sim/lib/billing/sandbox-pricing.ts b/apps/sim/lib/billing/sandbox-pricing.ts new file mode 100644 index 00000000000..fa5bc09f537 --- /dev/null +++ b/apps/sim/lib/billing/sandbox-pricing.ts @@ -0,0 +1,99 @@ +import { getCostMultiplier } from '@/lib/core/config/env-flags' +import { + FUNCTION_DAYTONA_DISK_GB, + FUNCTION_SANDBOX_CPU_COUNT, + FUNCTION_SANDBOX_MEMORY_GB, +} from '@/lib/execution/remote-sandbox/function-resources' +import type { SandboxProviderId } from '@/lib/execution/remote-sandbox/types' + +const E2B_CPU_USD_PER_VCPU_SECOND = 0.000014 +const E2B_MEMORY_USD_PER_GIB_SECOND = 0.0000045 +const DAYTONA_CPU_USD_PER_VCPU_SECOND = 0.0504 / 3600 +const DAYTONA_MEMORY_USD_PER_GIB_SECOND = 0.0162 / 3600 +const DAYTONA_DISK_USD_PER_GIB_SECOND = 0.000108 / 3600 + +export interface SandboxPricing { + provider: SandboxProviderId + multiplier: number + resources: { + vcpu: number + memoryGiB: number + diskGiB: number + } + rates: { + cpuUsdPerVcpuSecond: number + memoryUsdPerGiBSecond: number + diskUsdPerGiBSecond: number + } +} + +export interface PricedSandboxUsage { + durationMs: number + rawCost: number + billedCost: number +} + +const PRICING_BY_PROVIDER: Record< + SandboxProviderId, + Pick +> = { + e2b: { + resources: { + vcpu: FUNCTION_SANDBOX_CPU_COUNT, + memoryGiB: FUNCTION_SANDBOX_MEMORY_GB, + diskGiB: 0, + }, + rates: { + cpuUsdPerVcpuSecond: E2B_CPU_USD_PER_VCPU_SECOND, + memoryUsdPerGiBSecond: E2B_MEMORY_USD_PER_GIB_SECOND, + diskUsdPerGiBSecond: 0, + }, + }, + daytona: { + resources: { + vcpu: FUNCTION_SANDBOX_CPU_COUNT, + memoryGiB: FUNCTION_SANDBOX_MEMORY_GB, + diskGiB: FUNCTION_DAYTONA_DISK_GB, + }, + rates: { + cpuUsdPerVcpuSecond: DAYTONA_CPU_USD_PER_VCPU_SECOND, + memoryUsdPerGiBSecond: DAYTONA_MEMORY_USD_PER_GIB_SECOND, + diskUsdPerGiBSecond: DAYTONA_DISK_USD_PER_GIB_SECOND, + }, + }, +} + +export function createSandboxPricing( + provider: SandboxProviderId, + multiplier = getCostMultiplier() +): SandboxPricing { + if (!Number.isFinite(multiplier) || multiplier <= 0) { + throw new Error('Sandbox pricing multiplier must be a finite positive number') + } + const pricing = PRICING_BY_PROVIDER[provider] + return { + provider, + multiplier, + resources: { ...pricing.resources }, + rates: { ...pricing.rates }, + } +} + +export function priceSandboxUsage( + pricing: SandboxPricing, + observedDurationMs: number, + providerLifetimeMs: number +): PricedSandboxUsage { + const durationMs = Math.max(0, Math.min(observedDurationMs, providerLifetimeMs)) + const seconds = durationMs / 1000 + const rawCost = + seconds * pricing.resources.vcpu * pricing.rates.cpuUsdPerVcpuSecond + + seconds * pricing.resources.memoryGiB * pricing.rates.memoryUsdPerGiBSecond + + seconds * pricing.resources.diskGiB * pricing.rates.diskUsdPerGiBSecond + + return { + durationMs, + rawCost, + billedCost: Number.parseFloat((rawCost * pricing.multiplier).toFixed(8)), + } +} diff --git a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts index 0ea967c1742..c65c953eb28 100644 --- a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts @@ -123,8 +123,15 @@ import { SIM_RESULT_PREFIX, withPiSandbox, } from '@/lib/execution/remote-sandbox' -import { daytonaProvider } from '@/lib/execution/remote-sandbox/daytona' -import { E2B_MAX_SANDBOX_LIFETIME_MS, e2bProvider } from '@/lib/execution/remote-sandbox/e2b' +import { + daytonaProvider, + resolveDaytonaSandboxLifetimeMs, +} from '@/lib/execution/remote-sandbox/daytona' +import { + E2B_MAX_SANDBOX_LIFETIME_MS, + e2bProvider, + resolveE2BSandboxLifetimeMs, +} from '@/lib/execution/remote-sandbox/e2b' import { MAX_SANDBOX_OUTPUT_BYTES, MAX_SANDBOX_PROCESS_OUTPUT_BYTES, @@ -139,6 +146,27 @@ import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' type Provider = 'e2b' | 'daytona' const PROVIDERS: Provider[] = ['e2b', 'daytona'] +describe('provider-effective sandbox lifetimes', () => { + it('matches E2B second and Daytona minute rounding', () => { + expect(resolveE2BSandboxLifetimeMs(1001)).toBe(2000) + expect(resolveDaytonaSandboxLifetimeMs(1001)).toBe(60_000) + }) + + it.each(PROVIDERS)('reports the %s SDK create dispatch time', async (provider) => { + useProvider(provider) + const onProviderRequestStarted = vi.fn() + const createMock = provider === 'e2b' ? mockE2BCreate : mockDaytonaCreate + + await resolveProvider().create('code', { lifetimeMs: 1000, onProviderRequestStarted }) + + expect(onProviderRequestStarted).toHaveBeenCalledOnce() + expect(onProviderRequestStarted).toHaveBeenCalledWith(expect.any(Number)) + expect(onProviderRequestStarted.mock.invocationCallOrder[0]).toBeLessThan( + createMock.mock.invocationCallOrder[0] + ) + }) +}) + /** Points the shared layer at one provider via the SANDBOX_PROVIDER env var. */ function useProvider(provider: Provider) { mockEnv.SANDBOX_PROVIDER = provider @@ -337,6 +365,47 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { expect(res.result).toEqual({ ok: true }) expect(res.stdout).toBe('hello') expect(res.error).toBeUndefined() + expect(res.cost).toBeUndefined() + }) + + it('adds provider cost to a metered successful code result', async () => { + stubCodeRun(provider, `${SIM_RESULT_PREFIX}{"ok":true}`) + let now = 1_800_000_000_000 + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => ++now) + + try { + const res = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + meterUsage: true, + }) + + expect(res.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) + expect(res.cost?.total).toBeGreaterThan(0) + } finally { + nowSpy.mockRestore() + } + }) + + it('adds provider cost to a metered successful shell result', async () => { + stubShellCommand(provider, 'ok', '', 0) + let now = 1_800_000_000_000 + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => ++now) + + try { + const res = await executeShellInSandbox({ + code: 'echo ok', + envs: {}, + timeoutMs: 1000, + meterUsage: true, + }) + + expect(res.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) + expect(res.cost?.total).toBeGreaterThan(0) + } finally { + nowSpy.mockRestore() + } }) it('takes the LAST marker so user output cannot shadow the real result', async () => { @@ -502,11 +571,13 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { code: 'raise ValueError("boom")', language: CodeLanguage.Python, timeoutMs: 1000, + meterUsage: true, }) expect(res.error).toBe('ValueError: boom') expect(res.stdout).toContain('ValueError: boom') expect(res.result).toBeNull() + expect(res.cost).toBeUndefined() }) it('normalizes Python code budget expiry to a typed timeout abort', async () => { @@ -988,11 +1059,17 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { */ stubShellCommand(provider, provider === 'daytona' ? 'boom detail' : '', 'boom detail', 3) - const res = await executeShellInSandbox({ code: 'false', envs: {}, timeoutMs: 1000 }) + const res = await executeShellInSandbox({ + code: 'false', + envs: {}, + timeoutMs: 1000, + meterUsage: true, + }) expect(res.result).toBeNull() expect(res.error).toContain('boom detail') expect(res.stdout).toContain('boom detail') + expect(res.cost).toBeUndefined() }) it('terminates shell execution when streamed process output exceeds the byte budget', async () => { diff --git a/apps/sim/lib/execution/remote-sandbox/daytona.ts b/apps/sim/lib/execution/remote-sandbox/daytona.ts index 875fcb06856..f9f6912e725 100644 --- a/apps/sim/lib/execution/remote-sandbox/daytona.ts +++ b/apps/sim/lib/execution/remote-sandbox/daytona.ts @@ -43,6 +43,11 @@ const logger = createLogger('DaytonaSandboxProvider') const DAYTONA_DEFAULT_SANDBOX_TTL_MS = 24 * 60 * 60 * 1000 const DAYTONA_STREAM_READY_MARKER = '__SIM_DAYTONA_STREAM_READY__' +/** Daytona expresses sandbox TTLs as whole minutes. */ +export function resolveDaytonaSandboxLifetimeMs(lifetimeMs: number): number { + return Math.max(1, Math.ceil(lifetimeMs / 60_000)) * 60_000 +} + /** Daytona expresses every timeout in seconds; the rest of Sim works in milliseconds. */ function toSeconds(timeoutMs: number): number { return Math.max(1, Math.ceil(timeoutMs / 1000)) @@ -795,6 +800,7 @@ function shellQuote(value: string): string { export const daytonaProvider: SandboxProvider = { id: 'daytona', dependencyStrategy: 'runtime', + resolveLifetimeMs: resolveDaytonaSandboxLifetimeMs, async create(kind: SandboxKind, options?: CreateSandboxOptions): Promise { const apiKey = env.DAYTONA_API_KEY if (!apiKey) { @@ -810,11 +816,11 @@ export const daytonaProvider: SandboxProvider = { snapshot, language: toDaytonaLanguage(language), ephemeral: true, - ttlMinutes: Math.max( - 1, - Math.ceil((options?.lifetimeMs ?? DAYTONA_DEFAULT_SANDBOX_TTL_MS) / 60_000) - ), + ttlMinutes: + resolveDaytonaSandboxLifetimeMs(options?.lifetimeMs ?? DAYTONA_DEFAULT_SANDBOX_TTL_MS) / + 60_000, } + options?.onProviderRequestStarted?.(Date.now()) const sandbox = await daytona.create(createOptions) return new DaytonaSandboxHandle(sandbox, language) diff --git a/apps/sim/lib/execution/remote-sandbox/e2b.ts b/apps/sim/lib/execution/remote-sandbox/e2b.ts index 6a294713f6e..46d887c2118 100644 --- a/apps/sim/lib/execution/remote-sandbox/e2b.ts +++ b/apps/sim/lib/execution/remote-sandbox/e2b.ts @@ -97,6 +97,11 @@ export const E2B_SANDBOX_MATERIALIZER_REVISION = FUNCTION_SANDBOX_MATERIALIZER_R /** Maximum continuous sandbox lifetime supported by E2B. */ export const E2B_MAX_SANDBOX_LIFETIME_MS = 24 * 60 * 60 * 1000 +/** E2B sends sandbox lifetimes as whole seconds. */ +export function resolveE2BSandboxLifetimeMs(lifetimeMs: number): number { + return Math.min(Math.ceil(lifetimeMs / 1000) * 1000, E2B_MAX_SANDBOX_LIFETIME_MS) +} + const E2B_PROVIDER_LIMIT_ERROR = 'E2B reached its 24-hour limit for a single sandbox execution. The workflow timeout may be longer, but this Function call must finish within 24 hours.' const E2B_TIMEOUT_MESSAGE_PATTERN = @@ -863,6 +868,7 @@ export const e2bProvider: SandboxProvider = { id: 'e2b', dependencyStrategy: 'prebuilt', images: e2bImages, + resolveLifetimeMs: resolveE2BSandboxLifetimeMs, async create(kind: SandboxKind, options?: CreateSandboxOptions): Promise { const apiKey = env.E2B_API_KEY if (!apiKey) { @@ -881,7 +887,9 @@ export const e2bProvider: SandboxProvider = { // default — longer than the lifetime it asked for, which is the opposite of // what it requested. const effectiveLifetimeMs = - options?.lifetimeMs !== undefined ? e2bTimeoutMs(options.lifetimeMs) : undefined + options?.lifetimeMs !== undefined + ? resolveE2BSandboxLifetimeMs(options.lifetimeMs) + : undefined const createOptions = { apiKey, ...(effectiveLifetimeMs !== undefined ? { timeoutMs: effectiveLifetimeMs } : {}), @@ -889,6 +897,7 @@ export const e2bProvider: SandboxProvider = { const { Sandbox } = await import('@e2b/code-interpreter') const lifetimeStartedAtMs = Date.now() + options?.onProviderRequestStarted?.(lifetimeStartedAtMs) const sandbox = await Sandbox.create(templateName, createOptions) return new E2BSandboxHandle( diff --git a/apps/sim/lib/execution/remote-sandbox/function-resources.ts b/apps/sim/lib/execution/remote-sandbox/function-resources.ts index afbe013c32f..9061d043da9 100644 --- a/apps/sim/lib/execution/remote-sandbox/function-resources.ts +++ b/apps/sim/lib/execution/remote-sandbox/function-resources.ts @@ -2,6 +2,7 @@ export const FUNCTION_SANDBOX_CPU_COUNT = 2 export const FUNCTION_SANDBOX_MEMORY_GB = 4 export const FUNCTION_SANDBOX_MEMORY_MB = FUNCTION_SANDBOX_MEMORY_GB * 1024 +export const FUNCTION_DAYTONA_DISK_GB = 10 /** Bump when custom dependency-layer rendering changes without a semantic spec change. */ export const FUNCTION_SANDBOX_MATERIALIZER_REVISION = 2 diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index 7bf9f5bff1f..b7eead1ddd1 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -1,6 +1,11 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' +import { + createSandboxPricing, + priceSandboxUsage, + type SandboxPricing, +} from '@/lib/billing/sandbox-pricing' import { createTimeoutAbortController, getRemainingExecutionMs, @@ -46,6 +51,8 @@ import type { SandboxHandle, SandboxKind, SandboxPrivateInput, + SandboxProvider, + SandboxProviderId, SandboxShellExecutionRequest, } from '@/lib/execution/remote-sandbox/types' @@ -59,14 +66,41 @@ export type { const logger = createLogger('RemoteSandbox') +interface CreatedSandbox { + sandbox: SandboxHandle + providerId: SandboxProviderId + startedAtMs: number + effectiveLifetimeMs?: number + pricing?: SandboxPricing +} + async function createSandbox( kind: SandboxKind, - options?: CreateSandboxOptions -): Promise { - const provider = resolveProvider() - const sandbox = await provider.create(kind, options) + options?: CreateSandboxOptions, + meterUsage = false, + provider: SandboxProvider = resolveProvider() +): Promise { + const effectiveLifetimeMs = + options?.lifetimeMs !== undefined ? provider.resolveLifetimeMs(options.lifetimeMs) : undefined + if (meterUsage && effectiveLifetimeMs === undefined) { + throw new Error('Metered sandbox execution requires a provider lifetime') + } + const pricing = meterUsage ? createSandboxPricing(provider.id) : undefined + let startedAtMs = Date.now() + const providerOptions = { + ...options, + ...(effectiveLifetimeMs !== undefined ? { lifetimeMs: effectiveLifetimeMs } : {}), + ...(meterUsage ? { onProviderRequestStarted: (value: number) => (startedAtMs = value) } : {}), + } + const sandbox = await provider.create(kind, providerOptions) logger.info('Created sandbox', { provider: provider.id, kind, sandboxId: sandbox.sandboxId }) - return sandbox + return { + sandbox, + providerId: provider.id, + startedAtMs, + ...(effectiveLifetimeMs !== undefined ? { effectiveLifetimeMs } : {}), + ...(pricing ? { pricing } : {}), + } } /** @@ -83,10 +117,11 @@ async function createSelectedSandbox( kind: SandboxKind, options: CreateSandboxOptions, selected: ResolvedSandbox | null, - signal: AbortSignal -): Promise { + signal: AbortSignal, + meterUsage = false +): Promise { try { - return await createSandbox(kind, options) + return await createSandbox(kind, options, meterUsage) } catch (error) { throwIfAborted(signal) if (!selected) throw error @@ -166,10 +201,13 @@ function throwIfSandboxTimedOut(result: { timedOut?: boolean }): void { if (result.timedOut) throw new DOMException('timeout', 'AbortError') } -function bindSandboxAbort(sandbox: SandboxHandle, signal?: AbortSignal) { +function bindSandboxAbort( + sandbox: SandboxHandle, + provider: SandboxProviderId, + signal?: AbortSignal +) { let killed = false let killPromise: Promise | null = null - const provider = resolveProvider().id const kill = (reason: 'cleanup' | 'cancellation' | 'timeout'): Promise => { if (killed) return Promise.resolve() if (!killPromise) { @@ -211,6 +249,20 @@ function bindSandboxAbort(sandbox: SandboxHandle, signal?: AbortSignal) { } } +function attachSandboxCost( + result: SandboxExecutionResult | undefined, + created: CreatedSandbox, + cleanupStartedAtMs: number +): void { + if (!result || !created.pricing || created.effectiveLifetimeMs === undefined) return + const usage = priceSandboxUsage( + created.pricing, + cleanupStartedAtMs - created.startedAtMs, + created.effectiveLifetimeMs + ) + result.cost = { input: 0, output: 0, total: usage.billedCost } +} + /** * Fetches one URL mount inside the sandbox, bounded by MAX_BYTES. * @@ -753,7 +805,7 @@ async function executeInSandboxWithinBudget( }) throwIfAborted(signal) - const sandbox = await createSelectedSandbox( + const created = await createSelectedSandbox( kind, { language, @@ -761,10 +813,13 @@ async function executeInSandboxWithinBudget( lifetimeMs: remainingSandboxBudgetMs(signal), }, selected, - signal + signal, + req.meterUsage ) + const sandbox = created.sandbox const sandboxId = sandbox.sandboxId - const abortBinding = bindSandboxAbort(sandbox, signal) + const abortBinding = bindSandboxAbort(sandbox, created.providerId, signal) + let successfulResult: SandboxExecutionResult | undefined try { throwIfAborted(signal) @@ -850,7 +905,7 @@ async function executeInSandboxWithinBudget( ) throwIfAborted(signal) - return { + successfulResult = { result: extraction.result, stdout: cleanedStdout, sandboxId, @@ -858,7 +913,10 @@ async function executeInSandboxWithinBudget( exportedFiles, collectedFiles, } + return successfulResult } finally { + const cleanupStartedAtMs = Date.now() + attachSandboxCost(successfulResult, created, cleanupStartedAtMs) abortBinding.detach() await abortBinding.cleanup() } @@ -887,14 +945,17 @@ async function executeShellInSandboxWithinBudget( }) throwIfAborted(signal) - const sandbox = await createSelectedSandbox( + const created = await createSelectedSandbox( kind, { imageRef: selected?.imageRef, lifetimeMs: remainingSandboxBudgetMs(signal) }, selected, - signal + signal, + req.meterUsage ) + const sandbox = created.sandbox const sandboxId = sandbox.sandboxId - const abortBinding = bindSandboxAbort(sandbox, signal) + const abortBinding = bindSandboxAbort(sandbox, created.providerId, signal) + let successfulResult: SandboxExecutionResult | undefined try { throwIfAborted(signal) @@ -962,7 +1023,7 @@ async function executeShellInSandboxWithinBudget( ) throwIfAborted(signal) - return { + successfulResult = { result: parsed, stdout: extraction.cleanedStdout, sandboxId, @@ -970,7 +1031,10 @@ async function executeShellInSandboxWithinBudget( exportedFiles, collectedFiles, } + return successfulResult } finally { + const cleanupStartedAtMs = Date.now() + attachSandboxCost(successfulResult, created, cleanupStartedAtMs) abortBinding.detach() await abortBinding.cleanup() } @@ -1032,7 +1096,7 @@ export async function withPiSandbox( ): Promise { const lifetimeMs = options.lifetimeMs !== undefined ? options.lifetimeMs : resolvePiSandboxLifetimeMs() - const sandbox = await createSandbox('pi', { lifetimeMs }) + const { sandbox } = await createSandbox('pi', { lifetimeMs }) logger.info('Started Pi sandbox', { sandboxId: sandbox.sandboxId, lifetimeMs }) const runner: PiSandboxRunner = { diff --git a/apps/sim/lib/execution/remote-sandbox/types.ts b/apps/sim/lib/execution/remote-sandbox/types.ts index 2010c3cee2e..2b9584b1cf3 100644 --- a/apps/sim/lib/execution/remote-sandbox/types.ts +++ b/apps/sim/lib/execution/remote-sandbox/types.ts @@ -72,6 +72,8 @@ export interface SandboxExecutionRequest { sandboxId?: string /** Cancels the provider sandbox when the caller's execution budget expires. */ signal?: AbortSignal + /** Adds the remote provider cost to a successful Function result. */ + meterUsage?: boolean } export interface SandboxShellExecutionRequest { @@ -97,6 +99,8 @@ export interface SandboxShellExecutionRequest { sandboxId?: string /** Cancels the provider sandbox when the caller's execution budget expires. */ signal?: AbortSignal + /** Adds the remote provider cost to a successful Function result. */ + meterUsage?: boolean } export interface SandboxExecutionResult { @@ -116,6 +120,7 @@ export interface SandboxExecutionResult { * sequence, and the byte budget is enforced on the decoded length. */ collectedFiles?: SandboxCollectedFile[] + cost?: { input: number; output: number; total: number } } /** One harvested output file, carried as base64 with its decoded length. */ @@ -279,6 +284,8 @@ export interface CreateSandboxOptions { * and creates the sandbox as ephemeral. */ lifetimeMs?: number + /** Reports the instant immediately before the provider SDK create request is dispatched. */ + onProviderRequestStarted?: (startedAtMs: number) => void } /** @@ -366,5 +373,7 @@ export interface SandboxProvider { readonly dependencyStrategy: SandboxDependencyStrategy /** Present exactly when {@link dependencyStrategy} is `prebuilt`. */ readonly images?: SandboxImageBuilder + /** Resolves the provider's rounded lifetime for both creation and metering. */ + resolveLifetimeMs(lifetimeMs: number): number create(kind: SandboxKind, options?: CreateSandboxOptions): Promise } diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index cdba883bb08..0bbc72ab5e3 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -367,6 +367,74 @@ describe('Function execution request', () => { expect(mockExecuteShellInSandbox).not.toHaveBeenCalled() }) + it.each([ + { language: 'python', code: 'return 42', kind: 'code' }, + { language: 'shell', code: 'echo ready', kind: 'shell' }, + ])( + 'meters a standard workflow Function $kind sandbox and preserves its cost', + async ({ language, code, kind }) => { + envFlagsMock.isRemoteSandboxEnabled = true + const cost = { input: 0, output: 0, total: 0.00012345 } + const executeSandbox = kind === 'shell' ? mockExecuteShellInSandbox : mockExecuteInSandbox + executeSandbox.mockResolvedValueOnce({ + result: 42, + stdout: 'ready', + sandboxId: `sandbox-${kind}`, + cost, + }) + + const response = await POST( + createMockRequest('POST', { + code, + language, + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + }) + ) + const data = await response.json() + + expect(response.status).toBe(200) + expect(executeSandbox).toHaveBeenCalledWith(expect.objectContaining({ meterUsage: true })) + expect(data.output.cost).toEqual(cost) + } + ) + + it('does not meter a non-workflow remote Function call', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const response = await POST( + createMockRequest('POST', { + code: 'import path from "node:path"\nreturn path.sep', + language: 'javascript', + }) + ) + + expect(response.status).toBe(200) + expect(mockExecuteInSandbox).toHaveBeenCalledWith( + expect.objectContaining({ meterUsage: false }) + ) + }) + + it('keeps a custom Function tool local even when workflow context is present', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const response = await POST( + createMockRequest('POST', { + code: 'return 42', + language: 'python', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + isCustomTool: true, + }) + ) + + expect(response.status).toBe(200) + expect(mockExecuteInIsolatedVM).toHaveBeenCalledOnce() + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + }) + it('does not accept a Mothership sandbox profile from the request body', async () => { const req = createMockRequest('POST', { code: 'return "test"', @@ -422,6 +490,7 @@ describe('Function execution request', () => { expect.objectContaining({ language, sandboxKind: 'mothership', + meterUsage: false, }) ) expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() @@ -443,7 +512,7 @@ describe('Function execution request', () => { expect(response.status).toBe(200) expect(mockExecuteShellInSandbox).toHaveBeenCalledWith( - expect.objectContaining({ sandboxKind: 'mothership' }) + expect.objectContaining({ sandboxKind: 'mothership', meterUsage: false }) ) }) @@ -744,6 +813,7 @@ describe('Function execution request', () => { result: 'done', stdout: 'ok', sandboxId: 'sandbox-123', + cost: { input: 0, output: 0, total: 0.00023456 }, exportedFiles: { '/home/user/chart.png': 'iVBORw0KGgo=', '/home/user/summary.json': '{"ok":true}', @@ -754,6 +824,8 @@ describe('Function execution request', () => { code: 'print("done")', language: 'python', workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', outputs: { files: [ { @@ -800,6 +872,7 @@ describe('Function execution request', () => { }) ) expect(data.output.result.files).toHaveLength(2) + expect(data.output.cost).toEqual({ input: 0, output: 0, total: 0.00023456 }) expect(data.resources).toEqual([ expect.objectContaining({ path: 'files/reports/chart.png' }), expect.objectContaining({ path: 'files/reports/summary.json' }), @@ -1986,6 +2059,7 @@ describe('Function execution request', () => { result: null, stdout: 'generated 1 preview', sandboxId: 'sandbox-123', + cost: { input: 0, output: 0, total: 0.00034567 }, exportedFiles: { '/tmp/fellows-previews.zip': archiveBase64 }, }) @@ -1994,6 +2068,8 @@ describe('Function execution request', () => { code: source, language: 'python', workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', sandboxId: 'fellows-sandbox', envVars: { AIRTABLE_PAT: 'stub-airtable-token', @@ -2015,6 +2091,9 @@ describe('Function execution request', () => { ) expect(response.status).toBe(200) + await expect(response.clone().json()).resolves.toMatchObject({ + output: { cost: { input: 0, output: 0, total: 0.00034567 } }, + }) const sandboxRequest = mockExecuteInSandbox.mock.calls[0][0] expect(sandboxRequest.code).toContain("['bq', 'query'") expect(sandboxRequest.code).toContain('__sim_exec_globals__["__name__"] = "__main__"') diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index 73ef60c28d5..952a81d9ca7 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -128,6 +128,12 @@ const MAX_SANDBOX_OUTPUT_FILES = 20 const MAX_PRIVATE_FILE_SECRET_MATCH_EVENTS = 1_000_000 const SANDBOX_RUNTIME_PAYLOAD_PATH_ENV = '__SIM_RUNTIME_PAYLOAD_PATH' +interface FunctionExecutionCost { + input: number + output: number + total: number +} + interface SandboxRuntimePayload { params: Record environmentVariables: Record @@ -1458,6 +1464,7 @@ async function maybeExportSandboxFileToWorkspace(args: { exportedFileContent?: string stdout: string executionTime: number + cost?: FunctionExecutionCost }) { const { routeContext, @@ -1473,6 +1480,7 @@ async function maybeExportSandboxFileToWorkspace(args: { exportedFileContent, stdout, executionTime, + cost, } = args if (!outputSandboxPath) return null @@ -1595,6 +1603,7 @@ async function maybeExportSandboxFileToWorkspace(args: { }, stdout: cleanStdout(stdout), executionTime, + ...(cost ? { cost } : {}), }, resources: [{ type: 'file', id: written.id, title: written.name, path: written.vfsPath }], }) @@ -1618,6 +1627,7 @@ async function maybeExportSandboxFilesToWorkspace(args: { exportedFileContent?: string stdout: string executionTime: number + cost?: FunctionExecutionCost }) { const sandboxFiles = args.outputFiles.filter((file) => file.sandboxPath) if (sandboxFiles.length === 0) return null @@ -1647,6 +1657,7 @@ async function maybeExportSandboxFilesToWorkspace(args: { args.exportedFileContent, stdout: args.stdout, executionTime: args.executionTime, + cost: args.cost, }) } @@ -1844,6 +1855,7 @@ async function maybeExportSandboxFilesToWorkspace(args: { }, stdout: cleanStdout(args.stdout), executionTime: args.executionTime, + ...(args.cost ? { cost: args.cost } : {}), }, resources: writtenFiles.map((file) => ({ type: 'file', @@ -2131,6 +2143,8 @@ export async function executeFunctionRequest( _sandboxFiles, } = body + const meterRemoteSandboxUsage = Boolean(workflowId && !isCustomTool && !usesMothershipSandbox) + if (selectedSandboxId && !isRemoteSandboxEnabled) { return NextResponse.json( { success: false, error: 'The Function code sandbox is not configured' }, @@ -2500,6 +2514,7 @@ export async function executeFunctionRequest( exportedFileContent, exportedFiles, collectedFiles: shellCollectedFiles, + cost: shellCost, } = await executeShellInSandbox({ code: resolvedCode, envs: shellEnvs, @@ -2515,6 +2530,7 @@ export async function executeFunctionRequest( ? { sandboxKind: 'mothership' as const } : {}), signal: executionSignal, + meterUsage: meterRemoteSandboxUsage, }) const executionTime = Date.now() - execStart @@ -2547,6 +2563,7 @@ export async function executeFunctionRequest( exportedFileContent, stdout: shellStdout, executionTime, + cost: shellCost, }) if (fileExportResponse) { return appendResolvedSecretNames(fileExportResponse, routeContext) @@ -2575,6 +2592,7 @@ export async function executeFunctionRequest( stdout: cleanStdout(shellStdout), executionTime, files: shellOutputFiles.files, + ...(shellCost ? { cost: shellCost } : {}), }, }, routeContext @@ -2637,6 +2655,7 @@ export async function executeFunctionRequest( exportedFileContent, exportedFiles, collectedFiles: jsCollectedFiles, + cost: sandboxCost, } = await executeInSandbox({ code: codeForE2B, language: CodeLanguage.JavaScript, @@ -2653,6 +2672,7 @@ export async function executeFunctionRequest( ? { sandboxKind: 'mothership' as const } : {}), signal: executionSignal, + meterUsage: meterRemoteSandboxUsage, }) const executionTime = Date.now() - execStart stdout += e2bStdout @@ -2696,6 +2716,7 @@ export async function executeFunctionRequest( exportedFileContent, stdout, executionTime, + cost: sandboxCost, }) if (fileExportResponse) { return appendResolvedSecretNames(fileExportResponse, routeContext) @@ -2724,6 +2745,7 @@ export async function executeFunctionRequest( stdout: cleanStdout(stdout), executionTime, files: jsOutputFiles.files, + ...(sandboxCost ? { cost: sandboxCost } : {}), }, }, routeContext @@ -2749,6 +2771,7 @@ export async function executeFunctionRequest( exportedFileContent, exportedFiles, collectedFiles: pythonCollectedFiles, + cost: sandboxCost, } = await executeInSandbox({ code: codeForE2B, language: CodeLanguage.Python, @@ -2764,6 +2787,7 @@ export async function executeFunctionRequest( ? { sandboxKind: 'mothership' as const } : {}), signal: executionSignal, + meterUsage: meterRemoteSandboxUsage, }) const executionTime = Date.now() - execStart stdout += e2bStdout @@ -2807,6 +2831,7 @@ export async function executeFunctionRequest( exportedFileContent, stdout, executionTime, + cost: sandboxCost, }) if (fileExportResponse) { return appendResolvedSecretNames(fileExportResponse, routeContext) @@ -2835,6 +2860,7 @@ export async function executeFunctionRequest( stdout: cleanStdout(stdout), executionTime, files: pythonOutputFiles.files, + ...(sandboxCost ? { cost: sandboxCost } : {}), }, }, routeContext diff --git a/apps/sim/scripts/build-function-daytona-snapshot.ts b/apps/sim/scripts/build-function-daytona-snapshot.ts index 5dfae90c3cf..2c6b85b0be6 100644 --- a/apps/sim/scripts/build-function-daytona-snapshot.ts +++ b/apps/sim/scripts/build-function-daytona-snapshot.ts @@ -20,6 +20,7 @@ import { isImmutableDaytonaSnapshotRef, } from '@sim/utils/sandbox-references' import { + FUNCTION_DAYTONA_DISK_GB, FUNCTION_SANDBOX_CPU_COUNT, FUNCTION_SANDBOX_MEMORY_GB, } from '@/lib/execution/remote-sandbox/function-resources' @@ -53,7 +54,7 @@ const APT_INSTALL = 'DEBIAN_FRONTEND=noninteractive apt-get install -y --no-inst const RESOURCES = { cpu: FUNCTION_SANDBOX_CPU_COUNT, memory: FUNCTION_SANDBOX_MEMORY_GB, - disk: 10, + disk: FUNCTION_DAYTONA_DISK_GB, } as const export function createFunctionImage(manifest: FunctionSandboxParityManifest) { diff --git a/apps/sim/tools/function/execute.test.ts b/apps/sim/tools/function/execute.test.ts index 053ce3c3372..65f898ffab1 100644 --- a/apps/sim/tools/function/execute.test.ts +++ b/apps/sim/tools/function/execute.test.ts @@ -120,4 +120,20 @@ describe('Function Execute Tool', () => { expect(body[PRIVATE_SECRET_PROVENANCE_FIELD]).toEqual(bundle) expect(JSON.stringify(body)).not.toContain('plaintext') }) + + it('preserves sandbox cost in a successful Function result', async () => { + const cost = { input: 0, output: 0, total: 0.00012345 } + const result = await functionExecuteTool.transformResponse?.( + Response.json({ + success: true, + output: { result: 42, stdout: 'done', cost }, + }), + { code: 'return 42' } + ) + + expect(result).toMatchObject({ + success: true, + output: { result: 42, stdout: 'done', cost }, + }) + }) }) diff --git a/apps/sim/tools/function/execute.ts b/apps/sim/tools/function/execute.ts index 8da668e12a9..1279ac190f1 100644 --- a/apps/sim/tools/function/execute.ts +++ b/apps/sim/tools/function/execute.ts @@ -259,6 +259,7 @@ To return a file, write it to ${SANDBOX_OUTPUT_DIR}. Everything there comes back result: result.output.result, stdout: result.output.stdout, files: result.output.files ?? [], + ...(result.output.cost ? { cost: result.output.cost } : {}), }, resources: result.resources, largeValueKeys: result.largeValueKeys, diff --git a/apps/sim/tools/function/types.ts b/apps/sim/tools/function/types.ts index 9c590a7fa5c..9dd38298582 100644 --- a/apps/sim/tools/function/types.ts +++ b/apps/sim/tools/function/types.ts @@ -86,5 +86,10 @@ export interface CodeExecutionOutput extends ToolResponse { stdout: string /** Files harvested from the sandbox output directory, already persisted. */ files: UserFile[] + cost?: { + input: number + output: number + total: number + } } } diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index a16de831d9f..5f3847e9d48 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -800,7 +800,7 @@ describe('executeTool Function', () => { cleanupEnvVars() }) - it('executes trusted Function calls in process without dropping resolved execution context', async () => { + it('stamps standard Function identity and preserves trusted execution context', async () => { const fetchSpy = vi.fn() global.fetch = Object.assign(fetchSpy, { preconnect: vi.fn() }) as typeof fetch @@ -856,7 +856,7 @@ describe('executeTool Function', () => { workspaceId: 'workspace-456', body: { code: 'return [{{API_KEY}}, __blockRef_0.field]', - isCustomTool: true, + isCustomTool: false, inputs: { location: 'San Francisco' }, envVars: { API_KEY: 'resolved-secret' }, contextVariables: { diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index 847ac33a4e2..9a9ea23196e 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -2470,12 +2470,16 @@ async function executeDeclaredInternalOperation({ const operationParams = projectToolModelInputParams(tool, params, resolvedSecretTraceRegistry) let operationInput = tool.operation.input(operationParams) - const isFunctionOperation = toolId === 'function_execute' || isCustomTool(toolId) - if (isFunctionOperation && !isFunctionExecuteBody(operationInput)) { - throw new Error('Function operation input must be an object') + const isRegisteredCustomTool = isCustomTool(toolId) + const isFunctionOperation = toolId === 'function_execute' || isRegisteredCustomTool + if (isFunctionOperation) { + if (!isFunctionExecuteBody(operationInput)) { + throw new Error('Function operation input must be an object') + } + operationInput = { ...operationInput, isCustomTool: isRegisteredCustomTool } } if ( - isCustomTool(toolId) && + isRegisteredCustomTool && isFunctionExecuteBody(operationInput) && 'schema' in operationInput && 'params' in operationInput From 5a65b566e5eefb599b5787c7952bd6c5e2673688 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Fri, 28 Aug 2026 19:28:49 -0700 Subject: [PATCH 2/7] fix(billing): charge function user-code failures --- .../execution/block-executor.retry.test.ts | 64 +++++++++++++++++++ apps/sim/executor/execution/block-executor.ts | 58 +++++++++++++++-- .../function/function-handler.test.ts | 41 ++++++++++++ .../handlers/function/function-handler.ts | 12 ++-- .../handlers/pi/local/sim-tools.test.ts | 7 +- .../executor/handlers/pi/local/sim-tools.ts | 1 - apps/sim/executor/utils/errors.ts | 56 ++++++++++++++++ .../remote-sandbox/conformance.test.ts | 6 +- .../sim/lib/execution/remote-sandbox/index.ts | 22 ++++--- .../execute-request.test.ts | 39 +++++++++++ .../lib/function-execution/execute-request.ts | 21 +++++- apps/sim/providers/cost-policy.test.ts | 17 +++++ apps/sim/providers/cost-policy.ts | 15 ++++- apps/sim/providers/index.test.ts | 27 ++++++++ apps/sim/providers/index.ts | 27 +++++--- apps/sim/providers/runtime-context.test.ts | 29 +++++++++ apps/sim/providers/runtime-context.ts | 21 ++++++ apps/sim/tools/function/execute.test.ts | 21 ++++++ apps/sim/tools/function/execute.ts | 1 + apps/sim/tools/index.test.ts | 29 +++++++++ apps/sim/tools/index.ts | 38 ++++++++++- 21 files changed, 512 insertions(+), 40 deletions(-) diff --git a/apps/sim/executor/execution/block-executor.retry.test.ts b/apps/sim/executor/execution/block-executor.retry.test.ts index 54811c71f8f..e22c93104eb 100644 --- a/apps/sim/executor/execution/block-executor.retry.test.ts +++ b/apps/sim/executor/execution/block-executor.retry.test.ts @@ -5,11 +5,13 @@ * client has already seen and cannot re-run the deterministic post-processing. */ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { BlockType, EDGE } from '@/executor/constants' import type { DAGNode } from '@/executor/dag/builder' import { BlockExecutor } from '@/executor/execution/block-executor' import { ExecutionState } from '@/executor/execution/state' import type { BlockHandler, ExecutionContext } from '@/executor/types' +import { attachTrustedExecutionCost } from '@/executor/utils/errors' import { VariableResolver } from '@/executor/variables/resolver' import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types' @@ -137,6 +139,68 @@ describe('BlockExecutor retry', () => { expect(ctx.blockLogs[0]?.tries).toBe(2) }) + it('adds the trusted cost of failed Function tries to the successful result', async () => { + const block = createBlock(enabled) + const firstFailure = new Error('first attempt failed') + attachTrustedExecutionCost(firstFailure, { input: 0, output: 0, total: 0.125 }) + const successfulOutput = { + result: 'done', + cost: { input: 0, output: 0, total: 0.25 }, + } + attachTrustedExecutionCost(successfulOutput, successfulOutput.cost) + const execute = vi + .fn() + .mockRejectedValueOnce(firstFailure) + .mockResolvedValueOnce(successfulOutput) + const state = new ExecutionState() + const ctx = createContext(state) + const executor = buildExecutor(block, { canHandle: () => true, execute }, state) + + const output = await executor.execute(ctx, createNode(block), block) + + expect(execute).toHaveBeenCalledTimes(2) + expect(output.cost).toEqual({ input: 0, output: 0, total: 0.375 }) + expect(ctx.blockLogs[0]?.output?.cost).toEqual(output.cost) + }) + + it('keeps earlier trusted Function costs when the final try is an infrastructure error', async () => { + const block = createBlock(enabled) + const firstFailure = new Error('first Function attempt failed') + const secondFailure = new Error('second Function attempt failed') + const finalFailure = new Error('provider unavailable') + attachTrustedExecutionCost(firstFailure, { input: 0, output: 0, total: 0.125 }) + attachTrustedExecutionCost(secondFailure, { input: 0, output: 0, total: 0.25 }) + const execute = vi + .fn() + .mockRejectedValueOnce(firstFailure) + .mockRejectedValueOnce(secondFailure) + .mockRejectedValueOnce(finalFailure) + const state = new ExecutionState() + const ctx = createContext(state) + const executor = buildExecutor(block, { canHandle: () => true, execute }, state) + + await expect(executor.execute(ctx, createNode(block), block)).rejects.toThrow( + 'provider unavailable' + ) + + expect(execute).toHaveBeenCalledTimes(3) + expect(ctx.blockLogs[0]?.output).toEqual({ + error: 'provider unavailable', + cost: { input: 0, output: 0, total: 0.375 }, + }) + + const { traceSpans } = buildTraceSpans({ + success: false, + output: { error: 'provider unavailable' }, + error: 'provider unavailable', + logs: ctx.blockLogs, + }) + expect(traceSpans[0]).toMatchObject({ + status: 'error', + cost: { input: 0, output: 0, total: 0.375 }, + }) + }) + it('stops at maxTries and rethrows the final error unchanged', async () => { const block = createBlock(enabled) const failure = new Error('still failing') diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index bc4241208fd..fac437b3f78 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -48,7 +48,13 @@ import { type StreamingExecution, } from '@/executor/types' import { streamingResponseFormatProcessor } from '@/executor/utils' -import { buildBlockExecutionError, normalizeError } from '@/executor/utils/errors' +import { + attachTrustedExecutionCost, + buildBlockExecutionError, + normalizeError, + readTrustedExecutionCost, + type TrustedExecutionCost, +} from '@/executor/utils/errors' import { buildUnifiedParentIterations, getIterationContext, @@ -76,6 +82,20 @@ import { SYSTEM_SUBBLOCK_IDS } from '@/triggers/constants' const logger = createLogger('BlockExecutor') +function addTrustedExecutionCosts( + accumulated: TrustedExecutionCost | undefined, + current: TrustedExecutionCost | undefined +): TrustedExecutionCost | undefined { + if (!accumulated) return current + if (!current) return accumulated + + return { + input: accumulated.input + current.input, + output: accumulated.output + current.output, + total: accumulated.total + current.total, + } +} + export class BlockExecutor { private execLogger: Logger @@ -506,15 +526,40 @@ export class BlockExecutor { const policy = resolveBlockRetryPolicy(block) if (!policy) return invoke() + const shouldAccumulateFunctionCost = block.metadata?.id === BlockType.FUNCTION + let accumulatedFunctionCost: TrustedExecutionCost | undefined let tries = 0 try { for (;;) { tries++ try { - return await invoke() + const output = await invoke() + if (!shouldAccumulateFunctionCost || !accumulatedFunctionCost || !isRecordLike(output)) { + return output + } + + const totalCost = addTrustedExecutionCosts( + accumulatedFunctionCost, + readTrustedExecutionCost(output) + ) + if (!totalCost) return output + + const outputWithCost = { ...output, cost: totalCost } + attachTrustedExecutionCost(outputWithCost, totalCost) + return outputWithCost as T } catch (error) { + if (shouldAccumulateFunctionCost) { + accumulatedFunctionCost = addTrustedExecutionCosts( + accumulatedFunctionCost, + readTrustedExecutionCost(error) + ) + } + const isFinalTry = tries >= policy.maxTries - if (isFinalTry || ctx.abortSignal?.aborted || !isRetryableBlockError(error)) throw error + if (isFinalTry || ctx.abortSignal?.aborted || !isRetryableBlockError(error)) { + attachTrustedExecutionCost(error, accumulatedFunctionCost) + throw error + } this.execLogger.warn('Block failed; retrying', { blockId: block.id, @@ -528,7 +573,10 @@ export class BlockExecutor { if (policy.waitBetweenTriesMs > 0) await sleep(policy.waitBetweenTriesMs) /** `sleep` is not abort-aware, so a run stopped mid-wait must not start another try. */ - if (ctx.abortSignal?.aborted) throw error + if (ctx.abortSignal?.aborted) { + attachTrustedExecutionCost(error, accumulatedFunctionCost) + throw error + } } } } finally { @@ -620,8 +668,10 @@ export class BlockExecutor { return softOutput } + const trustedExecutionCost = readTrustedExecutionCost(error) const errorOutput: NormalizedBlockOutput = { error: errorMessage, + ...(trustedExecutionCost ? { cost: trustedExecutionCost } : {}), } // Keep any answer text already drained before timeout/failure so logs match diff --git a/apps/sim/executor/handlers/function/function-handler.test.ts b/apps/sim/executor/handlers/function/function-handler.test.ts index c914d2f05f9..72fe2ff4262 100644 --- a/apps/sim/executor/handlers/function/function-handler.test.ts +++ b/apps/sim/executor/handlers/function/function-handler.test.ts @@ -1,9 +1,11 @@ import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest' import { createTimeoutAbortController } from '@/lib/core/execution-limits' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/execution/constants' +import { NonRetryableExecutionError } from '@/lib/execution/non-retryable-error' import { BlockType } from '@/executor/constants' import { FunctionBlockHandler } from '@/executor/handlers/function/function-handler' import type { ExecutionContext } from '@/executor/types' +import { readTrustedExecutionCost } from '@/executor/utils/errors' import { FUNCTION_BLOCK_CONTEXT_VARS_KEY, FUNCTION_BLOCK_DISPLAY_CODE_KEY, @@ -254,6 +256,45 @@ describe('FunctionBlockHandler', () => { expect(mockExecuteTool).toHaveBeenCalled() }) + it.each([ + { retryable: true, nonRetryable: false }, + { retryable: false, nonRetryable: true }, + ])( + 'attaches trusted cost to a failed execution when retryable is $retryable', + async ({ retryable, nonRetryable }) => { + const cost = { input: 0, output: 0, total: 0.125 } + mockExecuteTool.mockResolvedValue({ + success: false, + error: 'Remote Function failed', + retryable, + output: { result: null, stdout: '', cost }, + }) + + let thrown: unknown + try { + await handler.execute(mockContext, mockBlock, { code: 'throw new Error("failed")' }) + } catch (error) { + thrown = error + } + + expect(thrown).toBeInstanceOf(Error) + expect(thrown instanceof NonRetryableExecutionError).toBe(nonRetryable) + expect(readTrustedExecutionCost(thrown)).toEqual(cost) + } + ) + + it('attaches trusted cost to a successful execution for retry aggregation', async () => { + const cost = { input: 0, output: 0, total: 0.25 } + mockExecuteTool.mockResolvedValue({ + success: true, + output: { result: 42, stdout: '', cost }, + }) + + const output = await handler.execute(mockContext, mockBlock, { code: 'return 42' }) + + expect(readTrustedExecutionCost(output)).toEqual(cost) + }) + it('should pass runtime context variables to function_execute', async () => { const contextVariables = { __blockRef_0: { result: 'from-block' } } diff --git a/apps/sim/executor/handlers/function/function-handler.ts b/apps/sim/executor/handlers/function/function-handler.ts index aefb5ab39d4..22d0b3b9938 100644 --- a/apps/sim/executor/handlers/function/function-handler.ts +++ b/apps/sim/executor/handlers/function/function-handler.ts @@ -12,6 +12,7 @@ import { mergeFileKeys, mergeLargeValueKeys } from '@/lib/execution/payloads/acc import { BlockType } from '@/executor/constants' import type { BlockHandler, ExecutionContext } from '@/executor/types' import { collectBlockData } from '@/executor/utils/block-data' +import { attachTrustedExecutionCost } from '@/executor/utils/errors' import { FUNCTION_BLOCK_CONTEXT_VARS_KEY, FUNCTION_BLOCK_DISPLAY_CODE_KEY, @@ -111,15 +112,18 @@ export class FunctionBlockHandler implements BlockHandler { const result = await executeTool('function_execute', toolParams, { executionContext: ctx }) if (!result.success) { - if (result.retryable === false) { - throw new NonRetryableExecutionError(result.error || 'Function execution is indeterminate') - } - throw new Error(result.error || 'Function execution failed') + const error = + result.retryable === false + ? new NonRetryableExecutionError(result.error || 'Function execution is indeterminate') + : new Error(result.error || 'Function execution failed') + attachTrustedExecutionCost(error, result.output?.cost) + throw error } mergeLargeValueKeys(ctx, result.largeValueKeys ?? []) mergeFileKeys(ctx, result.fileKeys ?? []) + attachTrustedExecutionCost(result.output, result.output?.cost) return result.output } } diff --git a/apps/sim/executor/handlers/pi/local/sim-tools.test.ts b/apps/sim/executor/handlers/pi/local/sim-tools.test.ts index 1ed6d359c2d..6427fc3cbf7 100644 --- a/apps/sim/executor/handlers/pi/local/sim-tools.test.ts +++ b/apps/sim/executor/handlers/pi/local/sim-tools.test.ts @@ -225,7 +225,7 @@ describe('buildSimToolSpecs', () => { }) }) - it('accumulates cost only from successful canonical Function tool results', async () => { + it('accumulates cost from canonical Function results while preserving failures', async () => { mockTransformBlockTool .mockResolvedValueOnce({ id: 'function_execute', @@ -268,9 +268,10 @@ describe('buildSimToolSpecs', () => { await functionSpec.execute({}) await searchSpec.execute({}) - await functionSpec.execute({}) + const failedResult = await functionSpec.execute({}) - expect(functionToolCost.total).toBe(0.125) + expect(functionToolCost.total).toBe(8.125) + expect(failedResult).toEqual({ text: 'execution failed', isError: true }) }) it('projects named provenance in successful Sim tool output', async () => { diff --git a/apps/sim/executor/handlers/pi/local/sim-tools.ts b/apps/sim/executor/handlers/pi/local/sim-tools.ts index 1ccf9b47318..be2d41a6fd7 100644 --- a/apps/sim/executor/handlers/pi/local/sim-tools.ts +++ b/apps/sim/executor/handlers/pi/local/sim-tools.ts @@ -182,7 +182,6 @@ function buildSimToolSpec( : undefined if ( toolId === 'function_execute' && - result.success && functionToolCost && typeof resultCostTotal === 'number' && Number.isFinite(resultCostTotal) && diff --git a/apps/sim/executor/utils/errors.ts b/apps/sim/executor/utils/errors.ts index deb1306e6ac..e3f0a9ee9b8 100644 --- a/apps/sim/executor/utils/errors.ts +++ b/apps/sim/executor/utils/errors.ts @@ -49,6 +49,22 @@ export function attachExecutionResult(error: Error, executionResult: ExecutionRe */ const attemptedExecutionIds = new WeakMap() +/** Cost emitted by a trusted execution boundary and safe to project into a block trace. */ +export interface TrustedExecutionCost { + readonly input: number + readonly output: number + readonly total: number +} + +/** + * Trusted execution costs, keyed by the value crossing the handler boundary. + * + * Cost stays in a side table until the executor deliberately copies it into block output. This + * prevents arbitrary properties on provider errors (or user-thrown values) from becoming billed + * trace data while still allowing a handler to preserve cost when it throws. + */ +const trustedExecutionCosts = new WeakMap() + /** * Names the run a failure belongs to once dispatch has been attempted. * @@ -72,6 +88,46 @@ export function readAttemptedExecutionId(error: unknown): string | undefined { return isRecordedThrown(error) ? attemptedExecutionIds.get(error) : undefined } +/** Attaches a validated, Sim-produced execution cost to an object crossing the handler boundary. */ +export function attachTrustedExecutionCost(subject: unknown, cost: unknown): void { + if (!isRecordedThrown(subject)) return + + const normalizedCost = normalizeTrustedExecutionCost(cost) + if (!normalizedCost) return + + trustedExecutionCosts.set(subject, normalizedCost) +} + +/** Reads execution cost only when a trusted caller previously attached it. */ +export function readTrustedExecutionCost(subject: unknown): TrustedExecutionCost | undefined { + return isRecordedThrown(subject) ? trustedExecutionCosts.get(subject) : undefined +} + +function normalizeTrustedExecutionCost(cost: unknown): TrustedExecutionCost | undefined { + if (!cost || typeof cost !== 'object' || Array.isArray(cost)) return undefined + + const candidate = cost as Record + if ( + typeof candidate.input !== 'number' || + !Number.isFinite(candidate.input) || + candidate.input < 0 || + typeof candidate.output !== 'number' || + !Number.isFinite(candidate.output) || + candidate.output < 0 || + typeof candidate.total !== 'number' || + !Number.isFinite(candidate.total) || + candidate.total < 0 + ) { + return undefined + } + + return { + input: candidate.input, + output: candidate.output, + total: candidate.total, + } +} + /** * Any non-null object, not only an `Error`. * diff --git a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts index c65c953eb28..370fe561464 100644 --- a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts @@ -430,10 +430,12 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { code: 'x', language: CodeLanguage.Python, timeoutMs: 1000, + meterUsage: true, }) expect(res.result).toBeNull() expect(res.error).toContain('corrupted in transport') + expect(res.cost).toBeUndefined() }) it('survives a large single-line payload without chunk corruption', async () => { @@ -577,7 +579,7 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { expect(res.error).toBe('ValueError: boom') expect(res.stdout).toContain('ValueError: boom') expect(res.result).toBeNull() - expect(res.cost).toBeUndefined() + expect(res.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) }) it('normalizes Python code budget expiry to a typed timeout abort', async () => { @@ -1069,7 +1071,7 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { expect(res.result).toBeNull() expect(res.error).toContain('boom detail') expect(res.stdout).toContain('boom detail') - expect(res.cost).toBeUndefined() + expect(res.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) }) it('terminates shell execution when streamed process output exceeds the byte budget', async () => { diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index b7eead1ddd1..8ed9c6e6003 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -819,7 +819,7 @@ async function executeInSandboxWithinBudget( const sandbox = created.sandbox const sandboxId = sandbox.sandboxId const abortBinding = bindSandboxAbort(sandbox, created.providerId, signal) - let successfulResult: SandboxExecutionResult | undefined + let billableResult: SandboxExecutionResult | undefined try { throwIfAborted(signal) @@ -864,12 +864,13 @@ async function executeInSandboxWithinBudget( sandboxId, hasTraceback: Boolean(execution.error.traceback), }) - return { + billableResult = { result: null, stdout: execution.error.traceback || errorMessage, error: errorMessage, sandboxId, } + return billableResult } // Distinct sources (final-expression text, stdout, stderr) join with '\n' so @@ -905,7 +906,7 @@ async function executeInSandboxWithinBudget( ) throwIfAborted(signal) - successfulResult = { + billableResult = { result: extraction.result, stdout: cleanedStdout, sandboxId, @@ -913,10 +914,10 @@ async function executeInSandboxWithinBudget( exportedFiles, collectedFiles, } - return successfulResult + return billableResult } finally { const cleanupStartedAtMs = Date.now() - attachSandboxCost(successfulResult, created, cleanupStartedAtMs) + attachSandboxCost(billableResult, created, cleanupStartedAtMs) abortBinding.detach() await abortBinding.cleanup() } @@ -955,7 +956,7 @@ async function executeShellInSandboxWithinBudget( const sandbox = created.sandbox const sandboxId = sandbox.sandboxId const abortBinding = bindSandboxAbort(sandbox, created.providerId, signal) - let successfulResult: SandboxExecutionResult | undefined + let billableResult: SandboxExecutionResult | undefined try { throwIfAborted(signal) @@ -1007,7 +1008,8 @@ async function executeShellInSandboxWithinBudget( sandboxId, exitCode: result.exitCode, }) - return { result: null, stdout, error: errorMessage, sandboxId } + billableResult = { result: null, stdout, error: errorMessage, sandboxId } + return billableResult } // Shell scripts have no wrapper: any __SIM_RESULT__ line is user-authored @@ -1023,7 +1025,7 @@ async function executeShellInSandboxWithinBudget( ) throwIfAborted(signal) - successfulResult = { + billableResult = { result: parsed, stdout: extraction.cleanedStdout, sandboxId, @@ -1031,10 +1033,10 @@ async function executeShellInSandboxWithinBudget( exportedFiles, collectedFiles, } - return successfulResult + return billableResult } finally { const cleanupStartedAtMs = Date.now() - attachSandboxCost(successfulResult, created, cleanupStartedAtMs) + attachSandboxCost(billableResult, created, cleanupStartedAtMs) abortBinding.detach() await abortBinding.cleanup() } diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index 0bbc72ab5e3..405c99ca41f 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -400,6 +400,45 @@ describe('Function execution request', () => { } ) + it.each([ + { + language: 'javascript', + code: 'import "node:path"\nthrow new Error("boom")', + kind: 'code', + }, + { language: 'python', code: 'raise ValueError("boom")', kind: 'code' }, + { language: 'shell', code: 'exit 1', kind: 'shell' }, + ])( + 'preserves sandbox cost in a failed remote $language Function response', + async ({ language, code, kind }) => { + envFlagsMock.isRemoteSandboxEnabled = true + const cost = { input: 0, output: 0, total: 0.00012345 } + const executeSandbox = kind === 'shell' ? mockExecuteShellInSandbox : mockExecuteInSandbox + executeSandbox.mockResolvedValueOnce({ + result: null, + stdout: 'boom', + error: 'boom', + sandboxId: `sandbox-${language}`, + cost, + }) + + const response = await POST( + createMockRequest('POST', { + code, + language, + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + }) + ) + const data = await response.json() + + expect(response.status).toBe(422) + expect(executeSandbox).toHaveBeenCalledWith(expect.objectContaining({ meterUsage: true })) + expect(data.output.cost).toEqual(cost) + } + ) + it('does not meter a non-workflow remote Function call', async () => { envFlagsMock.isRemoteSandboxEnabled = true diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index 952a81d9ca7..d184e6e026c 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -2545,7 +2545,12 @@ export async function executeFunctionRequest( { success: false, error: scrubInternalIdentifiers(shellError, compilerInternalIdentifiers), - output: { result: null, stdout: cleanStdout(shellStdout), executionTime }, + output: { + result: null, + stdout: cleanStdout(shellStdout), + executionTime, + ...(shellCost ? { cost: shellCost } : {}), + }, }, routeContext, { status: 422 } @@ -2698,7 +2703,12 @@ export async function executeFunctionRequest( { success: false, error: formattedError, - output: { result: null, stdout: cleanedOutput, executionTime }, + output: { + result: null, + stdout: cleanedOutput, + executionTime, + ...(sandboxCost ? { cost: sandboxCost } : {}), + }, }, routeContext, { status: 422 } @@ -2813,7 +2823,12 @@ export async function executeFunctionRequest( { success: false, error: formattedError, - output: { result: null, stdout: cleanedOutput, executionTime }, + output: { + result: null, + stdout: cleanedOutput, + executionTime, + ...(sandboxCost ? { cost: sandboxCost } : {}), + }, }, routeContext, { status: 422 } diff --git a/apps/sim/providers/cost-policy.test.ts b/apps/sim/providers/cost-policy.test.ts index 5d7ebd9869e..48ced0fffcd 100644 --- a/apps/sim/providers/cost-policy.test.ts +++ b/apps/sim/providers/cost-policy.test.ts @@ -226,6 +226,23 @@ describe('installStreamingCostPolicy', () => { expect(output.cost).toMatchObject({ input: 0, output: 0, total: 0.75, toolCost: 0.75 }) }) + it('adds late failed Function cost once without applying the model multiplier', () => { + const output = { + cost: { input: 1, output: 2, total: 3.25, toolCost: 0.25 }, + } as NormalizedBlockOutput + const failedFunctionToolCost = { total: 0 } + installStreamingCostPolicy( + output, + { billable: false, multiplier: 0 }, + () => failedFunctionToolCost.total + ) + + failedFunctionToolCost.total = 0.125 + + expect(output.cost).toMatchObject({ input: 0, output: 0, total: 0.375, toolCost: 0.375 }) + expect(output.cost).toMatchObject({ total: 0.375, toolCost: 0.375 }) + }) + it('zeroes model cost written by a provider for a model Sim does not host', () => { const output = { cost: { input: 0, output: 0, total: 0 } } as NormalizedBlockOutput installStreamingCostPolicy(output, resolveModelCostPolicy(SELF_KEYED_MODEL)) diff --git a/apps/sim/providers/cost-policy.ts b/apps/sim/providers/cost-policy.ts index 7c67d82dee8..b116a90c472 100644 --- a/apps/sim/providers/cost-policy.ts +++ b/apps/sim/providers/cost-policy.ts @@ -266,12 +266,23 @@ export function resolveProxiedModelCost(cost: unknown): ModelCost { */ export function installStreamingCostPolicy( output: NormalizedBlockOutput, - policy: ModelCostPolicy + policy: ModelCostPolicy, + additionalToolCost?: () => number ): void { let raw = output.cost as ModelCost | undefined Object.defineProperty(output, 'cost', { - get: () => applyModelCostPolicy(raw, policy), + get: () => { + const projected = applyModelCostPolicy(raw, policy) + const additional = additionalToolCost?.() ?? 0 + if (!Number.isFinite(additional) || additional <= 0) return projected + + return { + ...projected, + toolCost: roundCost((projected.toolCost ?? 0) + additional), + total: roundCost(projected.total + additional), + } + }, set: (value: ModelCost | undefined) => { raw = value }, diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 22efa428d51..f6f962b71e5 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -278,6 +278,33 @@ describe('executeProviderRequest — BYOK regression', () => { expect(result.cost?.total).toBeCloseTo(0.00675, 8) }) + it('adds failed Function cost once alongside successful tool results', async () => { + mockGetApiKeyWithBYOK.mockResolvedValue({ apiKey: 'sk-byok', isBYOK: true }) + mockExecuteTool.mockResolvedValueOnce({ + success: false, + output: { cost: { total: 0.004 } }, + error: 'execution failed', + }) + mockExecuteRequest.mockImplementationOnce(async () => { + const execution = await executeProviderTool('function_execute', {}) + expect(execution.rawResponse.success).toBe(false) + return { + ...makeAnthropicResponse(), + toolResults: [{ cost: { total: 0.005 } }], + } as ProviderResponse + }) + + const result = (await executeProviderRequest('anthropic', { + model: 'claude-opus-4-6', + workspaceId: 'ws-1', + tools: [makeProviderTool('function_execute', 'credential')], + })) as ProviderResponse + + expect(result.cost).toMatchObject({ input: 0, output: 0 }) + expect(result.cost?.toolCost).toBeCloseTo(0.009, 8) + expect(result.cost?.total).toBeCloseTo(0.009, 8) + }) + /** * Gemini hands the same cost object to its response and its model segment. * Adding tool cost by mutation would charge it to the segment too. diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index d8ecbb04388..f4db8ee535a 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -152,14 +152,18 @@ function isReadableStream(response: any): response is ReadableStream { * stream drain — long after this function returns — so the policy is installed * on the live output object rather than applied to a value. */ -function applyStreamingCostPolicy(response: StreamingExecution, policy: ModelCostPolicy): void { +function applyStreamingCostPolicy( + response: StreamingExecution, + policy: ModelCostPolicy, + additionalToolCost?: () => number +): void { const output = response.execution?.output if (!output || typeof output !== 'object') { logger.warn('Streaming output unavailable at intercept time; cost policy not applied') return } - installStreamingCostPolicy(output, policy) + installStreamingCostPolicy(output, policy, additionalToolCost) const segments = output.providerTiming?.timeSegments if (Array.isArray(segments)) { @@ -224,16 +228,19 @@ export async function executeProviderRequest( const provenanceSafeRequest = await omitUnsafeProviderFileAttachments(sanitizedRequest) const modelSafeRequest = provenanceSafeRequest const toolIdentities = assignProviderToolIdentities(modelSafeRequest.tools) - const requestRuntimeContext = - toolIdentities.toolIdByWireId.size > 0 + const failedFunctionToolCost = { total: 0 } + const requestRuntimeContext: ProviderRuntimeContext = { + ...runtimeContext, + failedFunctionToolCost, + ...(toolIdentities.toolIdByWireId.size > 0 ? { - ...runtimeContext, toolIdByWireId: new Map([ ...(runtimeContext?.toolIdByWireId ?? []), ...toolIdentities.toolIdByWireId, ]), } - : runtimeContext + : {}), + } if (modelSafeRequest.responseFormat) { const structuredOutputInstructions = generateStructuredOutputInstructions( @@ -254,7 +261,11 @@ export async function executeProviderRequest( if (isStreamingExecution(response)) { logger.info('Provider returned StreamingExecution', { isBYOK }) - applyStreamingCostPolicy(response, resolveModelCostPolicy(sanitizedRequest.model, isBYOK)) + applyStreamingCostPolicy( + response, + resolveModelCostPolicy(sanitizedRequest.model, isBYOK), + () => failedFunctionToolCost.total + ) projectStreamingExecutionToolIdentities(response, toolIdentities) return response } @@ -300,7 +311,7 @@ export async function executeProviderRequest( applySegmentCostPolicy(response.timing.timeSegments, costPolicy) } - const toolCost = sumToolCosts(response.toolResults) + const toolCost = sumToolCosts(response.toolResults) + failedFunctionToolCost.total if (toolCost > 0 && response.cost) { // Replaced rather than mutated: a provider-supplied cost can be the same // object it also handed to a time segment, and tool cost belongs only to diff --git a/apps/sim/providers/runtime-context.test.ts b/apps/sim/providers/runtime-context.test.ts index c4761eea0b6..6a7c718b2da 100644 --- a/apps/sim/providers/runtime-context.test.ts +++ b/apps/sim/providers/runtime-context.test.ts @@ -147,6 +147,35 @@ describe('provider runtime context', () => { ) }) + it('accumulates cost only for failed canonical Function results', async () => { + const failedFunctionToolCost = { total: 0 } + const context = { + failedFunctionToolCost, + toolIdByWireId: new Map([['function_execute__sim_2', 'function_execute']]), + } + + mockExecuteTool + .mockResolvedValueOnce({ + success: false, + output: { cost: { total: 0.125 } }, + error: 'execution failed', + }) + .mockResolvedValueOnce({ success: true, output: { cost: { total: 4 } } }) + .mockResolvedValueOnce({ + success: false, + output: { cost: { total: 8 } }, + error: 'other tool failed', + }) + + await runWithProviderRuntimeContext(context, () => + executeProviderTool('function_execute__sim_2', {}) + ) + await runWithProviderRuntimeContext(context, () => executeProviderTool('function_execute', {})) + await runWithProviderRuntimeContext(context, () => executeProviderTool('exa_search', {})) + + expect(failedFunctionToolCost.total).toBe(0.125) + }) + it('rebinds a prompt-exposed environment placeholder for the exact tool call', async () => { const sourceRegistry = new ResolvedSecretTraceRegistry([ { diff --git a/apps/sim/providers/runtime-context.ts b/apps/sim/providers/runtime-context.ts index 2e602a83ba0..3cfea3e1eab 100644 --- a/apps/sim/providers/runtime-context.ts +++ b/apps/sim/providers/runtime-context.ts @@ -19,6 +19,8 @@ export interface ProviderRuntimeContext { executionContext?: ExecutionContext /** Request-scoped provider wire ids mapped back to canonical tool registry ids. */ toolIdByWireId?: ReadonlyMap + /** Failed canonical Function cost omitted from provider tool-result collections. */ + failedFunctionToolCost?: { total: number } } export type ExecuteProviderToolOptions = ExecuteToolOptions @@ -88,6 +90,20 @@ function withoutChildTraceHandle(response: ToolResponse): ToolResponse { } } +function accumulateFailedFunctionToolCost( + toolId: string, + result: ToolResponse, + accumulator: ProviderRuntimeContext['failedFunctionToolCost'] +): void { + if (toolId !== 'function_execute' || result.success || !accumulator) return + if (!isRecordLike(result.output) || !isRecordLike(result.output.cost)) return + + const total = result.output.cost.total + if (typeof total === 'number' && Number.isFinite(total) && total > 0) { + accumulator.total += total + } +} + export async function executeProviderTool( toolId: string, params: Parameters[1], @@ -120,6 +136,11 @@ export async function executeProviderTool( ...(executionContext ? { executionContext } : {}), resolvedSecretTraceRegistry: toolCallRegistry, }) + accumulateFailedFunctionToolCost( + executionToolId, + result, + runtimeContext?.failedFunctionToolCost + ) if (!registry || !toolCallRegistry) { return { rawResponse: result, modelResponse: withoutChildTraceHandle(result) } } diff --git a/apps/sim/tools/function/execute.test.ts b/apps/sim/tools/function/execute.test.ts index 65f898ffab1..f4dc0573265 100644 --- a/apps/sim/tools/function/execute.test.ts +++ b/apps/sim/tools/function/execute.test.ts @@ -136,4 +136,25 @@ describe('Function Execute Tool', () => { output: { result: 42, stdout: 'done', cost }, }) }) + + it('preserves sandbox cost in a failed Function result', async () => { + const cost = { input: 0, output: 0, total: 0.00012345 } + const result = await functionExecuteTool.transformResponse?.( + Response.json( + { + success: false, + error: 'boom', + output: { result: null, stdout: 'trace', cost }, + }, + { status: 422 } + ), + { code: 'throw new Error("boom")' } + ) + + expect(result).toMatchObject({ + success: false, + output: { result: null, stdout: 'trace', cost }, + error: 'boom', + }) + }) }) diff --git a/apps/sim/tools/function/execute.ts b/apps/sim/tools/function/execute.ts index 1279ac190f1..ab2cde32a1d 100644 --- a/apps/sim/tools/function/execute.ts +++ b/apps/sim/tools/function/execute.ts @@ -244,6 +244,7 @@ To return a file, write it to ${SANDBOX_OUTPUT_DIR}. Everything there comes back // Always an array, never undefined: a declared `file[]` output that is // missing warns on every call, and this branch runs for every failure. files: result.output?.files ?? [], + ...(result.output?.cost ? { cost: result.output.cost } : {}), }, error: result.error, retryable: result.retryable, diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index 5f3847e9d48..e9d2b43c50b 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -1744,6 +1744,7 @@ describe('executeTool Function', () => { it('does not log plaintext or runtime aliases from Function errors', async () => { const secret = 'function-error-secret-value' const runtimeAlias = '__var_API_KEY' + const cost = { input: 0, output: 0, total: 0.00012345 } const registry = new ResolvedSecretTraceRegistry([ { name: 'API_KEY', @@ -1756,6 +1757,7 @@ describe('executeTool Function', () => { JSON.stringify({ success: false, error: `Execution failed with ${secret} via ${runtimeAlias}`, + output: { result: null, stdout: 'trace', cost }, __resolvedSecretNames: ['API_KEY'], }), { @@ -1782,6 +1784,7 @@ describe('executeTool Function', () => { expect(result.success).toBe(false) expect(result.error).toContain(secret) + expect(result.output?.cost).toEqual(cost) expect(JSON.stringify(mockToolsLogger.error.mock.calls)).not.toContain(secret) expect(JSON.stringify(mockToolsLogger.error.mock.calls)).not.toContain(runtimeAlias) expect(JSON.stringify(projectToolResultForCopilot(result, registry))).not.toContain(secret) @@ -1824,6 +1827,32 @@ describe('executeTool Function', () => { expect(JSON.stringify(mockToolsLogger.error.mock.calls)).not.toContain(runtimeAlias) }) + it('does not lift an invalid sandbox cost from a Function error response', async () => { + mockExecuteFunction.mockResolvedValueOnce( + Response.json( + { + success: false, + error: 'boom', + output: { + result: null, + stdout: 'trace', + cost: { input: 0, output: 0, total: -1 }, + }, + }, + { status: 422 } + ) + ) + + const result = await executeTool( + 'function_execute', + { code: 'throw new Error("boom")' }, + { executionContext: createToolExecutionContext({ userId: 'user-1' }) } + ) + + expect(result.success).toBe(false) + expect(result.output).not.toHaveProperty('cost') + }) + it('does not log a secret-bearing non-OK response stream error', async () => { const secret = 'function-body-stream-secret-value' const streamError = `${secret} __var_API_KEY __sim_code_0_binding_0` diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index 9a9ea23196e..72c8ba6f605 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -1629,11 +1629,12 @@ async function executeToolImplementation( const startTime = new Date() const startTimeISO = startTime.toISOString() const requestId = generateRequestId() + const normalizedToolId = normalizeToolId(toolId) const privateToolMetadataPolicy = resolvedSecretTraceRegistry ? getPrivateToolMetadataPolicy(toolId) : undefined const structuralOnlyToolLogs = - normalizeToolId(toolId) === 'function_execute' || + normalizedToolId === 'function_execute' || isCustomTool(toolId) || privateToolMetadataPolicy !== undefined @@ -1645,7 +1646,6 @@ async function executeToolImplementation( let tool: ExecutableToolConfig | undefined // Preserve direct-call compatibility with legacy resource-suffixed tool ids. - const normalizedToolId = normalizeToolId(toolId) if (internalSandboxProfile && normalizedToolId !== 'function_execute') { throw new Error('An internal sandbox profile may only be used with function_execute') } @@ -2283,9 +2283,14 @@ async function executeToolImplementation( const rawResponseData = error instanceof Error && 'data' in error ? (error as { data?: unknown }).data : undefined const responseData = isRecordLike(rawResponseData) ? rawResponseData : undefined + const functionSandboxCost = + normalizedToolId === 'function_execute' ? readFunctionSandboxCost(responseData) : undefined return { success: false, - output: errorDetails, + output: { + ...errorDetails, + ...(functionSandboxCost ? { cost: functionSandboxCost } : {}), + }, error: errorMessage, ...(responseData?.retryable === false ? { retryable: false } : {}), // Sim's own status (hosted-key 429/503) survives the flattening from a @@ -2446,6 +2451,33 @@ function isFunctionExecuteBody(value: unknown): value is FunctionExecuteBody { return isPlainRecord(value) && typeof value.code === 'string' } +interface FunctionSandboxCost { + input: number + output: number + total: number +} + +function readFunctionSandboxCost(value: unknown): FunctionSandboxCost | undefined { + if (!isRecordLike(value) || !isRecordLike(value.output) || !isRecordLike(value.output.cost)) { + return undefined + } + const { input, output, total } = value.output.cost + if ( + typeof input !== 'number' || + !Number.isFinite(input) || + input < 0 || + typeof output !== 'number' || + !Number.isFinite(output) || + output < 0 || + typeof total !== 'number' || + !Number.isFinite(total) || + total < 0 + ) { + return undefined + } + return { input, output, total } +} + function isToolResponse(value: unknown): value is ToolResponse { return isRecordLike(value) && typeof value.success === 'boolean' && isRecordLike(value.output) } From 9d689d45d0880815cd896181f7c44845dd555e65 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Fri, 28 Aug 2026 20:01:37 -0700 Subject: [PATCH 3/7] fix(billing): correct sandbox trace cost boundaries --- .../remote-sandbox/conformance.test.ts | 4 ++ apps/sim/lib/execution/remote-sandbox/e2b.ts | 10 +++- .../sim/lib/execution/remote-sandbox/index.ts | 10 ++-- .../sim/lib/execution/remote-sandbox/types.ts | 8 ++- .../execute-request.test.ts | 8 +++ .../lib/function-execution/execute-request.ts | 50 +++++++++++++------ 6 files changed, 68 insertions(+), 22 deletions(-) diff --git a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts index 370fe561464..cab38393a4f 100644 --- a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts @@ -2504,10 +2504,12 @@ describe('Pi sandbox lifetime', () => { code: 'x', language: CodeLanguage.Python, timeoutMs: 7 * 24 * 60 * 60 * 1000, + meterUsage: true, }) expect(result.error).toContain('E2B reached its 24-hour limit') expect(result.error).toContain('workflow timeout may be longer') + expect(result.cost).toBeUndefined() expect(mockRecordSandboxProviderLimit).toHaveBeenCalledWith({ provider: 'e2b', operation: 'code', @@ -2531,9 +2533,11 @@ describe('Pi sandbox lifetime', () => { const result = await executeShellInSandbox({ code: 'sleep infinity', timeoutMs: 7 * 24 * 60 * 60 * 1000, + meterUsage: true, }) expect(result.error).toContain('E2B reached its 24-hour limit') + expect(result.cost).toBeUndefined() expect(mockRecordSandboxProviderLimit).toHaveBeenCalledWith({ provider: 'e2b', operation: 'command', diff --git a/apps/sim/lib/execution/remote-sandbox/e2b.ts b/apps/sim/lib/execution/remote-sandbox/e2b.ts index 46d887c2118..c8922b2f3d6 100644 --- a/apps/sim/lib/execution/remote-sandbox/e2b.ts +++ b/apps/sim/lib/execution/remote-sandbox/e2b.ts @@ -372,7 +372,7 @@ class E2BSandboxHandle implements SandboxHandle { return { text: '', stdout: result.stdout, stderr: result.stderr, timedOut: true } } if (result.exitCode !== 0) { - if (result.stderr === E2B_PROVIDER_LIMIT_ERROR) { + if (result.providerFailure === 'provider_limit') { return { text: '', stdout: result.stdout, @@ -382,6 +382,7 @@ class E2BSandboxHandle implements SandboxHandle { value: E2B_PROVIDER_LIMIT_ERROR, traceback: E2B_PROVIDER_LIMIT_ERROR, }, + providerFailure: result.providerFailure, } } return processCodeFailure(result) @@ -541,7 +542,12 @@ class E2BSandboxHandle implements SandboxHandle { if (isNonRetryableExecutionError(error)) throw error if (reachedE2BProviderLimit(error, this.providerLimitAtMs, options.signal)) { recordSandboxProviderLimit({ provider: 'e2b', operation }) - return { stdout: '', stderr: E2B_PROVIDER_LIMIT_ERROR, exitCode: 1 } + return { + stdout: '', + stderr: E2B_PROVIDER_LIMIT_ERROR, + exitCode: 1, + providerFailure: 'provider_limit', + } } // The SDK throws on non-zero exit; callers want the streams, not a throw. const failure = error as { diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index 8ed9c6e6003..c51ca2b3303 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -864,13 +864,14 @@ async function executeInSandboxWithinBudget( sandboxId, hasTraceback: Boolean(execution.error.traceback), }) - billableResult = { + const executionResult = { result: null, stdout: execution.error.traceback || errorMessage, error: errorMessage, sandboxId, } - return billableResult + if (execution.providerFailure !== 'provider_limit') billableResult = executionResult + return executionResult } // Distinct sources (final-expression text, stdout, stderr) join with '\n' so @@ -1008,8 +1009,9 @@ async function executeShellInSandboxWithinBudget( sandboxId, exitCode: result.exitCode, }) - billableResult = { result: null, stdout, error: errorMessage, sandboxId } - return billableResult + const executionResult = { result: null, stdout, error: errorMessage, sandboxId } + if (result.providerFailure !== 'provider_limit') billableResult = executionResult + return executionResult } // Shell scripts have no wrapper: any __SIM_RESULT__ line is user-authored diff --git a/apps/sim/lib/execution/remote-sandbox/types.ts b/apps/sim/lib/execution/remote-sandbox/types.ts index 2b9584b1cf3..6ca5ab77ef8 100644 --- a/apps/sim/lib/execution/remote-sandbox/types.ts +++ b/apps/sim/lib/execution/remote-sandbox/types.ts @@ -72,7 +72,7 @@ export interface SandboxExecutionRequest { sandboxId?: string /** Cancels the provider sandbox when the caller's execution budget expires. */ signal?: AbortSignal - /** Adds the remote provider cost to a successful Function result. */ + /** Adds the remote provider cost to a completed, billable Function outcome. */ meterUsage?: boolean } @@ -99,7 +99,7 @@ export interface SandboxShellExecutionRequest { sandboxId?: string /** Cancels the provider sandbox when the caller's execution budget expires. */ signal?: AbortSignal - /** Adds the remote provider cost to a successful Function result. */ + /** Adds the remote provider cost to a completed, billable Function outcome. */ meterUsage?: boolean } @@ -138,6 +138,8 @@ export interface SandboxCommandResult { exitCode: number /** The provider stopped the command because its supplied execution budget elapsed. */ timedOut?: boolean + /** The provider ended execution for an infrastructure reason, not a user-process outcome. */ + providerFailure?: 'provider_limit' } /** @@ -161,6 +163,8 @@ export interface SandboxCodeResult { error?: SandboxCodeError /** The provider stopped the code runner because its supplied execution budget elapsed. */ timedOut?: boolean + /** The provider ended execution for an infrastructure reason, not a user-program outcome. */ + providerFailure?: 'provider_limit' } export interface RunCommandOptions { diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index 405c99ca41f..616a787b429 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -328,6 +328,7 @@ describe('Function execution request', () => { result: 'done', stdout: 'ok', sandboxId: 'sandbox-123', + cost: { input: 0, output: 0, total: 0.00012345 }, exportedFiles: { '/tmp/out.txt': 'owned by attacker' }, }) mockWriteWorkspaceFileByPath.mockRejectedValueOnce( @@ -338,6 +339,8 @@ describe('Function execution request', () => { code: 'print("done")', language: 'python', workspaceId: 'workspace-victim', + workflowId: 'workflow-1', + executionId: 'execution-1', outputs: { files: [{ path: 'files/README.md', mode: 'overwrite', sandboxPath: '/tmp/out.txt' }], }, @@ -348,6 +351,7 @@ describe('Function execution request', () => { expect(response.status).toBe(403) expect(data).toHaveProperty('error', 'Insufficient workspace permissions') + expect(data.output.cost).toEqual({ input: 0, output: 0, total: 0.00012345 }) expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(1) }) @@ -1543,6 +1547,7 @@ describe('Function execution request', () => { result: 'done', stdout: 'ok', sandboxId: 'sandbox-123', + cost: { input: 0, output: 0, total: 0.00023456 }, exportedFiles: { '/home/user/first.json': '{"first":true}', '/home/user/second.json': '{"second":true}', @@ -1556,6 +1561,8 @@ describe('Function execution request', () => { code: 'print("done")', language: 'python', workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', outputs: { files: [ { @@ -1578,6 +1585,7 @@ describe('Function execution request', () => { expect(response.status).toBe(400) expect(data.success).toBe(false) expect(data.error).toContain('Directory not yet created') + expect(data.output.cost).toEqual({ input: 0, output: 0, total: 0.00023456 }) expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() }) diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index d184e6e026c..c57cfc6030c 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -1438,10 +1438,20 @@ function exportFailure( error: string, status: number, stdout: string, - executionTime: number + executionTime: number, + cost: FunctionExecutionCost | undefined ): NextResponse { return NextResponse.json( - { success: false, error, output: { result: null, stdout: cleanStdout(stdout), executionTime } }, + { + success: false, + error, + output: { + result: null, + stdout: cleanStdout(stdout), + executionTime, + ...(cost ? { cost } : {}), + }, + }, { status } ) } @@ -1490,7 +1500,8 @@ async function maybeExportSandboxFileToWorkspace(args: { 'outputSandboxPath requires outputPath. Set outputPath to the destination workspace file, e.g. "files/result.csv".', 400, stdout, - executionTime + executionTime, + cost ) } @@ -1502,7 +1513,8 @@ async function maybeExportSandboxFileToWorkspace(args: { 'Workspace context required to save sandbox file to workspace', 400, stdout, - executionTime + executionTime, + cost ) } @@ -1511,7 +1523,8 @@ async function maybeExportSandboxFileToWorkspace(args: { `Sandbox file "${outputSandboxPath}" was not found or could not be read`, 500, stdout, - executionTime + executionTime, + cost ) } @@ -1529,7 +1542,8 @@ async function maybeExportSandboxFileToWorkspace(args: { `Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`, 400, stdout, - executionTime + executionTime, + cost ) } const fileBuffer = isBinary @@ -1612,7 +1626,8 @@ async function maybeExportSandboxFileToWorkspace(args: { getErrorMessage(error, 'Failed to export sandbox file'), workspaceFileExportErrorStatus(error), stdout, - executionTime + executionTime, + cost ) } } @@ -1636,7 +1651,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { `Too many sandbox output files requested (${sandboxFiles.length}). Maximum is ${MAX_SANDBOX_OUTPUT_FILES}.`, 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } @@ -1669,7 +1685,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { 'Workspace context required to save sandbox files to workspace', 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } @@ -1683,7 +1700,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { `Sandbox file "${sandboxPath}" was not found or could not be read`, 500, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } const outputPath = file.formatPath ?? file.path @@ -1700,7 +1718,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { `Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`, 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } const scanBuffer = isBinary ? Buffer.from(content, 'base64') : Buffer.from(content, 'utf-8') @@ -1749,7 +1768,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { getErrorMessage(error, 'Invalid sandbox output destination'), workspaceFileExportErrorStatus(error), args.stdout, - args.executionTime + args.executionTime, + args.cost ) } const duplicateDestination = validationPaths.find( @@ -1760,7 +1780,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { `Duplicate sandbox output destination: ${duplicateDestination}`, 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } @@ -1816,7 +1837,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { getErrorMessage(error, 'Failed to export sandbox files'), workspaceFileExportErrorStatus(error), args.stdout, - args.executionTime + args.executionTime, + args.cost ) } From dfc0e355c73d7a4f4d782ee3ba1140cd371832f0 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Sat, 29 Aug 2026 09:53:54 -0700 Subject: [PATCH 4/7] fix(billing): tighten sandbox completion boundaries --- apps/sim/lib/billing/sandbox-pricing.test.ts | 11 +- apps/sim/lib/billing/sandbox-pricing.ts | 8 +- .../remote-sandbox/conformance.test.ts | 182 ++++++++++++++++-- .../lib/execution/remote-sandbox/daytona.ts | 17 +- .../sim/lib/execution/remote-sandbox/index.ts | 84 +++++--- .../execution/remote-sandbox/output-limits.ts | 17 ++ .../sim/lib/execution/remote-sandbox/types.ts | 8 +- .../execute-request.test.ts | 40 +++- .../lib/function-execution/execute-request.ts | 36 +++- 9 files changed, 341 insertions(+), 62 deletions(-) diff --git a/apps/sim/lib/billing/sandbox-pricing.test.ts b/apps/sim/lib/billing/sandbox-pricing.test.ts index 658cd7a1cbe..e523120bc32 100644 --- a/apps/sim/lib/billing/sandbox-pricing.test.ts +++ b/apps/sim/lib/billing/sandbox-pricing.test.ts @@ -23,7 +23,14 @@ describe('sandbox pricing', () => { expect(priceSandboxUsage(pricing, 90_000, 60_000).durationMs).toBe(60_000) }) - it('rejects a non-positive multiplier', () => { - expect(() => createSandboxPricing('e2b', 0)).toThrow('finite positive') + it('allows a zero multiplier and rejects invalid multipliers', () => { + const freePricing = createSandboxPricing('e2b', 0) + + expect(priceSandboxUsage(freePricing, 1000, 1000).billedCost).toBe(0) + expect(() => createSandboxPricing('e2b', -1)).toThrow('finite nonnegative') + expect(() => createSandboxPricing('e2b', Number.NaN)).toThrow('finite nonnegative') + expect(() => createSandboxPricing('e2b', Number.POSITIVE_INFINITY)).toThrow( + 'finite nonnegative' + ) }) }) diff --git a/apps/sim/lib/billing/sandbox-pricing.ts b/apps/sim/lib/billing/sandbox-pricing.ts index fa5bc09f537..828fb83204a 100644 --- a/apps/sim/lib/billing/sandbox-pricing.ts +++ b/apps/sim/lib/billing/sandbox-pricing.ts @@ -10,6 +10,10 @@ const E2B_CPU_USD_PER_VCPU_SECOND = 0.000014 const E2B_MEMORY_USD_PER_GIB_SECOND = 0.0000045 const DAYTONA_CPU_USD_PER_VCPU_SECOND = 0.0504 / 3600 const DAYTONA_MEMORY_USD_PER_GIB_SECOND = 0.0162 / 3600 +/** + * Sim prices the full provisioned disk at the marginal list rate; provider free allowances, + * credits, and discounts are intentionally not subtracted. + */ const DAYTONA_DISK_USD_PER_GIB_SECOND = 0.000108 / 3600 export interface SandboxPricing { @@ -67,8 +71,8 @@ export function createSandboxPricing( provider: SandboxProviderId, multiplier = getCostMultiplier() ): SandboxPricing { - if (!Number.isFinite(multiplier) || multiplier <= 0) { - throw new Error('Sandbox pricing multiplier must be a finite positive number') + if (!Number.isFinite(multiplier) || multiplier < 0) { + throw new Error('Sandbox pricing multiplier must be a finite nonnegative number') } const pricing = PRICING_BY_PROVIDER[provider] return { diff --git a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts index cab38393a4f..7e502e34bf3 100644 --- a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts @@ -136,6 +136,7 @@ import { MAX_SANDBOX_OUTPUT_BYTES, MAX_SANDBOX_PROCESS_OUTPUT_BYTES, MAX_SANDBOX_STREAMED_OUTPUT_TAIL_BYTES, + readTrustedSandboxOutputCost, } from '@/lib/execution/remote-sandbox/output-limits' import { PI_SANDBOX_MIN_LIFETIME_MS, @@ -464,13 +465,19 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { }) } - await expect( - executeInSandbox({ code: 'x', language: CodeLanguage.Python, timeoutMs: 1000 }) - ).rejects.toMatchObject({ + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toMatchObject({ code: 'sandbox_output_limit_exceeded', outputKind: 'process', limitBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, }) + expect(readTrustedSandboxOutputCost(error)).toBeUndefined() expect(provider === 'e2b' ? mockE2BKill : mockDelete).toHaveBeenCalledTimes(1) }) @@ -1173,18 +1180,24 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) stubOutputFileSizes(provider, MAX_SANDBOX_OUTPUT_BYTES + 1) - await expect( - executeInSandbox({ - code: 'x', - language: CodeLanguage.Python, - timeoutMs: 1000, - outputSandboxPath: '/out/report.txt', - }) - ).rejects.toMatchObject({ + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPath: '/out/report.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toMatchObject({ code: 'sandbox_output_limit_exceeded', attemptedBytes: MAX_SANDBOX_OUTPUT_BYTES + 1, limitBytes: MAX_SANDBOX_OUTPUT_BYTES, }) + expect(readTrustedSandboxOutputCost(error)).toEqual({ + input: 0, + output: 0, + total: expect.any(Number), + }) expect(provider === 'e2b' ? mockE2BFilesRead : mockDownloadFileStream).not.toHaveBeenCalled() }) @@ -1237,15 +1250,96 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { mockGetFileDetails.mockResolvedValueOnce({ size: 1, isDir: false, mode: 'prw-r--r--' }) } - await expect( - executeInSandbox({ - code: 'x', - language: CodeLanguage.Python, + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPath: '/out/link.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toMatchObject({ code: 'sandbox_output_file_invalid' }) + expect(readTrustedSandboxOutputCost(error)).toEqual({ + input: 0, + output: 0, + total: expect.any(Number), + }) + expect(provider === 'e2b' ? mockE2BFilesRead : mockDownloadFileStream).not.toHaveBeenCalled() + }) + + it.each(['oversized', 'non-regular'] as const)( + 'retains metered shell cost for a completed execution with %s output', + async (failure) => { + stubShellCommand(provider, '', '', 0) + if (failure === 'oversized') { + stubOutputFileSizes(provider, MAX_SANDBOX_OUTPUT_BYTES + 1) + } else if (provider === 'e2b') { + mockE2BFilesGetInfo.mockResolvedValueOnce({ size: 1, type: 'symlink' }) + } else { + mockGetFileDetails.mockResolvedValueOnce({ size: 1, isDir: false, mode: 'prw-r--r--' }) + } + + const error = await executeShellInSandbox({ + code: 'echo done', + envs: {}, timeoutMs: 1000, - outputSandboxPath: '/out/link.txt', + outputSandboxPath: '/out/result.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toMatchObject({ + code: + failure === 'oversized' ? 'sandbox_output_limit_exceeded' : 'sandbox_output_file_invalid', }) - ).rejects.toMatchObject({ code: 'sandbox_output_file_invalid' }) - expect(provider === 'e2b' ? mockE2BFilesRead : mockDownloadFileStream).not.toHaveBeenCalled() + expect(readTrustedSandboxOutputCost(error)).toEqual({ + input: 0, + output: 0, + total: expect.any(Number), + }) + } + ) + + it('does not attach cost to a generic provider failure during output collection', async () => { + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + stubOutputFileSizes(provider, 1, 1) + const failure = new Error('provider file read failed') + if (provider === 'e2b') { + mockE2BFilesRead.mockRejectedValueOnce(failure) + } else { + mockDownloadFileStream.mockRejectedValueOnce(failure) + } + + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPath: '/out/result.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toBe(failure) + expect(readTrustedSandboxOutputCost(error)).toBeUndefined() + }) + + it('does not attach cost to a generic provider failure during output inspection', async () => { + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + const failure = new Error('provider file metadata failed') + if (provider === 'e2b') { + mockE2BFilesGetInfo.mockRejectedValueOnce(failure) + } else { + mockGetFileDetails.mockRejectedValueOnce(failure) + } + + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPath: '/out/result.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toBe(failure) + expect(readTrustedSandboxOutputCost(error)).toBeUndefined() }) it('does not return code results when cancellation arrives during output collection', async () => { @@ -1564,6 +1658,58 @@ describe('provider stream recovery', () => { expect(mockGetSessionCommand).toHaveBeenCalledWith(expect.any(String), 'cmd_1') }) + it('fails an at-most-once Daytona run closed when final status has no exit code', async () => { + mockGetSessionCommand.mockResolvedValueOnce({}) + const sandbox = await daytonaProvider.create('code') + + await expect( + sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true }) + ).rejects.toMatchObject({ + name: 'SandboxLaunchIndeterminateError', + retryable: false, + code: 'sandbox_launch_indeterminate', + }) + }) + + it('fails an at-most-once Daytona run closed when its readiness handshake never completes', async () => { + mockGetSessionCommandLogs.mockResolvedValueOnce(undefined) + mockGetSessionCommand.mockResolvedValueOnce({ exitCode: 78 }) + const sandbox = await daytonaProvider.create('code') + + await expect( + sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true }) + ).rejects.toMatchObject({ + name: 'SandboxLaunchIndeterminateError', + retryable: false, + code: 'sandbox_launch_indeterminate', + }) + expect(mockSendSessionCommandInput).not.toHaveBeenCalled() + }) + + it('fails an at-most-once Daytona run closed when final status lookup fails', async () => { + mockGetSessionCommand.mockRejectedValueOnce(new Error('control plane unavailable')) + const sandbox = await daytonaProvider.create('code') + + await expect( + sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true }) + ).rejects.toMatchObject({ + name: 'SandboxLaunchIndeterminateError', + retryable: false, + code: 'sandbox_launch_indeterminate', + }) + }) + + it('preserves a pre-dispatch Daytona failure for at-most-once runs', async () => { + const failure = new Error('session unavailable') + mockCreateSession.mockRejectedValueOnce(failure) + const sandbox = await daytonaProvider.create('code') + + await expect( + sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true }) + ).rejects.toBe(failure) + expect(mockExecuteSessionCommand).not.toHaveBeenCalled() + }) + it('keeps the original Daytona deadline while recovering a disconnected stream', async () => { mockGetSessionCommandLogs .mockRejectedValueOnce(new Error('stream disconnected')) diff --git a/apps/sim/lib/execution/remote-sandbox/daytona.ts b/apps/sim/lib/execution/remote-sandbox/daytona.ts index f9f6912e725..11df31c48f5 100644 --- a/apps/sim/lib/execution/remote-sandbox/daytona.ts +++ b/apps/sim/lib/execution/remote-sandbox/daytona.ts @@ -301,6 +301,7 @@ class DaytonaSandboxHandle implements SandboxHandle { // must never have. const finalStdout = () => (retainStdout ? stdout : tailStreamedSandboxOutput(stdout)) const finalStderr = () => (retainStderr ? stderr : tailStreamedSandboxOutput(stderr)) + let commandDispatched = false try { await this.sandbox.process.createSession(sessionId) sessionCreated = true @@ -339,6 +340,7 @@ class DaytonaSandboxHandle implements SandboxHandle { if (typeof commandId !== 'string' || commandId.length === 0) { throw new SandboxLaunchIndeterminateError('Daytona') } + commandDispatched = true // Accumulate the streamed chunks as well as forwarding them: callers read // markers out of stdout (the Pi cloud flow parses __BASE_SHA__/__CHANGED__) // and format failures from stderr, so returning empty strings here would @@ -660,7 +662,14 @@ class DaytonaSandboxHandle implements SandboxHandle { } const finished = await this.sandbox.process.getSessionCommand(sessionId, commandId) - const exitCode = finished.exitCode ?? 0 + if (options.atMostOnce && !releaseRequested) { + throw new SandboxLaunchIndeterminateError('Daytona') + } + const exitCode = finished.exitCode + if (typeof exitCode !== 'number' || !Number.isFinite(exitCode)) { + if (options.atMostOnce) throw new SandboxLaunchIndeterminateError('Daytona') + return { stdout: finalStdout(), stderr: finalStderr(), exitCode: 0 } + } return { stdout: finalStdout(), stderr: finalStderr(), exitCode } } catch (error) { if (isSandboxOutputLimitError(error)) { @@ -681,6 +690,12 @@ class DaytonaSandboxHandle implements SandboxHandle { timedOut: true, } } + if (options.atMostOnce) { + if (commandDispatched) { + throw new SandboxLaunchIndeterminateError('Daytona', { cause: error }) + } + throw error + } if (operation === 'code') throw error return { stdout: finalStdout(), stderr: finalStderr() || getErrorMessage(error), exitCode: 1 } } finally { diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index c51ca2b3303..3c962009d9f 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -15,6 +15,7 @@ import { recordSandboxTeardownFailure } from '@/lib/core/execution-limits/metric import { buildJavaScriptRuntimeBindingsSource } from '@/lib/execution/code-placeholders/javascript-runtime' import { SANDBOX_SYSTEM_PATH } from '@/lib/execution/remote-sandbox/cli-tools.server' import { + attachTrustedSandboxOutputCost, isSandboxOutputFileError, isSandboxOutputLimitError, MAX_SANDBOX_OUTPUT_BYTES, @@ -45,6 +46,7 @@ import type { SandboxCollectedFile, SandboxCommandResult, SandboxDirectoryEntry, + SandboxExecutionCost, SandboxExecutionRequest, SandboxExecutionResult, SandboxFile, @@ -249,18 +251,17 @@ function bindSandboxAbort( } } -function attachSandboxCost( - result: SandboxExecutionResult | undefined, +function calculateSandboxCost( created: CreatedSandbox, cleanupStartedAtMs: number -): void { - if (!result || !created.pricing || created.effectiveLifetimeMs === undefined) return +): SandboxExecutionCost | undefined { + if (!created.pricing || created.effectiveLifetimeMs === undefined) return undefined const usage = priceSandboxUsage( created.pricing, cleanupStartedAtMs - created.startedAtMs, created.effectiveLifetimeMs ) - result.cost = { input: 0, output: 0, total: usage.billedCost } + return { input: 0, output: 0, total: usage.billedCost } } /** @@ -510,14 +511,14 @@ async function readSandboxOutputFile( logger.warn('Failed to read requested sandbox output file', { sandboxId: sandbox.sandboxId, }) - return undefined + throw error } } async function inspectSandboxOutputFileSize( sandbox: SandboxHandle, outputSandboxPath: string -): Promise { +): Promise { try { const size = await sandbox.getFileSize(outputSandboxPath) if (!Number.isSafeInteger(size) || size < 0) { @@ -529,7 +530,7 @@ async function inspectSandboxOutputFileSize( logger.warn('Failed to inspect requested sandbox output file', { sandboxId: sandbox.sandboxId, }) - return undefined + throw error } } @@ -628,7 +629,6 @@ async function collectExportedFiles( for (const outputSandboxPath of requestedOutputSandboxPaths(req)) { const size = await inspectSandboxOutputFileSize(sandbox, outputSandboxPath) remainingSandboxBudgetMs(options.signal) - if (size === undefined) continue totalOutputBytes += size if (totalOutputBytes > MAX_SANDBOX_OUTPUT_BYTES) { throw new SandboxOutputLimitError(totalOutputBytes) @@ -820,6 +820,7 @@ async function executeInSandboxWithinBudget( const sandboxId = sandbox.sandboxId const abortBinding = bindSandboxAbort(sandbox, created.providerId, signal) let billableResult: SandboxExecutionResult | undefined + let billableOutputError: unknown try { throwIfAborted(signal) @@ -900,25 +901,35 @@ async function executeInSandboxWithinBudget( } } - const { exportedFiles, exportedFileContent, collectedFiles } = await collectExportedFiles( - sandbox, - req, - { signal } - ) - throwIfAborted(signal) - billableResult = { result: extraction.result, stdout: cleanedStdout, sandboxId, - exportedFileContent, - exportedFiles, - collectedFiles, + } + try { + const { exportedFiles, exportedFileContent, collectedFiles } = await collectExportedFiles( + sandbox, + req, + { signal } + ) + throwIfAborted(signal) + billableResult.exportedFileContent = exportedFileContent + billableResult.exportedFiles = exportedFiles + billableResult.collectedFiles = collectedFiles + } catch (error) { + if (isSandboxOutputLimitError(error) || isSandboxOutputFileError(error)) { + billableOutputError = error + } + throw error } return billableResult } finally { const cleanupStartedAtMs = Date.now() - attachSandboxCost(billableResult, created, cleanupStartedAtMs) + const cost = calculateSandboxCost(created, cleanupStartedAtMs) + if (cost && billableResult) billableResult.cost = cost + if (cost && billableOutputError) { + attachTrustedSandboxOutputCost(billableOutputError, cost) + } abortBinding.detach() await abortBinding.cleanup() } @@ -958,6 +969,7 @@ async function executeShellInSandboxWithinBudget( const sandboxId = sandbox.sandboxId const abortBinding = bindSandboxAbort(sandbox, created.providerId, signal) let billableResult: SandboxExecutionResult | undefined + let billableOutputError: unknown try { throwIfAborted(signal) @@ -1020,25 +1032,35 @@ async function executeShellInSandboxWithinBudget( const extraction = extractSimResult(stdout) const parsed = extraction.parseFailed ? extraction.rawPayload : extraction.result - const { exportedFiles, exportedFileContent, collectedFiles } = await collectExportedFiles( - sandbox, - req, - { signal } - ) - throwIfAborted(signal) - billableResult = { result: parsed, stdout: extraction.cleanedStdout, sandboxId, - exportedFileContent, - exportedFiles, - collectedFiles, + } + try { + const { exportedFiles, exportedFileContent, collectedFiles } = await collectExportedFiles( + sandbox, + req, + { signal } + ) + throwIfAborted(signal) + billableResult.exportedFileContent = exportedFileContent + billableResult.exportedFiles = exportedFiles + billableResult.collectedFiles = collectedFiles + } catch (error) { + if (isSandboxOutputLimitError(error) || isSandboxOutputFileError(error)) { + billableOutputError = error + } + throw error } return billableResult } finally { const cleanupStartedAtMs = Date.now() - attachSandboxCost(billableResult, created, cleanupStartedAtMs) + const cost = calculateSandboxCost(created, cleanupStartedAtMs) + if (cost && billableResult) billableResult.cost = cost + if (cost && billableOutputError) { + attachTrustedSandboxOutputCost(billableOutputError, cost) + } abortBinding.detach() await abortBinding.cleanup() } diff --git a/apps/sim/lib/execution/remote-sandbox/output-limits.ts b/apps/sim/lib/execution/remote-sandbox/output-limits.ts index 0110bf57215..e260660aa19 100644 --- a/apps/sim/lib/execution/remote-sandbox/output-limits.ts +++ b/apps/sim/lib/execution/remote-sandbox/output-limits.ts @@ -1,3 +1,5 @@ +import type { SandboxExecutionCost } from '@/lib/execution/remote-sandbox/types' + export const MAX_SANDBOX_OUTPUT_BYTES = 50 * 1024 * 1024 /** @@ -103,6 +105,21 @@ export class SandboxOutputDepthError extends Error { } } +const trustedSandboxOutputCosts = new WeakMap() + +/** Associates Sim-calculated cost with a trusted post-execution output error. */ +export function attachTrustedSandboxOutputCost(error: unknown, cost: SandboxExecutionCost): void { + if (typeof error !== 'object' || error === null) return + trustedSandboxOutputCosts.set(error, cost) +} + +/** Reads cost only when the sandbox lifecycle attached it after a completed execution. */ +export function readTrustedSandboxOutputCost(error: unknown): SandboxExecutionCost | undefined { + return typeof error === 'object' && error !== null + ? trustedSandboxOutputCosts.get(error) + : undefined +} + export class SandboxOutputFileError extends Error { readonly code = SANDBOX_OUTPUT_FILE_INVALID_CODE diff --git a/apps/sim/lib/execution/remote-sandbox/types.ts b/apps/sim/lib/execution/remote-sandbox/types.ts index 6ca5ab77ef8..5c21d0c1032 100644 --- a/apps/sim/lib/execution/remote-sandbox/types.ts +++ b/apps/sim/lib/execution/remote-sandbox/types.ts @@ -103,6 +103,12 @@ export interface SandboxShellExecutionRequest { meterUsage?: boolean } +export interface SandboxExecutionCost { + input: number + output: number + total: number +} + export interface SandboxExecutionResult { result: unknown stdout: string @@ -120,7 +126,7 @@ export interface SandboxExecutionResult { * sequence, and the byte budget is enforced on the decoded length. */ collectedFiles?: SandboxCollectedFile[] - cost?: { input: number; output: number; total: number } + cost?: SandboxExecutionCost } /** One harvested output file, carried as base64 with its decoded length. */ diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index 616a787b429..dab70ebe860 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -23,6 +23,7 @@ import { PRIVATE_SECRET_PROVENANCE_HEADER, } from '@/lib/execution/private-tool-metadata' import { + attachTrustedSandboxOutputCost, MAX_SANDBOX_OUTPUT_BYTES, SandboxOutputFileError, SandboxOutputLimitError, @@ -84,7 +85,11 @@ vi.mock('@/lib/copilot/request/tools/files', () => ({ md: 'text/markdown', html: 'text/html', }, - normalizeOutputWorkspaceFileName: vi.fn((p: string) => p.replace(/^files\//, '')), + normalizeOutputWorkspaceFileName: vi.fn((p: string) => { + const normalized = p.trim().replace(/^\/+|\/+$/g, '') + if (!normalized) throw new Error('Output path must include a file name') + return normalized.replace(/^files\//, '') + }), resolveOutputFormat: vi.fn(() => 'json'), getOutputFileDeclarations: vi.fn((params: Record) => { if (Array.isArray(params.outputs?.files)) { @@ -1494,9 +1499,10 @@ describe('Function execution request', () => { it('preserves output-limit classification from provider-side size inspection', async () => { envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockRejectedValueOnce( - new SandboxOutputLimitError(MAX_SANDBOX_OUTPUT_BYTES + 1) - ) + const error = new SandboxOutputLimitError(MAX_SANDBOX_OUTPUT_BYTES + 1) + const cost = { input: 0, output: 0, total: 0.00023456 } + attachTrustedSandboxOutputCost(error, cost) + mockExecuteInSandbox.mockRejectedValueOnce(error) const req = createMockRequest('POST', { code: 'print("done")', @@ -1517,6 +1523,7 @@ describe('Function execution request', () => { expect(response.status).toBe(400) expect(data.error).toBe(`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`) + expect(data.output.cost).toEqual(cost) expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() }) @@ -1538,9 +1545,34 @@ describe('Function execution request', () => { expect(response.status).toBe(400) expect(data.error).toContain('must reference a regular file') + expect(data.output.cost).toBeUndefined() expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() }) + it.each(['/', '///', ' / '])( + 'rejects malformed workspace output destination %j before sandbox execution', + async (path) => { + envFlagsMock.isRemoteSandboxEnabled = true + + const response = await POST( + createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [{ path, sandboxPath: '/out/report.json' }], + }, + }) + ) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data.error).toBe('Output path must include a file name') + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + expect(mockExecuteShellInSandbox).not.toHaveBeenCalled() + } + ) + it('prevalidates all sandbox output destinations before writing any files', async () => { envFlagsMock.isRemoteSandboxEnabled = true mockExecuteInSandbox.mockResolvedValueOnce({ diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index c57cfc6030c..1392221fbd0 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -82,6 +82,7 @@ import { isSandboxOutputLimitError, isSandboxOutputNotExportableError, MAX_SANDBOX_OUTPUT_BYTES, + readTrustedSandboxOutputCost, } from '@/lib/execution/remote-sandbox/output-limits' import { MAX_BLOCK_MOUNTED_FILES, @@ -1967,6 +1968,7 @@ async function collectExecutionOutputFiles(args: { collectedFiles: SandboxCollectedFile[] stdout: string executionTime: number + cost?: FunctionExecutionCost }): Promise<{ files: UserFile[] } | { response: NextResponse }> { const { routeContext, collectedFiles } = args if (collectedFiles.length === 0) return { files: [] } @@ -1983,7 +1985,8 @@ async function collectExecutionOutputFiles(args: { 'Workspace, workflow, and execution context are required to return files from the sandbox.', 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ), } } @@ -2023,7 +2026,8 @@ async function collectExecutionOutputFiles(args: { `Sandbox output file "${name}" contains a resolved secret value and was not returned. Write the file without embedding secret values, or export it to a workspace file where its provenance can be recorded.`, 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ), } } @@ -2225,6 +2229,23 @@ export async function executeFunctionRequest( privateResolvedSecretNamesMetadataType ) } + try { + for (const file of outputFiles) { + normalizeOutputWorkspaceFileName(file.formatPath ?? file.path) + } + } catch (error) { + return appendPrivateResolvedSecretNames( + NextResponse.json( + { + success: false, + error: getErrorMessage(error, 'Invalid sandbox output destination'), + }, + { status: 400 } + ), + includePrivateResolvedSecretNames ? [] : null, + privateResolvedSecretNamesMetadataType + ) + } // Planned before the runtime is chosen because it is pure: it decides whether // this execution needs a sandbox filesystem at all, without spending a presign @@ -2606,6 +2627,7 @@ export async function executeFunctionRequest( collectedFiles: shellCollectedFiles ?? [], stdout: shellStdout, executionTime, + cost: shellCost, }) if ('response' in shellOutputFiles) { return appendResolvedSecretNames(shellOutputFiles.response, routeContext) @@ -2764,6 +2786,7 @@ export async function executeFunctionRequest( collectedFiles: jsCollectedFiles ?? [], stdout, executionTime, + cost: sandboxCost, }) if ('response' in jsOutputFiles) { return appendResolvedSecretNames(jsOutputFiles.response, routeContext) @@ -2884,6 +2907,7 @@ export async function executeFunctionRequest( collectedFiles: pythonCollectedFiles ?? [], stdout, executionTime, + cost: sandboxCost, }) if ('response' in pythonOutputFiles) { return appendResolvedSecretNames(pythonOutputFiles.response, routeContext) @@ -3082,10 +3106,16 @@ export async function executeFunctionRequest( isSandboxOutputFileError(error) || isSandboxOutputNotExportableError(error) ) { + const cost = readTrustedSandboxOutputCost(error) const outputLimitResponse = { success: false, error: error.message, - output: { result: null, stdout: cleanStdout(stdout), executionTime }, + output: { + result: null, + stdout: cleanStdout(stdout), + executionTime, + ...(cost ? { cost } : {}), + }, } return routeContext ? functionJsonResponse(outputLimitResponse, routeContext, { status: 400 }) From 933a4bc2adb9df0ba5987e464aa6560760fde5fa Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Sat, 29 Aug 2026 12:21:55 -0700 Subject: [PATCH 5/7] test(billing): check the metered sandbox amount against a real provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pricing unit test pins the arithmetic and the conformance suite proves a cost is produced, attached to the right outcomes, and routed — but that suite stubs the provider and mocks Date.now() with a counter advancing one millisecond per call. Under that clock `total > 0` is the strongest claim available, and it holds equally well if the metered window is anchored to the wrong instants or the resource constants are wrong. Bounds the charge between what the sleep must cost and what the wall clock could justify, so a wrong rate, a wrong vCPU/memory constant, and a mis-anchored window all fail. Provider-agnostic via resolveProvider, opt-in behind SANDBOX_BILLING_SMOKE=1 like the sibling smoke suites. Verified against both providers: E2B billed 9.031s of a 9.302s call at $0.1656/hr, Daytona 8.435s of 8.721s at $0.16668/hr — both matching published rates, both excluding ~275ms of Sim-side overhead. Co-Authored-By: Claude Opus 5 (1M context) --- .../sandbox-billing.smoke.test.ts | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 apps/sim/lib/execution/remote-sandbox/sandbox-billing.smoke.test.ts diff --git a/apps/sim/lib/execution/remote-sandbox/sandbox-billing.smoke.test.ts b/apps/sim/lib/execution/remote-sandbox/sandbox-billing.smoke.test.ts new file mode 100644 index 00000000000..6b5ebfa63ee --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/sandbox-billing.smoke.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + * + * Checks the metered amount against a real provider run. + * + * `sandbox-pricing.test.ts` pins the arithmetic and the conformance suite proves + * a cost is produced, attached, and routed — but that suite stubs the provider + * and mocks `Date.now()`, so its clock advances one millisecond per call. Under + * those conditions `total > 0` is the strongest claim available, and it would + * hold just as well if the metered window measured the wrong instants. Only a + * real run can show that the window tracks the sandbox's actual lifetime. + * + * Enable with `SANDBOX_BILLING_SMOKE=1`. Runs against whichever provider + * `SANDBOX_PROVIDER` selects, so point it at each in turn to cover both. + */ +import { describe, expect, it } from 'vitest' +import { createSandboxPricing } from '@/lib/billing/sandbox-pricing' +import { CodeLanguage } from '@/lib/execution/languages' +import { executeInSandbox } from '@/lib/execution/remote-sandbox' +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' + +const smokeEnabled = process.env.SANDBOX_BILLING_SMOKE === '1' +const CASE_TIMEOUT_MS = 5 * 60_000 +const RUN_TIMEOUT_MS = 4 * 60_000 + +/** Long enough that provisioning jitter cannot dominate the measured runtime. */ +const SLEEP_SECONDS = 5 + +describe.skipIf(!smokeEnabled)('sandbox billing smoke', () => { + it( + 'bills the sandbox lifetime at the provider rate', + async () => { + const pricing = createSandboxPricing(resolveProvider().id) + const usdPerSecond = + pricing.resources.vcpu * pricing.rates.cpuUsdPerVcpuSecond + + pricing.resources.memoryGiB * pricing.rates.memoryUsdPerGiBSecond + + pricing.resources.diskGiB * pricing.rates.diskUsdPerGiBSecond + const usdPerBilledSecond = usdPerSecond * pricing.multiplier + + const wallClockStartedAtMs = Date.now() + const result = await executeInSandbox({ + code: `import time\ntime.sleep(${SLEEP_SECONDS})\nprint("slept")`, + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + meterUsage: true, + }) + const wallClockMs = Date.now() - wallClockStartedAtMs + + expect(result.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) + const billed = result.cost?.total ?? 0 + + /** + * The window opens immediately before the provider create call and closes + * before teardown, so it has to cover the sleep and cannot exceed the whole + * call measured from out here. A rate error, a wrong resource constant, or a + * window anchored to the wrong instant all land outside these bounds — which + * an `expect.any(Number)` assertion cannot see. + */ + expect(billed).toBeGreaterThanOrEqual(SLEEP_SECONDS * usdPerBilledSecond) + expect(billed).toBeLessThanOrEqual((wallClockMs / 1000) * usdPerBilledSecond) + }, + CASE_TIMEOUT_MS + ) + + it( + 'bills nothing when the caller did not ask for metering', + async () => { + const result = await executeInSandbox({ + code: 'print("unmetered")', + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + }) + + expect(result.cost).toBeUndefined() + }, + CASE_TIMEOUT_MS + ) +}) From 2beec084e3f09b5e04d91e1a28369aa1cf702b49 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Sat, 29 Aug 2026 12:34:32 -0700 Subject: [PATCH 6/7] fix(billing): meter the sandbox a cloud Pi session runs in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pi's own sandbox was never metered. withPiSandbox called createSandbox without the meterUsage argument, so only sandboxes created through executeFunctionRequest were charged — and Pi's is the larger consumer by an order of magnitude. A Function block holds one for seconds; a Pi session holds one for a minimum lifetime of 31 minutes. The gap was worst exactly where it was least visible. A Pi coding agent normally runs BYOK, so its model cost is zero by definition, and the ledger bills a model row on total > 0. With the sandbox unmetered, such a run produced a zero-cost model_unbilled row and Sim collected only the flat execution fee while paying its provider for the whole session. Threads a cost sink through PiRunContext, which is the seam backends already receive and the only one that reaches all four cloud modes. The handler owns one sink covering both sandbox sources — Function tools in local mode, the agent's own sandbox in cloud mode — so neither can be dropped where the cost is folded into the block's output. It rides in toolCost for the same reason the Function tool cost already does: that is what survives the BYOK zeroing. Unlike the Function path this charges on creation rather than on a completed session. A Function run is seconds long, so absorbing one the provider failed to deliver is cheap and reads as fair; tens of minutes of Pi compute is consumed whether the agent finished, errored, or was cancelled, and billing only clean endings would mean paying for every other one. A create that throws still costs nothing. Co-Authored-By: Claude Opus 5 (1M context) --- .../handlers/pi/cloud/authoring/backend.ts | 5 +- .../handlers/pi/cloud/babysit/backend.ts | 2 +- .../handlers/pi/cloud/plan/backend.ts | 2 +- .../handlers/pi/cloud/review/backend.ts | 2 +- apps/sim/executor/handlers/pi/core/backend.ts | 11 +++++ .../executor/handlers/pi/local/sim-tools.ts | 15 +++--- .../executor/handlers/pi/pi-handler.test.ts | 36 +++++++++++++++ apps/sim/executor/handlers/pi/pi-handler.ts | 46 +++++++++++++------ .../sim/lib/execution/remote-sandbox/index.ts | 19 +++++++- .../sim/lib/execution/remote-sandbox/types.ts | 12 +++++ 10 files changed, 120 insertions(+), 30 deletions(-) diff --git a/apps/sim/executor/handlers/pi/cloud/authoring/backend.ts b/apps/sim/executor/handlers/pi/cloud/authoring/backend.ts index d874c6ab477..40d19e23385 100644 --- a/apps/sim/executor/handlers/pi/cloud/authoring/backend.ts +++ b/apps/sim/executor/handlers/pi/cloud/authoring/backend.ts @@ -435,7 +435,10 @@ async function runCloudAuthoringPi( const lifetimeMs = resolvePiRunLifetimeMs(context.signal) const piTimeoutMs = resolvePiTimeoutMs(lifetimeMs) - const authored = await withPiSandbox({ lifetimeMs }, async (runner) => { + // Bound to a local so the call stays on one line: inlining the second option + // reflows this whole callback body and buries the change in re-indentation. + const sandboxOptions = { lifetimeMs, cost: context.sandboxCost } + const authored = await withPiSandbox(sandboxOptions, async (runner) => { try { const clone = await raceAbort( runner.run(params.mode === 'cloud' ? CREATE_PR_CLONE_SCRIPT : UPDATE_BRANCH_CLONE_SCRIPT, { diff --git a/apps/sim/executor/handlers/pi/cloud/babysit/backend.ts b/apps/sim/executor/handlers/pi/cloud/babysit/backend.ts index 6860dee6a4e..08d4af65ac5 100644 --- a/apps/sim/executor/handlers/pi/cloud/babysit/backend.ts +++ b/apps/sim/executor/handlers/pi/cloud/babysit/backend.ts @@ -784,7 +784,7 @@ export async function runBabysitPiWithOptions( const lifetimeMs = resolvePiRunLifetimeMs(context.signal) const piTimeoutMs = resolvePiTimeoutMs(lifetimeMs) - return await withPiSandbox({ lifetimeMs }, async (runner) => { + return await withPiSandbox({ lifetimeMs, cost: context.sandboxCost }, async (runner) => { const clone = await raceAbort( runner.run(BABYSIT_CLONE_SCRIPT, { envs: { diff --git a/apps/sim/executor/handlers/pi/cloud/plan/backend.ts b/apps/sim/executor/handlers/pi/cloud/plan/backend.ts index 27444a4493b..3997fac8eee 100644 --- a/apps/sim/executor/handlers/pi/cloud/plan/backend.ts +++ b/apps/sim/executor/handlers/pi/cloud/plan/backend.ts @@ -80,7 +80,7 @@ export const runCloudPlanPi: PiBackendRun = async (params, const thinking = mapThinkingLevel(params.thinkingLevel) ?? 'medium' const lifetimeMs = resolvePiRunLifetimeMs(context.signal) - return withPiSandbox({ lifetimeMs }, async (runner) => { + return withPiSandbox({ lifetimeMs, cost: context.sandboxCost }, async (runner) => { try { const clone = await raceAbort( runner.run(PLAN_CLONE_SCRIPT, { diff --git a/apps/sim/executor/handlers/pi/cloud/review/backend.ts b/apps/sim/executor/handlers/pi/cloud/review/backend.ts index cfd25533952..289416ce96b 100644 --- a/apps/sim/executor/handlers/pi/cloud/review/backend.ts +++ b/apps/sim/executor/handlers/pi/cloud/review/backend.ts @@ -218,7 +218,7 @@ export const runCloudReviewPi: PiBackendRun = async (par const lifetimeMs = resolvePiRunLifetimeMs(context.signal) try { - return await withPiSandbox({ lifetimeMs }, async (runner) => { + return await withPiSandbox({ lifetimeMs, cost: context.sandboxCost }, async (runner) => { await runner.writeFile(GIT_ASKPASS_PATH, GIT_ASKPASS_SCRIPT) const fetched = await raceAbort( runner.run(FETCH_PR_SCRIPT, { diff --git a/apps/sim/executor/handlers/pi/core/backend.ts b/apps/sim/executor/handlers/pi/core/backend.ts index 4fb3b3ee3d2..d50db15739b 100644 --- a/apps/sim/executor/handlers/pi/core/backend.ts +++ b/apps/sim/executor/handlers/pi/core/backend.ts @@ -9,6 +9,7 @@ */ import type { TSchema } from 'typebox' +import type { SandboxCostSink } from '@/lib/execution/remote-sandbox/types' import type { SSHConnectionConfig } from '@/lib/internal/ssh/client' import type { Message } from '@/executor/handlers/agent/types' import type { PiEvent, PiRunTotals } from '@/executor/handlers/pi/core/events' @@ -172,6 +173,16 @@ export type PiRunParams = export interface PiRunContext { onEvent: (event: PiEvent) => void signal?: AbortSignal + /** + * Where a backend reports the cost of Sim-provisioned compute it used. + * + * Only the cloud modes have any: they run the agent in a Sim-paid sandbox, + * while local mode drives the caller's own machine over SSH and costs Sim + * nothing. The handler folds whatever lands here into the block's `toolCost`, + * which is what keeps a BYOK Pi run — model unbilled by definition — from + * reporting no cost at all for a session that ran for tens of minutes. + */ + sandboxCost?: SandboxCostSink } /** Final result of a Pi run. */ diff --git a/apps/sim/executor/handlers/pi/local/sim-tools.ts b/apps/sim/executor/handlers/pi/local/sim-tools.ts index be2d41a6fd7..876bb7e99ee 100644 --- a/apps/sim/executor/handlers/pi/local/sim-tools.ts +++ b/apps/sim/executor/handlers/pi/local/sim-tools.ts @@ -9,6 +9,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import type { SandboxCostSink } from '@/lib/execution/remote-sandbox/types' import { readWorkflowInputFieldsForTool, readWorkflowMetadataForTool, @@ -47,10 +48,6 @@ type PiToolResultProjection = | { safe: true; result: PiToolResult } | { safe: false; result: PiToolResult } -export interface PiFunctionToolCostAccumulator { - total: number -} - function projectToolResult( result: ToolResponse, registry: ResolvedSecretTraceRegistry | undefined @@ -112,7 +109,7 @@ function buildSimToolSpec( inputTools: ToolInput[], provider: ProviderToolConfig, toolIndex: number, - functionToolCost?: PiFunctionToolCostAccumulator + sandboxCost?: SandboxCostSink ): PiToolSpec { const toolId = provider.canonicalId ?? provider.id const preseededParams = provider.params || {} @@ -182,12 +179,12 @@ function buildSimToolSpec( : undefined if ( toolId === 'function_execute' && - functionToolCost && + sandboxCost && typeof resultCostTotal === 'number' && Number.isFinite(resultCostTotal) && resultCostTotal > 0 ) { - functionToolCost.total += resultCostTotal + sandboxCost.total += resultCostTotal } const projection = projectToolResult(result, toolCallRegistry?.forkForPropagatedEntries()) if (projection.safe && registry && toolCallRegistry?.isComplete()) { @@ -219,7 +216,7 @@ function buildSimToolSpec( export async function buildSimToolSpecs( ctx: ExecutionContext, inputTools: unknown, - functionToolCost?: PiFunctionToolCostAccumulator + sandboxCost?: SandboxCostSink ): Promise { if (!Array.isArray(inputTools)) return [] @@ -263,6 +260,6 @@ export async function buildSimToolSpecs( await annotateDuplicateToolBindings(ctx, providers) assignProviderToolIdentities(providers) return configuredTools.map(({ provider, toolIndex }) => - buildSimToolSpec(ctx, inputTools, provider, toolIndex, functionToolCost) + buildSimToolSpec(ctx, inputTools, provider, toolIndex, sandboxCost) ) } diff --git a/apps/sim/executor/handlers/pi/pi-handler.test.ts b/apps/sim/executor/handlers/pi/pi-handler.test.ts index aee9b0fd0f6..1d4f42a3235 100644 --- a/apps/sim/executor/handlers/pi/pi-handler.test.ts +++ b/apps/sim/executor/handlers/pi/pi-handler.test.ts @@ -111,6 +111,7 @@ vi.mock('@/blocks/utils', () => ({ }, })) +import type { PiRunContext } from '@/executor/handlers/pi/core/backend' import { PiBlockHandler, parsePiReviewMentions } from '@/executor/handlers/pi/pi-handler' import type { ExecutionContext, StreamingExecution } from '@/executor/types' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -291,6 +292,41 @@ describe('PiBlockHandler', () => { expect(output.cost).toEqual({ input: 0, output: 0, toolCost: 0.125, total: 0.125 }) }) + it('bills the cloud sandbox a Pi session ran in, even when the model is BYOK', async () => { + // The regression this guards: the agent's own sandbox runs on Sim's provider + // account, so a BYOK run whose model cost is zero by definition would + // otherwise report no cost at all for tens of minutes of paid compute. + mockRunCloud.mockImplementation(async (_params: unknown, context: PiRunContext) => { + if (context.sandboxCost) context.sandboxCost.total += 0.0842 + return { totals: { finalText: 'done', inputTokens: 0, outputTokens: 0 } } + }) + + const output = (await handler.execute(ctx(), block, { + mode: 'cloud', + task: 'do it', + model: 'claude', + owner: 'o', + repo: 'r', + githubToken: 'ghp', + })) as { cost: unknown } + + expect(output.cost).toEqual({ input: 0, output: 0, toolCost: 0.0842, total: 0.0842 }) + }) + + it('leaves a cloud run that provisioned no sandbox uncharged', async () => { + const output = (await handler.execute(ctx(), block, { + mode: 'cloud', + task: 'do it', + model: 'claude', + owner: 'o', + repo: 'r', + githubToken: 'ghp', + })) as { cost: Record } + + expect(output.cost.toolCost).toBeUndefined() + expect(output.cost.total).toBe(0) + }) + it('routes Create PR to the cloud backend and surfaces PR output', async () => { const output = (await handler.execute(ctx(), block, { mode: 'cloud', diff --git a/apps/sim/executor/handlers/pi/pi-handler.ts b/apps/sim/executor/handlers/pi/pi-handler.ts index d152fa52cf4..319964e46a8 100644 --- a/apps/sim/executor/handlers/pi/pi-handler.ts +++ b/apps/sim/executor/handlers/pi/pi-handler.ts @@ -8,6 +8,7 @@ import { createLogger } from '@sim/logger' import { projectResolvedModelInput } from '@/lib/execution/model-input-provenance' +import type { SandboxCostSink } from '@/lib/execution/remote-sandbox/types' import type { BlockOutput } from '@/blocks/types' import { parseOptionalNumberInput } from '@/blocks/utils' import { @@ -45,10 +46,7 @@ import { resolvePiSearchKey, } from '@/executor/handlers/pi/core/keys' import { runLocalPi } from '@/executor/handlers/pi/local/backend' -import { - buildSimToolSpecs, - type PiFunctionToolCostAccumulator, -} from '@/executor/handlers/pi/local/sim-tools' +import { buildSimToolSpecs } from '@/executor/handlers/pi/local/sim-tools' import { buildPiSearchToolSpec } from '@/executor/handlers/pi/search/tool' import type { BlockHandler, @@ -270,8 +268,8 @@ export class PiBlockHandler implements BlockHandler { } const usePrivateKey = inputs.authMethod === 'privateKey' const port = parseOptionalNumberInput(inputs.port, 'port', { integer: true, min: 1 }) ?? 22 - const functionToolCost: PiFunctionToolCostAccumulator = { total: 0 } - const tools = await buildSimToolSpecs(ctx, inputs.tools, functionToolCost) + const sandboxCost: SandboxCostSink = { total: 0 } + const tools = await buildSimToolSpecs(ctx, inputs.tools, sandboxCost) const params: PiLocalRunParams = { ...contextualBase, mode: 'local', @@ -286,7 +284,7 @@ export class PiBlockHandler implements BlockHandler { passphrase: usePrivateKey ? asRawString(inputs.passphrase) : undefined, }, } - return this.runPi(ctx, block, runLocalPi, params, memoryConfig, functionToolCost) + return this.runPi(ctx, block, runLocalPi, params, memoryConfig, sandboxCost) } const owner = asOptString(inputs.owner) @@ -478,17 +476,23 @@ export class PiBlockHandler implements BlockHandler { isBYOK: boolean, startTime: number, startTimeISO: string, - functionToolCost = 0 + sandboxCost = 0 ): NormalizedBlockOutput { const { totals } = result const endTime = Date.now() const modelCost = computePiCost(model, totals.inputTokens, totals.outputTokens, isBYOK) + /* + * Sandbox compute rides in `toolCost` so it survives a BYOK run: the model + * side is zero by definition there, and the ledger bills a model row on + * `total > 0`. Folding it in is what makes a BYOK Pi session bill for the + * E2B time it actually consumed instead of nothing at all. + */ const cost = - functionToolCost > 0 + sandboxCost > 0 ? { ...modelCost, - toolCost: functionToolCost, - total: modelCost.total + functionToolCost, + toolCost: sandboxCost, + total: modelCost.total + sandboxCost, } : modelCost return { @@ -534,7 +538,14 @@ export class PiBlockHandler implements BlockHandler { backend: PiBackendRun

, params: P, memoryConfig?: PiMemoryConfig, - functionToolCost?: PiFunctionToolCostAccumulator + /** + * One sink for every Sim-paid sandbox this block touches. Local mode fills it + * from the Function tools it runs host-side; cloud modes fill it from the + * sandbox the agent itself runs in. They are mutually exclusive in practice, + * and sharing one total means neither can be forgotten at the point the cost + * is folded into the block's output. + */ + sandboxCost: SandboxCostSink = { total: 0 } ): Promise { const startTime = Date.now() const startTimeISO = new Date(startTime).toISOString() @@ -560,6 +571,7 @@ export class PiBlockHandler implements BlockHandler { if (text) controller.enqueue(encoder.encode(text)) }, signal: ctx.abortSignal, + sandboxCost, }) if (result.totals.errorMessage) { controller.error(new Error(result.totals.errorMessage)) @@ -577,7 +589,7 @@ export class PiBlockHandler implements BlockHandler { params.isBYOK, startTime, startTimeISO, - functionToolCost?.total + sandboxCost.total ) ) if (memoryConfig) { @@ -608,7 +620,11 @@ export class PiBlockHandler implements BlockHandler { } } - const result = await backend(params, { onEvent: () => {}, signal: ctx.abortSignal }) + const result = await backend(params, { + onEvent: () => {}, + signal: ctx.abortSignal, + sandboxCost, + }) if (result.totals.errorMessage) { throw new Error(result.totals.errorMessage) } @@ -627,7 +643,7 @@ export class PiBlockHandler implements BlockHandler { params.isBYOK, startTime, startTimeISO, - functionToolCost?.total + sandboxCost.total ) } } diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index 3c962009d9f..57432197bb8 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -45,6 +45,7 @@ import type { SandboxCodeResult, SandboxCollectedFile, SandboxCommandResult, + SandboxCostSink, SandboxDirectoryEntry, SandboxExecutionCost, SandboxExecutionRequest, @@ -59,6 +60,7 @@ import type { } from '@/lib/execution/remote-sandbox/types' export type { + SandboxCostSink, SandboxExecutionRequest, SandboxExecutionResult, SandboxFile, @@ -1117,12 +1119,13 @@ export interface PiSandboxRunner { * caller's sandbox body, which would have buried the change in whitespace. */ export async function withPiSandbox( - options: { lifetimeMs?: number }, + options: { lifetimeMs?: number; cost?: SandboxCostSink }, fn: (runner: PiSandboxRunner) => Promise ): Promise { const lifetimeMs = options.lifetimeMs !== undefined ? options.lifetimeMs : resolvePiSandboxLifetimeMs() - const { sandbox } = await createSandbox('pi', { lifetimeMs }) + const created = await createSandbox('pi', { lifetimeMs }, Boolean(options.cost)) + const { sandbox } = created logger.info('Started Pi sandbox', { sandboxId: sandbox.sandboxId, lifetimeMs }) const runner: PiSandboxRunner = { @@ -1142,6 +1145,18 @@ export async function withPiSandbox( try { return await fn(runner) } finally { + /* + * Charged on creation rather than on a successful session, which is where + * this departs from the Function path — and deliberately. A Function run is + * seconds long, so absorbing one that the provider failed to deliver costs + * little and reads as fair. A Pi session holds its sandbox for tens of + * minutes, and that compute is consumed whether the agent finished, errored, + * or was cancelled. Billing only the clean endings would mean paying for + * every other one. A create that throws never reaches here, so the one case + * where nothing was provisioned is still free. + */ + const cost = calculateSandboxCost(created, Date.now()) + if (cost && options.cost) options.cost.total += cost.total try { await sandbox.kill() } catch { diff --git a/apps/sim/lib/execution/remote-sandbox/types.ts b/apps/sim/lib/execution/remote-sandbox/types.ts index 5c21d0c1032..10732ad35ae 100644 --- a/apps/sim/lib/execution/remote-sandbox/types.ts +++ b/apps/sim/lib/execution/remote-sandbox/types.ts @@ -109,6 +109,18 @@ export interface SandboxExecutionCost { total: number } +/** + * Running total a caller accumulates sandbox charges into. + * + * A long-lived sandbox reports its cost when it is torn down, which is after the + * value its caller cares about has already been returned. Handing the layer a + * sink lets the charge land without reshaping every return type between here and + * the block that owns the bill. + */ +export interface SandboxCostSink { + total: number +} + export interface SandboxExecutionResult { result: unknown stdout: string From 8b1e69f83411be75a0651a3f7814a0a2f97691cc Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Sat, 29 Aug 2026 12:38:10 -0700 Subject: [PATCH 7/7] test(billing): check the Pi sandbox charge against a real provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The handler test mocks the backend and writes into the cost sink by hand, so it proves the wiring from a backend to the block's cost and nothing more — it would still pass if withPiSandbox never metered at all, which is precisely the bug that path had. Holds a real Pi sandbox open for a known interval and bounds the charge between what that interval must cost and what the whole session could justify. Verified to fail against the original unmetered call with "expected 0 to be greater than or equal to 0.00023", and to pass once the sink is threaded: 5.949s billed of a 6.141s session on E2B. The second case pins the other half of the contract — a caller that supplies no sink is not charged, which is what keeps mothership and other internal Pi sandboxes free. Co-Authored-By: Claude Opus 5 (1M context) --- .../pi-sandbox-billing.smoke.test.ts | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 apps/sim/lib/execution/remote-sandbox/pi-sandbox-billing.smoke.test.ts diff --git a/apps/sim/lib/execution/remote-sandbox/pi-sandbox-billing.smoke.test.ts b/apps/sim/lib/execution/remote-sandbox/pi-sandbox-billing.smoke.test.ts new file mode 100644 index 00000000000..927197c5252 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/pi-sandbox-billing.smoke.test.ts @@ -0,0 +1,76 @@ +/** + * @vitest-environment node + * + * Checks that a Pi session's sandbox is actually metered against a real provider. + * + * The handler-level test mocks the backend and writes into the sink by hand, so + * it proves the wiring from a backend to the block's cost and nothing else. It + * would still pass if `withPiSandbox` never metered at all — which is exactly + * the bug this path had. Only a real Pi sandbox shows that creation is metered, + * that teardown reports, and that the amount tracks the session's real lifetime. + * + * Enable with `SANDBOX_BILLING_SMOKE=1`, against whichever provider + * `SANDBOX_PROVIDER` selects. Needs that provider's Pi image configured + * (`E2B_PI_TEMPLATE_ID` / `DAYTONA_PI_SNAPSHOT_ID`). + */ +import { describe, expect, it } from 'vitest' +import { createSandboxPricing } from '@/lib/billing/sandbox-pricing' +import { withPiSandbox } from '@/lib/execution/remote-sandbox' +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' +import type { SandboxCostSink } from '@/lib/execution/remote-sandbox/types' + +const smokeEnabled = process.env.SANDBOX_BILLING_SMOKE === '1' +const CASE_TIMEOUT_MS = 5 * 60_000 + +/** Long enough that provisioning jitter cannot dominate the measured session. */ +const SLEEP_SECONDS = 5 +/** Well under any provider ceiling, so the lifetime cap never clamps the charge. */ +const LIFETIME_MS = 10 * 60_000 + +describe.skipIf(!smokeEnabled)('pi sandbox billing smoke', () => { + it( + 'bills the session a Pi sandbox was held for', + async () => { + const pricing = createSandboxPricing(resolveProvider().id) + const usdPerBilledSecond = + (pricing.resources.vcpu * pricing.rates.cpuUsdPerVcpuSecond + + pricing.resources.memoryGiB * pricing.rates.memoryUsdPerGiBSecond + + pricing.resources.diskGiB * pricing.rates.diskUsdPerGiBSecond) * + pricing.multiplier + + const sandboxCost: SandboxCostSink = { total: 0 } + const wallClockStartedAtMs = Date.now() + const exitCode = await withPiSandbox( + { lifetimeMs: LIFETIME_MS, cost: sandboxCost }, + async (runner) => { + const result = await runner.run(`sleep ${SLEEP_SECONDS}; echo held`, { + envs: {}, + timeoutMs: CASE_TIMEOUT_MS, + }) + return result.exitCode + } + ) + const wallClockMs = Date.now() - wallClockStartedAtMs + + expect(exitCode).toBe(0) + expect(sandboxCost.total).toBeGreaterThanOrEqual(SLEEP_SECONDS * usdPerBilledSecond) + expect(sandboxCost.total).toBeLessThanOrEqual((wallClockMs / 1000) * usdPerBilledSecond) + }, + CASE_TIMEOUT_MS + ) + + it( + 'bills nothing when no sink is supplied', + async () => { + // The mothership and any other internal caller must stay free, and the + // absence of a sink is the whole mechanism keeping them that way. + const held = await withPiSandbox({ lifetimeMs: LIFETIME_MS }, async (runner) => { + const result = await runner.run('echo held', { envs: {}, timeoutMs: CASE_TIMEOUT_MS }) + return result.exitCode + }) + + expect(held).toBe(0) + }, + CASE_TIMEOUT_MS + ) +})