diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index f20cae5f7d6..4d3fa2dbde4 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -245,4 +245,97 @@ jobs: AWS_REGION: 'us-west-2' ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only TURBO_CACHE_DIR: .turbo - run: bunx turbo run build --filter=sim \ No newline at end of file + run: bunx turbo run build --filter=sim + + settings-e2e: + name: Settings E2E (informational) + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + continue-on-error: true + + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: 22 + + - name: Mount Bun cache (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ~/.bun/install/cache + + - name: Mount node_modules (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ./node_modules + + - name: Mount Playwright browsers (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-playwright-browsers-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ~/.cache/ms-playwright + + - name: Restore E2E Next.js build cache + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: ./apps/sim/.next/cache + key: ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Configure E2E hostname + run: echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts + + - name: Install Chromium + working-directory: apps/sim + run: bun run test:e2e:install-browsers -- --with-deps + + - name: Run settings E2E foundation + timeout-minutes: 40 + working-directory: apps/sim + env: + CI: 'true' + E2E_PG_ADMIN_URL: 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' + run: bun run test:e2e + + - name: Upload E2E diagnostics + if: failure() || cancelled() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: settings-e2e-${{ github.run_id }} + path: | + apps/sim/playwright-report/ + apps/sim/test-results/ + apps/sim/e2e/.runs/ + !apps/sim/e2e/.runs/**/auth/** + if-no-files-found: ignore + include-hidden-files: true + retention-days: 7 \ No newline at end of file diff --git a/.gitignore b/.gitignore index a8a0d8e2fde..dc42d1c8e3f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,10 @@ package-lock.json # testing /coverage /apps/**/coverage +/apps/**/playwright-report/ +/apps/**/test-results/ +/apps/**/e2e/.runs/ +/apps/**/e2e/.auth/ # next.js /.next/ @@ -31,6 +35,7 @@ package-lock.json **/dist/ **/standalone/ sim-standalone.tar.gz +/.artifacts/ # redis dump.rdb diff --git a/apps/realtime/src/env.ts b/apps/realtime/src/env.ts index 5afdf3a20f3..2f75c4a595f 100644 --- a/apps/realtime/src/env.ts +++ b/apps/realtime/src/env.ts @@ -14,6 +14,7 @@ const EnvSchema = z.object({ INTERNAL_API_SECRET: z.string().min(32), NEXT_PUBLIC_APP_URL: z.string().url(), ALLOWED_ORIGINS: z.string().optional(), + REALTIME_HOST: z.string().min(1).default('0.0.0.0'), PORT: z.coerce.number().int().positive().default(3002), SIM_DB_ROLE: z.enum(['web', 'trigger', 'realtime']).optional(), DISABLE_AUTH: z diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index e43184c1f02..0c84867156b 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -31,6 +31,7 @@ async function createRoomManager(io: SocketIOServer): Promise { async function main() { const httpServer = createServer() const PORT = env.PORT + const HOST = env.REALTIME_HOST logger.info('Starting Socket.IO server...', { port: PORT, @@ -96,9 +97,9 @@ async function main() { await assertSchemaCompatibility() - httpServer.listen(PORT, '0.0.0.0', () => { - logger.info(`Socket.IO server running on port ${PORT}`) - logger.info(`Health check available at: http://localhost:${PORT}/health`) + httpServer.listen(PORT, HOST, () => { + logger.info(`Socket.IO server running on ${HOST}:${PORT}`) + logger.info(`Health check available at: http://${HOST}:${PORT}/health`) }) const shutdown = async () => { diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 0f8ed73cc52..0bfef282fcf 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -69,6 +69,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { status: 'ok', timestamp: new Date().toISOString(), connections, + ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}), }) ) } catch (error) { diff --git a/apps/sim/app/api/health/route.test.ts b/apps/sim/app/api/health/route.test.ts index 6abca827579..21d4d9d6a91 100644 --- a/apps/sim/app/api/health/route.test.ts +++ b/apps/sim/app/api/health/route.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { GET } from '@/app/api/health/route' describe('GET /api/health', () => { @@ -14,4 +14,17 @@ describe('GET /api/health', () => { timestamp: expect.any(String), }) }) + + it('returns the E2E run identity when configured', async () => { + vi.stubEnv('E2E_RUN_ID', 'run-health-check') + try { + const response = await GET() + await expect(response.json()).resolves.toMatchObject({ + status: 'ok', + runId: 'run-health-check', + }) + } finally { + vi.unstubAllEnvs() + } + }) }) diff --git a/apps/sim/app/api/health/route.ts b/apps/sim/app/api/health/route.ts index 5486272998c..517f9251724 100644 --- a/apps/sim/app/api/health/route.ts +++ b/apps/sim/app/api/health/route.ts @@ -1,11 +1,14 @@ /** * Health check endpoint for deployment platforms and container probes. */ +export const dynamic = 'force-dynamic' + export async function GET(): Promise { return Response.json( { status: 'ok', timestamp: new Date().toISOString(), + ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}), }, { status: 200 } ) diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md new file mode 100644 index 00000000000..49f108e8dd7 --- /dev/null +++ b/apps/sim/e2e/README.md @@ -0,0 +1,100 @@ +# Sim browser E2E + +This directory contains the reusable full-stack Playwright harness. It runs the +production Next.js app, realtime, deterministic external fakes, and a migrated +per-run pgvector database. + +## One-time setup + +0. Install Node 22 and Bun. Playwright workers require Node 22; set + `E2E_NODE_BINARY` to an alternate Node 22 executable when `node` on `PATH` + points elsewhere. + +1. Map the hosted E2E origin to loopback: + + ```bash + echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts + ``` + + The runner refuses to start unless every resolved address is loopback and an + IPv4 `127.0.0.1` result is present. Chromium and Node are configured to prefer + that IPv4 mapping, so CI environments that also synthesize `::1` remain safe. + +2. Start a local pgvector/Postgres admin instance: + + ```bash + docker run --rm --name sim-e2e-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=postgres \ + -p 5432:5432 \ + pgvector/pgvector:pg17 + ``` + + Cleanup uses `DROP DATABASE ... WITH (FORCE)`, which requires PostgreSQL 13 + or newer and is supported by the pinned pgvector/PostgreSQL 17 image. + +3. Install Chromium from `apps/sim`: + + ```bash + bun run test:e2e:install-browsers + ``` + +## Run the foundation + +From `apps/sim`: + +```bash +E2E_PG_ADMIN_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \ + bun run test:e2e +``` + +The runner creates a unique `sim_e2e_` database, migrates it, starts the +Stripe fake and realtime, builds and starts Next.js, runs Playwright on Node 22, +then stops services and drops only that guarded database. + +On interruption, the runner launches a detached cleanup supervisor before +exiting. It terminates managed process groups, force-drops the guarded database, +and removes temporary auth/cloud-config directories even if another Ctrl-C +terminates the foreground package runner. + +Pass Playwright arguments after `--`: + +```bash +bun run test:e2e -- --project=hosted-billing-chromium-navigation +bun run test:e2e -- --grep "unauthenticated" +``` + +Do not invoke `playwright test` directly. Raw Playwright bypasses environment, +database, process, sharding, and teardown guards; the config rejects runs that +were not launched by the orchestrator. Report and trace viewer commands remain +safe because they do not execute tests. + +Sharding is supported only for the navigation project. The runner rejects +`--shard` for `hosted-billing-chromium-workflows`. + +## Diagnostics + +- HTML report: `playwright-report/` +- Traces and screenshots: `test-results/` +- App, realtime, migration, and fake logs: `e2e/.runs//logs/` + +Open the report: + +```bash +node ../../node_modules/@playwright/test/cli.js show-report playwright-report +``` + +Open a trace: + +```bash +node ../../node_modules/@playwright/test/cli.js show-trace test-results//trace.zip +``` + +The runner starts every child process from a fresh environment. It allowlists +only deterministic E2E values and shadows keys found in local `.env*` files, so +developer credentials are not used as test state or written to reports. + +Provider log scans are diagnostic tripwires, not proof of zero egress. The +primary boundaries are the default-deny child environment, provider disabling, +loopback-only service bindings, and guarded Stripe transport. diff --git a/apps/sim/e2e/fakes/stripe/server.ts b/apps/sim/e2e/fakes/stripe/server.ts new file mode 100644 index 00000000000..8f89c48102e --- /dev/null +++ b/apps/sim/e2e/fakes/stripe/server.ts @@ -0,0 +1,530 @@ +import { createHash, timingSafeEqual } from 'node:crypto' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' +import type { AddressInfo } from 'node:net' + +export const STRIPE_FAKE_ENDPOINTS = { + health: '/health', + requestLog: '/__control/requests', + reset: '/__control/reset', +} as const + +const DEFAULT_MAX_BODY_BYTES = 64 * 1024 +const FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded' + +export interface StripeFakeRequestRecord { + sequence: number + method: string + path: string + unexpected: boolean +} + +export interface StripeFakeServerOptions { + apiKey: string + hostname?: '127.0.0.1' + maxBodyBytes?: number + port?: number +} + +export interface StripeFakeServer { + readonly baseUrl: string | null + readonly requestLog: readonly StripeFakeRequestRecord[] + start(): Promise + stop(): Promise + reset(): void +} + +interface FakeCustomer { + id: string + object: 'customer' + created: number + email: string | null + livemode: false + metadata: Record + name: string | null +} + +interface StripeError { + type: 'api_error' | 'invalid_request_error' + code: string + message: string +} + +class RequestBodyError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + } +} + +function isExpectedStripeRequest(method: string, path: string): boolean { + return ( + ((method === 'GET' || method === 'POST') && path === '/v1/customers/search') || + (method === 'GET' && path === '/v1/customers') || + (method === 'POST' && path === '/v1/customers') || + (method === 'GET' && /^\/v1\/customers\/cus_e2e_[a-f0-9]+$/.test(path)) + ) +} + +function secureEqual(actual: string | undefined, expected: string): boolean { + if (actual === undefined) return false + + const actualBuffer = Buffer.from(actual) + const expectedBuffer = Buffer.from(expected) + return ( + actualBuffer.length === expectedBuffer.length && timingSafeEqual(actualBuffer, expectedBuffer) + ) +} + +function cloneRequestLog(records: StripeFakeRequestRecord[]): StripeFakeRequestRecord[] { + return structuredClone(records) +} + +function sendJson( + response: ServerResponse, + status: number, + body: unknown, + requestId?: string +): void { + const serialized = JSON.stringify(body) + response.writeHead(status, { + 'content-length': Buffer.byteLength(serialized), + 'content-type': 'application/json; charset=utf-8', + ...(requestId ? { 'request-id': requestId } : {}), + }) + response.end(serialized) +} + +function sendStripeError( + response: ServerResponse, + status: number, + error: StripeError, + requestId: string +): void { + sendJson(response, status, { error }, requestId) +} + +async function readRequestBody(request: IncomingMessage, maxBodyBytes: number): Promise { + const chunks: Buffer[] = [] + let totalBytes = 0 + + for await (const chunk of request) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + totalBytes += buffer.length + if (totalBytes > maxBodyBytes) { + throw new RequestBodyError('Request body exceeds the Stripe fake limit', 413) + } + chunks.push(buffer) + } + + return Buffer.concat(chunks).toString('utf8') +} + +function parseFormBody(request: IncomingMessage, rawBody: string): URLSearchParams { + const contentType = request.headers['content-type']?.split(';', 1)[0]?.trim().toLowerCase() + if (contentType !== FORM_CONTENT_TYPE) { + throw new RequestBodyError(`Stripe fake requires ${FORM_CONTENT_TYPE}`, 400) + } + return new URLSearchParams(rawBody) +} + +function metadataFrom(parameters: URLSearchParams): Record { + const metadata: Record = {} + for (const [key, value] of parameters) { + const match = /^metadata\[([^\]]+)\]$/.exec(key) + if (match) metadata[match[1]] = value + } + return metadata +} + +function stableCustomerIdentity(parameters: URLSearchParams): string { + const metadata = metadataFrom(parameters) + const preferredIdentity = + metadata.userId || metadata.organizationId || parameters.get('email') || undefined + if (preferredIdentity) return preferredIdentity + + return [...parameters.entries()] + .sort(([leftKey, leftValue], [rightKey, rightValue]) => + `${leftKey}\u0000${leftValue}`.localeCompare(`${rightKey}\u0000${rightValue}`) + ) + .map(([key, value]) => `${key}=${value}`) + .join('&') +} + +function createDeterministicCustomer(parameters: URLSearchParams): FakeCustomer { + const digest = createHash('sha256').update(stableCustomerIdentity(parameters)).digest('hex') + return { + id: `cus_e2e_${digest.slice(0, 24)}`, + object: 'customer', + created: 1_700_000_000 + (Number.parseInt(digest.slice(0, 6), 16) % 1_000_000), + email: parameters.get('email'), + livemode: false, + metadata: metadataFrom(parameters), + name: parameters.get('name'), + } +} + +function unescapeSearchValue(value: string): string { + return value.replace(/\\(["\\])/g, '$1') +} + +function parseSupportedCustomerSearchEmail(query: string): string { + const match = /^email:"((?:\\.|[^"])*)" AND -metadata\["customerType"\]:"organization"$/.exec( + query + ) + if (!match) { + throw new RequestBodyError( + `Stripe fake does not implement customer search query: ${query}`, + 501 + ) + } + return unescapeSearchValue(match[1]) +} + +function parseLimit(parameters: URLSearchParams): number { + const rawLimit = parameters.get('limit') + if (rawLimit === null) return 10 + + const limit = Number(rawLimit) + if (!Number.isInteger(limit) || limit < 1 || limit > 100) { + throw new RequestBodyError('Stripe fake requires limit to be an integer from 1 to 100', 400) + } + return limit +} + +function validateTestApiKey(apiKey: string): void { + if (!apiKey.startsWith('sk_test_') || apiKey.length === 'sk_test_'.length) { + throw new Error('Stripe fake apiKey must be a non-empty sk_test_ secret key') + } +} + +function validateServerOptions(options: StripeFakeServerOptions): void { + validateTestApiKey(options.apiKey) + if ( + options.port !== undefined && + (!Number.isInteger(options.port) || options.port < 0 || options.port > 65535) + ) { + throw new Error('Stripe fake port must be an integer from 0 to 65535') + } + if ( + options.maxBodyBytes !== undefined && + (!Number.isInteger(options.maxBodyBytes) || options.maxBodyBytes < 1) + ) { + throw new Error('Stripe fake maxBodyBytes must be a positive integer') + } +} + +/** + * Creates a loopback-only Stripe fake. The returned server is inert until start() + * is called, allowing an orchestrator to own lifecycle and failure handling. + */ +export function createStripeFakeServer(options: StripeFakeServerOptions): StripeFakeServer { + validateServerOptions(options) + + const hostname = options.hostname ?? '127.0.0.1' + const port = options.port ?? 0 + const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES + const expectedAuthorization = `Bearer ${options.apiKey}` + const records: StripeFakeRequestRecord[] = [] + const customers = new Map() + let sequence = 0 + let baseUrl: string | null = null + + const reset = (): void => { + records.length = 0 + customers.clear() + sequence = 0 + } + + const handleControlRequest = ( + request: IncomingMessage, + response: ServerResponse, + method: string, + path: string + ): boolean => { + if (method === 'GET' && path === STRIPE_FAKE_ENDPOINTS.health) { + sendJson(response, 200, { status: 'ok', service: 'stripe-fake' }) + return true + } + + const isRequestLog = method === 'GET' && path === STRIPE_FAKE_ENDPOINTS.requestLog + const isReset = method === 'POST' && path === STRIPE_FAKE_ENDPOINTS.reset + if (!isRequestLog && !isReset) return false + + if (!secureEqual(request.headers.authorization, expectedAuthorization)) { + sendStripeError( + response, + 401, + { + type: 'invalid_request_error', + code: 'api_key_invalid', + message: 'Invalid test Bearer authorization for Stripe fake control endpoint.', + }, + 'req_e2e_control' + ) + return true + } + + if (isRequestLog) { + sendJson(response, 200, { requests: cloneRequestLog(records) }) + } else { + reset() + sendJson(response, 200, { reset: true }) + } + return true + } + + const handleStripeRequest = async ( + request: IncomingMessage, + response: ServerResponse, + method: string, + url: URL + ): Promise => { + sequence += 1 + const requestId = `req_e2e_${String(sequence).padStart(6, '0')}` + const expected = isExpectedStripeRequest(method, url.pathname) + let formBody: URLSearchParams | null = null + + try { + if (method !== 'GET' && method !== 'HEAD') { + const rawBody = await readRequestBody(request, maxBodyBytes) + if (rawBody) { + if (request.headers['content-type']?.startsWith(FORM_CONTENT_TYPE)) { + formBody = new URLSearchParams(rawBody) + } + } + } + } catch (error) { + records.push({ + sequence, + method, + path: url.pathname, + unexpected: !expected, + }) + const bodyError = + error instanceof RequestBodyError + ? error + : new RequestBodyError('Unable to read Stripe fake request body', 400) + sendStripeError( + response, + bodyError.status, + { + type: 'invalid_request_error', + code: 'invalid_request_body', + message: bodyError.message, + }, + requestId + ) + return + } + + records.push({ + sequence, + method, + path: url.pathname, + unexpected: !expected, + }) + + if (!secureEqual(request.headers.authorization, expectedAuthorization)) { + sendStripeError( + response, + 401, + { + type: 'invalid_request_error', + code: 'api_key_invalid', + message: 'Invalid test Bearer authorization for Stripe fake.', + }, + requestId + ) + return + } + + if (!expected) { + sendStripeError( + response, + 501, + { + type: 'api_error', + code: 'stripe_fake_unimplemented', + message: `Stripe fake does not implement ${method} ${url.pathname}.`, + }, + requestId + ) + return + } + + try { + if ((method === 'GET' || method === 'POST') && url.pathname === '/v1/customers/search') { + const parameters = + method === 'GET' ? url.searchParams : (formBody ?? parseFormBody(request, '')) + const query = parameters.get('query') + if (!query) throw new RequestBodyError('Stripe fake customer search requires query', 400) + const email = parseSupportedCustomerSearchEmail(query) + + const data = [...customers.values()] + .filter( + (customer) => + customer.email === email && customer.metadata.customerType !== 'organization' + ) + .slice(0, parseLimit(parameters)) + sendJson( + response, + 200, + { + object: 'search_result', + data, + has_more: false, + next_page: null, + url: '/v1/customers/search', + }, + requestId + ) + return + } + + if (method === 'GET' && url.pathname === '/v1/customers') { + const email = url.searchParams.get('email') + const data = [...customers.values()] + .filter((customer) => email === null || customer.email === email) + .slice(0, parseLimit(url.searchParams)) + sendJson( + response, + 200, + { + object: 'list', + data, + has_more: false, + url: '/v1/customers', + }, + requestId + ) + return + } + + if (method === 'GET' && url.pathname.startsWith('/v1/customers/')) { + const customerId = decodeURIComponent(url.pathname.slice('/v1/customers/'.length)) + const customer = customers.get(customerId) + if (!customer) { + sendStripeError( + response, + 404, + { + type: 'invalid_request_error', + code: 'resource_missing', + message: `No such customer: ${customerId}`, + }, + requestId + ) + return + } + sendJson(response, 200, customer, requestId) + return + } + + const parameters = formBody ?? parseFormBody(request, '') + const candidate = createDeterministicCustomer(parameters) + const customer = customers.get(candidate.id) ?? candidate + customers.set(customer.id, customer) + sendJson(response, 200, customer, requestId) + } catch (error) { + const bodyError = + error instanceof RequestBodyError + ? error + : new RequestBodyError('Stripe fake rejected malformed request parameters', 400) + if (bodyError.status === 501) { + const recorded = records.at(-1) + if (recorded) recorded.unexpected = true + } + sendStripeError( + response, + bodyError.status, + { + type: 'invalid_request_error', + code: 'invalid_request', + message: bodyError.message, + }, + requestId + ) + } + } + + const server: Server = createServer((request, response) => { + const method = request.method?.toUpperCase() ?? 'UNKNOWN' + const url = new URL(request.url ?? '/', 'http://stripe-fake.invalid') + if (handleControlRequest(request, response, method, url.pathname)) return + + void handleStripeRequest(request, response, method, url).catch(() => { + if (!response.headersSent) { + sendStripeError( + response, + 500, + { + type: 'api_error', + code: 'stripe_fake_internal_error', + message: 'Stripe fake encountered an internal error.', + }, + 'req_e2e_internal' + ) + } else { + response.destroy() + } + }) + }) + + return { + get baseUrl() { + return baseUrl + }, + get requestLog() { + return cloneRequestLog(records) + }, + async start() { + if (baseUrl) return baseUrl + + await new Promise((resolve, reject) => { + const handleError = (error: Error) => { + server.off('listening', handleListening) + reject(error) + } + const handleListening = () => { + server.off('error', handleError) + resolve() + } + server.once('error', handleError) + server.once('listening', handleListening) + server.listen(port, hostname) + }) + + const address = server.address() as AddressInfo + const urlHostname = address.family === 'IPv6' ? `[${address.address}]` : address.address + baseUrl = `http://${urlHostname}:${address.port}` + return baseUrl + }, + async stop() { + if (!server.listening) { + baseUrl = null + return + } + + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) reject(error) + else resolve() + }) + server.closeIdleConnections() + }) + baseUrl = null + }, + reset, + } +} + +/** Starts a Stripe fake in one call for orchestration code. */ +export async function startStripeFakeServer( + options: StripeFakeServerOptions +): Promise { + const server = createStripeFakeServer(options) + await server.start() + return server +} diff --git a/apps/sim/e2e/foundation/provider-isolation.spec.ts b/apps/sim/e2e/foundation/provider-isolation.spec.ts new file mode 100644 index 00000000000..f31c7a62644 --- /dev/null +++ b/apps/sim/e2e/foundation/provider-isolation.spec.ts @@ -0,0 +1,7 @@ +import { expect, test } from '@playwright/test' + +test('blacklisted local providers skip outbound model discovery', async ({ request }) => { + const response = await request.get('/api/providers/ollama/models') + expect(response.status()).toBe(200) + expect(await response.json()).toEqual({ models: [] }) +}) diff --git a/apps/sim/e2e/foundation/runtime.spec.ts b/apps/sim/e2e/foundation/runtime.spec.ts new file mode 100644 index 00000000000..0e49d712bb7 --- /dev/null +++ b/apps/sim/e2e/foundation/runtime.spec.ts @@ -0,0 +1,8 @@ +import { expect, test } from '@playwright/test' + +test('Playwright workers run on Node 22', () => { + expect(process.release.name).toBe('node') + expect(Number(process.versions.node.split('.')[0])).toBe(22) + expect(process.execPath.toLowerCase()).not.toContain('bun') + expect(process.env.NODE_OPTIONS).toContain('--dns-result-order=ipv4first') +}) diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts new file mode 100644 index 00000000000..261180d16ed --- /dev/null +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -0,0 +1,178 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer } from 'node:net' +import os from 'node:os' +import path from 'node:path' +import { loadEnvConfig } from '@next/env' +import { expect, test } from '@playwright/test' +import { parseRunOptions } from '../scripts/options' +import { + assertLoopbackPostgresUrl, + assertSafeDatabaseName, + buildRunDatabaseUrl, +} from '../support/database' +import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' +import { areValidE2eHostAddresses, isLoopbackAddress } from '../support/hosts' +import { + assertPortAvailable, + spawnManagedProcess, + waitForManagedProcessReady, +} from '../support/process' +import { parseProcessGroupIds } from '../support/signal-cleanup' + +test.describe('foundation safety guards', () => { + test('discovers env keys without leaking values and shadows unknown keys', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-env-')) + try { + writeFileSync( + path.join(directory, '.env'), + 'OPENAI_API_KEY=do-not-leak-this\nSAFE_VALUE=from-file\n' + ) + expect(discoverEnvFileKeys(directory)).toEqual(['OPENAI_API_KEY', 'SAFE_VALUE']) + + const result = buildChildEnvironment({ + values: { REQUIRED_VALUE: 'configured' }, + required: ['REQUIRED_VALUE'], + allowedSensitiveKeys: new Set(), + envDirectory: directory, + }) + expect(result.env.OPENAI_API_KEY).toBe('') + expect(result.env.SAFE_VALUE).toBe('') + expect(JSON.stringify(result)).not.toContain('do-not-leak-this') + expect(JSON.stringify(result)).not.toContain('from-file') + } finally { + rmSync(directory, { recursive: true, force: true }) + } + }) + + test('@next/env preserves non-empty and empty shadow values over an env file', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-next-env-')) + const key = `SIM_E2E_CANARY_${Date.now()}` + const emptyKey = `${key}_EMPTY` + try { + writeFileSync(path.join(directory, '.env'), `${key}=file-value\n${emptyKey}=must-not-load\n`) + process.env[key] = 'shadowed-value' + process.env[emptyKey] = '' + loadEnvConfig(directory, false, console, true) + expect(process.env[key]).toBe('shadowed-value') + expect(process.env[emptyKey]).toBe('') + } finally { + delete process.env[key] + delete process.env[emptyKey] + rmSync(directory, { recursive: true, force: true }) + } + }) + + test('database guards reject shared or remote targets', () => { + expect(() => assertSafeDatabaseName('simstudio')).toThrow() + expect(() => assertSafeDatabaseName('sim_e2e_valid_run')).not.toThrow() + expect(() => + assertLoopbackPostgresUrl('postgresql://postgres:postgres@example.com/postgres') + ).toThrow() + expect(() => + assertLoopbackPostgresUrl('postgresql://postgres:postgres@localhost/postgres') + ).toThrow() + expect(() => + assertLoopbackPostgresUrl('postgresql://postgres:postgres@127.0.0.1/postgres?sslmode=disable') + ).toThrow() + expect( + buildRunDatabaseUrl( + 'postgresql://postgres:postgres@127.0.0.1:5432/postgres', + 'sim_e2e_valid_run' + ) + ).toContain('/sim_e2e_valid_run') + }) + + test('loopback detection rejects public addresses', () => { + expect(isLoopbackAddress('127.0.0.1')).toBe(true) + expect(isLoopbackAddress('127.10.20.30')).toBe(true) + expect(isLoopbackAddress('::1')).toBe(true) + expect(isLoopbackAddress('8.8.8.8')).toBe(false) + expect(areValidE2eHostAddresses(['127.0.0.1'])).toBe(true) + expect(areValidE2eHostAddresses(['127.0.0.1', '::1'])).toBe(true) + expect(areValidE2eHostAddresses(['::1'])).toBe(false) + expect(areValidE2eHostAddresses(['127.0.0.2'])).toBe(false) + }) + + test('sharding is limited to the navigation project', () => { + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-navigation', '--shard=1/2']) + ).not.toThrow() + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-workflows', '--shard=1/2']) + ).toThrow(/coupled workflows must remain unsharded/) + }) + + test('Playwright CLI arguments cannot override orchestration invariants', () => { + expect(() => parseRunOptions(['--workers=8'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--config=other.config.ts'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--project', 'hosted-billing-chromium-navigation'])).toThrow( + /canonical/ + ) + expect(() => parseRunOptions(['--pass-with-no-tests'])).toThrow(/cannot override/) + }) + + test('empty signal cleanup groups never become PID zero', () => { + expect(parseProcessGroupIds(undefined)).toEqual([]) + expect(parseProcessGroupIds('')).toEqual([]) + expect(parseProcessGroupIds(' 123, 0, -1, nope, 456 ')).toEqual([123, 456]) + }) + + test('port preflight rejects an existing listener', async () => { + const server = createServer() + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + try { + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Expected TCP listener') + await expect(assertPortAvailable(address.port)).rejects.toThrow(/to be free/) + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ) + } + }) + + test('spawn failures finalize without hanging cleanup', async () => { + const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-spawn-')) + try { + const managed = spawnManagedProcess({ + name: 'missing-command', + command: path.join(logsDirectory, 'does-not-exist'), + args: [], + cwd: logsDirectory, + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + logsDirectory, + }) + const completion = await managed.completion + expect(completion.error).toBeTruthy() + await expect(managed.stop()).resolves.toBeUndefined() + } finally { + rmSync(logsDirectory, { recursive: true, force: true }) + } + }) + + test('process exit after readiness does not create an unhandled rejection', async () => { + const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-ready-')) + let unhandled: unknown + const onUnhandled = (error: unknown) => { + unhandled = error + } + process.once('unhandledRejection', onUnhandled) + try { + const managed = spawnManagedProcess({ + name: 'ready-process', + command: 'sleep', + args: ['10'], + cwd: logsDirectory, + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + logsDirectory, + }) + await waitForManagedProcessReady(managed, async () => {}) + await managed.stop() + await new Promise((resolve) => setImmediate(resolve)) + expect(unhandled).toBeUndefined() + } finally { + process.off('unhandledRejection', onUnhandled) + rmSync(logsDirectory, { recursive: true, force: true }) + } + }) +}) diff --git a/apps/sim/e2e/foundation/stripe-fake.spec.ts b/apps/sim/e2e/foundation/stripe-fake.spec.ts new file mode 100644 index 00000000000..8eec71d6870 --- /dev/null +++ b/apps/sim/e2e/foundation/stripe-fake.spec.ts @@ -0,0 +1,55 @@ +import { expect, test } from '@playwright/test' +import { startStripeFakeServer } from '../fakes/stripe/server' + +test('Stripe fake records allowlisted calls and rejects unknown routes', async () => { + const apiKey = 'sk_test_foundation_fake_spec' + const fake = await startStripeFakeServer({ apiKey }) + try { + const baseUrl = fake.baseUrl + expect(baseUrl).toBeTruthy() + + const health = await fetch(`${baseUrl}/health`) + expect(health.status).toBe(200) + + const customer = await fetch(`${baseUrl}/v1/customers`, { + method: 'POST', + headers: { + authorization: `Bearer ${apiKey}`, + 'content-type': 'application/x-www-form-urlencoded', + }, + body: new URLSearchParams({ + email: 'foundation@example.com', + 'metadata[userId]': 'user-foundation', + }), + }) + expect(customer.status).toBe(200) + expect((await customer.json()) as { id: string }).toMatchObject({ + id: expect.stringMatching(/^cus_e2e_/), + }) + + const search = await fetch( + `${baseUrl}/v1/customers/search?${new URLSearchParams({ + query: 'email:"foundation@example.com" AND -metadata["customerType"]:"organization"', + limit: '1', + })}`, + { headers: { authorization: `Bearer ${apiKey}` } } + ) + expect(search.status).toBe(200) + + const unsupportedSearch = await fetch( + `${baseUrl}/v1/customers/search?${new URLSearchParams({ + query: 'name:"Foundation"', + })}`, + { headers: { authorization: `Bearer ${apiKey}` } } + ) + expect(unsupportedSearch.status).toBe(501) + + const unknown = await fetch(`${baseUrl}/v1/invoices`, { + headers: { authorization: `Bearer ${apiKey}` }, + }) + expect(unknown.status).toBe(501) + expect(fake.requestLog.some(({ unexpected }) => unexpected)).toBe(true) + } finally { + await fake.stop() + } +}) diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts new file mode 100644 index 00000000000..fb8bdde3e0f --- /dev/null +++ b/apps/sim/e2e/scripts/options.ts @@ -0,0 +1,64 @@ +const NAVIGATION_PROJECT = 'hosted-billing-chromium-navigation' +const WORKFLOWS_PROJECT = 'hosted-billing-chromium-workflows' +const FORBIDDEN_OPTIONS = [ + '--config', + '-c', + '--workers', + '-j', + '--retries', + '--fully-parallel', + '--pass-with-no-tests', + '--list', +] as const + +export interface E2eRunOptions { + playwrightArgs: string[] +} + +export function parseRunOptions(argv: string[]): E2eRunOptions { + const normalizedArgs = argv[0] === '--' ? argv.slice(1) : [...argv] + if (normalizedArgs.includes('--skip-build')) { + throw new Error( + '--skip-build remains disabled until the planned profile/source-keyed build reuse experiment proves it safe' + ) + } + for (const option of FORBIDDEN_OPTIONS) { + if (hasOption(normalizedArgs, option)) { + throw new Error(`${option} cannot override E2E orchestration invariants`) + } + } + if (normalizedArgs.includes('--project')) { + throw new Error('Use canonical --project= syntax') + } + if (normalizedArgs.includes('--shard')) { + throw new Error('Use canonical --shard= syntax') + } + const playwrightArgs = [...normalizedArgs] + const projects = getEqualsOptionValues(playwrightArgs, '--project') + const unknownProject = projects.find( + (project) => project !== NAVIGATION_PROJECT && project !== WORKFLOWS_PROJECT + ) + if (unknownProject) throw new Error(`Unknown E2E Playwright project: ${unknownProject}`) + const hasShard = hasOption(playwrightArgs, '--shard') + + if ( + hasShard && + (projects.length === 0 || + projects.includes(WORKFLOWS_PROJECT) || + projects.some((project) => project !== NAVIGATION_PROJECT)) + ) { + throw new Error( + `--shard is supported only with --project=${NAVIGATION_PROJECT}; coupled workflows must remain unsharded` + ) + } + + return { playwrightArgs } +} + +function hasOption(args: string[], name: string): boolean { + return args.some((arg) => arg === name || arg.startsWith(`${name}=`)) +} + +function getEqualsOptionValues(args: string[], name: string): string[] { + return args.filter((arg) => arg.startsWith(`${name}=`)).map((arg) => arg.slice(name.length + 1)) +} diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts new file mode 100644 index 00000000000..9a1f102e921 --- /dev/null +++ b/apps/sim/e2e/scripts/run.ts @@ -0,0 +1,355 @@ +import { spawn, spawnSync } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs' +import path from 'node:path' +import { type StripeFakeServer, startStripeFakeServer } from '../fakes/stripe/server' +import { + buildRunDatabaseUrl, + createRunDatabase, + createRunDatabaseName, + dropRunDatabase, + type RunDatabase, +} from '../support/database' +import { createHostedBillingProfile, E2E_ORIGIN, E2E_PROFILE } from '../support/deployment-profile' +import { + assertNoForbiddenProviderInitialization, + assertNoForbiddenProviderTraffic, +} from '../support/diagnostics' +import { E2E_OS_PASSTHROUGH_KEYS, formatRedactedEnvironmentSummary } from '../support/env' +import { assertE2eHostResolvesToLoopback } from '../support/hosts' +import { getRunDirectory, SIM_APP_DIR } from '../support/paths' +import { + assertAdminApiBoundary, + type FoundationProvisioningResult, + inspectFoundationUsers, +} from '../support/probes' +import { + assertPortAvailable, + getActiveManagedProcessGroupIds, + type ManagedProcess, + stopAllManagedProcesses, +} from '../support/process' +import { buildApp, runMigrations, runPlaywright, startApp, startRealtime } from '../support/stack' +import { parseRunOptions } from './options' + +const DEFAULT_ADMIN_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' +const STRIPE_TEST_KEY = 'sk_test_sim_e2e_foundation' + +async function main(): Promise { + const options = parseRunOptions(process.argv.slice(2)) + const runId = createRunId() + const runDirectory = getRunDirectory(runId) + const logsDirectory = path.join(runDirectory, 'logs') + const storageStateDirectory = path.join(runDirectory, 'auth') + const markerDirectory = path.join(runDirectory, 'markers') + const homeDirectory = path.join(runDirectory, 'home') + mkdirSync(logsDirectory, { recursive: true }) + mkdirSync(storageStateDirectory, { recursive: true }) + mkdirSync(markerDirectory, { recursive: true }) + mkdirSync(homeDirectory, { recursive: true }) + + const nodeExecutable = resolveNode22() + const bunExecutable = resolveBunExecutable() + const adminDatabaseUrl = process.env.E2E_PG_ADMIN_URL ?? DEFAULT_ADMIN_DATABASE_URL + + let runDatabase: RunDatabase | null = null + let databaseCreationComplete = false + let stripeFake: StripeFakeServer | null = null + let realtime: ManagedProcess | null = null + let app: ManagedProcess | null = null + let cleanupPromise: Promise | null = null + let failed = false + + const cleanup = (): Promise => { + if (cleanupPromise) return cleanupPromise + cleanupPromise = (async () => { + const failures: unknown[] = [] + try { + await stopAllManagedProcesses() + } catch (error) { + failures.push(error) + } + + if (stripeFake) { + try { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + } catch (error) { + failures.push(error) + } + try { + await stripeFake.stop() + } catch (error) { + failures.push(error) + } + } + + for (const sensitiveDirectory of [storageStateDirectory, homeDirectory]) { + try { + rmSync(sensitiveDirectory, { recursive: true, force: true }) + } catch (error) { + failures.push(error) + } + } + + try { + if (runDatabase) await dropRunDatabase(adminDatabaseUrl, runDatabase.name) + } catch (error) { + failures.push(error) + } + + if (failures.length > 0) { + throw new AggregateError(failures, 'One or more E2E cleanup stages failed') + } + })() + return cleanupPromise + } + + const handleSignal = (signal: NodeJS.Signals): void => { + failed = true + const exitCode = signal === 'SIGINT' ? 130 : 143 + process.exitCode = exitCode + console.error(`Received ${signal}; cleaning up the E2E run`) + if (stripeFake) { + try { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + } catch {} + } + if (runDatabase) { + const cleanupLogFd = openSync(path.join(logsDirectory, 'signal-cleanup.log'), 'a') + const cleanupProcess = spawn( + bunExecutable, + ['--no-env-file', path.join(SIM_APP_DIR, 'e2e/scripts/signal-cleanup.ts')], + { + cwd: SIM_APP_DIR, + detached: true, + env: { + NODE_ENV: 'test', + PATH: process.env.PATH ?? '', + HOME: homeDirectory, + E2E_PG_ADMIN_URL: adminDatabaseUrl, + E2E_DATABASE_NAME: runDatabase.name, + E2E_DATABASE_CREATION_COMPLETE: String(databaseCreationComplete), + E2E_CLEANUP_PROCESS_GROUPS: getActiveManagedProcessGroupIds().join(','), + E2E_CLEANUP_DIRECTORIES: JSON.stringify([storageStateDirectory, homeDirectory]), + }, + stdio: ['ignore', cleanupLogFd, cleanupLogFd], + } + ) + cleanupProcess.unref() + closeSync(cleanupLogFd) + console.error(`Detached E2E cleanup supervisor started as PID ${cleanupProcess.pid}`) + } + process.exit(exitCode) + } + // `once` is intentional: a second signal uses the OS default force termination. + process.once('SIGINT', handleSignal) + process.once('SIGTERM', handleSignal) + + try { + const hostAddresses = await assertE2eHostResolvesToLoopback() + console.info(`E2E host resolved to loopback: ${hostAddresses.join(', ')}`) + await Promise.all([assertPortAvailable(3000), assertPortAvailable(3002)]) + + const runDatabaseName = createRunDatabaseName(runId) + runDatabase = { + name: runDatabaseName, + url: buildRunDatabaseUrl(adminDatabaseUrl, runDatabaseName), + } + await createRunDatabase(adminDatabaseUrl, runDatabaseName) + databaseCreationComplete = true + stripeFake = await startStripeFakeServer({ + apiKey: STRIPE_TEST_KEY, + hostname: '127.0.0.1', + port: 0, + }) + if (!stripeFake.baseUrl) throw new Error('Stripe fake did not expose a base URL') + + const profile = createHostedBillingProfile({ + runId, + databaseUrl: runDatabase.url, + stripeApiBaseUrl: stripeFake.baseUrl, + homeDirectory, + playwrightBrowsersPath: resolvePlaywrightBrowsersPath(), + ci: process.env.CI === 'true', + }) + console.info(formatRedactedEnvironmentSummary(profile.id, profile.childEnvironment)) + + const stackOptions = { + bunExecutable, + nodeExecutable, + env: profile.childEnvironment.env, + logsDirectory, + } + + await runMigrations(stackOptions) + await buildApp(stackOptions) + realtime = await startRealtime(stackOptions) + app = await startApp(stackOptions) + await assertAdminApiBoundary( + 'http://127.0.0.1:3000', + profile.childEnvironment.env.ADMIN_API_KEY + ) + assertNoForbiddenProviderInitialization([app.logPath, realtime.logPath]) + + const playwrightEnvironment = createPlaywrightEnvironment( + profile.childEnvironment.env, + runId, + storageStateDirectory, + markerDirectory + ) + await runPlaywright( + { + nodeExecutable, + env: playwrightEnvironment, + logsDirectory, + }, + options.playwrightArgs + ) + const provisioning = await inspectFoundationUsers(runDatabase.url, runId) + assertAuthenticatedSmokeEffectsIfPresent( + stripeFake, + provisioning, + hasFoundationCompletionMarker(markerDirectory) + ) + } catch (error) { + failed = true + console.error(error) + process.exitCode = 1 + } finally { + try { + assertNoForbiddenProviderTraffic( + [app?.logPath, realtime?.logPath].filter((value): value is string => Boolean(value)) + ) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } + try { + await cleanup() + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } + process.off('SIGINT', handleSignal) + process.off('SIGTERM', handleSignal) + console.info(`E2E ${failed ? 'failed' : 'completed'}; diagnostics: ${runDirectory}`) + } +} + +function createRunId(): string { + const timestamp = new Date().toISOString().replace(/\D/g, '').slice(0, 14) + const random = randomUUID().replace(/-/g, '').slice(0, 10) + return `${timestamp}_${random}` +} + +function resolveBunExecutable(): string { + if (!process.versions.bun) { + throw new Error('The E2E orchestrator must be started with Bun') + } + return process.execPath +} + +function resolveNode22(): string { + const executable = process.env.E2E_NODE_BINARY ?? 'node' + const result = spawnSync(executable, ['--version'], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (result.status !== 0) { + throw new Error(`Unable to execute Node for Playwright: ${result.stderr}`) + } + const version = result.stdout.trim() + if (!/^v22\./.test(version)) { + throw new Error(`Playwright E2E requires Node 22, received ${version}`) + } + return executable +} + +function resolvePlaywrightBrowsersPath(): string { + if (process.env.PLAYWRIGHT_BROWSERS_PATH) return process.env.PLAYWRIGHT_BROWSERS_PATH + const parentHome = process.env.HOME + if (!parentHome) throw new Error('HOME is required to locate installed Playwright browsers') + return process.platform === 'darwin' + ? path.join(parentHome, 'Library/Caches/ms-playwright') + : path.join(parentHome, '.cache/ms-playwright') +} + +function createPlaywrightEnvironment( + stackEnvironment: Record, + runId: string, + storageStateDirectory: string, + markerDirectory: string +): Record { + const keys = [...E2E_OS_PASSTHROUGH_KEYS, 'HOME', 'NODE_OPTIONS', 'PLAYWRIGHT_BROWSERS_PATH'] + const env: Record = {} + for (const key of keys) { + const value = stackEnvironment[key] + if (value !== undefined) env[key] = value + } + return { + ...env, + E2E_PROFILE, + E2E_ORCHESTRATED: '1', + E2E_RUN_ID: runId, + E2E_BASE_URL: E2E_ORIGIN, + E2E_STORAGE_STATE_DIR: storageStateDirectory, + E2E_MARKER_DIR: markerDirectory, + } +} + +function assertAuthenticatedSmokeEffectsIfPresent( + stripeFake: StripeFakeServer, + provisioning: FoundationProvisioningResult, + completed: boolean +): void { + const created = stripeFake.requestLog.some( + ({ method, path }) => method === 'POST' && path === '/v1/customers' + ) + const unexpected = stripeFake.requestLog.filter((request) => request.unexpected) + if (unexpected.length > 0) { + throw new Error( + `Stripe fake received unsupported requests: ${unexpected + .map(({ method, path }) => `${method} ${path}`) + .join(', ')}` + ) + } + if (!completed) { + console.info('Authenticated foundation smoke was filtered out; skipping its post-run probes') + return + } + if (!created) throw new Error('Billing-enabled signup did not create a fake Stripe customer') + if (provisioning.count === 0) { + throw new Error('Authenticated smoke did not create a foundation user') + } + if (!provisioning.allHaveStripeCustomers) { + throw new Error('A foundation user was not persisted with its fake Stripe customer') + } + if (!provisioning.allHaveStats) { + throw new Error('A foundation user was not initialized with user_stats') + } +} + +function hasFoundationCompletionMarker(markerDirectory: string): boolean { + return ( + existsSync(markerDirectory) && + readdirSync(markerDirectory).some((name) => name.startsWith('foundation-authenticated-')) + ) +} + +await main() diff --git a/apps/sim/e2e/scripts/signal-cleanup.ts b/apps/sim/e2e/scripts/signal-cleanup.ts new file mode 100644 index 00000000000..5d24f82df5a --- /dev/null +++ b/apps/sim/e2e/scripts/signal-cleanup.ts @@ -0,0 +1,71 @@ +import { rmSync } from 'node:fs' +import { sleep } from '@sim/utils/helpers' +import { dropRunDatabase, dropRunDatabaseWithRetries } from '../support/database' +import { parseProcessGroupIds } from '../support/signal-cleanup' + +const adminUrl = process.env.E2E_PG_ADMIN_URL +const databaseName = process.env.E2E_DATABASE_NAME +const processGroupIds = parseProcessGroupIds(process.env.E2E_CLEANUP_PROCESS_GROUPS) +const sensitiveDirectories = JSON.parse(process.env.E2E_CLEANUP_DIRECTORIES ?? '[]') as string[] +const databaseCreationComplete = process.env.E2E_DATABASE_CREATION_COMPLETE === 'true' + +if (!adminUrl || !databaseName) { + console.error('Signal cleanup requires E2E_PG_ADMIN_URL and E2E_DATABASE_NAME') + process.exit(1) +} + +const failures: unknown[] = [] +signalProcessGroups(processGroupIds, 'SIGTERM', failures) +await sleep(500) + +try { + if (databaseCreationComplete) { + await dropRunDatabase(adminUrl, databaseName) + } else { + await dropRunDatabaseWithRetries(adminUrl, databaseName) + } +} catch (error) { + failures.push(error) +} + +signalProcessGroups(processGroupIds, 'SIGKILL', failures) +await sleep(250) + +for (const directory of sensitiveDirectories) { + try { + rmSync(directory, { recursive: true, force: true }) + } catch (error) { + failures.push(error) + } +} + +for (const error of failures) console.error(error) +process.exit(failures.length > 0 ? 1 : 0) + +function signalProcessGroups( + groupIds: number[], + signal: NodeJS.Signals, + failures: unknown[] +): void { + for (const groupId of groupIds) { + if (!Number.isInteger(groupId) || groupId <= 0) continue + try { + if (process.platform !== 'win32') process.kill(-groupId, signal) + else process.kill(groupId, signal) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === 'ESRCH') continue + if (code === 'EPERM' && process.platform !== 'win32') { + try { + process.kill(groupId, signal) + } catch (fallbackError) { + if ((fallbackError as NodeJS.ErrnoException).code !== 'ESRCH') { + failures.push(fallbackError) + } + } + } else { + failures.push(error) + } + } + } +} diff --git a/apps/sim/e2e/settings/smoke/authenticated.spec.ts b/apps/sim/e2e/settings/smoke/authenticated.spec.ts new file mode 100644 index 00000000000..a64c7e9a1db --- /dev/null +++ b/apps/sim/e2e/settings/smoke/authenticated.spec.ts @@ -0,0 +1,74 @@ +import { createHash } from 'node:crypto' +import { writeFileSync } from 'node:fs' +import path from 'node:path' +import { expect, test } from '@playwright/test' + +test('billing-enabled signup, login, and settings use real Sim boundaries', async ({ + browser, + page, +}, testInfo) => { + test.slow() + + const runId = process.env.E2E_RUN_ID ?? `${Date.now()}` + const testIdentity = createHash('sha256') + .update(`${runId}:${testInfo.project.name}:${testInfo.workerIndex}:${testInfo.repeatEachIndex}`) + .digest('hex') + .slice(0, 16) + const email = `e2e-foundation-${runId}-${testIdentity}@example.com` + const password = 'E2eFoundation1!' + const storageStatePath = path.join(requiredEnv('E2E_STORAGE_STATE_DIR'), `${testIdentity}.json`) + + await page.goto('/signup') + await expect(page.getByRole('heading', { name: 'Create an account' })).toBeVisible() + await page.getByLabel('Full name').fill('Playwright Foundation') + await page.getByLabel('Email').fill(email) + await page.getByRole('textbox', { name: 'Password' }).fill(password) + await page.getByRole('button', { name: 'Create account' }).click() + + await expect(page).toHaveURL(/\/workspace\/[^/]+\/(?:home|w(?:\/|$))/, { + timeout: 60_000, + }) + const workspaceId = new URL(page.url()).pathname.split('/')[2] + expect(workspaceId).toBeTruthy() + + const settingsPath = `/workspace/${workspaceId}/settings/general` + await page.goto(settingsPath) + await assertGeneralSettings(page) + + await page.getByRole('button', { name: 'Sign out' }).click() + await expect(page).toHaveURL(/\/login\?fromLogout=true$/) + + await page.getByLabel('Email').fill(email) + await page.getByRole('textbox', { name: 'Password' }).fill(password) + await page.getByRole('button', { name: 'Sign in' }).click() + await expect(page).toHaveURL(/\/workspace(?:\/|$)/) + + await page.context().storageState({ path: storageStatePath }) + const restoredContext = await browser.newContext({ storageState: storageStatePath }) + try { + const restoredPage = await restoredContext.newPage() + await restoredPage.goto(`${requiredEnv('E2E_BASE_URL')}${settingsPath}`) + await assertGeneralSettings(restoredPage) + } finally { + await restoredContext.close() + } + writeFileSync( + path.join(requiredEnv('E2E_MARKER_DIR'), `foundation-authenticated-${testIdentity}.json`), + JSON.stringify({ runId, testIdentity }) + ) +}) + +async function assertGeneralSettings(page: import('@playwright/test').Page): Promise { + await expect(page).toHaveURL(/\/settings\/general$/) + await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() + await expect( + page.getByText('Manage your profile, appearance, and preferences.', { exact: true }) + ).toBeVisible() + await expect(page.getByText('Profile', { exact: true })).toBeVisible() +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing required Playwright environment value: ${key}`) + return value +} diff --git a/apps/sim/e2e/settings/smoke/unauthenticated.spec.ts b/apps/sim/e2e/settings/smoke/unauthenticated.spec.ts new file mode 100644 index 00000000000..ab3aaca910f --- /dev/null +++ b/apps/sim/e2e/settings/smoke/unauthenticated.spec.ts @@ -0,0 +1,8 @@ +import { expect, test } from '@playwright/test' + +test('protected settings redirect unauthenticated users to login', async ({ page }) => { + await page.goto('/account/settings/general') + + await expect(page).toHaveURL(/\/login(?:\?|$)/) + await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible() +}) diff --git a/apps/sim/e2e/support/database.ts b/apps/sim/e2e/support/database.ts new file mode 100644 index 00000000000..8284e707791 --- /dev/null +++ b/apps/sim/e2e/support/database.ts @@ -0,0 +1,120 @@ +import { sleep } from '@sim/utils/helpers' +import postgres from 'postgres' +import { isLoopbackAddress } from './hosts' + +const DATABASE_NAME_PATTERN = /^sim_e2e_[a-z0-9_]+$/ +const PROTECTED_DATABASES = new Set(['postgres', 'simstudio', 'template0', 'template1']) + +export interface RunDatabase { + name: string + url: string +} + +export function createRunDatabaseName(runId: string): string { + const suffix = runId + .toLowerCase() + .replace(/[^a-z0-9]+/g, '_') + .replace(/^_+|_+$/g, '') + const name = `sim_e2e_${suffix}` + assertSafeDatabaseName(name) + return name +} + +export function assertSafeDatabaseName(name: string): void { + if (!DATABASE_NAME_PATTERN.test(name) || PROTECTED_DATABASES.has(name)) { + throw new Error(`Refusing unsafe E2E database name: ${name}`) + } +} + +export function assertLoopbackPostgresUrl(rawUrl: string): URL { + const url = new URL(rawUrl) + const hostname = url.hostname.replace(/^\[|\]$/g, '') + if (!['postgres:', 'postgresql:'].includes(url.protocol)) { + throw new Error(`E2E PostgreSQL admin URL requires postgres protocol, received ${url.protocol}`) + } + if (url.search || url.hash) { + throw new Error('E2E PostgreSQL admin URL must not contain query parameters or a fragment') + } + if (!isLoopbackAddress(hostname)) { + throw new Error(`E2E PostgreSQL admin URL must be loopback, received ${url.hostname}`) + } + return url +} + +export function buildRunDatabaseUrl(adminUrl: string, databaseName: string): string { + assertSafeDatabaseName(databaseName) + const url = assertLoopbackPostgresUrl(adminUrl) + url.pathname = `/${databaseName}` + return url.toString() +} + +export async function createRunDatabase( + adminUrl: string, + databaseName: string +): Promise { + assertSafeDatabaseName(databaseName) + assertLoopbackPostgresUrl(adminUrl) + const sql = postgres(adminUrl, { max: 1, connect_timeout: 10 }) + try { + await sql.unsafe(`CREATE DATABASE "${databaseName}"`) + } finally { + await sql.end() + } + return { name: databaseName, url: buildRunDatabaseUrl(adminUrl, databaseName) } +} + +export async function dropRunDatabase(adminUrl: string, databaseName: string): Promise { + assertSafeDatabaseName(databaseName) + assertLoopbackPostgresUrl(adminUrl) + const sql = postgres(adminUrl, { + max: 1, + connect_timeout: 10, + onnotice: () => {}, + }) + try { + await sql.unsafe(`DROP DATABASE IF EXISTS "${databaseName}" WITH (FORCE)`) + } finally { + await sql.end() + } +} + +export async function dropRunDatabaseWithRetries( + adminUrl: string, + databaseName: string, + deadlineMs = 5_000, + intervalMs = 100 +): Promise { + const deadline = Date.now() + deadlineMs + let lastError: unknown + do { + try { + await dropRunDatabase(adminUrl, databaseName) + lastError = undefined + } catch (error) { + lastError = error + } + await sleep(intervalMs) + } while (Date.now() < deadline) + if (await runDatabaseExists(adminUrl, databaseName)) { + throw ( + lastError ?? + new Error(`Guarded E2E database still exists after forced-drop retry window: ${databaseName}`) + ) + } +} + +async function runDatabaseExists(adminUrl: string, databaseName: string): Promise { + assertSafeDatabaseName(databaseName) + assertLoopbackPostgresUrl(adminUrl) + const sql = postgres(adminUrl, { max: 1, connect_timeout: 10, onnotice: () => {} }) + try { + const rows = await sql>` + SELECT EXISTS( + SELECT 1 FROM pg_database WHERE datname = ${databaseName} + ) AS "exists" + ` + return rows[0]?.exists ?? false + } finally { + await sql.end() + } +} diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts new file mode 100644 index 00000000000..badcd7bb054 --- /dev/null +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -0,0 +1,134 @@ +import { buildChildEnvironment, type ChildEnvironment } from './env' + +export const E2E_PROFILE = 'hosted-billing-chromium' +export const E2E_HOST = 'e2e.sim.ai' +export const E2E_ORIGIN = `http://${E2E_HOST}:3000` +export const E2E_SOCKET_ORIGIN = `http://${E2E_HOST}:3002` + +const REQUIRED_KEYS = [ + 'NODE_ENV', + 'NEXT_PUBLIC_APP_URL', + 'BETTER_AUTH_URL', + 'BETTER_AUTH_SECRET', + 'DATABASE_URL', + 'MIGRATION_DATABASE_URL', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'ADMIN_API_KEY', + 'BILLING_ENABLED', + 'NEXT_PUBLIC_BILLING_ENABLED', + 'STRIPE_SECRET_KEY', + 'STRIPE_API_BASE_URL', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'HOME', + 'PLAYWRIGHT_BROWSERS_PATH', +] as const + +const ALLOWED_SENSITIVE_KEYS = new Set([ + 'BETTER_AUTH_SECRET', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'ADMIN_API_KEY', + 'EMAIL_PASSWORD_SIGNUP_ENABLED', + 'NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED', + 'STRIPE_SECRET_KEY', + 'STRIPE_WEBHOOK_SECRET', +]) + +export interface HostedBillingProfileOptions { + runId: string + databaseUrl: string + stripeApiBaseUrl: string + homeDirectory: string + playwrightBrowsersPath: string + ci: boolean +} + +export interface HostedBillingProfile { + id: typeof E2E_PROFILE + origin: typeof E2E_ORIGIN + childEnvironment: ChildEnvironment +} + +export function createHostedBillingProfile({ + runId, + databaseUrl, + stripeApiBaseUrl, + homeDirectory, + playwrightBrowsersPath, + ci, +}: HostedBillingProfileOptions): HostedBillingProfile { + const values: Record = { + NODE_ENV: 'production', + NODE_OPTIONS: '--no-warnings --max-old-space-size=8192 --dns-result-order=ipv4first', + NEXT_TELEMETRY_DISABLED: '1', + HOME: homeDirectory, + XDG_CONFIG_HOME: `${homeDirectory}/xdg`, + AWS_EC2_METADATA_DISABLED: 'true', + AWS_SHARED_CREDENTIALS_FILE: '/dev/null', + AWS_CONFIG_FILE: '/dev/null', + CLOUDSDK_CONFIG: `${homeDirectory}/gcloud`, + AZURE_CONFIG_DIR: `${homeDirectory}/azure`, + PLAYWRIGHT_BROWSERS_PATH: playwrightBrowsersPath, + E2E_PROFILE, + E2E_RUN_ID: runId, + E2E_BASE_URL: E2E_ORIGIN, + NEXT_PUBLIC_APP_URL: E2E_ORIGIN, + BETTER_AUTH_URL: E2E_ORIGIN, + BETTER_AUTH_SECRET: 'e2e-better-auth-secret-at-least-32-characters-long', + DATABASE_URL: databaseUrl, + MIGRATION_DATABASE_URL: databaseUrl, + ENCRYPTION_KEY: '11'.repeat(32), + API_ENCRYPTION_KEY: '22'.repeat(32), + INTERNAL_API_SECRET: 'e2e-internal-api-secret-at-least-32-characters', + ADMIN_API_KEY: 'e2e-admin-api-key-at-least-32-characters-long', + BILLING_ENABLED: 'true', + NEXT_PUBLIC_BILLING_ENABLED: 'true', + EMAIL_VERIFICATION_ENABLED: 'false', + EMAIL_PASSWORD_SIGNUP_ENABLED: 'true', + NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED: 'true', + DISABLE_REGISTRATION: 'false', + DISABLE_EMAIL_SIGNUP: 'false', + SIGNUP_MX_VALIDATION_ENABLED: 'false', + NEXT_PUBLIC_POSTHOG_ENABLED: 'false', + BLACKLISTED_PROVIDERS: 'ollama,ollama-cloud,vllm,litellm,openrouter,together,fireworks,baseten', + STRIPE_SECRET_KEY: 'sk_test_sim_e2e_foundation', + STRIPE_WEBHOOK_SECRET: 'whsec_sim_e2e_foundation', + STRIPE_FREE_PRICE_ID: 'price_e2e_free', + STRIPE_API_BASE_URL: stripeApiBaseUrl, + SOCKET_SERVER_URL: 'http://127.0.0.1:3002', + NEXT_PUBLIC_SOCKET_URL: E2E_SOCKET_ORIGIN, + CI: ci ? 'true' : 'false', + } + + validateProfileValues(values) + + return { + id: E2E_PROFILE, + origin: E2E_ORIGIN, + childEnvironment: buildChildEnvironment({ + values, + required: REQUIRED_KEYS, + allowedSensitiveKeys: ALLOWED_SENSITIVE_KEYS, + }), + } +} + +function validateProfileValues(values: Record): void { + if (values.NEXT_PUBLIC_APP_URL !== E2E_ORIGIN || values.BETTER_AUTH_URL !== E2E_ORIGIN) { + throw new Error('E2E app and Better Auth origins must exactly match') + } + if (values.BILLING_ENABLED !== values.NEXT_PUBLIC_BILLING_ENABLED) { + throw new Error('BILLING_ENABLED and NEXT_PUBLIC_BILLING_ENABLED must match') + } + if (!values.DATABASE_URL.includes('/sim_e2e_')) { + throw new Error('E2E profile requires a sim_e2e_ database') + } + const stripeUrl = new URL(values.STRIPE_API_BASE_URL) + if (stripeUrl.hostname !== '127.0.0.1') { + throw new Error('E2E Stripe API must use numeric IPv4 loopback 127.0.0.1') + } +} diff --git a/apps/sim/e2e/support/diagnostics.ts b/apps/sim/e2e/support/diagnostics.ts new file mode 100644 index 00000000000..370b3212153 --- /dev/null +++ b/apps/sim/e2e/support/diagnostics.ts @@ -0,0 +1,52 @@ +import { existsSync, readFileSync } from 'node:fs' + +const FORBIDDEN_PROVIDER_TRAFFIC_PATTERNS = [ + /api\.stripe\.com/i, + /api\.agentmail\.to/i, + /telemetry\.simstudio\.ai/i, + /Failed to fetch Ollama models/i, +] + +const FORBIDDEN_PROVIDER_STARTUP_PATTERNS = [ + ...FORBIDDEN_PROVIDER_TRAFFIC_PATTERNS, + /Failed to initialize .*provider/i, + /Failed to initialize .*client/i, +] + +/** + * Scan only service startup logs. Later settings tests intentionally exercise + * URL-validation denials whose error copy must not be treated as provider boot. + */ +export function assertNoForbiddenProviderInitialization(logPaths: string[]): void { + assertLogsDoNotMatch( + logPaths, + FORBIDDEN_PROVIDER_STARTUP_PATTERNS, + 'Shadowed or forbidden provider initialization was detected' + ) +} + +/** + * Re-scan after browser activity using only concrete outbound signatures. + * Intentional URL-validation errors in later workflow suites remain allowed. + */ +export function assertNoForbiddenProviderTraffic(logPaths: string[]): void { + assertLogsDoNotMatch( + logPaths, + FORBIDDEN_PROVIDER_TRAFFIC_PATTERNS, + 'Forbidden provider traffic was detected' + ) +} + +function assertLogsDoNotMatch(logPaths: string[], patterns: RegExp[], message: string): void { + const violations: string[] = [] + for (const logPath of logPaths) { + if (!existsSync(logPath)) continue + const contents = readFileSync(logPath, 'utf8') + for (const pattern of patterns) { + if (pattern.test(contents)) violations.push(`${logPath}: ${pattern}`) + } + } + if (violations.length > 0) { + throw new Error(`${message}:\n${violations.join('\n')}`) + } +} diff --git a/apps/sim/e2e/support/env.ts b/apps/sim/e2e/support/env.ts new file mode 100644 index 00000000000..7527596b41e --- /dev/null +++ b/apps/sim/e2e/support/env.ts @@ -0,0 +1,102 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs' +import path from 'node:path' +import { SIM_APP_DIR } from './paths' + +const ENV_KEY_PATTERN = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/ +const SENSITIVE_KEY_PATTERN = + /(?:^|_)(?:API_?KEY|SECRET|TOKEN|PASSWORD|PRIVATE_?KEY|CLIENT_?SECRET|WEBHOOK)(?:_|$)/i + +const OS_PASSTHROUGH_KEYS = [ + 'PATH', + 'USER', + 'SHELL', + 'TMPDIR', + 'TMP', + 'TEMP', + 'SYSTEMROOT', + 'CI', + 'GITHUB_ACTIONS', +] as const + +export interface ChildEnvironment { + env: Record + discoveredKeys: string[] + shadowedKeys: string[] +} + +export interface BuildChildEnvironmentOptions { + values: Record + required: readonly string[] + allowedSensitiveKeys: ReadonlySet + envDirectory?: string +} + +export function discoverEnvFileKeys(directory = SIM_APP_DIR): string[] { + if (!existsSync(directory)) return [] + + const keys = new Set() + const files = readdirSync(directory) + .filter((name) => name === '.env' || name.startsWith('.env.')) + .sort() + + for (const file of files) { + const contents = readFileSync(path.join(directory, file), 'utf8') + for (const line of contents.split(/\r?\n/)) { + const match = ENV_KEY_PATTERN.exec(line) + if (match) keys.add(match[1]) + } + } + + return [...keys].sort() +} + +export function buildChildEnvironment({ + values, + required, + allowedSensitiveKeys, + envDirectory = SIM_APP_DIR, +}: BuildChildEnvironmentOptions): ChildEnvironment { + const missing = required.filter((key) => !values[key]?.trim()) + if (missing.length > 0) { + throw new Error(`Missing required E2E environment values: ${missing.join(', ')}`) + } + + const env: Record = {} + for (const key of OS_PASSTHROUGH_KEYS) { + const value = process.env[key] + if (value !== undefined) env[key] = value + } + + for (const [key, value] of Object.entries(values)) { + if (SENSITIVE_KEY_PATTERN.test(key) && !allowedSensitiveKeys.has(key)) { + throw new Error(`Sensitive E2E environment key is not explicitly allowed: ${key}`) + } + env[key] = value + } + + const discoveredKeys = discoverEnvFileKeys(envDirectory) + const shadowedKeys: string[] = [] + for (const key of discoveredKeys) { + if (key in env) continue + env[key] = '' + shadowedKeys.push(key) + } + + return { env, discoveredKeys, shadowedKeys } +} + +export function formatRedactedEnvironmentSummary( + profile: string, + childEnvironment: ChildEnvironment +): string { + return [ + `E2E profile: ${profile}`, + `Allowed keys: ${Object.keys(childEnvironment.env) + .filter((key) => !childEnvironment.shadowedKeys.includes(key)) + .sort() + .join(', ')}`, + `Shadowed local keys: ${childEnvironment.shadowedKeys.join(', ') || '(none)'}`, + ].join('\n') +} + +export const E2E_OS_PASSTHROUGH_KEYS = OS_PASSTHROUGH_KEYS diff --git a/apps/sim/e2e/support/hosts.ts b/apps/sim/e2e/support/hosts.ts new file mode 100644 index 00000000000..81e6127755c --- /dev/null +++ b/apps/sim/e2e/support/hosts.ts @@ -0,0 +1,41 @@ +import { promises as dns } from 'node:dns' +import { isIP } from 'node:net' +import { E2E_HOST } from './deployment-profile' + +export function isLoopbackAddress(address: string): boolean { + if (address === '::1' || address === '0:0:0:0:0:0:0:1') return true + if (isIP(address) === 4) { + const first = Number(address.split('.')[0]) + return first === 127 + } + return false +} + +export function areValidE2eHostAddresses(addresses: string[]): boolean { + return ( + addresses.length > 0 && addresses.every(isLoopbackAddress) && addresses.includes('127.0.0.1') + ) +} + +export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Promise { + let records: Array<{ address: string }> + try { + records = await dns.lookup(hostname, { all: true, verbatim: true }) + } catch { + throw new Error(getHostMappingError(hostname, [])) + } + + const addresses = [...new Set(records.map(({ address }) => address))] + if (!areValidE2eHostAddresses(addresses)) { + throw new Error(getHostMappingError(hostname, addresses)) + } + return addresses +} + +function getHostMappingError(hostname: string, addresses: string[]): string { + const observed = addresses.length > 0 ? addresses.join(', ') : 'no addresses' + return [ + `${hostname} must resolve only to loopback and include IPv4 127.0.0.1; observed ${observed}.`, + `Add it once with: echo "127.0.0.1 ${hostname}" | sudo tee -a /etc/hosts`, + ].join(' ') +} diff --git a/apps/sim/e2e/support/paths.ts b/apps/sim/e2e/support/paths.ts new file mode 100644 index 00000000000..44ec033e0e4 --- /dev/null +++ b/apps/sim/e2e/support/paths.ts @@ -0,0 +1,11 @@ +import path from 'node:path' + +export const SIM_APP_DIR = path.resolve(process.cwd()) +export const REPO_ROOT = path.resolve(SIM_APP_DIR, '../..') +export const DB_PACKAGE_DIR = path.join(REPO_ROOT, 'packages/db') +export const REALTIME_APP_DIR = path.join(REPO_ROOT, 'apps/realtime') +export const PLAYWRIGHT_CLI = path.join(REPO_ROOT, 'node_modules/@playwright/test/cli.js') + +export function getRunDirectory(runId: string): string { + return path.join(SIM_APP_DIR, 'e2e/.runs', runId) +} diff --git a/apps/sim/e2e/support/probes.ts b/apps/sim/e2e/support/probes.ts new file mode 100644 index 00000000000..40f4e42bae1 --- /dev/null +++ b/apps/sim/e2e/support/probes.ts @@ -0,0 +1,47 @@ +import postgres from 'postgres' + +export async function assertAdminApiBoundary(origin: string, adminKey: string): Promise { + const endpoint = `${origin}/api/v1/admin/users?limit=1&offset=0` + const unauthorized = await fetch(endpoint) + if (unauthorized.status !== 401) { + throw new Error(`Admin API missing-key probe returned ${unauthorized.status}, expected 401`) + } + + const authorized = await fetch(endpoint, { + headers: { 'x-admin-key': adminKey }, + }) + if (!authorized.ok) { + throw new Error(`Configured Admin API probe returned ${authorized.status}`) + } +} + +export interface FoundationProvisioningResult { + count: number + allHaveStripeCustomers: boolean + allHaveStats: boolean +} + +export async function inspectFoundationUsers( + databaseUrl: string, + runId: string +): Promise { + const sql = postgres(databaseUrl, { max: 1, connect_timeout: 10 }) + try { + const prefix = `e2e-foundation-${runId}-` + const rows = await sql>` + SELECT + u.stripe_customer_id AS "stripeCustomerId", + (s.user_id IS NOT NULL) AS "hasStats" + FROM "user" u + LEFT JOIN user_stats s ON s.user_id = u.id + WHERE starts_with(u.email, ${prefix}) + ` + return { + count: rows.length, + allHaveStripeCustomers: rows.every((row) => row.stripeCustomerId?.startsWith('cus_e2e_')), + allHaveStats: rows.every((row) => row.hasStats), + } + } finally { + await sql.end() + } +} diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts new file mode 100644 index 00000000000..406502275af --- /dev/null +++ b/apps/sim/e2e/support/process.ts @@ -0,0 +1,222 @@ +import { type ChildProcess, spawn, spawnSync } from 'node:child_process' +import { closeSync, mkdirSync, openSync } from 'node:fs' +import { createServer } from 'node:net' +import path from 'node:path' + +export interface CommandOptions { + name: string + command: string + args: string[] + cwd: string + env: Record + logsDirectory: string +} + +export interface ManagedProcess { + name: string + child: ChildProcess + completion: Promise + logPath: string + stop(): Promise +} + +interface ProcessCompletion { + code: number | null + signal: NodeJS.Signals | null + error?: Error +} + +const activeProcesses = new Set() + +export function spawnManagedProcess(options: CommandOptions): ManagedProcess { + mkdirSync(options.logsDirectory, { recursive: true }) + const logPath = path.join(options.logsDirectory, `${options.name}.log`) + const logFd = openSync(logPath, 'a') + const child: ChildProcess = spawn(options.command, options.args, { + cwd: options.cwd, + detached: process.platform !== 'win32', + env: options.env as NodeJS.ProcessEnv, + stdio: ['ignore', logFd, logFd], + }) + let resolveCompletion!: (completion: ProcessCompletion) => void + const completion = new Promise((resolve) => { + resolveCompletion = resolve + }) + let finalized = false + const managed: ManagedProcess = { + name: options.name, + child, + completion, + logPath, + stop: () => stopProcess(managed), + } + const finalize = (result: ProcessCompletion): void => { + if (finalized) return + finalized = true + activeProcesses.delete(managed) + closeSync(logFd) + resolveCompletion(result) + } + activeProcesses.add(managed) + child.once('error', (error) => finalize({ code: null, signal: null, error })) + child.once('exit', (code, signal) => finalize({ code, signal })) + return managed +} + +export async function runCommand(options: CommandOptions): Promise { + const managed = spawnManagedProcess(options) + const result = await managed.completion + if (result.error) { + throw new Error(`${options.name} failed to spawn. See ${managed.logPath}`, { + cause: result.error, + }) + } + if (result.code !== 0) { + throw new Error( + `${options.name} exited with ${result.code ?? result.signal ?? 'unknown status'}. See ${managed.logPath}` + ) + } +} + +export async function stopProcesses(processes: ManagedProcess[]): Promise { + const results = await Promise.allSettled( + [...processes].reverse().map((process) => process.stop()) + ) + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [] + ) + if (failures.length > 0) { + throw new AggregateError(failures, 'Failed to stop one or more managed E2E processes') + } +} + +export async function stopAllManagedProcesses(): Promise { + await stopProcesses([...activeProcesses]) +} + +export function getActiveManagedProcessGroupIds(): number[] { + return [...new Set([...activeProcesses].flatMap(({ child }) => (child.pid ? [child.pid] : [])))] +} + +export async function waitForManagedProcessReady( + process: ManagedProcess, + readiness: (signal: AbortSignal) => Promise +): Promise { + if (process.child.exitCode !== null || process.child.signalCode !== null) { + throw new Error(`${process.name} exited before readiness. See ${process.logPath}`) + } + + const controller = new AbortController() + let ready = false + const exited = process.completion.then((result) => { + if (ready) return + throw new Error( + `${process.name} exited before readiness with ${result.error?.message ?? result.code ?? result.signal ?? 'unknown status'}. See ${process.logPath}` + ) + }) + const becameReady = readiness(controller.signal).then(() => { + ready = true + }) + + try { + await Promise.race([becameReady, exited]) + } finally { + controller.abort(new Error(`${process.name} readiness polling cancelled`)) + } +} + +export async function assertPortAvailable(port: number, host = '127.0.0.1'): Promise { + await new Promise((resolve, reject) => { + const server = createServer() + server.once('error', (error) => { + reject(new Error(`E2E requires ${host}:${port} to be free: ${String(error)}`)) + }) + server.listen({ host, port, exclusive: true }, () => { + server.close((error) => { + if (error) reject(error) + else resolve() + }) + }) + }) +} + +async function stopProcess(managed: ManagedProcess): Promise { + const { child } = managed + if (!child.pid) { + await managed.completion + return + } + if (child.exitCode !== null || child.signalCode !== null) return + const processIds = [child.pid] + sendSignalToPids(processIds, 'SIGTERM') + + const stopped = await Promise.race([ + managed.completion.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), + ]) + if (stopped) return + + sendSignalToPids(processIds.filter(isPidRunning), 'SIGKILL') + const killed = await Promise.race([ + managed.completion.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 2_000)), + ]) + if (!killed) { + throw new Error(`Managed process ${managed.name} did not exit after SIGKILL`) + } +} + +function sendSignalToPids(processIds: number[], signal: NodeJS.Signals): void { + for (const processId of processIds) { + try { + if (process.platform !== 'win32') process.kill(-processId, signal) + else process.kill(processId, signal) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === 'ESRCH') continue + if (code === 'EPERM' && process.platform !== 'win32') { + try { + process.kill(processId, signal) + } catch (fallbackError) { + if ((fallbackError as NodeJS.ErrnoException).code !== 'ESRCH') { + throw fallbackError + } + } + continue + } + throw error + } + } +} + +function isPidRunning(processId: number): boolean { + return getRunningPids([processId]).length > 0 +} + +function getRunningPids(processIds: number[]): number[] { + if (processIds.length === 0) return [] + if (process.platform === 'win32') { + return processIds.filter((processId) => { + try { + process.kill(processId, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false + throw error + } + }) + } + + const status = spawnSync('ps', ['-o', 'pid=,stat=', '-p', processIds.join(',')], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (status.status !== 0 && status.status !== 1) { + throw new Error(`Unable to inspect managed E2E processes: ${status.stderr}`) + } + return status.stdout.split('\n').flatMap((line) => { + const [pidText, processStatus] = line.trim().split(/\s+/) + const pid = Number(pidText) + return Number.isInteger(pid) && !processStatus?.startsWith('Z') ? [pid] : [] + }) +} diff --git a/apps/sim/e2e/support/readiness.ts b/apps/sim/e2e/support/readiness.ts new file mode 100644 index 00000000000..628976bfa14 --- /dev/null +++ b/apps/sim/e2e/support/readiness.ts @@ -0,0 +1,55 @@ +export interface ReadinessOptions { + name: string + url: string + timeoutMs?: number + intervalMs?: number + signal?: AbortSignal + validate?: (response: Response) => Promise | boolean +} + +export async function waitForHttpReady({ + name, + url, + timeoutMs = 120_000, + intervalMs = 500, + signal, + validate = (response) => response.ok, +}: ReadinessOptions): Promise { + const deadline = Date.now() + timeoutMs + let lastError: unknown + + while (Date.now() < deadline) { + if (signal?.aborted) throw signal.reason + try { + const timeoutSignal = AbortSignal.timeout(5_000) + const response = await fetch(url, { + redirect: 'manual', + signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal, + }) + if (await validate(response)) return + lastError = new Error(`${name} returned ${response.status}`) + } catch (error) { + if (signal?.aborted) throw signal.reason + lastError = error + } + await waitForInterval(intervalMs, signal) + } + + throw new Error(`${name} did not become ready at ${url}: ${String(lastError)}`) +} + +async function waitForInterval(milliseconds: number, signal?: AbortSignal): Promise { + await new Promise((resolve, reject) => { + const complete = () => { + signal?.removeEventListener('abort', abort) + resolve() + } + const timeout = setTimeout(complete, milliseconds) + const abort = () => { + clearTimeout(timeout) + reject(signal?.reason) + } + if (signal?.aborted) abort() + else signal?.addEventListener('abort', abort, { once: true }) + }) +} diff --git a/apps/sim/e2e/support/signal-cleanup.ts b/apps/sim/e2e/support/signal-cleanup.ts new file mode 100644 index 00000000000..18fcfe1e0d8 --- /dev/null +++ b/apps/sim/e2e/support/signal-cleanup.ts @@ -0,0 +1,8 @@ +export function parseProcessGroupIds(rawValue: string | undefined): number[] { + return (rawValue ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean) + .map(Number) + .filter((value) => Number.isInteger(value) && value > 0) +} diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts new file mode 100644 index 00000000000..d1ce203d6b6 --- /dev/null +++ b/apps/sim/e2e/support/stack.ts @@ -0,0 +1,136 @@ +import path from 'node:path' +import { DB_PACKAGE_DIR, PLAYWRIGHT_CLI, REALTIME_APP_DIR, REPO_ROOT, SIM_APP_DIR } from './paths' +import { + type ManagedProcess, + runCommand, + spawnManagedProcess, + waitForManagedProcessReady, +} from './process' +import { waitForHttpReady } from './readiness' + +export interface StackCommandOptions { + bunExecutable: string + nodeExecutable: string + env: Record + logsDirectory: string +} + +export async function runMigrations(options: StackCommandOptions): Promise { + await runCommand({ + name: 'migrate', + command: options.bunExecutable, + args: ['--no-env-file', path.join(DB_PACKAGE_DIR, 'scripts/migrate.ts')], + cwd: DB_PACKAGE_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} + +export async function buildApp(options: StackCommandOptions): Promise { + await runCommand({ + name: 'sandbox-bundles', + command: options.bunExecutable, + args: ['--no-env-file', 'run', 'build:sandbox-bundles'], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) + await runCommand({ + name: 'next-build', + command: options.nodeExecutable, + args: [path.join(REPO_ROOT, 'node_modules/next/dist/bin/next'), 'build'], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} + +export async function startRealtime(options: StackCommandOptions): Promise { + const realtime = spawnManagedProcess({ + name: 'realtime', + command: options.bunExecutable, + args: ['--no-env-file', 'src/index.ts'], + cwd: REALTIME_APP_DIR, + env: { + ...options.env, + SIM_DB_ROLE: 'realtime', + DB_APP_NAME: 'sim-realtime', + REALTIME_HOST: '127.0.0.1', + PORT: '3002', + }, + logsDirectory: options.logsDirectory, + }) + try { + await waitForManagedProcessReady(realtime, (signal) => + waitForHttpReady({ + name: 'Realtime', + url: 'http://127.0.0.1:3002/health', + signal, + validate: async (response) => { + if (!response.ok) return false + const body = (await response.json()) as { status?: string; runId?: string } + return body.status === 'ok' && body.runId === options.env.E2E_RUN_ID + }, + }) + ) + return realtime + } catch (error) { + await realtime.stop() + throw error + } +} + +export async function startApp(options: StackCommandOptions): Promise { + const app = spawnManagedProcess({ + name: 'app', + command: options.nodeExecutable, + args: [ + path.join(REPO_ROOT, 'node_modules/next/dist/bin/next'), + 'start', + '-p', + '3000', + '-H', + '127.0.0.1', + ], + cwd: SIM_APP_DIR, + env: { + ...options.env, + SIM_DB_ROLE: 'web', + DB_APP_NAME: 'sim-app', + PORT: '3000', + }, + logsDirectory: options.logsDirectory, + }) + try { + await waitForManagedProcessReady(app, (signal) => + waitForHttpReady({ + name: 'Next.js', + url: 'http://127.0.0.1:3000/api/health', + signal, + validate: async (response) => { + if (!response.ok) return false + const body = (await response.json()) as { status?: string; runId?: string } + return body.status === 'ok' && body.runId === options.env.E2E_RUN_ID + }, + }) + ) + return app + } catch (error) { + await app.stop() + throw error + } +} + +export async function runPlaywright( + options: Omit, + args: string[] +): Promise { + await runCommand({ + name: 'playwright', + command: options.nodeExecutable, + args: [PLAYWRIGHT_CLI, 'test', '--config=playwright.config.ts', ...args], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index ea2d1f0b227..b0dba3441aa 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -54,6 +54,7 @@ import { import { pauseProSubscriptionForOrgCoverage } from '@/lib/billing/organizations/membership' import { isPro, isTeam } from '@/lib/billing/plan-helpers' import { getPlans, resolvePlanFromStripeSubscription } from '@/lib/billing/plans' +import { buildStripeClientConfig } from '@/lib/billing/stripe-client-config' import { syncSeatsFromStripeQuantity } from '@/lib/billing/validation/seat-management' import { handleAbandonedCheckout } from '@/lib/billing/webhooks/checkout' import { handleChargeDispute, handleDisputeClosed } from '@/lib/billing/webhooks/disputes' @@ -188,9 +189,7 @@ const validStripeKey = env.STRIPE_SECRET_KEY let stripeClient = null if (validStripeKey) { - stripeClient = new Stripe(env.STRIPE_SECRET_KEY || '', { - apiVersion: '2025-08-27.basil', - }) + stripeClient = new Stripe(validStripeKey, buildStripeClientConfig(env)) } export const auth = betterAuth({ diff --git a/apps/sim/lib/billing/stripe-client-config.test.ts b/apps/sim/lib/billing/stripe-client-config.test.ts new file mode 100644 index 00000000000..e15d41f203b --- /dev/null +++ b/apps/sim/lib/billing/stripe-client-config.test.ts @@ -0,0 +1,123 @@ +/** @vitest-environment node */ + +import { describe, expect, it } from 'vitest' +import { + buildStripeClientConfig, + STRIPE_API_VERSION, + STRIPE_E2E_PROFILE, + type StripeClientConfigEnvironment, +} from '@/lib/billing/stripe-client-config' + +const guardedEnvironment: StripeClientConfigEnvironment = { + DATABASE_URL: 'postgresql://sim:sim@127.0.0.1:5432/sim_e2e_config_test', + E2E_PROFILE: STRIPE_E2E_PROFILE, + STRIPE_API_BASE_URL: 'http://127.0.0.1:12111', + STRIPE_SECRET_KEY: 'sk_test_e2e_config', +} + +describe('buildStripeClientConfig', () => { + it('preserves the normal Stripe configuration without an override', () => { + expect( + buildStripeClientConfig({ + DATABASE_URL: 'postgresql://db.example.com/sim', + STRIPE_SECRET_KEY: 'sk_live_normal_deployment', + }) + ).toEqual({ + apiVersion: STRIPE_API_VERSION, + }) + }) + + it('builds an HTTP transport for a guarded loopback fake', () => { + expect(buildStripeClientConfig(guardedEnvironment)).toEqual({ + apiVersion: STRIPE_API_VERSION, + host: '127.0.0.1', + port: '12111', + protocol: 'http', + }) + }) + + it('uses the HTTP default port when the override omits one', () => { + expect( + buildStripeClientConfig({ + ...guardedEnvironment, + STRIPE_API_BASE_URL: 'http://localhost', + }) + ).toMatchObject({ + host: 'localhost', + port: 80, + protocol: 'http', + }) + }) + + it.each(['sk_test_e2e_config', 'sk_live_e2e_config'])( + 'fails closed when the E2E profile configures %s without an override', + (stripeSecretKey) => { + expect(() => + buildStripeClientConfig({ + ...guardedEnvironment, + STRIPE_API_BASE_URL: undefined, + STRIPE_SECRET_KEY: stripeSecretKey, + }) + ).toThrow('STRIPE_API_BASE_URL is required') + } + ) + + it.each([ + { + name: 'a missing E2E profile', + environment: { E2E_PROFILE: undefined }, + message: 'requires E2E_PROFILE', + }, + { + name: 'a different E2E profile', + environment: { E2E_PROFILE: 'self-hosted-chromium' }, + message: 'requires E2E_PROFILE', + }, + { + name: 'a live Stripe key', + environment: { STRIPE_SECRET_KEY: 'sk_live_not_redirectable' }, + message: 'requires a Stripe test secret key', + }, + { + name: 'a public database', + environment: { DATABASE_URL: 'postgresql://db.example.com/sim' }, + message: 'requires a guarded sim_e2e_* database', + }, + { + name: 'an empty E2E database suffix', + environment: { DATABASE_URL: 'postgresql://127.0.0.1/sim_e2e_' }, + message: 'requires a guarded sim_e2e_* database', + }, + { + name: 'a malformed database URL', + environment: { DATABASE_URL: 'not-a-database-url' }, + message: 'requires a guarded sim_e2e_* database', + }, + ])('rejects an override with $name', ({ environment, message }) => { + expect(() => + buildStripeClientConfig({ + ...guardedEnvironment, + ...environment, + }) + ).toThrow(message) + }) + + it.each([ + 'https://127.0.0.1:12111', + 'http://stripe.example.com:12111', + 'http://0.0.0.0:12111', + 'http://user:password@127.0.0.1:12111', + 'http://127.0.0.1:12111/v1', + 'http://127.0.0.1:12111?mode=test', + 'http://127.0.0.1:12111#fragment', + ' http://127.0.0.1:12111', + 'not-a-url', + ])('rejects an unsafe or malformed override: %s', (stripeApiBaseUrl) => { + expect(() => + buildStripeClientConfig({ + ...guardedEnvironment, + STRIPE_API_BASE_URL: stripeApiBaseUrl, + }) + ).toThrow() + }) +}) diff --git a/apps/sim/lib/billing/stripe-client-config.ts b/apps/sim/lib/billing/stripe-client-config.ts new file mode 100644 index 00000000000..320cee74b77 --- /dev/null +++ b/apps/sim/lib/billing/stripe-client-config.ts @@ -0,0 +1,114 @@ +import type Stripe from 'stripe' + +export const STRIPE_API_VERSION = '2025-08-27.basil' as const +export const STRIPE_E2E_PROFILE = 'hosted-billing-chromium' as const + +export interface StripeClientConfigEnvironment { + DATABASE_URL?: string + E2E_PROFILE?: string + STRIPE_API_BASE_URL?: string + STRIPE_SECRET_KEY?: string +} + +const E2E_DATABASE_NAME_PATTERN = /^sim_e2e_[A-Za-z0-9_-]+$/ + +function getDatabaseName(databaseUrl: string | undefined): string | null { + if (!databaseUrl) return null + + try { + const parsed = new URL(databaseUrl) + if (parsed.protocol !== 'postgres:' && parsed.protocol !== 'postgresql:') return null + + const pathSegments = parsed.pathname.split('/').filter(Boolean) + if (pathSegments.length !== 1) return null + + return decodeURIComponent(pathSegments[0]) + } catch { + return null + } +} + +function isLoopbackHostname(hostname: string): boolean { + const normalized = hostname.toLowerCase() + if (normalized === 'localhost') return true + + const octets = normalized.split('.') + return ( + octets.length === 4 && + octets.every((octet) => /^\d+$/.test(octet) && Number(octet) <= 255) && + Number(octets[0]) === 127 + ) +} + +function parseStripeApiBaseUrl(rawUrl: string): URL { + if (rawUrl.trim() !== rawUrl) { + throw new Error('STRIPE_API_BASE_URL must not contain surrounding whitespace') + } + + let parsed: URL + try { + parsed = new URL(rawUrl) + } catch { + throw new Error('STRIPE_API_BASE_URL must be a valid absolute URL') + } + + if (parsed.protocol !== 'http:') { + throw new Error('STRIPE_API_BASE_URL must use HTTP for the loopback E2E fake') + } + if (!isLoopbackHostname(parsed.hostname)) { + throw new Error('STRIPE_API_BASE_URL must use a loopback hostname') + } + if (parsed.username || parsed.password) { + throw new Error('STRIPE_API_BASE_URL must not contain credentials') + } + if (parsed.search || parsed.hash) { + throw new Error('STRIPE_API_BASE_URL must not contain a query or hash') + } + if (parsed.pathname !== '/') { + throw new Error('STRIPE_API_BASE_URL must not contain a non-root path') + } + + return parsed +} + +/** + * Builds the Stripe SDK configuration while keeping the E2E transport override + * unavailable to normal deployments and production credentials. + */ +export function buildStripeClientConfig( + environment: StripeClientConfigEnvironment +): Stripe.StripeConfig { + const baseConfig: Stripe.StripeConfig = { + apiVersion: STRIPE_API_VERSION, + } + const override = environment.STRIPE_API_BASE_URL + + if (override === undefined || override === '') { + if (environment.E2E_PROFILE === STRIPE_E2E_PROFILE && environment.STRIPE_SECRET_KEY) { + throw new Error( + 'STRIPE_API_BASE_URL is required when the hosted billing E2E profile configures Stripe' + ) + } + return baseConfig + } + + if (environment.E2E_PROFILE !== STRIPE_E2E_PROFILE) { + throw new Error(`STRIPE_API_BASE_URL requires E2E_PROFILE=${STRIPE_E2E_PROFILE}`) + } + if (!environment.STRIPE_SECRET_KEY?.startsWith('sk_test_')) { + throw new Error('STRIPE_API_BASE_URL requires a Stripe test secret key') + } + + const databaseName = getDatabaseName(environment.DATABASE_URL) + if (!databaseName || !E2E_DATABASE_NAME_PATTERN.test(databaseName)) { + throw new Error('STRIPE_API_BASE_URL requires a guarded sim_e2e_* database') + } + + const endpoint = parseStripeApiBaseUrl(override) + return { + ...baseConfig, + host: endpoint.hostname, + port: endpoint.port || 80, + protocol: 'http', + } +} diff --git a/apps/sim/lib/billing/stripe-client.ts b/apps/sim/lib/billing/stripe-client.ts index 13bb089845b..9f1d6edc808 100644 --- a/apps/sim/lib/billing/stripe-client.ts +++ b/apps/sim/lib/billing/stripe-client.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import Stripe from 'stripe' +import { buildStripeClientConfig } from '@/lib/billing/stripe-client-config' import { env } from '@/lib/core/config/env' const logger = createLogger('StripeClient') @@ -37,9 +38,7 @@ const createStripeClientSingleton = () => { try { isInitializing = true - stripeClient = new Stripe(env.STRIPE_SECRET_KEY || '', { - apiVersion: '2025-08-27.basil', - }) + stripeClient = new Stripe(env.STRIPE_SECRET_KEY || '', buildStripeClientConfig(env)) logger.info('Stripe client initialized successfully') return stripeClient diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index c86eb9e7ff4..0972bd75634 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -68,7 +68,9 @@ export const env = createEnv({ REDIS_TLS_SERVERNAME: z.string().min(1).optional(), // TLS SNI override; required when REDIS_URL targets an IP over rediss:// (e.g. trigger.dev PrivateLink VPCE IP) so cert hostname verification matches the ElastiCache cert's CN // Payment & Billing + E2E_PROFILE: z.string().min(1).optional(), // Server-only E2E deployment profile selector STRIPE_SECRET_KEY: z.string().min(1).optional(), // Stripe secret key for payment processing + STRIPE_API_BASE_URL: z.string().url().optional(), // Guarded server-only Stripe API override for E2E fakes STRIPE_WEBHOOK_SECRET: z.string().min(1).optional(), // General Stripe webhook secret STRIPE_FREE_PRICE_ID: z.string().min(1).optional(), // Stripe price ID for free tier FREE_TIER_COST_LIMIT: z.number().optional(), // Cost limit for free tier users diff --git a/apps/sim/package.json b/apps/sim/package.json index a7800a9f904..288bf38b3c7 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -24,6 +24,8 @@ "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", + "test:e2e": "exec bun --no-env-file e2e/scripts/run.ts", + "test:e2e:install-browsers": "node ../../node_modules/@playwright/test/cli.js install chromium", "email:dev": "email dev --dir components/emails", "type-check": "NODE_OPTIONS='--max-old-space-size=8192' tsc --noEmit", "lint": "biome check --write --unsafe .", @@ -220,6 +222,7 @@ "zustand": "^5.0.13" }, "devDependencies": { + "@playwright/test": "1.61.1", "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@tailwindcss/typography": "0.5.19", diff --git a/apps/sim/playwright.config.ts b/apps/sim/playwright.config.ts new file mode 100644 index 00000000000..70cb2641644 --- /dev/null +++ b/apps/sim/playwright.config.ts @@ -0,0 +1,58 @@ +import { defineConfig, devices } from '@playwright/test' + +if (process.env.E2E_ORCHESTRATED !== '1') { + throw new Error( + 'Playwright tests must run through `bun run test:e2e` so database, environment, and teardown guards are active' + ) +} + +const isCI = process.env.CI === 'true' +const baseURL = process.env.E2E_BASE_URL ?? 'http://e2e.sim.ai:3000' + +export default defineConfig({ + testDir: './e2e', + testMatch: '**/*.spec.ts', + fullyParallel: true, + forbidOnly: isCI, + retries: 0, + workers: 1, + timeout: 60_000, + expect: { + timeout: 10_000, + }, + reporter: [ + ['list'], + ['html', { open: 'never', outputFolder: 'playwright-report' }], + ...(isCI ? ([['github']] as const) : []), + ], + outputDir: 'test-results', + use: { + ...devices['Desktop Chrome'], + baseURL, + launchOptions: { + args: ['--host-resolver-rules=MAP e2e.sim.ai 127.0.0.1'], + }, + actionTimeout: 15_000, + navigationTimeout: 30_000, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'off', + }, + projects: [ + { + name: 'hosted-billing-chromium-navigation', + testMatch: [ + '**/foundation/**/*.spec.ts', + '**/settings/smoke/unauthenticated.spec.ts', + '**/settings/navigation/**/*.spec.ts', + ], + workers: 1, + }, + { + name: 'hosted-billing-chromium-workflows', + testMatch: ['**/settings/smoke/authenticated.spec.ts', '**/settings/workflows/**/*.spec.ts'], + fullyParallel: false, + workers: 1, + }, + ], +}) diff --git a/apps/sim/vitest.config.ts b/apps/sim/vitest.config.ts index a966ad5233e..02425c6f712 100644 --- a/apps/sim/vitest.config.ts +++ b/apps/sim/vitest.config.ts @@ -16,7 +16,7 @@ export default defineConfig({ globals: true, environment: 'node', include: ['**/*.test.{ts,tsx}'], - exclude: [...configDefaults.exclude, '**/node_modules/**', '**/dist/**'], + exclude: [...configDefaults.exclude, '**/node_modules/**', '**/dist/**', '**/e2e/**'], setupFiles: ['./vitest.setup.ts'], pool: 'threads', isolate: true, diff --git a/biome.json b/biome.json index 271e701c8b4..3271ed0dde1 100644 --- a/biome.json +++ b/biome.json @@ -22,6 +22,9 @@ "!**/.env", "!**/.vercel", "!**/coverage", + "!**/playwright-report", + "!**/test-results", + "!**/e2e/.runs", "!**/public/sw.js", "!**/public/workbox-*.js", "!**/public/worker-*.js", diff --git a/bun.lock b/bun.lock index f3ad75a78dd..51ef90840cb 100644 --- a/bun.lock +++ b/bun.lock @@ -288,6 +288,7 @@ "zustand": "^5.0.13", }, "devDependencies": { + "@playwright/test": "1.61.1", "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@tailwindcss/typography": "0.5.19", @@ -1331,6 +1332,8 @@ "@pkgjs/parseargs": ["@pkgjs/parseargs@0.11.0", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="], + "@playwright/test": ["@playwright/test@1.61.1", "", { "dependencies": { "playwright": "1.61.1" }, "bin": { "playwright": "cli.js" } }, "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig=="], + "@posthog/core": ["@posthog/core@1.24.4", "", { "dependencies": { "cross-spawn": "^7.0.6" } }, "sha512-S+TolwBHSSJz7WWtgaELQWQqXviSm3uf1e+qorWUts0bZcgPwWzhnmhCUZAhvn0NVpTQHDJ3epv+hHbPLl5dHg=="], "@posthog/types": ["@posthog/types@1.364.4", "", {}, "sha512-U7NpIy9XWrzz1q/66xyDu8Wm12a7avNRKRn5ISPT5kuCJQRaeAaHuf+dpgrFnuqjCCgxg+oIY/ReJdlZ+8/z4Q=="], @@ -3467,6 +3470,10 @@ "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], + "playwright": ["playwright@1.61.1", "", { "dependencies": { "playwright-core": "1.61.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ=="], + + "playwright-core": ["playwright-core@1.61.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg=="], + "points-on-curve": ["points-on-curve@0.2.0", "", {}, "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A=="], "points-on-path": ["points-on-path@0.2.1", "", { "dependencies": { "path-data-parser": "0.1.0", "points-on-curve": "0.2.0" } }, "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g=="], @@ -4781,6 +4788,8 @@ "pino-pretty/pino-abstract-transport": ["pino-abstract-transport@3.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg=="], + "playwright/fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + "postcss-nested/postcss-selector-parser": ["postcss-selector-parser@6.1.4", "", { "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" } }, "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ=="], "posthog-js/@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.208.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg=="],