Add durable settings browser test suite

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ on:
   push:
     branches: [main, staging, dev]
   pull_request:
-    branches: [main, staging, dev]
+    branches: [main, staging, dev, e2e/settings-playwright]
     # Docs content and markdown don't affect the app build or images; push
     # runs stay unfiltered because they feed the deploy pipeline.
     paths-ignore:

.github/workflows/test-build.yml

@@ -258,4 +258,104 @@ jobs:
           AWS_REGION: 'us-west-2'
           ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only
           TURBO_CACHE_DIR: .turbo
-        run: bunx turbo run build --filter=sim
+        run: bunx turbo run build --filter=sim
+
+  settings-e2e:
+    name: Settings E2E (informational)
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    timeout-minutes: 45
+    continue-on-error: true
+
+    services:
+      postgres:
+        image: pgvector/pgvector:pg17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.13
+
+      - name: Setup Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: 22
+
+      - name: Mount Bun cache (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
+          path: ~/.bun/install/cache
+
+      - name: Mount node_modules (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
+          path: ./node_modules
+
+      - name: Mount Playwright browsers (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-playwright-browsers-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
+          path: ~/.cache/ms-playwright
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Configure E2E hostname
+        run: echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts
+
+      - name: Install Chromium
+        working-directory: apps/sim
+        run: bun run test:e2e:install-browsers -- --with-deps
+
+      - name: Run settings E2E foundation
+        timeout-minutes: 40
+        working-directory: apps/sim
+        env:
+          CI: 'true'
+          E2E_PG_ADMIN_URL: 'postgresql://postgres:postgres@127.0.0.1:5432/postgres'
+        run: bun run test:e2e
+
+      - name: Check E2E diagnostics eligibility
+        id: e2e_diagnostics
+        if: failure()
+        shell: bash
+        run: |
+          shopt -s nullglob
+          markers=(apps/sim/e2e/.runs/*/markers/leak-scan-complete.json)
+          if (( ${#markers[@]} > 0 )); then
+            echo "safe=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "safe=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Upload E2E diagnostics
+        if: failure() && steps.e2e_diagnostics.outputs.safe == 'true'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: settings-e2e-${{ github.run_id }}
+          path: |
+            apps/sim/playwright-report/
+            apps/sim/test-results/
+            apps/sim/e2e/.runs/
+            !apps/sim/e2e/.runs/**/auth/**
+            !apps/sim/e2e/.runs/**/private/**
+            !apps/sim/e2e/.runs/**/homes/**
+          if-no-files-found: ignore
+          include-hidden-files: true
+          retention-days: 7

.gitignore

@@ -18,6 +18,11 @@ package-lock.json
 # testing
 /coverage
 /apps/**/coverage
+/apps/**/playwright-report/
+/apps/**/test-results/
+/apps/**/e2e/.runs/
+/apps/**/e2e/.auth/
+/apps/**/e2e/.cache/
 
 # next.js
 /.next/
@@ -31,6 +36,7 @@ package-lock.json
 **/dist/
 **/standalone/
 sim-standalone.tar.gz
+/.artifacts/
 
 # redis
 dump.rdb

apps/realtime/src/env.ts

@@ -14,6 +14,7 @@ const EnvSchema = z.object({
   INTERNAL_API_SECRET: z.string().min(32),
   NEXT_PUBLIC_APP_URL: z.string().url(),
   ALLOWED_ORIGINS: z.string().optional(),
+  REALTIME_HOST: z.string().min(1).default('0.0.0.0'),
   PORT: z.coerce.number().int().positive().default(3002),
   SIM_DB_ROLE: z.enum(['web', 'trigger', 'realtime']).optional(),
   DISABLE_AUTH: z

apps/realtime/src/index.ts

@@ -31,6 +31,7 @@ async function createRoomManager(io: SocketIOServer): Promise<IRoomManager> {
 async function main() {
   const httpServer = createServer()
   const PORT = env.PORT
+  const HOST = env.REALTIME_HOST
 
   logger.info('Starting Socket.IO server...', {
     port: PORT,
@@ -96,9 +97,9 @@ async function main() {
 
   await assertSchemaCompatibility()
 
-  httpServer.listen(PORT, '0.0.0.0', () => {
-    logger.info(`Socket.IO server running on port ${PORT}`)
-    logger.info(`Health check available at: http://localhost:${PORT}/health`)
+  httpServer.listen(PORT, HOST, () => {
+    logger.info(`Socket.IO server running on ${HOST}:${PORT}`)
+    logger.info(`Health check available at: http://${HOST}:${PORT}/health`)
   })
 
   const shutdown = async () => {

apps/realtime/src/routes/http.ts

@@ -69,6 +69,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) {
             status: 'ok',
             timestamp: new Date().toISOString(),
             connections,
+            ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}),
           })
         )
       } catch (error) {

apps/sim/app/(landing)/layout.tsx

@@ -11,6 +11,7 @@ import { XPageViewTracker } from '@/app/(landing)/x-page-view-tracker'
 const HUBSPOT_SCRIPT_SRC = 'https://js-na2.hs-scripts.com/246720681.js' as const
 
 const X_PIXEL_ID = 'q5xbl' as const
+const isMarketingAnalyticsEnabled = isHosted && !process.env.E2E_PROFILE
 
 /** X (Twitter) conversion tracking base code — loads uwt.js and fires the initial PageView. */
 const X_PIXEL_BASE_CODE = `!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
@@ -43,7 +44,7 @@ export default function LandingLayout({ children }: { children: ReactNode }) {
     <LandingShell>
       {children}
       {/* HubSpot + X pixel tracking — hosted only */}
-      {isHosted && (
+      {isMarketingAnalyticsEnabled && (
         <>
           <Script id='hs-script-loader' src={HUBSPOT_SCRIPT_SRC} strategy='afterInteractive' />
           <Script id='x-pixel-base' strategy='afterInteractive'>

apps/sim/app/api/health/route.test.ts

@@ -1,7 +1,7 @@
 /**
  * @vitest-environment node
  */
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
 import { GET } from '@/app/api/health/route'
 
 describe('GET /api/health', () => {
@@ -14,4 +14,17 @@ describe('GET /api/health', () => {
       timestamp: expect.any(String),
     })
   })
+
+  it('returns the E2E run identity when configured', async () => {
+    vi.stubEnv('E2E_RUN_ID', 'run-health-check')
+    try {
+      const response = await GET()
+      await expect(response.json()).resolves.toMatchObject({
+        status: 'ok',
+        runId: 'run-health-check',
+      })
+    } finally {
+      vi.unstubAllEnvs()
+    }
+  })
 })

apps/sim/app/api/health/route.ts

@@ -1,11 +1,14 @@
 /**
  * Health check endpoint for deployment platforms and container probes.
  */
+export const dynamic = 'force-dynamic'
+
 export async function GET(): Promise<Response> {
   return Response.json(
     {
       status: 'ok',
       timestamp: new Date().toISOString(),
+      ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}),
     },
     { status: 200 }
   )

apps/sim/app/api/users/me/settings/route.ts

@@ -60,6 +60,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest) => {
       .values({
         id: generateShortId(),
         userId,
+        ...defaultUserSettings,
         ...validatedData,
         updatedAt: new Date(),
       })

apps/sim/app/layout.tsx

@@ -28,6 +28,7 @@ export const metadata: Metadata = generateBrandedMetadata()
 
 const GTM_ID = 'GTM-T7PHSRX5' as const
 const GA_ID = 'G-DR7YBE70VS' as const
+const isMarketingAnalyticsEnabled = isHosted && !process.env.E2E_PROFILE
 
 export default function RootLayout({ children }: { children: React.ReactNode }) {
   const themeCSS = generateThemeCSS()
@@ -196,7 +197,7 @@ export default function RootLayout({ children }: { children: React.ReactNode })
         <meta httpEquiv='x-ua-compatible' content='ie=edge' />
 
         {/* Google Tag Manager — hosted only */}
-        {isHosted && (
+        {isMarketingAnalyticsEnabled && (
           <Script
             id='gtm'
             strategy='afterInteractive'
@@ -211,7 +212,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
         )}
 
         {/* Google Analytics (gtag.js) — hosted only */}
-        {isHosted && (
+        {isMarketingAnalyticsEnabled && (
           <>
             <Script
               id='gtag-src'
@@ -232,7 +233,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
       </head>
       <body className={`${season.variable} font-season`} suppressHydrationWarning>
         {/* Google Tag Manager (noscript) — hosted only */}
-        {isHosted && (
+        {isMarketingAnalyticsEnabled && (
           <noscript>
             <iframe
               src={`https://www.googletagmanager.com/ns.html?id=${GTM_ID}`}