v0.8.15: ci improvements, trigger machine resizing, connectors fixes

Author: waleedlatif1

.github/actions/docker-build/action.yml

@@ -19,6 +19,13 @@ inputs:
   tags:
     description: Comma-separated list of tags to push.
     required: true
+  max-cache-size-mb:
+    description: >-
+      Layer cache to retain after the post-job prune, in MB. Must stay above one
+      build's working set (base + dependency layers + RUN --mount=type=cache
+      dirs) or every build evicts what the next one needs. Falls back to the
+      small-image default below when empty.
+    required: false
 
 # Registry logins must precede this action. provenance/sbom stay off: attestation
 # manifests break `imagetools create` retagging in promote-images.
@@ -42,11 +49,24 @@ runs:
         PLATFORMS: ${{ inputs.platforms }}
       run: echo "value=${GITHUB_REPOSITORY##*/}/${FILE#./}/${PLATFORMS//\//-}" >> "$GITHUB_OUTPUT"
 
+    # max-cache-size-mb is what bounds the disk: BuildKit's default GC is
+    # time-based only (layers unused for 8 days), and setup-docker-builder skips
+    # pruning altogether when the value is empty. On a repo that builds this
+    # often nothing ever ages out, so the disks grew without limit —
+    # app.Dockerfile/linux-amd64 reached 351 GB inside a day, and realtime, whose
+    # image is under 300 MB, sat at 249 GB. Sticky disks bill at ~$0.51/GB-month,
+    # so that was real money for layers no build would ever read again.
+    #
+    # The fallback is here rather than an input `default:` because callers pass
+    # this from a matrix field, and an unset matrix key arrives as the empty
+    # string — which counts as "provided", so a `default:` would never apply and
+    # a row that forgot the field would silently go back to unbounded growth.
     - name: Set up Blacksmith builder
       if: inputs.provider == '' || inputs.provider == 'blacksmith'
       uses: useblacksmith/setup-docker-builder@a5256a73e30f09e37e3eceb8ca36043d17621d24 # v2
       with:
         cache-key: ${{ steps.cache-key.outputs.value }}
+        max-cache-size-mb: ${{ inputs.max-cache-size-mb || '25600' }}
 
     - name: Build and push (Blacksmith)
       if: inputs.provider == '' || inputs.provider == 'blacksmith'

.github/codeql/codeql-config.yml

@@ -31,3 +31,11 @@ paths-ignore:
   - '**/dist/**'
   - '**/.next/**'
   - 'apps/docs/content/**'
+
+# Do NOT add `queries:`, `packs:`, `query-filters:`, or `disable-default-queries`
+# here to try to speed the scan up. Under the code-scanning feature flag the
+# action's checkOverlayAnalysisFeatureEnabled treats any of those as
+# OverlayDisabledReason.NonDefaultQueries and permanently turns off overlay
+# (incremental) analysis. Extraction is ~53% of a run and is exactly what overlay
+# skips, so scoping the queries trades a documented up-to-10x win for a few
+# percent off the 27% query phase.

.github/workflows/ci.yml

@@ -76,7 +76,7 @@ jobs:
   # (/api/desktop/update) starts offering automatically.
   detect-desktop-changes:
     name: Detect Desktop Changes
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 5
     if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/staging')
     outputs:
@@ -165,7 +165,15 @@ jobs:
           # build` ~260s). The same `next build` runs on 16 vCPU in the separate
           # Build App verification job, which does not gate anything; this one
           # was doing comparable work on half the cores.
+          #
+          # cache_mb is the layer cache the post-job prune retains, and it is the
+          # only reason the sticky disks stay bounded — see docker-build's
+          # action.yml. Rows that omit it take the small-image default there. The
+          # app image overrides because it carries ~34 layers plus apt and bun
+          # cache mounts for the whole monorepo; 100 GB is several builds' worth
+          # of headroom over that working set.
           - dockerfile: ./docker/app.Dockerfile
+            cache_mb: '102400'
             ecr_repo_secret: ECR_APP
             gh_runner: linux-x64-8-core
             bs_runner: blacksmith-16vcpu-ubuntu-2404
@@ -176,11 +184,11 @@ jobs:
           - dockerfile: ./docker/realtime.Dockerfile
             ecr_repo_secret: ECR_REALTIME
             gh_runner: ubuntu-latest
-            bs_runner: blacksmith-4vcpu-ubuntu-2404
+            bs_runner: blacksmith-2vcpu-ubuntu-2404
           - dockerfile: ./docker/pii.Dockerfile
             ecr_repo_secret: ECR_PII
             gh_runner: ubuntu-latest
-            bs_runner: blacksmith-4vcpu-ubuntu-2404
+            bs_runner: blacksmith-2vcpu-ubuntu-2404
     steps:
       - name: Checkout code
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -214,6 +222,7 @@ jobs:
           file: ${{ matrix.dockerfile }}
           platforms: linux/amd64
           tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev
+          max-cache-size-mb: ${{ matrix.cache_mb }}
 
   # Dev: deploy Trigger.dev background tasks to the preview "dev-sim" branch.
   # Gated after migrate-dev for the same reason as build-dev — the new task
@@ -280,6 +289,7 @@ jobs:
       matrix:
         include:
           - dockerfile: ./docker/app.Dockerfile
+            cache_mb: '102400'
             ghcr_image: ghcr.io/simstudioai/simstudio
             ecr_repo_secret: ECR_APP
             gh_runner: linux-x64-8-core
@@ -293,12 +303,12 @@ jobs:
             ghcr_image: ghcr.io/simstudioai/realtime
             ecr_repo_secret: ECR_REALTIME
             gh_runner: ubuntu-latest
-            bs_runner: blacksmith-4vcpu-ubuntu-2404
+            bs_runner: blacksmith-2vcpu-ubuntu-2404
           - dockerfile: ./docker/pii.Dockerfile
             ghcr_image: ghcr.io/simstudioai/pii
             ecr_repo_secret: ECR_PII
             gh_runner: ubuntu-latest
-            bs_runner: blacksmith-4vcpu-ubuntu-2404
+            bs_runner: blacksmith-2vcpu-ubuntu-2404
           # No ECR repo is provisioned for cron, so it publishes to GHCR only.
           # The tag step below omits the ECR tag when the repo name is empty.
           - dockerfile: ./docker/cron.Dockerfile
@@ -382,6 +392,7 @@ jobs:
           file: ${{ matrix.dockerfile }}
           platforms: linux/amd64
           tags: ${{ steps.meta.outputs.tags }}
+          max-cache-size-mb: ${{ matrix.cache_mb }}
 
   # Promote the sha-tagged ECR images to the deploy tags once tests and
   # migrations pass. Pushing the ECR latest/staging tag is what triggers
@@ -484,6 +495,7 @@ jobs:
         # hang a release in `queued` rather than fail a PR.
         include:
           - dockerfile: ./docker/app.Dockerfile
+            cache_mb: '102400'
             image: ghcr.io/simstudioai/simstudio
             gh_runner: linux-arm64-8-core
             bs_runner: blacksmith-8vcpu-ubuntu-2404-arm
@@ -522,6 +534,7 @@ jobs:
           file: ${{ matrix.dockerfile }}
           platforms: linux/arm64
           tags: ${{ matrix.image }}:${{ github.sha }}-arm64
+          max-cache-size-mb: ${{ matrix.cache_mb }}
 
   # Publish all mutable GHCR tags (latest, latest-amd64/arm64, version tags)
   # and the multi-arch manifests from the immutable sha tags — only on main,
@@ -675,7 +688,7 @@ jobs:
   # Job-level `if:` cannot read the secrets context, hence the probe job.
   check-desktop-signing:
     name: Check Desktop Signing Secrets
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 2
     needs: [detect-version, detect-desktop-changes]
     # !cancelled(): detect-desktop-changes is skipped on main (and
@@ -724,7 +737,7 @@ jobs:
   # remains testable end to end with a manual download.
   create-desktop-prerelease:
     name: Create Desktop Prerelease
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 5
     needs: [detect-desktop-changes, check-desktop-signing]
     # Requires the signing probe to have actually succeeded (not just "not
@@ -813,7 +826,7 @@ jobs:
   # point of view.
   publish-desktop-prerelease:
     name: Publish Desktop Prerelease
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 5
     needs: [create-desktop-prerelease, desktop-prerelease]
     permissions:
@@ -837,7 +850,7 @@ jobs:
   # are always garbage by this point — the current run's release is published.
   prune-desktop-prereleases:
     name: Prune Desktop Prereleases
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 5
     needs: [publish-desktop-prerelease]
     permissions:

.github/workflows/codeql.yml

@@ -20,8 +20,16 @@ on:
   # created, prevents developers from introducing new vulnerabilities."
   push:
     branches: [main]
+  # main only, not staging. Feature PRs land on staging and are ~90% of PR scan
+  # volume, and every one of them is scanned again — against the exact tree being
+  # promoted — when the staging->main PR opens. Scanning at the promotion
+  # boundary defers the signal rather than dropping it.
+  #
+  # Deliberately a branch cut and not an activity-type cut: dropping
+  # `synchronize` would have scanned each PR's first commit and never its final
+  # state, which is backwards, since review fixups land in later pushes.
   pull_request:
-    branches: [main, staging]
+    branches: [main]
     # `ready_for_review` is not a default activity type, so it has to be listed
     # alongside the defaults it replaces. Without it, a PR opened as a draft and
     # then marked ready is skipped by the job-level draft guard and never
@@ -41,7 +49,15 @@ on:
     # Safety net behind the push trigger, and the thing that keeps the
     # default-branch alert view fresh when main is quiet. Only fires once this
     # file is on the default branch — schedule events ignore other branches.
-    - cron: '17 8 * * 1'
+    #
+    # Daily rather than weekly. Pushes to main are rare, and with PR scans now
+    # limited to main the alert view leans on this more than it used to; a week
+    # is too long to leave it stale. It also reseeds the overlay-base database
+    # that PR runs restore from — that cache key embeds the CodeQL bundle
+    # version, so a bundle bump invalidates it, and an unused Actions cache is
+    # evicted after 7 days. One 8 vCPU default-branch scan a day is a few
+    # dollars a month against a PR scan that halves when the base is warm.
+    - cron: '17 8 * * *'
   workflow_dispatch:
 
 concurrency:
@@ -54,7 +70,12 @@ permissions:
 jobs:
   analyze:
     name: Analyze ${{ matrix.language }}
-    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
+    # Sized per language, not per workflow. The two analyses are nothing alike:
+    # javascript-typescript peaks at 19.5 GB (p95 over 3090 runs), so it needs
+    # the 8 vCPU tier's 30.4 GB and would OOM on the 4 vCPU tier's 15.2 GB; the
+    # actions analysis peaks at 1.3 GB and averages 22% CPU over a 39s median
+    # run, so 8 vCPU was 4x more machine than it ever used.
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && matrix.bs_runner || 'ubuntu-latest' }}
     timeout-minutes: 60
     if: github.event.pull_request.draft != true
     permissions:
@@ -71,7 +92,11 @@ jobs:
         # entries default setup listed were one analysis, not three.
         # `javascript-typescript` is the documented spelling. Python dropped:
         # 7 files in the tree.
-        language: [javascript-typescript, actions]
+        include:
+          - language: javascript-typescript
+            bs_runner: blacksmith-8vcpu-ubuntu-2404
+          - language: actions
+            bs_runner: blacksmith-4vcpu-ubuntu-2404
 
     steps:
       - name: Checkout repository

apps/sim/background/knowledge-connector-sync.ts

@@ -98,7 +98,14 @@ export async function executeConnectorSyncJob(payload: unknown) {
 export const knowledgeConnectorSync = task({
   id: 'knowledge-connector-sync',
   maxDuration: CONNECTOR_SYNC_MAX_DURATION_SECONDS,
-  machine: 'large-2x',
+  /**
+   * Sized from production telemetry: peak sampled RSS 2.6 GB and peak 1.4 vCPU,
+   * so `large-1x` holds ~3x memory and ~2.8x CPU headroom. No `outOfMemory`
+   * escalation: an OOM is a SIGKILL, so the run never reaches the terminal
+   * write that clears `syncLockToken`, and the escalated attempt would find the
+   * row still `syncing` and skip. The stale-lock reaper owns that recovery.
+   */
+  machine: 'large-1x',
   retry: {
     maxAttempts: 3,
     factor: 2,

apps/sim/background/knowledge-processing.ts

@@ -135,7 +135,13 @@ export async function runDocumentProcessing(
 export const processDocument = task({
   id: 'knowledge-process-document',
   maxDuration: envNumber(env.KB_CONFIG_MAX_DURATION, 600),
-  machine: 'large-1x', // 4 vCPU, 8GB RAM - needed for large PDF processing
+  /**
+   * Sized from production telemetry: peak sampled RSS 902 MB and peak 1.2 vCPU
+   * across a corpus where no document exceeded 2 GB, so `medium-2x` holds ~4x
+   * memory and ~1.7x CPU headroom over the observed worst case. The prior
+   * `large-1x` reserved 8 GB against a worst case using an eighth of it.
+   */
+  machine: 'medium-2x',
   retry: {
     maxAttempts: envNumber(env.KB_CONFIG_MAX_ATTEMPTS, 3),
     factor: envNumber(env.KB_CONFIG_RETRY_FACTOR, 2),

apps/sim/lib/api/contracts/knowledge/documents.ts

@@ -319,16 +319,23 @@ export const listKnowledgeDocumentsContract = defineRouteContract({
   },
 })
 
-export const createKnowledgeDocumentsContract = defineRouteContract({
-  method: 'POST',
-  path: '/api/knowledge/[id]/documents',
+/**
+ * Document creation from inline content has no HTTP route: `POST
+ * /api/knowledge/[id]/documents` was retired when tool operations moved
+ * in-process, and the surviving `GET`/`PATCH` on that path would answer a `POST`
+ * with 405. So these stay plain schemas rather than a `defineRouteContract` —
+ * `lib/internal/knowledge/execute-tool.ts` validates `knowledge_create_document`
+ * against them directly. Callers wanting an HTTP upload use v1 or v2, both of
+ * which take multipart file bodies rather than inline content.
+ */
+export const createKnowledgeDocumentsSchemas = {
   params: knowledgeBaseParamsSchema,
   body: createKnowledgeDocumentsBodySchema,
-  response: {
-    mode: 'json',
-    schema: successResponseSchema(z.union([bulkCreateDocumentsResponseSchema, documentDataSchema])),
-  },
-})
+} as const
+
+export const createKnowledgeDocumentsResponseSchema = successResponseSchema(
+  z.union([bulkCreateDocumentsResponseSchema, documentDataSchema])
+)
 
 export const updateKnowledgeDocumentContract = defineRouteContract({
   method: 'PUT',

apps/sim/lib/api/contracts/selectors/confluence.ts

@@ -399,14 +399,16 @@ export const confluencePageSelectorContract = definePostSelector(
   z.object({ id: z.string(), title: z.string() }).passthrough()
 )
 
-export const confluenceUpdatePageContract = defineConfluencePutContract(
-  '/api/tools/confluence/page',
-  confluenceUpdatePageBodySchema
-)
-export const confluenceDeletePageContract = defineConfluenceDeleteContract(
-  '/api/tools/confluence/page',
-  confluenceDeletePageBodySchema
-)
+/**
+ * Page update and delete have no contract because they have no route: the
+ * `PUT`/`DELETE` handlers on `/api/tools/confluence/page` were retired when the
+ * tool moved in process, and the surviving selector `POST` on that path would
+ * answer either verb with 405. `lib/internal/confluence/execute-tool.ts`
+ * validates both against `confluenceUpdatePageBodySchema` /
+ * `confluenceDeletePageBodySchema` directly.
+ */
+export type ConfluenceUpdatePageBody = z.output<typeof confluenceUpdatePageBodySchema>
+export type ConfluenceDeletePageBody = z.output<typeof confluenceDeletePageBodySchema>
 export const confluenceDeleteAttachmentContract = defineConfluenceDeleteContract(
   '/api/tools/confluence/attachment',
   confluenceDeleteAttachmentBodySchema
@@ -562,8 +564,6 @@ export const confluenceUserContract = defineConfluencePostContract(
 
 export type ConfluencePagesBody = ContractBody<typeof confluencePagesSelectorContract>
 export type ConfluencePageBody = ContractBody<typeof confluencePageSelectorContract>
-export type ConfluenceUpdatePageBody = ContractBody<typeof confluenceUpdatePageContract>
-export type ConfluenceDeletePageBody = ContractBody<typeof confluenceDeletePageContract>
 export type ConfluenceDeleteAttachmentBody = ContractBody<typeof confluenceDeleteAttachmentContract>
 export type ConfluenceListAttachmentsQuery = ContractQuery<typeof confluenceListAttachmentsContract>
 export type ConfluenceListBlogPostsQuery = ContractQuery<typeof confluenceListBlogPostsContract>

apps/sim/lib/api/contracts/tools/docusign.ts

@@ -4,7 +4,6 @@ export * from './communication'
 export * from './crowdstrike'
 export * from './custom'
 export * from './databases'
-export * from './docusign'
 export * from './file'
 export * from './google'
 export * from './imap'