diff --git a/CHANGELOG.md b/CHANGELOG.md index bb4f9e8..bc80033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,25 @@ new version heading in the same commit. ## [Unreleased] +## [0.293.0] — 2026-08-03 +### Added +- **A ceiling on how long a session may sit blocked on an unanswered question.** The idle janitor skips a + session that's waiting on a person — rightly, that wait is real — but nothing expires an Inbox card, so + the exemption had no floor and the wait could be permanent. Live expresstech was holding a `support` + session **66 hours** after its question was asked, alongside two more questions unanswered since 07-28; + each such session pins a `claude` process (~300 MB) and a concurrency-cap slot indefinitely. New setting + **Settings → Runtime → "Close a session waiting on an unanswered question after (hours)"**, default + **72 h** — the same age at which `escalateStalePrompts` already gives up nagging and treats a prompt as + dead. Past it the session is closed and its card **cancelled**, which is also what makes the card + dismissable instead of hanging in the Inbox. `0` restores the old wait-for-ever. + - The clock runs from when the **oldest pending card was raised**, not from session idleness — the claim + being made is "nobody answered this in three days", and a blocked session is quiet by definition. + - A session with **someone attached is never cut**: a human is right there and can answer. + - Applies to the interactive lane. Unattended runs already had a ceiling (`unattendedMaxHours`, 24 h) + that overrides a pending block. + - Audited as `session.reaped` with `reason: 'blocked-timeout'` and how long it had waited, so this is + distinguishable from an ordinary idle reap. `blockedMaxHours` on `GET`/`PUT /api/settings/concurrency`. + ## [0.292.5] — 2026-08-03 ### Changed - **The router now uses the LLM to pick from the FULL roster when keyword routing isn't confident — not diff --git a/package-lock.json b/package-lock.json index 1fa2bf6..9ab5f52 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.292.5", + "version": "0.293.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.292.5", + "version": "0.293.0", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index 83b59e6..f9dd828 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.292.5", + "version": "0.293.0", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", diff --git a/scripts/idle-reaper-test.cjs b/scripts/idle-reaper-test.cjs index 4471696..c727ecf 100644 --- a/scripts/idle-reaper-test.cjs +++ b/scripts/idle-reaper-test.cjs @@ -115,6 +115,56 @@ assert(!killed.includes('aos-' + runningBlocked), 'a still-running run blocked o assert(statusOf(runningBlocked) === 'running', '…and still running'); assert(qStatus(qst) === 'pending', '…with its question still open for an answer'); +/* Sweep 3 — the BLOCKED ceiling. An interactive session waiting on a question/approval is exempt from the + * idle reaper, and nothing expires an Inbox card, so with no ceiling it waits for ever (live expresstech: + * 66 h on a question raised three days earlier). Past `blockedMaxHours`, measured from when the card was + * RAISED, it is closed and the card cancelled — but a recent block, and an attached session, are left. */ +console.log('\n\x1b[1m5) sweep 3: the blocked-on-a-card ceiling\x1b[0m'); +// Section 4 pinned aliveNames to its own two panes; restore the all-alive stub or sweep 0 (crash +// detection) marks every session created below as `crashed` before the idle sweep ever sees it. +tm.backend.aliveNames = () => new Set(aos.db.prepare('SELECT tmux FROM term_sessions').all().map((r) => r.tmux)); +aos.settings.setInteractiveIdleTimeoutHours(48); +assert(aos.settings.blockedMaxHours() === 72, 'unset → 72h default'); +assert(aos.settings.setBlockedMaxHours(0) === 0, 'set 0 → 0 (disabled)'); +assert(aos.settings.setBlockedMaxHours(99999) === 24 * 30, 'clamps to 30 days max'); +aos.settings.setBlockedMaxHours(72); + +const askAt = (id, ageH) => { + const qid = 'qst_b_' + id; + aos.db.prepare('INSERT INTO questions (id, run_id, tenant, agent, prompt, status, created_at) VALUES (?,?,?,?,?,?,?)') + .run(qid, id, aos.tenant, 'website-bot', 'which one?', 'pending', Date.now() - ageH * H); + return qid; +}; +// All four are far past the 48h IDLE cutoff — the only thing that differs is the block. +const blockedOld = mkSession({ created_at: Date.now() - 96 * H }); // card raised 90h ago → reap +const qOld = askAt(blockedOld, 90); +const blockedFresh = mkSession({ created_at: Date.now() - 96 * H }); // card raised 2h ago → keep waiting +const qFresh = askAt(blockedFresh, 2); +const blockedAttached = mkSession({ created_at: Date.now() - 96 * H, tmux: 'aos-WATCHED' }); attached.add('aos-WATCHED'); +askAt(blockedAttached, 90); // someone is there to answer → keep +const blockedApproval = mkSession({ created_at: Date.now() - 96 * H }); // an APPROVAL counts the same way +const { req: oldApr } = aos.approvals.request({ runId: blockedApproval, tenant: aos.tenant, level: 'owner', + reason: 'test', attempt: { capabilityId: 'shell.exec', args: {}, reasoning: '' } }); +aos.db.prepare('UPDATE approvals SET created_at = ? WHERE id = ?').run(Date.now() - 90 * H, oldApr.id); +killed.length = 0; + +tm.reapIdleSessions(); + +assert(statusOf(blockedOld) === 'stopped', 'blocked 90h on a 72h ceiling → closed'); +assert(qStatus(qOld) === 'cancelled', '…and its question cancelled, so the card is dismissable'); +assert(statusOf(blockedFresh) === 'running', 'blocked only 2h → still waiting'); +assert(qStatus(qFresh) === 'pending', '…its question still open'); +assert(statusOf(blockedAttached) === 'running', 'blocked 90h but a human is attached → left alone'); +assert(statusOf(blockedApproval) === 'stopped', 'a 90h-old pending APPROVAL blocks the same way'); +assert(aos.approvals.statusOf(oldApr.id) === 'cancelled', '…and is cancelled on teardown'); + +console.log('\n\x1b[1m6) blocked ceiling disabled (0) restores the old wait-for-ever\x1b[0m'); +aos.settings.setBlockedMaxHours(0); +const blockedForever = mkSession({ created_at: Date.now() - 200 * H }); +askAt(blockedForever, 150); +tm.reapIdleSessions(); +assert(statusOf(blockedForever) === 'running', '0 → a 150h-old block is never cut'); + console.log(`\n${fail === 0 ? '\x1b[32m' : '\x1b[31m'}IDLE REAPER: ${pass}/${pass + fail} passed\x1b[0m`); try { fs.rmSync(HOME, { recursive: true, force: true }); } catch {} process.exit(fail === 0 ? 0 : 1); diff --git a/src/governance/settings.ts b/src/governance/settings.ts index 79b7c63..c0aaa87 100644 --- a/src/governance/settings.ts +++ b/src/governance/settings.ts @@ -80,6 +80,7 @@ const CHAT_IDLE_MIN_KEY = 'chat_idle_timeout_min'; // resident (warm) chat sessi const MAX_CONCURRENT_KEY = 'max_concurrent_sessions'; // whole-box concurrency cap override; unset → RAM-derived default, 0 → unlimited const INTERACTIVE_IDLE_HOURS_KEY = 'interactive_idle_timeout_hours'; // auto-close a detached member session idle past this; unset → 48h, 0 → off const UNATTENDED_MAX_HOURS_KEY = 'unattended_max_runtime_hours'; // hard runtime ceiling for a headless/unattended run (stuck-mid-turn backstop); unset → 24h, 0 → off +const BLOCKED_MAX_HOURS_KEY = 'blocked_max_hours'; // force-close an interactive session waiting this long on an unanswered card; unset → 72h, 0 → off const UNATTENDED_NO_PROGRESS_MIN_KEY = 'unattended_no_progress_minutes'; // reap a headless run that never made a tool call (never-started: rate-limit/trust-hang/lost-prompt); unset → 30m, 0 → off const KILL_SWITCH_KEY = 'kill_switch'; // workspace-wide emergency stop (JSON KillSwitchState) const SUPPRESSED_BUILTINS_KEY = 'suppressed_builtins'; // built-in agent ids an admin deleted (JSON string[]); boot won't re-seed them @@ -974,6 +975,33 @@ export class SettingsStore { return this.unattendedMaxHours(); } + /** + * Hours an INTERACTIVE session may sit blocked on an unanswered question/approval before the janitor + * closes it anyway. The idle reaper deliberately skips a session that is waiting on a person — but + * nothing expires an Inbox card, so that exemption had no floor: a session blocked on a question nobody + * answers waits for ever, holding a `claude` process and a concurrency-cap slot (live expresstech: 66 h + * and counting on a question raised three days earlier). Past this ceiling the wait is an abandonment, + * so the session is closed and its card cancelled — which is also what makes the card dismissable + * instead of hanging in the Inbox. Measured from when the OLDEST pending card was raised. + * + * Default **72 h**, matching `STALE_PROMPT_MAX_MS` — the age at which the reminder sweep already gives + * up on a prompt and stops nagging. Clamped 1 h–30 d; `0` disables (restoring the old wait-for-ever). + * A session with someone attached is never cut by this: a human is right there to answer. + */ + blockedMaxHours(): number { + const n = Number(this.getRow(BLOCKED_MAX_HOURS_KEY)?.value); + if (!Number.isFinite(n)) return 72; // unset → default + if (n <= 0) return 0; // explicit 0 → disabled + return Math.min(Math.max(Math.round(n), 1), 24 * 30); + } + + setBlockedMaxHours(hours: number, by?: string): number { + const n = Number(hours); + const clamped = !Number.isFinite(n) || n < 0 ? 72 : n === 0 ? 0 : Math.min(Math.max(Math.round(n), 1), 24 * 30); + this.set(BLOCKED_MAX_HOURS_KEY, String(clamped), by); + return this.blockedMaxHours(); + } + /** Minutes after which a headless/unattended run that has made ZERO progress — never a completed turn * (`last_activity` NULL) AND never a single governed tool call (`gate.attempt`) — is reaped as * `stuck-no-progress`. This is the fast net for a run that never actually STARTED: the account hit its diff --git a/src/server.ts b/src/server.ts index 129ce2b..3c96bdc 100644 --- a/src/server.ts +++ b/src/server.ts @@ -4105,11 +4105,11 @@ async function handle(os: AgentOS, tm: TerminalManager, autos: Automations, req: const value = os.settings.maxConcurrentSessions(); // operator override (null = unset) const resolved = autos.concurrencyCap(); // effective cap the scheduler enforces (0 = unlimited) const source = envLocked ? 'env' : value != null ? 'setting' : 'derived'; - return sendJson(res, 200, { value, resolved, derived: derivedConcurrencyCap(), source, envLocked, alive: tm.aliveSessionCount(), idleHours: os.settings.interactiveIdleTimeoutHours(), unattendedMaxHours: os.settings.unattendedMaxHours(), unattendedNoProgressMinutes: os.settings.unattendedNoProgressMinutes() }); + return sendJson(res, 200, { value, resolved, derived: derivedConcurrencyCap(), source, envLocked, alive: tm.aliveSessionCount(), idleHours: os.settings.interactiveIdleTimeoutHours(), unattendedMaxHours: os.settings.unattendedMaxHours(), unattendedNoProgressMinutes: os.settings.unattendedNoProgressMinutes(), blockedMaxHours: os.settings.blockedMaxHours() }); } if (method === 'PUT' && p === '/api/settings/concurrency') { if (!isAdmin(me)) return sendJson(res, 403, { error: 'owner or admin required' }); - const b = await readBody(req) as { value?: unknown; idleHours?: unknown; unattendedMaxHours?: unknown; unattendedNoProgressMinutes?: unknown }; + const b = await readBody(req) as { value?: unknown; idleHours?: unknown; unattendedMaxHours?: unknown; unattendedNoProgressMinutes?: unknown; blockedMaxHours?: unknown }; // Cap: `null`/'' clears the override (→ derived default); 0 = unlimited; N>0 = cap. Only touched when the // key is present, so a PUT that only sets idleHours leaves the cap alone. if ('value' in b) { @@ -4141,7 +4141,7 @@ async function handle(os: AgentOS, tm: TerminalManager, autos: Automations, req: const savedM = os.settings.setUnattendedNoProgressMinutes(m, me.email); os.audit.append({ ts: Date.now(), runId: '-', tenant: os.tenant, principal: me.email, type: 'settings.noProgress.updated', data: { unattendedNoProgressMinutes: savedM } }); } - return sendJson(res, 200, { ok: true, value: os.settings.maxConcurrentSessions(), resolved: autos.concurrencyCap(), derived: derivedConcurrencyCap(), idleHours: os.settings.interactiveIdleTimeoutHours(), unattendedMaxHours: os.settings.unattendedMaxHours(), unattendedNoProgressMinutes: os.settings.unattendedNoProgressMinutes() }); + return sendJson(res, 200, { ok: true, value: os.settings.maxConcurrentSessions(), resolved: autos.concurrencyCap(), derived: derivedConcurrencyCap(), idleHours: os.settings.interactiveIdleTimeoutHours(), unattendedMaxHours: os.settings.unattendedMaxHours(), unattendedNoProgressMinutes: os.settings.unattendedNoProgressMinutes(), blockedMaxHours: os.settings.blockedMaxHours() }); } // ── Runtime account POOL (launch-time credential rotation) — owner-managed ────────────── diff --git a/src/terminal.ts b/src/terminal.ts index 8871048..a75158d 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -2258,9 +2258,20 @@ export class TerminalManager { // console re-open clears `blockResume`), so this is a janitor, not a guillotine. Skip claimed take-overs — // a human owns that lifecycle. `0` disables. Uses COALESCE(last_activity, created_at): a member session // rarely stamps last_activity, so age is the fallback clock. + // + // BLOCKED CEILING. "No pending human block" was an unconditional exemption, and that is the same + // mistake the done-orphan leak was: nothing expires an unanswered card, so a session waiting on one + // waits FOREVER. Live expresstech: a `support` session blocked on a question asked 2026-07-31 was + // still holding its pane 66 h later, with two more questions unanswered since 07-28. Past + // `blockedMaxHours` (default 72 h — the age at which `escalateStalePrompts` already stops nagging and + // treats a prompt as dead) the wait is not a wait, it's an abandonment, so reap and cancel the card. + // Measured from when the OLDEST pending card was RAISED, not from session idleness — "nobody answered + // this in three days" is the actual claim. Attached sessions are still skipped: someone is right there. const idleHours = this.os.settings.interactiveIdleTimeoutHours(); if (idleHours > 0) { const idleCutoff = Date.now() - idleHours * 3600_000; + const blockedHours = this.os.settings.blockedMaxHours(); + const blockedCutoff = blockedHours > 0 ? Date.now() - blockedHours * 3600_000 : null; const stale = this.db.prepare("SELECT id, tmux, run_as, spawned_by, agent, status FROM term_sessions WHERE headless = 0 AND resident = 0 AND claimed_by IS NULL AND status IN ('running','done') AND COALESCE(last_activity, created_at) < ?") .all<{ id: string; tmux: string; run_as: string | null; spawned_by: string | null; agent: string; status: string }>(idleCutoff); for (const r of stale) { @@ -2271,14 +2282,23 @@ export class TerminalManager { if (r.status === 'done' && alive && !alive.has(r.tmux)) continue; const space = this.spaceFor(r.run_as ?? r.spawned_by); if (this.backend.hasClient(space, r.tmux) === true) continue; // someone's attached — it's in use - if (this.hasPendingHumanBlock(r.id)) continue; // blocked on a person — leave it + // Blocked on a person: leave it — unless nobody has answered inside the ceiling above, at which + // point it is abandoned, not waiting. + let reason = 'idle-interactive'; + const blockedAt = this.oldestPendingBlockAt(r.id); + if (blockedAt !== undefined) { + if (blockedCutoff == null || blockedAt >= blockedCutoff) continue; + reason = 'blocked-timeout'; + } this.backend.kill(space, r.tmux); // Preserve a completed session's outcome — only a still-running one becomes 'stopped'. this.db.prepare("UPDATE term_sessions SET status = ?, updated_at = ? WHERE id = ?").run(r.status === 'done' ? 'done' : 'stopped', Date.now(), r.id); this.cancelPendingQuestions(r.id, 'system'); this.cancelPendingApprovals(r.id, 'system'); this.blockResume(r.id); // stay reaped against a ttyd auto-reconnect; a deliberate Resume clears it - this.audit(r.id, r.agent, 'session.reaped', { reason: 'idle-interactive', idleHours, status: r.status }); + this.audit(r.id, r.agent, 'session.reaped', reason === 'blocked-timeout' + ? { reason, blockedHours, blockedForMs: Date.now() - (blockedAt as number), status: r.status } + : { reason, idleHours, status: r.status }); } catch { /* one bad row must not stop the sweep */ } } } @@ -2342,6 +2362,20 @@ export class TerminalManager { return this.os.approvals.pending(this.os.tenant).some((a) => a.runId === sessionId); } + /** WHEN this session started waiting on a person — the creation time of its OLDEST still-pending question + * or approval, or undefined when it isn't blocked. The clock for the blocked ceiling (sweep 3): what + * matters is how long the CARD has gone unanswered, not how long the session has been quiet. */ + private oldestPendingBlockAt(sessionId: string): number | undefined { + const q = this.db.prepare("SELECT MIN(created_at) AS at FROM questions WHERE run_id = ? AND status = 'pending'") + .get<{ at: number | null }>(sessionId)?.at ?? undefined; + const a = this.os.approvals.pending(this.os.tenant) + .filter((x) => x.runId === sessionId) + .reduce((min, x) => (min === undefined || x.createdAt < min ? x.createdAt : min), undefined); + if (q === undefined) return a; + if (a === undefined) return q; + return Math.min(q, a); + } + /** Has this run made any real PROGRESS — i.e. attempted at least one governed tool call (a `gate.attempt` * audit event)? The no-progress backstop (sweep 2) uses this to tell a run that never STARTED (usage-limit * refusal / trust-hang / lost prompt → zero tools) apart from a genuinely-busy long first turn (which fires diff --git a/web/src/App.tsx b/web/src/App.tsx index 80ef1d0..3a82b05 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -13618,6 +13618,7 @@ function ConcurrencySettings({ me }: { me: Member }) { const [idle, setIdle] = useState('') // hours; '0' = off const [maxRun, setMaxRun] = useState('') // headless hard runtime ceiling, hours; '0' = off const [noProg, setNoProg] = useState('') // headless no-progress reap, minutes; '0' = off + const [blocked, setBlocked] = useState('') // interactive blocked-on-a-card ceiling, hours; '0' = off const [busy, setBusy] = useState(false) const [hint, setHint] = useState('') const canEdit = me.role === 'owner' || me.role === 'admin' @@ -13629,6 +13630,7 @@ function ConcurrencySettings({ me }: { me: Member }) { setIdle(String(r.idleHours)) setMaxRun(String(r.unattendedMaxHours)) setNoProg(String(r.unattendedNoProgressMinutes)) + setBlocked(String(r.blockedMaxHours)) }).catch(() => {}) useEffect(() => { load() }, []) @@ -13636,14 +13638,16 @@ function ConcurrencySettings({ me }: { me: Member }) { const idleDirty = data != null && idle.trim() !== String(data.idleHours) const maxRunDirty = data != null && maxRun.trim() !== String(data.unattendedMaxHours) const noProgDirty = data != null && noProg.trim() !== String(data.unattendedNoProgressMinutes) - const dirty = capDirty || idleDirty || maxRunDirty || noProgDirty + const blockedDirty = data != null && blocked.trim() !== String(data.blockedMaxHours) + const dirty = capDirty || idleDirty || maxRunDirty || noProgDirty || blockedDirty const save = async () => { setBusy(true); setHint('') - const body: { value?: number | null; idleHours?: number; unattendedMaxHours?: number; unattendedNoProgressMinutes?: number } = {} + const body: { value?: number | null; idleHours?: number; unattendedMaxHours?: number; unattendedNoProgressMinutes?: number; blockedMaxHours?: number } = {} if (capDirty) body.value = input.trim() === '' ? null : Number(input) if (idleDirty) body.idleHours = Number(idle) if (maxRunDirty) body.unattendedMaxHours = Number(maxRun) if (noProgDirty) body.unattendedNoProgressMinutes = Number(noProg) + if (blockedDirty) body.blockedMaxHours = Number(blocked) const r = await api.saveConcurrency(body) setBusy(false) if (r.error) return setHint('⚠ ' + r.error) @@ -13742,6 +13746,25 @@ function ConcurrencySettings({ me }: { me: Member }) { is treated as working and is never cut by this.

+
+ +
+ setBlocked(e.target.value)} + placeholder="72" + disabled={!canEdit} + className="h-8 w-40 font-mono text-xs" + /> + 0 = wait for ever · nobody attached +
+

+ A session blocked on a question or approval is normally left alone — that wait is real. But nothing expires an Inbox card, so + with no ceiling it waits for ever, holding a claude process and a cap slot. Past this age + (measured from when the card was raised) the session is closed and the card cancelled, which is what makes it dismissable + instead of hanging. Someone attached to the session is never cut — they can answer. +

+
{hint && {hint}} diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 9374031..8f041c7 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -64,6 +64,8 @@ export interface Concurrency { unattendedMaxHours: number /** Reap a headless run that never made a tool call after this many minutes — never-started net (0 = off; default 30). */ unattendedNoProgressMinutes: number + /** Close an interactive session waiting this many hours on an unanswered question/approval (0 = off; default 72). */ + blockedMaxHours: number } /** One credential set in the runtime rotation pool (never carries the api-key value, only its vault ref). */ @@ -1572,7 +1574,7 @@ export const api = { saveSubagentDefault: (mode: 'all' | 'none') => call<{ ok: boolean; mode?: 'all' | 'none'; error?: string }>('PUT', '/api/settings/subagent-default', { mode }), saveSessionMetrics: (value: SessionMetrics) => call<{ ok: boolean; sessionMetrics?: SessionMetrics; error?: string }>('PUT', '/api/settings/session-metrics', { value }), concurrency: () => call('GET', '/api/settings/concurrency'), - saveConcurrency: (body: { value?: number | null; idleHours?: number; unattendedMaxHours?: number; unattendedNoProgressMinutes?: number }) => call<{ ok: boolean; error?: string; value?: number | null; resolved?: number; derived?: number; idleHours?: number; unattendedMaxHours?: number; unattendedNoProgressMinutes?: number }>('PUT', '/api/settings/concurrency', body), + saveConcurrency: (body: { value?: number | null; idleHours?: number; unattendedMaxHours?: number; unattendedNoProgressMinutes?: number; blockedMaxHours?: number }) => call<{ ok: boolean; error?: string; value?: number | null; resolved?: number; derived?: number; idleHours?: number; unattendedMaxHours?: number; unattendedNoProgressMinutes?: number; blockedMaxHours?: number }>('PUT', '/api/settings/concurrency', body), runtimeAccounts: () => call('GET', '/api/runtime-accounts'), addRuntimeAccount: (body: { runtime: string; name: string; kind: RuntimeAccountKind; configDir?: string; apiKeyRef?: string; token?: string }) => call<{ ok: boolean; error?: string; account?: RuntimeAccount }>('POST', '/api/runtime-accounts', body), setRuntimeAccountEnabled: (runtime: string, name: string, enabled: boolean) => call<{ ok: boolean; error?: string }>('PATCH', `/api/runtime-accounts/${encodeURIComponent(runtime)}/${encodeURIComponent(name)}`, { enabled }),