Files
grok-glance/server/src/state.ts
T
iceBear67 6d34a17d9d Supervise several agents at once
One daemon already saw every session; the dashboard only ever showed one of
them well. Under four parallel agents it failed in specific ways, each fixed
here:

- Sessions were identified by workspace basename, so two agents in one repo
  were indistinguishable. The daemon now hands out a small ordinal badge in
  arrival order and the web UI colours each agent by it — rows, timeline
  lines, and approval cards, which previously asked you to approve `rm -rf`
  "in remote-grok" without saying which one.
- `currentTool` held a single call, so parallel tools overwrote each other.
  It is now a list; durations are matched FIFO per tool name, since hook
  payloads carry no call id.
- One 400-event ring, evicted oldest-first, let a chatty agent blank
  everyone else's history. Eviction now takes from whichever session holds
  the most of the ring.
- Sessions sorted by recency jumped under a moving thumb. They are ordered
  waiting -> error -> working -> idle -> ended, ties on badge, so an agent
  keeps its slot.
- Events carry no workspace root, so a restart came back with a full
  timeline and an empty roster. The session map is persisted to
  sessions.json (debounced, flushed on shutdown, 12h cutoff on load).
  In-flight tools are dropped on the way out: they belonged to a process
  that no longer exists.
- Snapshot pushes now back off to 1s once a snapshot exceeds 24KB, since
  full-snapshot SSE cost scales with agent count.
- Pending approvals are ordered by which expires first, not by arrival.

`glance status` and /local/status report the roster by state. e2e suite:
220 passed, 0 failed.
2026-08-09 05:27:00 +00:00

474 lines
15 KiB
TypeScript

import { VERSION, type Config } from "./config.js";
import { appendEventLog, readRecentEvents, readSessions, writeSessions } from "./store.js";
import {
labelForWorkspace,
summarizeNotification,
summarizePrompt,
summarizeTool,
truncateDetail,
truncateTitle,
} from "./summarize.js";
import type {
EventKind,
GlanceEvent,
PendingApproval,
SessionState,
SessionView,
Snapshot,
} from "./protocol.js";
/** A session that has said nothing for this long is treated as idle, not working. */
const STALE_WORKING_MS = 10 * 60_000;
/** Sessions quieter than this are not restored on start — they are last week's agents. */
const RESTORE_MAX_AGE_MS = 12 * 60 * 60_000;
/**
* A PreToolUse whose PostToolUse never arrives (crash, kill, timeout) would otherwise sit in
* the running list forever, so the list is bounded and the oldest entry falls off.
*/
const MAX_RUNNING_PER_SESSION = 8;
/** Persisting the session map on every hook would mean a file write per tool call. */
const PERSIST_DEBOUNCE_MS = 2_000;
/**
* Which agent you want to look at first. Sorting purely by recency — the obvious choice with
* one session — makes every row jump under your thumb once four agents are working at once.
*/
const ATTENTION_RANK: Record<SessionState, number> = {
waiting: 0,
error: 1,
working: 2,
idle: 3,
ended: 4,
};
const EVENT_KIND_BY_HOOK: Record<string, EventKind> = {
SessionStart: "session_start",
SessionEnd: "session_end",
UserPromptSubmit: "prompt",
PreToolUse: "tool_start",
PostToolUse: "tool_end",
PostToolUseFailure: "tool_fail",
PermissionDenied: "permission_denied",
Stop: "turn_end",
StopFailure: "turn_error",
Notification: "notification",
SubagentStart: "subagent_start",
SubagentStop: "subagent_end",
PreCompact: "compact",
PostCompact: "compact",
};
export interface HookPayload {
hookEventName?: string;
sessionId?: string;
cwd?: string;
workspaceRoot?: string;
toolName?: string;
toolInput?: unknown;
[key: string]: unknown;
}
export class GlanceState {
private events: GlanceEvent[] = [];
private sessions = new Map<string, SessionView>();
/**
* sessionId|toolName -> start timestamps, oldest first, so PostToolUse can report a
* duration. An array rather than a single stamp because an agent runs tools in parallel
* and the payload carries no call id: matching FIFO within a tool name is the closest
* thing to one we have.
*/
private toolStarts = new Map<string, number[]>();
private nextId = 1;
private nextBadge = 1;
private persistTimer: NodeJS.Timeout | null = null;
private readonly listeners = new Set<() => void>();
constructor(private readonly cfg: Config) {
// Warm start: keep recent history across daemon restarts.
const recent = readRecentEvents(cfg.retainEvents);
this.events = recent;
this.nextId = recent.reduce((max, e) => Math.max(max, e.id), 0) + 1;
// …and the agents themselves, so a restart mid-supervision does not blank the overview.
const cutoff = Date.now() - RESTORE_MAX_AGE_MS;
for (const stored of readSessions()) {
const session = restoreSession(stored);
if (!session || session.lastActivity < cutoff) continue;
this.sessions.set(session.id, session);
this.nextBadge = Math.max(this.nextBadge, session.badge + 1);
}
}
onChange(listener: () => void): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
private notify(): void {
for (const listener of this.listeners) {
try {
listener();
} catch {
/* a broken listener must not break ingestion */
}
}
}
private session(id: string, payload: HookPayload): SessionView {
let existing = this.sessions.get(id);
if (!existing) {
existing = {
id,
label: labelForWorkspace(payload.workspaceRoot, payload.cwd ?? ""),
badge: this.nextBadge++,
cwd: payload.workspaceRoot ?? payload.cwd ?? "",
state: "idle",
startedAt: Date.now(),
lastActivity: Date.now(),
running: [],
counts: { tools: 0, failures: 0, denials: 0 },
};
this.sessions.set(id, existing);
} else if (payload.workspaceRoot || payload.cwd) {
// Keep the label fresh if the session moved.
existing.label = labelForWorkspace(payload.workspaceRoot, payload.cwd ?? existing.cwd);
existing.cwd = payload.workspaceRoot ?? payload.cwd ?? existing.cwd;
}
return existing;
}
private push(event: GlanceEvent): void {
this.events.push(event);
this.trim();
appendEventLog(event);
}
/**
* Evict from whichever session is using most of the ring rather than simply dropping the
* oldest event. A single agent grinding through a build would otherwise push every other
* agent's history out, and the timeline would silently become a one-agent timeline.
*/
private trim(): void {
while (this.events.length > this.cfg.retainEvents) {
const perSession = new Map<string, number>();
for (const event of this.events) {
perSession.set(event.sessionId, (perSession.get(event.sessionId) ?? 0) + 1);
}
let greediest = this.events[0].sessionId;
let most = 0;
for (const [sessionId, count] of perSession) {
if (count > most) {
most = count;
greediest = sessionId;
}
}
const oldest = this.events.findIndex((e) => e.sessionId === greediest);
this.events.splice(oldest < 0 ? 0 : oldest, 1);
}
}
/** Forget what a session had in flight — nothing survives a turn ending or a crash. */
private clearRunning(sessionId: string, session: SessionView): void {
session.running = [];
for (const key of this.toolStarts.keys()) {
if (key.startsWith(`${sessionId}|`)) this.toolStarts.delete(key);
}
}
/**
* Write the session map out. Debounced, because the alternative is a file write per hook —
* and with several agents running that is several writes a second.
*/
private schedulePersist(): void {
if (this.persistTimer) return;
this.persistTimer = setTimeout(() => {
this.persistTimer = null;
this.persist();
}, PERSIST_DEBOUNCE_MS);
this.persistTimer.unref?.();
}
/** Flush that write now — called on shutdown so the last few seconds are not lost. */
flush(): void {
if (this.persistTimer) {
clearTimeout(this.persistTimer);
this.persistTimer = null;
}
this.persist();
}
private persist(): void {
// `running` is dropped on the way out: those tool calls belong to a process that is about
// to stop existing, and a restored session claiming three live tools would be a lie the
// dashboard has no way to disprove.
writeSessions([...this.sessions.values()].map((s) => ({ ...s, running: [] })));
}
/**
* Make sure a session is known without recording anything for it. The approval gate and the
* recorder are two separate hooks on the same PreToolUse, so the gate can easily be the
* first to hear about an agent — and an approval card that cannot say which agent is asking
* is worthless when four of them are running.
*/
ensureSession(sessionId: string, payload: HookPayload = {}): SessionView {
return this.session(sessionId, payload);
}
/** Record a raw hook payload. Returns the event it produced, if any. */
ingest(payload: HookPayload): GlanceEvent | null {
const hookName = payload.hookEventName ?? "";
const kind = EVENT_KIND_BY_HOOK[hookName];
if (!kind) return null;
const sessionId = payload.sessionId ?? "unknown";
const session = this.session(sessionId, payload);
const now = Date.now();
session.lastActivity = now;
const tool = typeof payload.toolName === "string" ? payload.toolName : undefined;
let title = hookName;
let detail: string | undefined;
let durationMs: number | undefined;
switch (kind) {
case "session_start":
session.state = "idle";
title = `Session started in ${session.label}`;
detail = session.cwd || undefined;
break;
case "session_end":
session.state = "ended";
this.clearRunning(sessionId, session);
title = "Session ended";
break;
case "prompt":
session.state = "working";
session.lastPrompt = summarizePrompt(payload);
title = session.lastPrompt;
break;
case "tool_start": {
const name = tool ?? "tool";
const summary = summarizeTool(name, payload.toolInput);
session.state = "working";
session.running.push({ name, title: summary.title, startedAt: now });
if (session.running.length > MAX_RUNNING_PER_SESSION) session.running.shift();
const starts = this.toolStarts.get(`${sessionId}|${name}`) ?? [];
starts.push(now);
if (starts.length > MAX_RUNNING_PER_SESSION) starts.shift();
this.toolStarts.set(`${sessionId}|${name}`, starts);
title = summary.title;
detail = summary.detail;
break;
}
case "tool_end":
case "tool_fail": {
const name = tool ?? "tool";
const summary = summarizeTool(name, payload.toolInput);
const key = `${sessionId}|${name}`;
const starts = this.toolStarts.get(key);
if (starts?.length) {
durationMs = now - starts.shift()!;
if (!starts.length) this.toolStarts.delete(key);
}
const running = session.running.findIndex((t) => t.name === name);
if (running >= 0) session.running.splice(running, 1);
session.state = "working";
title = summary.title;
detail = summary.detail;
if (kind === "tool_end") {
session.counts.tools += 1;
} else {
session.counts.failures += 1;
detail = truncateDetail(String(payload["error"] ?? payload["message"] ?? "")) || detail;
}
break;
}
case "permission_denied":
session.counts.denials += 1;
title = tool ? `Permission denied: ${tool}` : "Permission denied";
detail = summarizeTool(tool ?? "tool", payload.toolInput).title;
break;
case "turn_end":
session.state = "idle";
this.clearRunning(sessionId, session);
title = "Turn finished";
break;
case "turn_error":
session.state = "error";
this.clearRunning(sessionId, session);
title = "Turn failed";
detail = truncateDetail(String(payload["error"] ?? payload["message"] ?? "")) || undefined;
break;
case "notification":
title = summarizeNotification(payload);
break;
case "subagent_start":
title = "Subagent started";
detail = truncateDetail(String(payload["description"] ?? payload["subagentType"] ?? "")) || undefined;
break;
case "subagent_end":
title = "Subagent finished";
break;
case "compact":
title = hookName === "PreCompact" ? "Compacting conversation" : "Compaction done";
break;
default:
break;
}
const event: GlanceEvent = {
id: this.nextId++,
ts: now,
sessionId,
kind,
tool,
title: truncateTitle(title),
detail,
durationMs,
};
this.push(event);
this.schedulePersist();
this.notify();
return event;
}
/** Record something the daemon itself decided, e.g. an approval outcome. */
record(
sessionId: string,
kind: EventKind,
title: string,
opts: { tool?: string; detail?: string } = {},
): GlanceEvent {
const now = Date.now();
const session = this.sessions.get(sessionId);
if (session) {
session.lastActivity = now;
if (kind === "approval_request") session.state = "waiting";
else if (kind === "approval_allowed" || kind === "approval_denied") session.state = "working";
}
const event: GlanceEvent = {
id: this.nextId++,
ts: now,
sessionId,
kind,
tool: opts.tool,
title: truncateTitle(title),
detail: opts.detail,
};
this.push(event);
this.schedulePersist();
this.notify();
return event;
}
private effectiveState(session: SessionView, now: number): SessionState {
if (session.state === "working" && now - session.lastActivity > STALE_WORKING_MS) {
return "idle";
}
return session.state;
}
snapshot(pending: PendingApproval[]): Snapshot {
const now = Date.now();
const waiting = new Set(pending.map((p) => p.sessionId));
const sessions = [...this.sessions.values()]
.map((s) => ({
...s,
state: waiting.has(s.id) ? ("waiting" as SessionState) : this.effectiveState(s, now),
// A tool that has been "running" for ten minutes lost its PostToolUse somewhere.
running: s.running.filter((t) => now - t.startedAt < STALE_WORKING_MS),
}))
// Whatever needs you first, then a fixed slot per agent so rows stay where you left them.
.sort((a, b) => ATTENTION_RANK[a.state] - ATTENTION_RANK[b.state] || a.badge - b.badge);
return {
now,
version: VERSION,
sessions,
events: [...this.events].sort((a, b) => b.ts - a.ts || b.id - a.id),
pending,
approval: this.cfg.approval,
};
}
sessionLabel(sessionId: string): string {
return this.sessions.get(sessionId)?.label ?? "workspace";
}
sessionBadge(sessionId: string): number {
return this.sessions.get(sessionId)?.badge ?? 0;
}
/** How many agents are in each state — what `glance status` prints from the terminal. */
stateSummary(pending: PendingApproval[] = []): Record<SessionState, number> {
const now = Date.now();
const waiting = new Set(pending.map((p) => p.sessionId));
const counts: Record<SessionState, number> = {
working: 0,
idle: 0,
waiting: 0,
error: 0,
ended: 0,
};
for (const session of this.sessions.values()) {
const state = waiting.has(session.id) ? "waiting" : this.effectiveState(session, now);
counts[state] += 1;
}
return counts;
}
get sessionCount(): number {
return this.sessions.size;
}
get eventCount(): number {
return this.events.length;
}
}
/**
* Accept a session read back from disk, or reject it. Written by a previous version, edited
* by hand, truncated by a full disk — none of that may take the daemon down, and a session
* with a broken shape is better dropped than rendered as `undefined` on a phone.
*/
function restoreSession(raw: unknown): SessionView | null {
if (typeof raw !== "object" || raw === null) return null;
const s = raw as Partial<SessionView>;
if (typeof s.id !== "string" || !s.id) return null;
if (typeof s.badge !== "number" || !Number.isFinite(s.badge)) return null;
const counts = s.counts ?? { tools: 0, failures: 0, denials: 0 };
return {
id: s.id,
label: typeof s.label === "string" && s.label ? s.label : "workspace",
badge: Math.max(1, Math.floor(s.badge)),
cwd: typeof s.cwd === "string" ? s.cwd : "",
state: s.state && s.state in ATTENTION_RANK ? s.state : "idle",
startedAt: typeof s.startedAt === "number" ? s.startedAt : Date.now(),
lastActivity: typeof s.lastActivity === "number" ? s.lastActivity : 0,
lastPrompt: typeof s.lastPrompt === "string" ? s.lastPrompt : undefined,
// Nothing survives the restart: whatever reports the end of a tool call was talking to
// the process that just died.
running: [],
counts: {
tools: Number(counts.tools) || 0,
failures: Number(counts.failures) || 0,
denials: Number(counts.denials) || 0,
},
};
}