[codex] Harden heartbeat scheduling and runtime controls (#4223)

## Thinking Path

> - Paperclip orchestrates AI agents through issue checkout, heartbeat
runs, routines, and auditable control-plane state
> - The runtime path has to recover from lost local processes, transient
adapter failures, blocked dependencies, and routine coalescing without
stranding work
> - The existing branch carried several reliability fixes across
heartbeat scheduling, issue runtime controls, routine dispatch, and
operator-facing run state
> - These changes belong together because they share backend contracts,
migrations, and runtime status semantics
> - This pull request groups the control-plane/runtime slice so it can
merge independently from board UI polish and adapter sandbox work
> - The benefit is safer heartbeat recovery, clearer runtime controls,
and more predictable recurring execution behavior

## What Changed

- Adds bounded heartbeat retry scheduling, scheduled retry state, and
Codex transient failure recovery handling.
- Tightens heartbeat process recovery, blocker wake behavior, issue
comment wake handling, routine dispatch coalescing, and
activity/dashboard bounds.
- Adds runtime-control MCP tools and Paperclip skill docs for issue
workspace runtime management.
- Adds migrations `0061_lively_thor_girl.sql` and
`0062_routine_run_dispatch_fingerprint.sql`.
- Surfaces retry state in run ledger/agent UI and keeps related shared
types synchronized.

## Verification

- `pnpm exec vitest run
server/src/__tests__/heartbeat-retry-scheduling.test.ts
server/src/__tests__/heartbeat-process-recovery.test.ts
server/src/__tests__/routines-service.test.ts`
- `pnpm exec vitest run src/tools.test.ts` from `packages/mcp-server`

## Risks

- Medium risk: this touches heartbeat recovery and routine dispatch,
which are central execution paths.
- Migration order matters if split branches land out of order: merge
this PR before branches that assume the new runtime/routine fields.
- Runtime retry behavior should be watched in CI and in local operator
smoke tests because it changes how transient failures are resumed.

> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.

## Model Used

- OpenAI Codex, GPT-5-based coding agent runtime, shell/git tool use
enabled. Exact hosted model build and context window are not exposed in
this Paperclip heartbeat environment.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [ ] If this change affects the UI, I have included before/after
screenshots
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge

ui/src/lib/runRetryState.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import { describeRunRetryState, formatRetryReason } from "./runRetryState";
+
+describe("runRetryState", () => {
+  it("formats internal retry reasons for operators", () => {
+    expect(formatRetryReason("transient_failure")).toBe("Transient failure");
+    expect(formatRetryReason("issue_continuation_needed")).toBe("Continuation needed");
+    expect(formatRetryReason("custom_reason")).toBe("custom reason");
+  });
+
+  it("describes scheduled retries", () => {
+    expect(
+      describeRunRetryState({
+        status: "scheduled_retry",
+        retryOfRunId: "run-1",
+        scheduledRetryAttempt: 2,
+        scheduledRetryReason: "transient_failure",
+        scheduledRetryAt: "2026-04-18T20:15:00.000Z",
+      }),
+    ).toMatchObject({
+      kind: "scheduled",
+      badgeLabel: "Retry scheduled",
+      detail: "Attempt 2 · Transient failure",
+    });
+  });
+
+  it("describes exhausted retries", () => {
+    expect(
+      describeRunRetryState({
+        status: "failed",
+        retryOfRunId: "run-1",
+        scheduledRetryAttempt: 4,
+        scheduledRetryReason: "transient_failure",
+        retryExhaustedReason: "Bounded retry exhausted after 4 scheduled attempts; no further automatic retry will be queued",
+      }),
+    ).toMatchObject({
+      kind: "exhausted",
+      badgeLabel: "Retry exhausted",
+      detail: "Attempt 4 · Transient failure · No further automatic retry queued",
+    });
+  });
+});

ui/src/lib/runRetryState.ts

@@ -0,0 +1,93 @@
+import { formatDateTime } from "./utils";
+
+type RetryAwareRun = {
+  status: string;
+  retryOfRunId?: string | null;
+  scheduledRetryAt?: string | Date | null;
+  scheduledRetryAttempt?: number | null;
+  scheduledRetryReason?: string | null;
+  retryExhaustedReason?: string | null;
+};
+
+export type RunRetryStateSummary = {
+  kind: "scheduled" | "exhausted" | "attempted";
+  badgeLabel: string;
+  tone: string;
+  detail: string | null;
+  secondary: string | null;
+  retryOfRunId: string | null;
+};
+
+const RETRY_REASON_LABELS: Record<string, string> = {
+  transient_failure: "Transient failure",
+  missing_issue_comment: "Missing issue comment",
+  process_lost: "Process lost",
+  assignment_recovery: "Assignment recovery",
+  issue_continuation_needed: "Continuation needed",
+};
+
+function readNonEmptyString(value: unknown) {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
+}
+
+function joinFragments(parts: Array<string | null>) {
+  const filtered = parts.filter((part): part is string => Boolean(part));
+  return filtered.length > 0 ? filtered.join(" · ") : null;
+}
+
+export function formatRetryReason(reason: string | null | undefined) {
+  const normalized = readNonEmptyString(reason);
+  if (!normalized) return null;
+  return RETRY_REASON_LABELS[normalized] ?? normalized.replace(/_/g, " ");
+}
+
+export function describeRunRetryState(run: RetryAwareRun): RunRetryStateSummary | null {
+  const attempt =
+    typeof run.scheduledRetryAttempt === "number" && Number.isFinite(run.scheduledRetryAttempt) && run.scheduledRetryAttempt > 0
+      ? run.scheduledRetryAttempt
+      : null;
+  const attemptLabel = attempt ? `Attempt ${attempt}` : null;
+  const reasonLabel = formatRetryReason(run.scheduledRetryReason);
+  const retryOfRunId = readNonEmptyString(run.retryOfRunId);
+  const exhaustedReason = readNonEmptyString(run.retryExhaustedReason);
+  const dueAt = run.scheduledRetryAt ? formatDateTime(run.scheduledRetryAt) : null;
+  const hasRetryMetadata =
+    Boolean(retryOfRunId)
+    || Boolean(reasonLabel)
+    || Boolean(dueAt)
+    || Boolean(attemptLabel)
+    || Boolean(exhaustedReason);
+
+  if (!hasRetryMetadata) return null;
+
+  if (run.status === "scheduled_retry") {
+    return {
+      kind: "scheduled",
+      badgeLabel: "Retry scheduled",
+      tone: "border-cyan-500/30 bg-cyan-500/10 text-cyan-700 dark:text-cyan-300",
+      detail: joinFragments([attemptLabel, reasonLabel]),
+      secondary: dueAt ? `Next retry ${dueAt}` : "Next retry pending schedule",
+      retryOfRunId,
+    };
+  }
+
+  if (exhaustedReason) {
+    return {
+      kind: "exhausted",
+      badgeLabel: "Retry exhausted",
+      tone: "border-amber-500/30 bg-amber-500/10 text-amber-700 dark:text-amber-300",
+      detail: joinFragments([attemptLabel, reasonLabel, "No further automatic retry queued"]),
+      secondary: exhaustedReason,
+      retryOfRunId,
+    };
+  }
+
+  return {
+    kind: "attempted",
+    badgeLabel: "Retried run",
+    tone: "border-slate-500/20 bg-slate-500/10 text-slate-700 dark:text-slate-300",
+    detail: joinFragments([attemptLabel, reasonLabel]),
+    secondary: null,
+    retryOfRunId,
+  };
+}

ui/src/lib/status-colors.ts

@@ -43,6 +43,7 @@ export const statusBadge: Record<string, string> = {
   // Agent statuses
   active: "bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300",
   running: "bg-cyan-100 text-cyan-700 dark:bg-cyan-900/50 dark:text-cyan-300",
+  scheduled_retry: "bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300",
   paused: "bg-orange-100 text-orange-700 dark:bg-orange-900/50 dark:text-orange-300",
   idle: "bg-yellow-100 text-yellow-700 dark:bg-yellow-900/50 dark:text-yellow-300",
   archived: "bg-muted text-muted-foreground",