[codex] Harden heartbeat scheduling and runtime controls (#4223)

## Thinking Path

> - Paperclip orchestrates AI agents through issue checkout, heartbeat
runs, routines, and auditable control-plane state
> - The runtime path has to recover from lost local processes, transient
adapter failures, blocked dependencies, and routine coalescing without
stranding work
> - The existing branch carried several reliability fixes across
heartbeat scheduling, issue runtime controls, routine dispatch, and
operator-facing run state
> - These changes belong together because they share backend contracts,
migrations, and runtime status semantics
> - This pull request groups the control-plane/runtime slice so it can
merge independently from board UI polish and adapter sandbox work
> - The benefit is safer heartbeat recovery, clearer runtime controls,
and more predictable recurring execution behavior

## What Changed

- Adds bounded heartbeat retry scheduling, scheduled retry state, and
Codex transient failure recovery handling.
- Tightens heartbeat process recovery, blocker wake behavior, issue
comment wake handling, routine dispatch coalescing, and
activity/dashboard bounds.
- Adds runtime-control MCP tools and Paperclip skill docs for issue
workspace runtime management.
- Adds migrations `0061_lively_thor_girl.sql` and
`0062_routine_run_dispatch_fingerprint.sql`.
- Surfaces retry state in run ledger/agent UI and keeps related shared
types synchronized.

## Verification

- `pnpm exec vitest run
server/src/__tests__/heartbeat-retry-scheduling.test.ts
server/src/__tests__/heartbeat-process-recovery.test.ts
server/src/__tests__/routines-service.test.ts`
- `pnpm exec vitest run src/tools.test.ts` from `packages/mcp-server`

## Risks

- Medium risk: this touches heartbeat recovery and routine dispatch,
which are central execution paths.
- Migration order matters if split branches land out of order: merge
this PR before branches that assume the new runtime/routine fields.
- Runtime retry behavior should be watched in CI and in local operator
smoke tests because it changes how transient failures are resumed.

> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.

## Model Used

- OpenAI Codex, GPT-5-based coding agent runtime, shell/git tool use
enabled. Exact hosted model build and context window are not exposed in
this Paperclip heartbeat environment.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [ ] If this change affects the UI, I have included before/after
screenshots
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge
This commit is contained in:
Dotta 2026-04-21 12:24:11 -05:00 committed by GitHub
parent ab9051b595
commit 09d0678840
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
61 changed files with 17622 additions and 456 deletions

View file

@ -1,3 +1,4 @@
import { Buffer } from "node:buffer";
import { and, asc, desc, eq, inArray, isNull, ne, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
@ -79,6 +80,7 @@ export interface IssueFilters {
inboxArchivedByUserId?: string;
unreadForUserId?: string;
projectId?: string;
workspaceId?: string;
executionWorkspaceId?: string;
parentId?: string;
labelId?: string;
@ -168,6 +170,7 @@ function sameRunLock(checkoutRunId: string | null, actorRunId: string | null) {
const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]);
const ISSUE_LIST_DESCRIPTION_MAX_CHARS = 1200;
const ISSUE_LIST_DESCRIPTION_MAX_BYTES = ISSUE_LIST_DESCRIPTION_MAX_CHARS * 4;
function escapeLikePattern(value: string): string {
return value.replace(/[\\%_]/g, "\\$&");
@ -191,6 +194,16 @@ function truncateInlineSummary(value: string | null | undefined, maxChars = CHIL
return normalized.length > maxChars ? `${normalized.slice(0, Math.max(0, maxChars - 15)).trimEnd()} [truncated]` : normalized;
}
function truncateByCodePoint(value: string, maxChars: number): string {
if (value.length <= maxChars) return value;
return Array.from(value).slice(0, maxChars).join("");
}
function decodeDatabaseTextPreview(value: string | null | undefined, maxChars: number): string | null {
if (value == null) return null;
return truncateByCodePoint(Buffer.from(value, "base64").toString("utf8"), maxChars);
}
function appendAcceptanceCriteriaToDescription(description: string | null | undefined, acceptanceCriteria: string[] | undefined) {
const criteria = (acceptanceCriteria ?? []).map((item) => item.trim()).filter(Boolean);
if (criteria.length === 0) return description ?? null;
@ -275,7 +288,6 @@ async function listUnresolvedBlockerIssueIds(
)
.then((rows) => rows.map((row) => row.id));
}
async function getProjectDefaultGoalId(
db: ProjectGoalReader,
companyId: string,
@ -681,7 +693,13 @@ const issueListSelect = {
description: sql<string | null>`
CASE
WHEN ${issues.description} IS NULL THEN NULL
ELSE substring(${issues.description} FROM 1 FOR ${ISSUE_LIST_DESCRIPTION_MAX_CHARS})
ELSE encode(
substring(
convert_to(${issues.description}, current_setting('server_encoding'))
FROM 1 FOR ${ISSUE_LIST_DESCRIPTION_MAX_BYTES}
),
'base64'
)
END
`,
status: issues.status,
@ -699,6 +717,7 @@ const issueListSelect = {
originKind: issues.originKind,
originId: issues.originId,
originRunId: issues.originRunId,
originFingerprint: issues.originFingerprint,
requestDepth: issues.requestDepth,
billingCode: issues.billingCode,
assigneeAdapterOverrides: issues.assigneeAdapterOverrides,
@ -1275,6 +1294,12 @@ export function issueService(db: Db) {
conditions.push(unreadForUserCondition(companyId, unreadForUserId));
}
if (filters?.projectId) conditions.push(eq(issues.projectId, filters.projectId));
if (filters?.workspaceId) {
conditions.push(or(
eq(issues.executionWorkspaceId, filters.workspaceId),
eq(issues.projectWorkspaceId, filters.workspaceId),
)!);
}
if (filters?.executionWorkspaceId) {
conditions.push(eq(issues.executionWorkspaceId, filters.executionWorkspaceId));
}
@ -1327,7 +1352,10 @@ export function issueService(db: Db) {
desc(canonicalLastActivityAt),
desc(issues.updatedAt),
);
const rows = limit === undefined ? await baseQuery : await baseQuery.limit(limit);
const rows = (limit === undefined ? await baseQuery : await baseQuery.limit(limit)).map((row) => ({
...row,
description: decodeDatabaseTextPreview(row.description, ISSUE_LIST_DESCRIPTION_MAX_CHARS),
}));
const withLabels = await withIssueLabels(db, rows);
const runMap = await activeRunMapForIssues(db, withLabels);
const withRuns = withActiveRuns(withLabels, runMap);