mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-16 19:00:38 +09:00
[codex] Harden heartbeat scheduling and runtime controls (#4223)
## Thinking Path > - Paperclip orchestrates AI agents through issue checkout, heartbeat runs, routines, and auditable control-plane state > - The runtime path has to recover from lost local processes, transient adapter failures, blocked dependencies, and routine coalescing without stranding work > - The existing branch carried several reliability fixes across heartbeat scheduling, issue runtime controls, routine dispatch, and operator-facing run state > - These changes belong together because they share backend contracts, migrations, and runtime status semantics > - This pull request groups the control-plane/runtime slice so it can merge independently from board UI polish and adapter sandbox work > - The benefit is safer heartbeat recovery, clearer runtime controls, and more predictable recurring execution behavior ## What Changed - Adds bounded heartbeat retry scheduling, scheduled retry state, and Codex transient failure recovery handling. - Tightens heartbeat process recovery, blocker wake behavior, issue comment wake handling, routine dispatch coalescing, and activity/dashboard bounds. - Adds runtime-control MCP tools and Paperclip skill docs for issue workspace runtime management. - Adds migrations `0061_lively_thor_girl.sql` and `0062_routine_run_dispatch_fingerprint.sql`. - Surfaces retry state in run ledger/agent UI and keeps related shared types synchronized. ## Verification - `pnpm exec vitest run server/src/__tests__/heartbeat-retry-scheduling.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/routines-service.test.ts` - `pnpm exec vitest run src/tools.test.ts` from `packages/mcp-server` ## Risks - Medium risk: this touches heartbeat recovery and routine dispatch, which are central execution paths. - Migration order matters if split branches land out of order: merge this PR before branches that assume the new runtime/routine fields. - Runtime retry behavior should be watched in CI and in local operator smoke tests because it changes how transient failures are resumed. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5-based coding agent runtime, shell/git tool use enabled. Exact hosted model build and context window are not exposed in this Paperclip heartbeat environment. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge
This commit is contained in:
Dotta
GitHub
parent
ab9051b595
commit
09d0678840
61 changed files with 17622 additions and 456 deletions
|
|
@ -2,7 +2,7 @@ import { Router } from "express";
|
|||
import { z } from "zod";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { validate } from "../middleware/validate.js";
|
||||
import { activityService } from "../services/activity.js";
|
||||
import { activityService, normalizeActivityLimit } from "../services/activity.js";
|
||||
import { assertAuthenticated, assertBoard, assertCompanyAccess } from "./authz.js";
|
||||
import { heartbeatService, issueService } from "../services/index.js";
|
||||
import { sanitizeRecord } from "../redaction.js";
|
||||
|
|
@ -39,6 +39,7 @@ export function activityRoutes(db: Db) {
|
|||
agentId: req.query.agentId as string | undefined,
|
||||
entityType: req.query.entityType as string | undefined,
|
||||
entityId: req.query.entityId as string | undefined,
|
||||
limit: normalizeActivityLimit(Number(req.query.limit)),
|
||||
};
|
||||
const result = await svc.list(filters);
|
||||
res.json(result);
|
||||
|
|
|
|||
|
|
@ -2155,7 +2155,6 @@ export function agentRoutes(db: Db) {
|
|||
res.status(409).json({ error: "Only pending approval agents can be approved" });
|
||||
return;
|
||||
}
|
||||
|
||||
const approval = await svc.activatePendingApproval(id);
|
||||
if (!approval) {
|
||||
res.status(404).json({ error: "Agent not found" });
|
||||
|
|
@ -2515,7 +2514,13 @@ export function agentRoutes(db: Db) {
|
|||
return;
|
||||
}
|
||||
assertCompanyAccess(req, run.companyId);
|
||||
res.json(redactCurrentUserValue(run, await getCurrentUserRedactionOptions()));
|
||||
const retryExhaustedReason = await heartbeat.getRetryExhaustedReason(runId);
|
||||
res.json(
|
||||
redactCurrentUserValue(
|
||||
{ ...run, retryExhaustedReason },
|
||||
await getCurrentUserRedactionOptions(),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
router.post("/heartbeat-runs/:runId/cancel", async (req, res) => {
|
||||
|
|
|
|||
|
|
@ -173,13 +173,13 @@ function isClosedIssueStatus(status: string | null | undefined): status is "done
|
|||
return status === "done" || status === "cancelled";
|
||||
}
|
||||
|
||||
function shouldImplicitlyReopenCommentForAgent(input: {
|
||||
function shouldImplicitlyMoveCommentedIssueToTodoForAgent(input: {
|
||||
issueStatus: string | null | undefined;
|
||||
assigneeAgentId: string | null | undefined;
|
||||
actorType: "agent" | "user";
|
||||
actorId: string;
|
||||
}) {
|
||||
if (!isClosedIssueStatus(input.issueStatus)) return false;
|
||||
if (!isClosedIssueStatus(input.issueStatus) && input.issueStatus !== "blocked") return false;
|
||||
if (typeof input.assigneeAgentId !== "string" || input.assigneeAgentId.length === 0) return false;
|
||||
if (input.actorType === "agent" && input.actorId === input.assigneeAgentId) return false;
|
||||
return true;
|
||||
|
|
@ -721,6 +721,7 @@ export function issueRoutes(
|
|||
inboxArchivedByUserId,
|
||||
unreadForUserId,
|
||||
projectId: req.query.projectId as string | undefined,
|
||||
workspaceId: req.query.workspaceId as string | undefined,
|
||||
executionWorkspaceId: req.query.executionWorkspaceId as string | undefined,
|
||||
parentId: req.query.parentId as string | undefined,
|
||||
labelId: req.query.labelId as string | undefined,
|
||||
|
|
@ -804,16 +805,29 @@ export function issueRoutes(
|
|||
? req.query.wakeCommentId.trim()
|
||||
: null;
|
||||
|
||||
const [{ project, goal }, ancestors, commentCursor, wakeComment, relations, attachments, continuationSummary] =
|
||||
const currentExecutionWorkspacePromise = issue.executionWorkspaceId
|
||||
? executionWorkspacesSvc.getById(issue.executionWorkspaceId)
|
||||
: Promise.resolve(null);
|
||||
const [
|
||||
{ project, goal },
|
||||
ancestors,
|
||||
commentCursor,
|
||||
wakeComment,
|
||||
relations,
|
||||
attachments,
|
||||
continuationSummary,
|
||||
currentExecutionWorkspace,
|
||||
] =
|
||||
await Promise.all([
|
||||
resolveIssueProjectAndGoal(issue),
|
||||
svc.getAncestors(issue.id),
|
||||
svc.getCommentCursor(issue.id),
|
||||
wakeCommentId ? svc.getComment(wakeCommentId) : null,
|
||||
svc.getRelationSummaries(issue.id),
|
||||
svc.listAttachments(issue.id),
|
||||
documentsSvc.getIssueDocumentByKey(issue.id, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY),
|
||||
]);
|
||||
resolveIssueProjectAndGoal(issue),
|
||||
svc.getAncestors(issue.id),
|
||||
svc.getCommentCursor(issue.id),
|
||||
wakeCommentId ? svc.getComment(wakeCommentId) : null,
|
||||
svc.getRelationSummaries(issue.id),
|
||||
svc.listAttachments(issue.id),
|
||||
documentsSvc.getIssueDocumentByKey(issue.id, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY),
|
||||
currentExecutionWorkspacePromise,
|
||||
]);
|
||||
|
||||
res.json({
|
||||
issue: {
|
||||
|
|
@ -879,6 +893,7 @@ export function issueRoutes(
|
|||
updatedAt: continuationSummary.updatedAt,
|
||||
}
|
||||
: null,
|
||||
currentExecutionWorkspace,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1590,6 +1605,7 @@ export function issueRoutes(
|
|||
|
||||
const actor = getActorInfo(req);
|
||||
const isClosed = isClosedIssueStatus(existing.status);
|
||||
const isBlocked = existing.status === "blocked";
|
||||
const normalizedAssigneeAgentId = await normalizeIssueAssigneeAgentReference(
|
||||
existing.companyId,
|
||||
req.body.assigneeAgentId as string | null | undefined,
|
||||
|
|
@ -1608,10 +1624,10 @@ export function issueRoutes(
|
|||
} = req.body;
|
||||
const requestedAssigneeAgentId =
|
||||
normalizedAssigneeAgentId === undefined ? existing.assigneeAgentId : normalizedAssigneeAgentId;
|
||||
const effectiveReopenRequested =
|
||||
const effectiveMoveToTodoRequested =
|
||||
reopenRequested ||
|
||||
(!!commentBody &&
|
||||
shouldImplicitlyReopenCommentForAgent({
|
||||
shouldImplicitlyMoveCommentedIssueToTodoForAgent({
|
||||
issueStatus: existing.status,
|
||||
assigneeAgentId: requestedAssigneeAgentId,
|
||||
actorType: actor.actorType,
|
||||
|
|
@ -1620,6 +1636,10 @@ export function issueRoutes(
|
|||
const updateReferenceSummaryBefore = titleOrDescriptionChanged
|
||||
? await issueReferencesSvc.listIssueReferenceSummary(existing.id)
|
||||
: null;
|
||||
const hasUnresolvedFirstClassBlockers =
|
||||
isBlocked && effectiveMoveToTodoRequested
|
||||
? (await svc.getDependencyReadiness(existing.id)).unresolvedBlockerCount > 0
|
||||
: false;
|
||||
let interruptedRunId: string | null = null;
|
||||
const closedExecutionWorkspace = await getClosedIssueExecutionWorkspace(existing);
|
||||
const isAgentWorkUpdate = req.actor.type === "agent" && Object.keys(updateFields).length > 0;
|
||||
|
|
@ -1662,7 +1682,12 @@ export function issueRoutes(
|
|||
if (hiddenAtRaw !== undefined) {
|
||||
updateFields.hiddenAt = hiddenAtRaw ? new Date(hiddenAtRaw) : null;
|
||||
}
|
||||
if (commentBody && effectiveReopenRequested && isClosed && updateFields.status === undefined) {
|
||||
if (
|
||||
commentBody &&
|
||||
effectiveMoveToTodoRequested &&
|
||||
(isClosed || (isBlocked && !hasUnresolvedFirstClassBlockers)) &&
|
||||
updateFields.status === undefined
|
||||
) {
|
||||
updateFields.status = "todo";
|
||||
}
|
||||
if (req.body.executionPolicy !== undefined) {
|
||||
|
|
@ -1836,8 +1861,8 @@ export function issueRoutes(
|
|||
const hasFieldChanges = Object.keys(previous).length > 0;
|
||||
const reopened =
|
||||
commentBody &&
|
||||
effectiveReopenRequested &&
|
||||
isClosed &&
|
||||
effectiveMoveToTodoRequested &&
|
||||
(isClosed || (isBlocked && !hasUnresolvedFirstClassBlockers)) &&
|
||||
previous.status !== undefined &&
|
||||
issue.status === "todo";
|
||||
const reopenFromStatus = reopened ? existing.status : null;
|
||||
|
|
@ -2025,7 +2050,7 @@ export function issueRoutes(
|
|||
const statusChangedFromBlockedToTodo =
|
||||
existing.status === "blocked" &&
|
||||
issue.status === "todo" &&
|
||||
req.body.status !== undefined;
|
||||
(req.body.status !== undefined || reopened);
|
||||
const previousExecutionState = parseIssueExecutionState(existing.executionState);
|
||||
const nextExecutionState = parseIssueExecutionState(issue.executionState);
|
||||
const executionStageWakeup = buildExecutionStageWakeup({
|
||||
|
|
@ -2596,21 +2621,26 @@ export function issueRoutes(
|
|||
const reopenRequested = req.body.reopen === true;
|
||||
const interruptRequested = req.body.interrupt === true;
|
||||
const isClosed = isClosedIssueStatus(issue.status);
|
||||
const effectiveReopenRequested =
|
||||
const isBlocked = issue.status === "blocked";
|
||||
const effectiveMoveToTodoRequested =
|
||||
reopenRequested ||
|
||||
shouldImplicitlyReopenCommentForAgent({
|
||||
shouldImplicitlyMoveCommentedIssueToTodoForAgent({
|
||||
issueStatus: issue.status,
|
||||
assigneeAgentId: issue.assigneeAgentId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
});
|
||||
const hasUnresolvedFirstClassBlockers =
|
||||
isBlocked && effectiveMoveToTodoRequested
|
||||
? (await svc.getDependencyReadiness(issue.id)).unresolvedBlockerCount > 0
|
||||
: false;
|
||||
let reopened = false;
|
||||
let reopenFromStatus: string | null = null;
|
||||
let interruptedRunId: string | null = null;
|
||||
let currentIssue = issue;
|
||||
const commentReferenceSummaryBefore = await issueReferencesSvc.listIssueReferenceSummary(issue.id);
|
||||
|
||||
if (effectiveReopenRequested && isClosed) {
|
||||
if (effectiveMoveToTodoRequested && (isClosed || (isBlocked && !hasUnresolvedFirstClassBlockers))) {
|
||||
const reopenedIssue = await svc.update(id, { status: "todo" });
|
||||
if (!reopenedIssue) {
|
||||
res.status(404).json({ error: "Issue not found" });
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue