mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-17 11:20:37 +09:00
[codex] Harden heartbeat scheduling and runtime controls (#4223)
## Thinking Path > - Paperclip orchestrates AI agents through issue checkout, heartbeat runs, routines, and auditable control-plane state > - The runtime path has to recover from lost local processes, transient adapter failures, blocked dependencies, and routine coalescing without stranding work > - The existing branch carried several reliability fixes across heartbeat scheduling, issue runtime controls, routine dispatch, and operator-facing run state > - These changes belong together because they share backend contracts, migrations, and runtime status semantics > - This pull request groups the control-plane/runtime slice so it can merge independently from board UI polish and adapter sandbox work > - The benefit is safer heartbeat recovery, clearer runtime controls, and more predictable recurring execution behavior ## What Changed - Adds bounded heartbeat retry scheduling, scheduled retry state, and Codex transient failure recovery handling. - Tightens heartbeat process recovery, blocker wake behavior, issue comment wake handling, routine dispatch coalescing, and activity/dashboard bounds. - Adds runtime-control MCP tools and Paperclip skill docs for issue workspace runtime management. - Adds migrations `0061_lively_thor_girl.sql` and `0062_routine_run_dispatch_fingerprint.sql`. - Surfaces retry state in run ledger/agent UI and keeps related shared types synchronized. ## Verification - `pnpm exec vitest run server/src/__tests__/heartbeat-retry-scheduling.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/routines-service.test.ts` - `pnpm exec vitest run src/tools.test.ts` from `packages/mcp-server` ## Risks - Medium risk: this touches heartbeat recovery and routine dispatch, which are central execution paths. - Migration order matters if split branches land out of order: merge this PR before branches that assume the new runtime/routine fields. - Runtime retry behavior should be watched in CI and in local operator smoke tests because it changes how transient failures are resumed. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5-based coding agent runtime, shell/git tool use enabled. Exact hosted model build and context window are not exposed in this Paperclip heartbeat environment. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge
This commit is contained in:
Dotta
GitHub
parent
ab9051b595
commit
09d0678840
61 changed files with 17622 additions and 456 deletions
|
|
@ -47,7 +47,7 @@ import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "./is
|
|||
import { logActivity } from "./activity-log.js";
|
||||
|
||||
const OPEN_ISSUE_STATUSES = ["backlog", "todo", "in_progress", "in_review", "blocked"];
|
||||
const LIVE_HEARTBEAT_RUN_STATUSES = ["queued", "running"];
|
||||
const LIVE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
|
||||
const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]);
|
||||
const MAX_CATCH_UP_RUNS = 25;
|
||||
const WEEKDAY_INDEX: Record<string, number> = {
|
||||
|
|
@ -320,6 +320,37 @@ function mergeRoutineRunPayload(
|
|||
};
|
||||
}
|
||||
|
||||
function normalizeRoutineDispatchFingerprintValue(value: unknown): unknown {
|
||||
if (value === undefined) return null;
|
||||
if (value == null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
||||
return value;
|
||||
}
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
if (Array.isArray(value)) return value.map((item) => normalizeRoutineDispatchFingerprintValue(item));
|
||||
if (isPlainRecord(value)) {
|
||||
return Object.fromEntries(
|
||||
Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => [key, normalizeRoutineDispatchFingerprintValue(value[key])]),
|
||||
);
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function createRoutineDispatchFingerprint(input: {
|
||||
payload: Record<string, unknown> | null;
|
||||
projectId: string | null;
|
||||
assigneeAgentId: string | null;
|
||||
executionWorkspaceId?: string | null;
|
||||
executionWorkspacePreference?: string | null;
|
||||
executionWorkspaceSettings?: Record<string, unknown> | null;
|
||||
title: string;
|
||||
description: string | null;
|
||||
}) {
|
||||
const canonical = JSON.stringify(normalizeRoutineDispatchFingerprintValue(input));
|
||||
return crypto.createHash("sha256").update(canonical).digest("hex");
|
||||
}
|
||||
|
||||
function routineUsesWorkspaceBranch(routine: typeof routines.$inferSelect) {
|
||||
return (routine.variables ?? []).some((variable) => variable.name === WORKSPACE_BRANCH_ROUTINE_VARIABLE)
|
||||
|| extractRoutineVariableNames([routine.title, routine.description]).includes(WORKSPACE_BRANCH_ROUTINE_VARIABLE);
|
||||
|
|
@ -426,6 +457,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
triggeredAt: routineRuns.triggeredAt,
|
||||
idempotencyKey: routineRuns.idempotencyKey,
|
||||
triggerPayload: routineRuns.triggerPayload,
|
||||
dispatchFingerprint: routineRuns.dispatchFingerprint,
|
||||
linkedIssueId: routineRuns.linkedIssueId,
|
||||
coalescedIntoRunId: routineRuns.coalescedIntoRunId,
|
||||
failureReason: routineRuns.failureReason,
|
||||
|
|
@ -458,6 +490,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
triggeredAt: row.triggeredAt,
|
||||
idempotencyKey: row.idempotencyKey,
|
||||
triggerPayload: row.triggerPayload as Record<string, unknown> | null,
|
||||
dispatchFingerprint: row.dispatchFingerprint,
|
||||
linkedIssueId: row.linkedIssueId,
|
||||
coalescedIntoRunId: row.coalescedIntoRunId,
|
||||
failureReason: row.failureReason,
|
||||
|
|
@ -606,7 +639,22 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
}
|
||||
}
|
||||
|
||||
async function findLiveExecutionIssue(routine: typeof routines.$inferSelect, executor: Db = db) {
|
||||
function routineExecutionFingerprintCondition(dispatchFingerprint?: string | null) {
|
||||
if (!dispatchFingerprint) return null;
|
||||
// The "default" arm preserves coalescing against pre-migration open issues.
|
||||
// It becomes inert once those legacy routine execution issues drain out.
|
||||
return or(
|
||||
eq(issues.originFingerprint, dispatchFingerprint),
|
||||
eq(issues.originFingerprint, "default"),
|
||||
);
|
||||
}
|
||||
|
||||
async function findLiveExecutionIssue(
|
||||
routine: typeof routines.$inferSelect,
|
||||
executor: Db = db,
|
||||
dispatchFingerprint?: string | null,
|
||||
) {
|
||||
const fingerprintCondition = routineExecutionFingerprintCondition(dispatchFingerprint);
|
||||
const executionBoundIssue = await executor
|
||||
.select()
|
||||
.from(issues)
|
||||
|
|
@ -624,6 +672,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
eq(issues.originId, routine.id),
|
||||
inArray(issues.status, OPEN_ISSUE_STATUSES),
|
||||
isNull(issues.hiddenAt),
|
||||
...(fingerprintCondition ? [fingerprintCondition] : []),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(issues.updatedAt), desc(issues.createdAt))
|
||||
|
|
@ -649,6 +698,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
eq(issues.originId, routine.id),
|
||||
inArray(issues.status, OPEN_ISSUE_STATUSES),
|
||||
isNull(issues.hiddenAt),
|
||||
...(fingerprintCondition ? [fingerprintCondition] : []),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(issues.updatedAt), desc(issues.createdAt))
|
||||
|
|
@ -745,6 +795,16 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
const title = interpolateRoutineTemplate(input.routine.title, allVariables) ?? input.routine.title;
|
||||
const description = interpolateRoutineTemplate(input.routine.description, allVariables);
|
||||
const triggerPayload = mergeRoutineRunPayload(input.payload, { ...automaticVariables, ...resolvedVariables });
|
||||
const dispatchFingerprint = createRoutineDispatchFingerprint({
|
||||
payload: triggerPayload,
|
||||
projectId,
|
||||
assigneeAgentId,
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
||||
executionWorkspacePreference: input.executionWorkspacePreference ?? null,
|
||||
executionWorkspaceSettings: input.executionWorkspaceSettings ?? null,
|
||||
title,
|
||||
description,
|
||||
});
|
||||
const run = await db.transaction(async (tx) => {
|
||||
const txDb = tx as unknown as Db;
|
||||
await tx.execute(
|
||||
|
|
@ -782,6 +842,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
triggeredAt,
|
||||
idempotencyKey: input.idempotencyKey ?? null,
|
||||
triggerPayload,
|
||||
dispatchFingerprint,
|
||||
})
|
||||
.returning();
|
||||
|
||||
|
|
@ -791,7 +852,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
|
||||
let createdIssue: Awaited<ReturnType<typeof issueSvc.create>> | null = null;
|
||||
try {
|
||||
const activeIssue = await findLiveExecutionIssue(input.routine, txDb);
|
||||
const activeIssue = await findLiveExecutionIssue(input.routine, txDb, dispatchFingerprint);
|
||||
if (activeIssue && input.routine.concurrencyPolicy !== "always_enqueue") {
|
||||
const status = input.routine.concurrencyPolicy === "skip_if_active" ? "skipped" : "coalesced";
|
||||
const updated = await finalizeRun(createdRun.id, {
|
||||
|
|
@ -824,6 +885,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
originKind: "routine_execution",
|
||||
originId: input.routine.id,
|
||||
originRunId: createdRun.id,
|
||||
originFingerprint: dispatchFingerprint,
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
||||
executionWorkspacePreference: input.executionWorkspacePreference ?? null,
|
||||
executionWorkspaceSettings: input.executionWorkspaceSettings ?? null,
|
||||
|
|
@ -840,7 +902,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
throw error;
|
||||
}
|
||||
|
||||
const existingIssue = await findLiveExecutionIssue(input.routine, txDb);
|
||||
const existingIssue = await findLiveExecutionIssue(input.routine, txDb, dispatchFingerprint);
|
||||
if (!existingIssue) throw error;
|
||||
const status = input.routine.concurrencyPolicy === "skip_if_active" ? "skipped" : "coalesced";
|
||||
const updated = await finalizeRun(createdRun.id, {
|
||||
|
|
@ -994,6 +1056,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
triggeredAt: routineRuns.triggeredAt,
|
||||
idempotencyKey: routineRuns.idempotencyKey,
|
||||
triggerPayload: routineRuns.triggerPayload,
|
||||
dispatchFingerprint: routineRuns.dispatchFingerprint,
|
||||
linkedIssueId: routineRuns.linkedIssueId,
|
||||
coalescedIntoRunId: routineRuns.coalescedIntoRunId,
|
||||
failureReason: routineRuns.failureReason,
|
||||
|
|
@ -1025,6 +1088,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
triggeredAt: run.triggeredAt,
|
||||
idempotencyKey: run.idempotencyKey,
|
||||
triggerPayload: run.triggerPayload as Record<string, unknown> | null,
|
||||
dispatchFingerprint: run.dispatchFingerprint,
|
||||
linkedIssueId: run.linkedIssueId,
|
||||
coalescedIntoRunId: run.coalescedIntoRunId,
|
||||
failureReason: run.failureReason,
|
||||
|
|
@ -1437,6 +1501,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
triggeredAt: routineRuns.triggeredAt,
|
||||
idempotencyKey: routineRuns.idempotencyKey,
|
||||
triggerPayload: routineRuns.triggerPayload,
|
||||
dispatchFingerprint: routineRuns.dispatchFingerprint,
|
||||
linkedIssueId: routineRuns.linkedIssueId,
|
||||
coalescedIntoRunId: routineRuns.coalescedIntoRunId,
|
||||
failureReason: routineRuns.failureReason,
|
||||
|
|
@ -1468,6 +1533,7 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
|||
triggeredAt: row.triggeredAt,
|
||||
idempotencyKey: row.idempotencyKey,
|
||||
triggerPayload: row.triggerPayload as Record<string, unknown> | null,
|
||||
dispatchFingerprint: row.dispatchFingerprint,
|
||||
linkedIssueId: row.linkedIssueId,
|
||||
coalescedIntoRunId: row.coalescedIntoRunId,
|
||||
failureReason: row.failureReason,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue