mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-15 18:30:39 +09:00
Cancel stale retries when issue ownership changes (#4445)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies > - Issue execution is guarded by run locks and bounded retry scheduling > - A failed run can schedule a retry, but the issue may be reassigned before that retry becomes due > - The old assignee's scheduled retry should not continue to hold or reclaim execution for the issue > - This pull request cancels stale scheduled retries when ownership changes and cancels live work when an issue is explicitly cancelled > - The benefit is cleaner issue handoff semantics and fewer stranded or incorrect execution locks ## What Changed - Cancel scheduled retry runs when their issue has been reassigned before the retry is promoted. - Clear stale issue execution locks and cancel the associated wakeup request when a stale retry is cancelled. - Avoid deferring a new assignee behind a previous assignee's scheduled retry. - Cancel an active run when an issue status is explicitly changed to `cancelled`, while leaving `done` transitions alone. - Added route and heartbeat regressions for reassignment and cancellation behavior. ## Verification - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/heartbeat-retry-scheduling.test.ts server/src/__tests__/issue-comment-reopen-routes.test.ts --pool=forks --poolOptions.forks.isolate=true` - `issue-comment-reopen-routes.test.ts`: 28 passed. - `heartbeat-retry-scheduling.test.ts`: skipped by the existing embedded Postgres host guard (`Postgres init script exited with code null`). - `pnpm --filter @paperclipai/server typecheck` ## Risks - Medium risk because this changes heartbeat retry lifecycle behavior. - The cancellation path is scoped to scheduled retries whose issue assignee no longer matches the retrying agent, and logs a lifecycle event for auditability. - No migrations. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5 coding agent, tool-enabled with shell/GitHub/Paperclip API access. Context window was not reported by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
0c6961a03e
commit
6916e30f8e
4 changed files with 676 additions and 15 deletions
|
|
@ -3545,6 +3545,90 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
|||
const promotedRunIds: string[] = [];
|
||||
|
||||
for (const dueRun of dueRuns) {
|
||||
const dueRunIssueId = readNonEmptyString(parseObject(dueRun.contextSnapshot).issueId);
|
||||
if (dueRunIssueId) {
|
||||
const issue = await db
|
||||
.select({
|
||||
id: issues.id,
|
||||
status: issues.status,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionRunId: issues.executionRunId,
|
||||
})
|
||||
.from(issues)
|
||||
.where(and(eq(issues.id, dueRunIssueId), eq(issues.companyId, dueRun.companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
if (issue && (issue.assigneeAgentId !== dueRun.agentId || issue.status === "cancelled")) {
|
||||
const issueCancelled = issue.status === "cancelled";
|
||||
const reason = issueCancelled
|
||||
? "Cancelled because the issue was cancelled before the scheduled retry became due"
|
||||
: "Cancelled because the issue was reassigned before the scheduled retry became due";
|
||||
const cancelled = await db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
status: "cancelled",
|
||||
finishedAt: now,
|
||||
error: reason,
|
||||
errorCode: issueCancelled ? "issue_cancelled" : "issue_reassigned",
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.id, dueRun.id),
|
||||
eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
lte(heartbeatRuns.scheduledRetryAt, now),
|
||||
),
|
||||
)
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
if (!cancelled) continue;
|
||||
|
||||
if (cancelled.wakeupRequestId) {
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
status: "cancelled",
|
||||
finishedAt: now,
|
||||
error: reason,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(eq(agentWakeupRequests.id, cancelled.wakeupRequestId));
|
||||
}
|
||||
|
||||
if (issue.executionRunId === cancelled.id) {
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(eq(issues.id, issue.id), eq(issues.executionRunId, cancelled.id)));
|
||||
}
|
||||
|
||||
await appendRunEvent(cancelled, await nextRunEventSeq(cancelled.id), {
|
||||
eventType: "lifecycle",
|
||||
stream: "system",
|
||||
level: "warn",
|
||||
message: issueCancelled
|
||||
? "Scheduled retry cancelled because issue was cancelled before it became due"
|
||||
: "Scheduled retry cancelled because issue ownership changed before it became due",
|
||||
payload: {
|
||||
issueId: issue.id,
|
||||
issueStatus: issue.status,
|
||||
scheduledRetryAttempt: cancelled.scheduledRetryAttempt,
|
||||
scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null,
|
||||
scheduledRetryReason: cancelled.scheduledRetryReason,
|
||||
previousRetryAgentId: cancelled.agentId,
|
||||
currentAssigneeAgentId: issue.assigneeAgentId,
|
||||
},
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const promoted = await db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
|
|
@ -6228,6 +6312,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
|||
.select({
|
||||
id: issues.id,
|
||||
companyId: issues.companyId,
|
||||
status: issues.status,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
})
|
||||
|
|
@ -6252,6 +6338,88 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
|||
return { kind: "skipped" as const };
|
||||
}
|
||||
|
||||
const cancelStaleScheduledRetry = async (scheduledRun: typeof heartbeatRuns.$inferSelect) => {
|
||||
const issueCancelled = issue.status === "cancelled";
|
||||
if (
|
||||
scheduledRun.status !== "scheduled_retry" ||
|
||||
(scheduledRun.agentId === issue.assigneeAgentId && !issueCancelled)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const reason = issueCancelled
|
||||
? "Cancelled because the issue was cancelled before the scheduled retry became due"
|
||||
: "Cancelled because the issue was reassigned before the scheduled retry became due";
|
||||
const cancelled = await tx
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
status: "cancelled",
|
||||
finishedAt: now,
|
||||
error: reason,
|
||||
errorCode: issueCancelled ? "issue_cancelled" : "issue_reassigned",
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(eq(heartbeatRuns.id, scheduledRun.id), eq(heartbeatRuns.status, "scheduled_retry")))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
if (!cancelled) return false;
|
||||
|
||||
if (scheduledRun.wakeupRequestId) {
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
status: "cancelled",
|
||||
finishedAt: now,
|
||||
error: reason,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(eq(agentWakeupRequests.id, scheduledRun.wakeupRequestId));
|
||||
}
|
||||
|
||||
if (issue.executionRunId === scheduledRun.id) {
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(eq(issues.id, issue.id), eq(issues.executionRunId, scheduledRun.id)));
|
||||
}
|
||||
|
||||
const [eventSeq] = await tx
|
||||
.select({ maxSeq: sql<number | null>`max(${heartbeatRunEvents.seq})` })
|
||||
.from(heartbeatRunEvents)
|
||||
.where(eq(heartbeatRunEvents.runId, cancelled.id));
|
||||
|
||||
await tx.insert(heartbeatRunEvents).values({
|
||||
companyId: cancelled.companyId,
|
||||
runId: cancelled.id,
|
||||
agentId: cancelled.agentId,
|
||||
seq: Number(eventSeq?.maxSeq ?? 0) + 1,
|
||||
eventType: "lifecycle",
|
||||
stream: "system",
|
||||
level: "warn",
|
||||
message: issueCancelled
|
||||
? "Scheduled retry cancelled because issue was cancelled before it became due"
|
||||
: "Scheduled retry cancelled because issue ownership changed before it became due",
|
||||
payload: {
|
||||
issueId: issue.id,
|
||||
issueStatus: issue.status,
|
||||
scheduledRetryAttempt: cancelled.scheduledRetryAttempt,
|
||||
scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null,
|
||||
scheduledRetryReason: cancelled.scheduledRetryReason,
|
||||
previousRetryAgentId: cancelled.agentId,
|
||||
currentAssigneeAgentId: issue.assigneeAgentId,
|
||||
},
|
||||
});
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
let activeExecutionRun = issue.executionRunId
|
||||
? await tx
|
||||
.select()
|
||||
|
|
@ -6269,6 +6437,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
|||
activeExecutionRun = null;
|
||||
}
|
||||
|
||||
if (activeExecutionRun && await cancelStaleScheduledRetry(activeExecutionRun)) {
|
||||
activeExecutionRun = null;
|
||||
}
|
||||
|
||||
if (!activeExecutionRun && issue.executionRunId) {
|
||||
await tx
|
||||
.update(issues)
|
||||
|
|
@ -6300,21 +6472,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
|||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
if (legacyRun) {
|
||||
activeExecutionRun = legacyRun;
|
||||
const legacyAgent = await tx
|
||||
.select({ name: agents.name })
|
||||
.from(agents)
|
||||
.where(eq(agents.id, legacyRun.agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: legacyRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(legacyAgent?.name),
|
||||
executionLockedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, issue.id));
|
||||
if (await cancelStaleScheduledRetry(legacyRun)) {
|
||||
activeExecutionRun = null;
|
||||
} else {
|
||||
activeExecutionRun = legacyRun;
|
||||
const legacyAgent = await tx
|
||||
.select({ name: agents.name })
|
||||
.from(agents)
|
||||
.where(eq(agents.id, legacyRun.agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: legacyRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(legacyAgent?.name),
|
||||
executionLockedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, issue.id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue