mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-14 01:50:39 +09:00
[codex] Add source-scoped recovery actions (#5599)
## Thinking Path > - Paperclip is a control plane for autonomous AI companies, where work must end with a clear disposition rather than ambiguous agent liveness. > - Recovery currently detects stalled or missing-next-step issues, but source issue recovery can become split across child recovery issues, blockers, and comments. > - That makes it harder for operators and agents to see who owns recovery and what exact action is needed on the original issue. > - Source-scoped recovery actions give the original issue a first-class active recovery state with owner, evidence, wake policy, and resolution outcome. > - This pull request adds the recovery-action data model, backend reconciliation and resolution APIs, and board UI indicators/actions. > - The benefit is clearer stalled-work recovery without losing source issue context or relying on comments as the liveness path. ## What Changed - Added the `issue_recovery_actions` schema, shared types/constants/validators, and an idempotent `0084_issue_recovery_actions` migration ordered after current `master` migrations. - Updated stranded/missing-disposition recovery to create source-scoped recovery actions, wake the recovery owner on the source issue, and avoid locking the source issue for recovery-action wakes. - Added API support for reading active recovery actions on issue detail/list surfaces and resolving them with restored, blocked, cancelled, or false-positive outcomes. - Require blocked recovery resolutions to have an unresolved first-class blocker, and removed the UI shortcut that could mark recovery blocked without a blocker selection path. - Surfaced recovery indicators/actions in the issue UI, blocker notices, active run panels, issue rows, and Storybook coverage. - Updated docs and focused tests for recovery semantics, ownership, races, stale comments, and UI behavior. ## Verification - `pnpm exec vitest run server/src/__tests__/issue-recovery-actions.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts ui/src/components/IssueRecoveryActionCard.test.tsx ui/src/components/IssueBlockedNotice.test.tsx ui/src/api/issues.test.ts` — 5 files, 72 tests passed. - `pnpm --filter @paperclipai/shared typecheck` — passed. - `pnpm --filter @paperclipai/db typecheck` — passed, including migration numbering check. - `pnpm --filter @paperclipai/server typecheck` — passed. - `pnpm --filter @paperclipai/ui typecheck` — passed. - Follow-up verification after blocker-resolution guard: `pnpm exec vitest run server/src/__tests__/issue-recovery-actions.test.ts ui/src/components/IssueRecoveryActionCard.test.tsx ui/src/api/issues.test.ts` — 3 files, 27 tests passed. - Follow-up `pnpm --filter @paperclipai/server typecheck` — passed. - Follow-up `pnpm --filter @paperclipai/ui typecheck` — passed. - UI states are available in `ui/storybook/stories/source-issue-recovery.stories.tsx`; screenshot capture helper is `scripts/screenshot-recovery-card.cjs`. ## Risks - Medium: recovery behavior changes from child recovery issue ownership toward source-scoped actions, so operators may see stalled-work state in new places. - Migration risk is mitigated by using the next migration slot after `master` and making the table/constraints/index creation idempotent for anyone who previously applied the old branch-local `0082_dizzy_master_mold` migration. - Existing child recovery issue paths are still guarded for already-created recovery issues, but new source-scoped flows should be watched in CI and Greptile review. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5 coding agent, tool use enabled for shell, Git, GitHub, and local test execution. Context window not exposed by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
c445e59256
commit
0808b388ee
57 changed files with 3947 additions and 224 deletions
|
|
@ -80,6 +80,10 @@ vi.mock("../services/index.js", () => ({
|
|||
listApprovalsForIssue: vi.fn(),
|
||||
unlink: vi.fn(),
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
documentService: () => ({}),
|
||||
routineService: () => ({}),
|
||||
workProductService: () => ({}),
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import {
|
|||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
issueRecoveryActions,
|
||||
issueRelations,
|
||||
issueTreeHoldMembers,
|
||||
issueTreeHolds,
|
||||
|
|
@ -328,6 +329,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
await db.delete(documentRevisions);
|
||||
await db.delete(documents);
|
||||
await db.delete(issueRelations);
|
||||
await db.delete(issueRecoveryActions);
|
||||
await db.delete(issueTreeHoldMembers);
|
||||
await db.delete(issueTreeHolds);
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
|
|
@ -692,67 +694,76 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
return { companyId, agentId, issueId };
|
||||
}
|
||||
|
||||
async function expectStrandedRecoveryArtifacts(input: {
|
||||
async function expectSourceScopedStrandedRecoveryAction(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
issueId: string;
|
||||
runId: string;
|
||||
previousStatus: "todo" | "in_progress";
|
||||
retryReason: "assignment_recovery" | "issue_continuation_needed";
|
||||
retryReason?: "assignment_recovery" | "issue_continuation_needed" | null;
|
||||
cause?: string;
|
||||
kind?: string;
|
||||
}) {
|
||||
const recovery = await waitForValue(async () =>
|
||||
db.select().from(issues).where(
|
||||
const action = await waitForValue(async () =>
|
||||
db.select().from(issueRecoveryActions).where(
|
||||
and(
|
||||
eq(issues.companyId, input.companyId),
|
||||
eq(issues.originKind, "stranded_issue_recovery"),
|
||||
eq(issues.originId, input.issueId),
|
||||
eq(issueRecoveryActions.companyId, input.companyId),
|
||||
eq(issueRecoveryActions.sourceIssueId, input.issueId),
|
||||
),
|
||||
).then((rows) => rows[0] ?? null),
|
||||
);
|
||||
if (!recovery) throw new Error("Expected stranded issue recovery issue to be created");
|
||||
if (!action) throw new Error("Expected source-scoped stranded recovery action to be created");
|
||||
|
||||
expect(recovery).toMatchObject({
|
||||
expect(action).toMatchObject({
|
||||
companyId: input.companyId,
|
||||
parentId: input.issueId,
|
||||
assigneeAgentId: input.agentId,
|
||||
originKind: "stranded_issue_recovery",
|
||||
originId: input.issueId,
|
||||
originRunId: input.runId,
|
||||
priority: "medium",
|
||||
assigneeAdapterOverrides: { modelProfile: "cheap" },
|
||||
sourceIssueId: input.issueId,
|
||||
recoveryIssueId: null,
|
||||
kind: input.kind ?? "stranded_assigned_issue",
|
||||
status: "active",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: input.agentId,
|
||||
previousOwnerAgentId: input.agentId,
|
||||
returnOwnerAgentId: input.agentId,
|
||||
cause: input.cause ?? "stranded_assigned_issue",
|
||||
attemptCount: 1,
|
||||
maxAttempts: null,
|
||||
});
|
||||
expect(recovery.title).toContain("Recover stalled issue");
|
||||
expect(recovery.description).toContain(`Previous source status: \`${input.previousStatus}\``);
|
||||
expect(recovery.description).toContain(`Retry reason: \`${input.retryReason}\``);
|
||||
expect(recovery.description).toContain("Fix the runtime/adapter problem");
|
||||
expect(action.evidence).toMatchObject({
|
||||
sourceIssueId: input.issueId,
|
||||
previousStatus: input.previousStatus,
|
||||
latestRunId: input.runId,
|
||||
retryReason: input.retryReason ?? null,
|
||||
});
|
||||
expect(action.nextAction).toContain(
|
||||
input.kind === "missing_disposition" ? "valid issue disposition" : "Restore a live execution path",
|
||||
);
|
||||
|
||||
const relation = await db
|
||||
const recoveryIssues = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, input.companyId),
|
||||
eq(issueRelations.issueId, recovery.id),
|
||||
eq(issueRelations.relatedIssueId, input.issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(relation).toBeTruthy();
|
||||
.from(issues)
|
||||
.where(and(
|
||||
eq(issues.companyId, input.companyId),
|
||||
eq(issues.originKind, "stranded_issue_recovery"),
|
||||
eq(issues.originId, input.issueId),
|
||||
));
|
||||
expect(recoveryIssues).toHaveLength(0);
|
||||
|
||||
const wakeups = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.agentId, input.agentId));
|
||||
const recoveryWakeup = wakeups.find((wakeup) => {
|
||||
const payload = wakeup.payload as Record<string, unknown> | null;
|
||||
return payload?.issueId === recovery.id &&
|
||||
payload?.sourceIssueId === input.issueId &&
|
||||
payload?.strandedRunId === input.runId;
|
||||
const recoveryWakeup = await waitForValue(async () => {
|
||||
const wakeups = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.agentId, input.agentId));
|
||||
return wakeups.find((wakeup) => {
|
||||
const payload = wakeup.payload as Record<string, unknown> | null;
|
||||
return payload?.issueId === input.issueId &&
|
||||
payload?.sourceIssueId === input.issueId &&
|
||||
payload?.recoveryActionId === action.id &&
|
||||
payload?.strandedRunId === input.runId;
|
||||
}) ?? null;
|
||||
});
|
||||
expect(recoveryWakeup).toMatchObject({
|
||||
companyId: input.companyId,
|
||||
reason: "issue_assigned",
|
||||
reason: "source_scoped_recovery_action",
|
||||
source: "assignment",
|
||||
payload: expect.objectContaining({ modelProfile: "cheap" }),
|
||||
});
|
||||
|
|
@ -765,15 +776,23 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
.then((rows) => rows[0] ?? null)
|
||||
: null;
|
||||
expect(recoveryRun?.contextSnapshot).toMatchObject({
|
||||
issueId: recovery.id,
|
||||
taskId: recovery.id,
|
||||
source: "stranded_issue_recovery",
|
||||
issueId: input.issueId,
|
||||
taskId: input.issueId,
|
||||
source: "issue_recovery_action",
|
||||
recoveryActionId: action.id,
|
||||
sourceIssueId: input.issueId,
|
||||
strandedRunId: input.runId,
|
||||
modelProfile: "cheap",
|
||||
});
|
||||
await waitForHeartbeatIdle(db);
|
||||
const sourceIssue = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(eq(issues.id, input.issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(sourceIssue?.status).toBe("blocked");
|
||||
|
||||
return recovery;
|
||||
return action;
|
||||
}
|
||||
|
||||
async function sourceBlockerIssueIds(companyId: string, sourceIssueId: string) {
|
||||
|
|
@ -1056,7 +1075,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
expect(blockedIssue?.checkoutRunId).toBeNull();
|
||||
if (!continuationRun?.id) throw new Error("Expected continuation recovery run to exist");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
|
|
@ -1065,17 +1084,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
retryReason: "issue_continuation_needed",
|
||||
});
|
||||
|
||||
const blockerRelations = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.relatedIssueId, issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
);
|
||||
expect(blockerRelations.map((relation) => relation.issueId)).toEqual([recovery.id]);
|
||||
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([]);
|
||||
|
||||
const comments = await waitForValue(async () => {
|
||||
const rows = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
|
|
@ -1083,7 +1092,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
});
|
||||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("retried continuation");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
|
||||
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
|
||||
});
|
||||
|
||||
it("blocks failed recovery work in place during immediate terminal-run cleanup", async () => {
|
||||
|
|
@ -1600,27 +1610,28 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
expect(result.successfulRunHandoffEscalated).toBe(1);
|
||||
expect(result.issueIds).toEqual([issueId]);
|
||||
|
||||
const recovery = await waitForValue(async () =>
|
||||
db.select().from(issues).where(
|
||||
and(
|
||||
eq(issues.companyId, companyId),
|
||||
eq(issues.originKind, "stranded_issue_recovery"),
|
||||
eq(issues.originId, issueId),
|
||||
),
|
||||
).then((rows) => rows[0] ?? null),
|
||||
);
|
||||
expect(recovery?.assigneeAgentId).toBe(agentId);
|
||||
expect(recovery?.title).toContain("Recover missing next step");
|
||||
expect(recovery?.description).toContain("Normalized cause: `successful_run_missing_state`");
|
||||
expect(recovery?.description).toContain("not a runtime/adapter crash report");
|
||||
expect(recovery?.description).toContain(`Source run: [\`${sourceRunId}\`]`);
|
||||
expect(recovery?.description).toContain("Missing disposition: `clear_next_step`");
|
||||
expect(recovery?.description).toContain("Source assignee: [CodexCoder]");
|
||||
expect(recovery?.description).not.toContain("sk-test-successful-handoff-secret");
|
||||
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
runId,
|
||||
previousStatus: "in_progress",
|
||||
retryReason: null,
|
||||
cause: SUCCESSFUL_RUN_MISSING_STATE_REASON,
|
||||
kind: "missing_disposition",
|
||||
});
|
||||
expect(recoveryAction.evidence).toMatchObject({
|
||||
sourceRunId,
|
||||
missingDisposition: "clear_next_step",
|
||||
latestRunStatus: "failed",
|
||||
latestRunErrorCode: "adapter_failed",
|
||||
recoveryCause: SUCCESSFUL_RUN_MISSING_STATE_REASON,
|
||||
});
|
||||
expect(JSON.stringify(recoveryAction.evidence)).not.toContain("sk-test-successful-handoff-secret");
|
||||
|
||||
const sourceIssue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(sourceIssue?.status).toBe("blocked");
|
||||
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([recovery?.id]);
|
||||
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([]);
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments[0]?.body).toBe(SUCCESSFUL_RUN_HANDOFF_EXHAUSTED_NOTICE_BODY);
|
||||
|
|
@ -1636,7 +1647,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
expect.objectContaining({
|
||||
title: "Recovery owner",
|
||||
rows: expect.arrayContaining([
|
||||
expect.objectContaining({ type: "issue_link", identifier: recovery?.identifier }),
|
||||
expect.objectContaining({ type: "key_value", label: "Recovery action", value: recoveryAction.id }),
|
||||
expect.objectContaining({ type: "agent_link", label: "Recovery owner", name: "CodexCoder" }),
|
||||
]),
|
||||
}),
|
||||
|
|
@ -1657,7 +1668,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
});
|
||||
|
||||
it("escalates an exhausted successful handoff run that still leaves no disposition", async () => {
|
||||
const { companyId, runId, issueId } = await seedStrandedIssueFixture({
|
||||
const { companyId, agentId, runId, issueId } = await seedStrandedIssueFixture({
|
||||
status: "in_progress",
|
||||
runStatus: "succeeded",
|
||||
livenessState: "advanced",
|
||||
|
|
@ -1687,17 +1698,21 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
expect(result.successfulContinuationObserved).toBe(0);
|
||||
expect(result.successfulRunHandoffEscalated).toBe(1);
|
||||
|
||||
const recovery = await waitForValue(async () =>
|
||||
db.select().from(issues).where(
|
||||
and(
|
||||
eq(issues.companyId, companyId),
|
||||
eq(issues.originKind, "stranded_issue_recovery"),
|
||||
eq(issues.originId, issueId),
|
||||
),
|
||||
).then((rows) => rows[0] ?? null),
|
||||
);
|
||||
expect(recovery?.description).toContain("Latest handoff run status: `succeeded`");
|
||||
expect(recovery?.description).toContain("Suggested");
|
||||
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
runId,
|
||||
previousStatus: "in_progress",
|
||||
retryReason: null,
|
||||
cause: SUCCESSFUL_RUN_MISSING_STATE_REASON,
|
||||
kind: "missing_disposition",
|
||||
});
|
||||
expect(recoveryAction.evidence).toMatchObject({
|
||||
sourceRunId,
|
||||
latestRunStatus: "succeeded",
|
||||
missingDisposition: "clear_next_step",
|
||||
});
|
||||
});
|
||||
|
||||
it("clears the detached warning when the run reports activity again", async () => {
|
||||
|
|
@ -2063,7 +2078,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("blocked");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
|
|
@ -2071,13 +2086,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
previousStatus: "todo",
|
||||
retryReason: "assignment_recovery",
|
||||
});
|
||||
expect(recovery.description ?? "").not.toContain("sk-test-recovery-secret");
|
||||
expect(JSON.stringify(recoveryAction.evidence)).not.toContain("sk-test-recovery-secret");
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("retried dispatch");
|
||||
expect(comments[0]?.body).toContain("Latest retry failure details were withheld from the issue thread");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
|
||||
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
|
||||
});
|
||||
|
||||
it("blocks an already stranded recovery issue without creating a recovery child", async () => {
|
||||
|
|
@ -2457,7 +2473,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("blocked");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
|
|
@ -2470,7 +2486,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("retried continuation");
|
||||
expect(comments[0]?.body).toContain("Latest retry failure details were withheld from the issue thread");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
|
||||
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
|
||||
});
|
||||
|
||||
it("redacts error-code-only stranded recovery failures in issue copy", async () => {
|
||||
|
|
@ -2486,7 +2503,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
const result = await heartbeat.reconcileStrandedAssignedIssues();
|
||||
expect(result.escalated).toBe(1);
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
|
|
@ -2494,8 +2511,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
previousStatus: "in_progress",
|
||||
retryReason: "issue_continuation_needed",
|
||||
});
|
||||
expect(recovery.description).toContain("Latest retry failure details were withheld from the issue thread");
|
||||
expect(recovery.description).not.toContain("- Failure: none recorded");
|
||||
expect(recoveryAction.evidence).toMatchObject({
|
||||
latestRunErrorCode: "adapter_exit_code",
|
||||
});
|
||||
expect(JSON.stringify(recoveryAction.evidence)).not.toContain("- Failure: none recorded");
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(1);
|
||||
|
|
@ -2516,6 +2535,15 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
);
|
||||
expect(results.every((result) => result.status === "fulfilled")).toBe(true);
|
||||
|
||||
const actions = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(and(
|
||||
eq(issueRecoveryActions.companyId, companyId),
|
||||
eq(issueRecoveryActions.sourceIssueId, issueId),
|
||||
));
|
||||
expect(actions).toHaveLength(1);
|
||||
expect(actions[0]?.attemptCount).toBeGreaterThanOrEqual(1);
|
||||
const recoveries = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
|
|
@ -2524,8 +2552,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
eq(issues.originKind, "stranded_issue_recovery"),
|
||||
eq(issues.originId, issueId),
|
||||
));
|
||||
expect(recoveries).toHaveLength(1);
|
||||
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([recoveries[0]?.id]);
|
||||
expect(recoveries).toHaveLength(0);
|
||||
await expect(sourceBlockerIssueIds(companyId, issueId)).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("blocks stranded recovery issues in place instead of creating nested recovery issues", async () => {
|
||||
|
|
@ -2783,7 +2811,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("blocked");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
const recoveryAction = await expectSourceScopedStrandedRecoveryAction({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
|
|
@ -2796,7 +2824,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("automatically retried continuation");
|
||||
expect(comments[0]?.body).toContain("still has no live execution path");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
|
||||
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
|
||||
});
|
||||
|
||||
it("allows one productive-terminal recovery after regular continuation recovery made progress", async () => {
|
||||
|
|
|
|||
|
|
@ -89,6 +89,10 @@ function registerModuleMocks() {
|
|||
heartbeatService: () => mockHeartbeatService,
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
issueApprovalService: () => ({}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueReferenceService: () => ({
|
||||
deleteDocumentSource: async () => undefined,
|
||||
diffIssueReferenceSummary: () => ({
|
||||
|
|
|
|||
|
|
@ -60,6 +60,9 @@ const mockIssueThreadInteractionService = vi.hoisted(() => ({
|
|||
expireRequestConfirmationsSupersededByComment: vi.fn(async () => []),
|
||||
expireStaleRequestConfirmationsForIssueDocument: vi.fn(async () => []),
|
||||
}));
|
||||
const mockIssueRecoveryActionService = vi.hoisted(() => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
function registerRouteMocks() {
|
||||
vi.doMock("@paperclipai/shared/telemetry", () => ({
|
||||
|
|
@ -124,6 +127,7 @@ function registerRouteMocks() {
|
|||
listCompanyIds: vi.fn(async () => [companyId]),
|
||||
}),
|
||||
issueApprovalService: () => ({}),
|
||||
issueRecoveryActionService: () => mockIssueRecoveryActionService,
|
||||
issueReferenceService: () => ({
|
||||
deleteDocumentSource: async () => undefined,
|
||||
diffIssueReferenceSummary: () => ({
|
||||
|
|
@ -259,6 +263,8 @@ describe("agent issue mutation checkout ownership", () => {
|
|||
mockIssueService.getWakeableParentAfterChildCompletion.mockReset();
|
||||
mockIssueService.listAttachments.mockReset();
|
||||
mockIssueService.listWakeableBlockedDependents.mockReset();
|
||||
mockIssueRecoveryActionService.getActiveForIssue.mockReset();
|
||||
mockIssueRecoveryActionService.getActiveForIssue.mockResolvedValue(null);
|
||||
mockIssueService.remove.mockReset();
|
||||
mockIssueService.removeAttachment.mockReset();
|
||||
mockIssueService.update.mockReset();
|
||||
|
|
|
|||
|
|
@ -59,6 +59,10 @@ vi.mock("../services/index.js", () => ({
|
|||
listCompanyIds: vi.fn(async () => ["company-1"]),
|
||||
}),
|
||||
issueApprovalService: () => ({}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueReferenceService: () => ({
|
||||
deleteDocumentSource: async () => undefined,
|
||||
diffIssueReferenceSummary: () => ({
|
||||
|
|
|
|||
|
|
@ -81,6 +81,10 @@ function registerRouteMocks() {
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
logActivity: mockLogActivity,
|
||||
projectService: () => ({}),
|
||||
|
|
|
|||
|
|
@ -116,6 +116,10 @@ function registerServiceMocks() {
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
logActivity: mockLogActivity,
|
||||
projectService: () => mockProjectService,
|
||||
|
|
|
|||
|
|
@ -86,6 +86,10 @@ function registerModuleMocks() {
|
|||
heartbeatService: () => mockHeartbeatService,
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
issueApprovalService: () => ({}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueReferenceService: () => ({
|
||||
deleteDocumentSource: async () => undefined,
|
||||
diffIssueReferenceSummary: () => ({
|
||||
|
|
|
|||
|
|
@ -67,6 +67,9 @@ const mockIssueThreadInteractionService = vi.hoisted(() => ({
|
|||
expireRequestConfirmationsSupersededByComment: vi.fn(async () => []),
|
||||
expireStaleRequestConfirmationsForIssueDocument: vi.fn(async () => []),
|
||||
}));
|
||||
const mockIssueRecoveryActionService = vi.hoisted(() => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
}));
|
||||
const mockIssueTreeControlService = vi.hoisted(() => ({
|
||||
getActivePauseHoldGate: vi.fn(async () => null),
|
||||
}));
|
||||
|
|
@ -125,6 +128,7 @@ vi.mock("../services/index.js", () => ({
|
|||
heartbeatService: () => mockHeartbeatService,
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
issueApprovalService: () => ({}),
|
||||
issueRecoveryActionService: () => mockIssueRecoveryActionService,
|
||||
issueReferenceService: () => ({
|
||||
deleteDocumentSource: async () => undefined,
|
||||
diffIssueReferenceSummary: () => ({
|
||||
|
|
@ -238,6 +242,7 @@ describe.sequential("issue comment reopen routes", () => {
|
|||
mockInstanceSettingsService.get.mockReset();
|
||||
mockInstanceSettingsService.listCompanyIds.mockReset();
|
||||
mockRoutineService.syncRunStatusForIssue.mockReset();
|
||||
mockIssueRecoveryActionService.getActiveForIssue.mockReset();
|
||||
mockIssueTreeControlService.getActivePauseHoldGate.mockReset();
|
||||
mockTxInsertValues.mockReset();
|
||||
mockTxInsert.mockReset();
|
||||
|
|
@ -274,6 +279,7 @@ describe.sequential("issue comment reopen routes", () => {
|
|||
});
|
||||
mockInstanceSettingsService.listCompanyIds.mockResolvedValue(["company-1"]);
|
||||
mockRoutineService.syncRunStatusForIssue.mockResolvedValue(undefined);
|
||||
mockIssueRecoveryActionService.getActiveForIssue.mockResolvedValue(null);
|
||||
mockIssueTreeControlService.getActivePauseHoldGate.mockResolvedValue(null);
|
||||
mockIssueService.addComment.mockResolvedValue({
|
||||
id: "comment-1",
|
||||
|
|
|
|||
|
|
@ -61,6 +61,10 @@ vi.mock("../services/index.js", () => ({
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
logActivity: vi.fn(async () => undefined),
|
||||
projectService: () => ({
|
||||
|
|
|
|||
|
|
@ -133,6 +133,10 @@ function registerModuleMocks() {
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockIssueThreadInteractionService,
|
||||
logActivity: mockLogActivity,
|
||||
|
|
|
|||
|
|
@ -82,6 +82,10 @@ function registerModuleMocks() {
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockIssueThreadInteractionService,
|
||||
logActivity: mockLogActivity,
|
||||
|
|
|
|||
|
|
@ -93,6 +93,10 @@ function registerModuleMocks() {
|
|||
heartbeatService: () => mockHeartbeatService,
|
||||
issueApprovalService: () => ({}),
|
||||
issueReferenceService: () => mockIssueReferenceService,
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockIssueThreadInteractionService,
|
||||
logActivity: mockLogActivity,
|
||||
|
|
|
|||
767
server/src/__tests__/issue-recovery-actions.test.ts
Normal file
767
server/src/__tests__/issue-recovery-actions.test.ts
Normal file
|
|
@ -0,0 +1,767 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
activityLog,
|
||||
companies,
|
||||
createDb,
|
||||
issueComments,
|
||||
issueRecoveryActions,
|
||||
issueRelations,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
|
||||
import { recoveryService } from "../services/recovery/service.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
function makeRecoveryActionRow(overrides: Record<string, unknown> = {}) {
|
||||
const now = new Date("2026-05-09T19:30:00.000Z");
|
||||
return {
|
||||
id: randomUUID(),
|
||||
companyId: "company-1",
|
||||
sourceIssueId: "source-1",
|
||||
recoveryIssueId: null,
|
||||
kind: "missing_disposition",
|
||||
status: "active",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: "agent-1",
|
||||
ownerUserId: null,
|
||||
previousOwnerAgentId: null,
|
||||
returnOwnerAgentId: null,
|
||||
cause: "successful_run_missing_issue_disposition",
|
||||
fingerprint: "missing-disposition:fingerprint",
|
||||
evidence: {},
|
||||
nextAction: "Choose a valid issue disposition.",
|
||||
wakePolicy: null,
|
||||
monitorPolicy: null,
|
||||
attemptCount: 1,
|
||||
maxAttempts: null,
|
||||
timeoutAt: null,
|
||||
lastAttemptAt: now,
|
||||
outcome: null,
|
||||
resolutionNote: null,
|
||||
resolvedAt: null,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("issueRecoveryActionService", () => {
|
||||
it("does not reactivate an action resolved between the active read and update", async () => {
|
||||
const existingRow = makeRecoveryActionRow({ id: "existing-action", attemptCount: 1 });
|
||||
const createdRow = makeRecoveryActionRow({ id: "new-action", attemptCount: 1 });
|
||||
const selectResults = [[existingRow], []];
|
||||
|
||||
const makeSelectQuery = (rows: unknown[]) => ({
|
||||
from() {
|
||||
return this;
|
||||
},
|
||||
where() {
|
||||
return this;
|
||||
},
|
||||
orderBy() {
|
||||
return this;
|
||||
},
|
||||
limit() {
|
||||
return Promise.resolve(rows);
|
||||
},
|
||||
});
|
||||
|
||||
const fakeDb = {
|
||||
select: vi.fn(() => makeSelectQuery(selectResults.shift() ?? [])),
|
||||
update: vi.fn(() => ({
|
||||
set: vi.fn(() => ({
|
||||
where: vi.fn(() => ({
|
||||
returning: vi.fn(async () => []),
|
||||
})),
|
||||
})),
|
||||
})),
|
||||
insert: vi.fn(() => ({
|
||||
values: vi.fn(() => ({
|
||||
returning: vi.fn(async () => [createdRow]),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
|
||||
const result = await issueRecoveryActionService(fakeDb as never).upsertSourceScoped({
|
||||
companyId: "company-1",
|
||||
sourceIssueId: "source-1",
|
||||
kind: "missing_disposition",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: "agent-1",
|
||||
cause: "successful_run_missing_issue_disposition",
|
||||
fingerprint: "missing-disposition:fingerprint",
|
||||
nextAction: "Choose a valid issue disposition.",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ id: "new-action", status: "active" });
|
||||
expect(fakeDb.update).toHaveBeenCalledTimes(1);
|
||||
expect(fakeDb.insert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres issue recovery action tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("issue recovery actions", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-recovery-actions-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 30_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(issueRecoveryActions);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(activityLog);
|
||||
await db.delete(issues);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedCompany() {
|
||||
const companyId = randomUUID();
|
||||
const managerId = randomUUID();
|
||||
const coderId = randomUUID();
|
||||
const sourceIssueId = randomUUID();
|
||||
const prefix = `RA${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}`;
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Recovery Co",
|
||||
issuePrefix: prefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values([
|
||||
{
|
||||
id: managerId,
|
||||
companyId,
|
||||
name: "CTO",
|
||||
role: "cto",
|
||||
status: "idle",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
id: coderId,
|
||||
companyId,
|
||||
name: "Coder",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
reportsTo: managerId,
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
await db.insert(issues).values({
|
||||
id: sourceIssueId,
|
||||
companyId,
|
||||
title: "Implement backend recovery",
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: coderId,
|
||||
issueNumber: 1,
|
||||
identifier: `${prefix}-1`,
|
||||
});
|
||||
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
|
||||
return { companyId, managerId, coderId, sourceIssueId, prefix, sourceIssue: sourceIssue! };
|
||||
}
|
||||
|
||||
function createApp(actor: any = { type: "board", source: "local_implicit" }) {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = actor;
|
||||
next();
|
||||
});
|
||||
app.use("/api", issueRoutes(db, {} as any));
|
||||
app.use(errorHandler);
|
||||
return app;
|
||||
}
|
||||
|
||||
it("upserts one active source-scoped action per issue and keeps company scoping explicit", async () => {
|
||||
const { companyId, managerId, sourceIssueId } = await seedCompany();
|
||||
const svc = issueRecoveryActionService(db);
|
||||
|
||||
const first = await svc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "stranded_assigned_issue",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "stranded_assigned_issue",
|
||||
fingerprint: "recovery:fingerprint",
|
||||
evidence: { latestRunId: "run-1" },
|
||||
nextAction: "Restore a live execution path.",
|
||||
wakePolicy: { type: "wake_owner" },
|
||||
});
|
||||
const second = await svc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "stranded_assigned_issue",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "stranded_assigned_issue",
|
||||
fingerprint: "recovery:fingerprint",
|
||||
evidence: { latestRunId: "run-2" },
|
||||
nextAction: "Restore a live execution path.",
|
||||
wakePolicy: { type: "wake_owner" },
|
||||
});
|
||||
|
||||
expect(second.id).toBe(first.id);
|
||||
expect(second.attemptCount).toBe(2);
|
||||
expect(second.evidence).toMatchObject({ latestRunId: "run-2" });
|
||||
expect(await svc.getActiveForIssue(companyId, sourceIssueId)).toMatchObject({ id: first.id });
|
||||
expect(await svc.getActiveForIssue(randomUUID(), sourceIssueId)).toBeNull();
|
||||
});
|
||||
|
||||
it("escalates stranded assigned work into a source action instead of a recovery issue", async () => {
|
||||
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
|
||||
const enqueueWakeup = vi.fn(async () => null);
|
||||
const recovery = recoveryService(db, { enqueueWakeup });
|
||||
const latestRun = {
|
||||
id: randomUUID(),
|
||||
agentId: coderId,
|
||||
status: "failed",
|
||||
error: "adapter failed",
|
||||
errorCode: "adapter_failed",
|
||||
contextSnapshot: { retryReason: "issue_continuation_needed" },
|
||||
livenessState: "needs_followup",
|
||||
} as const;
|
||||
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: sourceIssue,
|
||||
previousStatus: "in_progress",
|
||||
latestRun,
|
||||
comment: "Automatic continuation recovery failed.",
|
||||
});
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: sourceIssue,
|
||||
previousStatus: "in_progress",
|
||||
latestRun,
|
||||
comment: "Automatic continuation recovery failed.",
|
||||
});
|
||||
|
||||
const actionRows = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
|
||||
expect(actionRows).toHaveLength(1);
|
||||
expect(actionRows[0]).toMatchObject({
|
||||
companyId,
|
||||
kind: "stranded_assigned_issue",
|
||||
status: "active",
|
||||
previousOwnerAgentId: coderId,
|
||||
returnOwnerAgentId: coderId,
|
||||
cause: "stranded_assigned_issue",
|
||||
attemptCount: 2,
|
||||
});
|
||||
|
||||
const [updatedIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
|
||||
expect(updatedIssue).toMatchObject({
|
||||
status: "blocked",
|
||||
});
|
||||
const recoveryIssues = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
|
||||
expect(recoveryIssues).toHaveLength(0);
|
||||
expect(enqueueWakeup).toHaveBeenCalledTimes(2);
|
||||
expect(enqueueWakeup.mock.calls[0]?.[1]?.payload).toMatchObject({
|
||||
issueId: sourceIssue.id,
|
||||
sourceIssueId: sourceIssue.id,
|
||||
recoveryCause: "stranded_assigned_issue",
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses the same source-scoped action when latest run IDs change while the cause stays the same", async () => {
|
||||
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
|
||||
const enqueueWakeup = vi.fn(async () => null);
|
||||
const recovery = recoveryService(db, { enqueueWakeup });
|
||||
const firstLatestRun = {
|
||||
id: randomUUID(),
|
||||
agentId: coderId,
|
||||
status: "failed",
|
||||
error: "adapter failed",
|
||||
errorCode: "adapter_failed",
|
||||
contextSnapshot: { retryReason: "issue_continuation_needed" },
|
||||
livenessState: "needs_followup",
|
||||
} as const;
|
||||
const secondLatestRun = {
|
||||
...firstLatestRun,
|
||||
id: randomUUID(),
|
||||
};
|
||||
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: sourceIssue,
|
||||
previousStatus: "in_progress",
|
||||
latestRun: firstLatestRun,
|
||||
comment: "Automatic continuation recovery failed.",
|
||||
});
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: sourceIssue,
|
||||
previousStatus: "in_progress",
|
||||
latestRun: secondLatestRun,
|
||||
comment: "Automatic continuation recovery failed.",
|
||||
});
|
||||
|
||||
const actionRows = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
|
||||
expect(actionRows).toHaveLength(1);
|
||||
expect(actionRows[0]).toMatchObject({
|
||||
companyId,
|
||||
kind: "stranded_assigned_issue",
|
||||
status: "active",
|
||||
previousOwnerAgentId: coderId,
|
||||
returnOwnerAgentId: coderId,
|
||||
cause: "stranded_assigned_issue",
|
||||
attemptCount: 2,
|
||||
});
|
||||
expect(actionRows[0]?.evidence).toMatchObject({ latestRunId: secondLatestRun.id });
|
||||
expect(enqueueWakeup).toHaveBeenCalledTimes(2);
|
||||
expect(enqueueWakeup.mock.calls[1]?.[1]?.payload).toMatchObject({
|
||||
issueId: sourceIssue.id,
|
||||
sourceIssueId: sourceIssue.id,
|
||||
strandedRunId: secondLatestRun.id,
|
||||
recoveryCause: "stranded_assigned_issue",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the source issue blocked when source-scoped wakeup is claimed synchronously", async () => {
|
||||
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
|
||||
await db.update(agents).set({ status: "paused" }).where(eq(agents.id, managerId));
|
||||
const enqueueWakeup = vi.fn(async () => {
|
||||
await db
|
||||
.update(issues)
|
||||
.set({ status: "in_progress" })
|
||||
.where(eq(issues.id, sourceIssue.id));
|
||||
return null;
|
||||
});
|
||||
const recovery = recoveryService(db, { enqueueWakeup });
|
||||
const firstLatestRun = {
|
||||
id: randomUUID(),
|
||||
agentId: coderId,
|
||||
status: "failed",
|
||||
error: "adapter failed",
|
||||
errorCode: "adapter_failed",
|
||||
contextSnapshot: { retryReason: "issue_continuation_needed" },
|
||||
livenessState: "needs_followup",
|
||||
} as const;
|
||||
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: sourceIssue,
|
||||
previousStatus: "in_progress",
|
||||
latestRun: firstLatestRun,
|
||||
comment: "Automatic continuation recovery failed.",
|
||||
});
|
||||
|
||||
const [afterFirst] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
|
||||
expect(afterFirst?.status).toBe("blocked");
|
||||
expect(afterFirst?.assigneeAgentId).toBe(coderId);
|
||||
|
||||
const secondLatestRun = {
|
||||
...firstLatestRun,
|
||||
id: randomUUID(),
|
||||
};
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: sourceIssue,
|
||||
previousStatus: "in_progress",
|
||||
latestRun: secondLatestRun,
|
||||
comment: "Automatic continuation recovery failed.",
|
||||
});
|
||||
|
||||
const actionRows = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
|
||||
expect(actionRows).toHaveLength(1);
|
||||
expect(actionRows[0]).toMatchObject({
|
||||
companyId,
|
||||
kind: "stranded_assigned_issue",
|
||||
status: "active",
|
||||
previousOwnerAgentId: coderId,
|
||||
returnOwnerAgentId: coderId,
|
||||
cause: "stranded_assigned_issue",
|
||||
attemptCount: 2,
|
||||
});
|
||||
const [afterSecond] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
|
||||
expect(afterSecond?.status).toBe("blocked");
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, sourceIssue.id));
|
||||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("Recovery action:");
|
||||
});
|
||||
|
||||
it("does not create nested recovery artifacts when issue-backed fallback work itself fails", async () => {
|
||||
const { companyId, managerId, sourceIssueId, prefix } = await seedCompany();
|
||||
const recoveryIssueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: recoveryIssueId,
|
||||
companyId,
|
||||
title: "Recover stalled issue",
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: managerId,
|
||||
parentId: sourceIssueId,
|
||||
issueNumber: 2,
|
||||
identifier: `${prefix}-2`,
|
||||
originKind: "stranded_issue_recovery",
|
||||
originId: sourceIssueId,
|
||||
originFingerprint: `stranded_issue_recovery:${sourceIssueId}`,
|
||||
});
|
||||
const [recoveryIssue] = await db.select().from(issues).where(eq(issues.id, recoveryIssueId));
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn(async () => null) });
|
||||
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: recoveryIssue!,
|
||||
previousStatus: "in_progress",
|
||||
latestRun: {
|
||||
id: randomUUID(),
|
||||
agentId: managerId,
|
||||
status: "failed",
|
||||
error: "adapter failed",
|
||||
errorCode: "adapter_failed",
|
||||
contextSnapshot: { retryReason: "issue_continuation_needed" },
|
||||
livenessState: "needs_followup",
|
||||
},
|
||||
});
|
||||
|
||||
const actionRows = await db.select().from(issueRecoveryActions);
|
||||
expect(actionRows).toHaveLength(0);
|
||||
const recoveryIssues = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
|
||||
expect(recoveryIssues).toHaveLength(1);
|
||||
expect(recoveryIssues[0]?.status).toBe("blocked");
|
||||
});
|
||||
|
||||
it("exposes active recovery actions on the issue read API", async () => {
|
||||
const { companyId, managerId, sourceIssueId } = await seedCompany();
|
||||
const recoveryActionSvc = issueRecoveryActionService(db);
|
||||
const action = await recoveryActionSvc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "missing_disposition",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "successful_run_missing_issue_disposition",
|
||||
fingerprint: "missing-disposition:fingerprint",
|
||||
evidence: { sourceRunId: "run-1" },
|
||||
nextAction: "Choose a valid issue disposition.",
|
||||
wakePolicy: { type: "wake_owner" },
|
||||
});
|
||||
const app = createApp();
|
||||
|
||||
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
|
||||
expect(detail.body.activeRecoveryAction).toMatchObject({
|
||||
id: action.id,
|
||||
sourceIssueId,
|
||||
kind: "missing_disposition",
|
||||
ownerAgentId: managerId,
|
||||
});
|
||||
|
||||
const list = await request(app).get(`/api/issues/${sourceIssueId}/recovery-actions`).expect(200);
|
||||
expect(list.body.active).toMatchObject({ id: action.id });
|
||||
expect(list.body.actions).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("resolves an active recovery action and removes it from active projections", async () => {
|
||||
const { companyId, managerId, sourceIssueId } = await seedCompany();
|
||||
const recoveryActionSvc = issueRecoveryActionService(db);
|
||||
const action = await recoveryActionSvc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "missing_disposition",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "successful_run_missing_issue_disposition",
|
||||
fingerprint: "missing-disposition:fingerprint",
|
||||
evidence: { sourceRunId: "run-1" },
|
||||
nextAction: "Choose a valid issue disposition.",
|
||||
wakePolicy: { type: "wake_owner" },
|
||||
});
|
||||
const app = createApp();
|
||||
|
||||
const resolved = await request(app)
|
||||
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
|
||||
.send({
|
||||
actionId: action.id,
|
||||
outcome: "restored",
|
||||
sourceIssueStatus: "done",
|
||||
resolutionNote: "Operator confirmed the source issue is complete.",
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
expect(resolved.body.issue).toMatchObject({
|
||||
id: sourceIssueId,
|
||||
status: "done",
|
||||
activeRecoveryAction: null,
|
||||
});
|
||||
expect(resolved.body.recoveryAction).toMatchObject({
|
||||
id: action.id,
|
||||
status: "resolved",
|
||||
outcome: "restored",
|
||||
resolutionNote: "Operator confirmed the source issue is complete.",
|
||||
});
|
||||
expect(resolved.body.recoveryAction.resolvedAt).toBeTruthy();
|
||||
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toBeNull();
|
||||
|
||||
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
|
||||
expect(detail.body.activeRecoveryAction).toBeNull();
|
||||
|
||||
const activityRows = await db
|
||||
.select()
|
||||
.from(activityLog)
|
||||
.where(eq(activityLog.entityId, sourceIssueId));
|
||||
expect(activityRows.map((row) => row.action)).toEqual(
|
||||
expect.arrayContaining(["issue.updated", "issue.recovery_action_resolved"]),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects blocked recovery resolution when the source issue has no first-class blockers", async () => {
|
||||
const { companyId, managerId, sourceIssueId } = await seedCompany();
|
||||
const recoveryActionSvc = issueRecoveryActionService(db);
|
||||
const action = await recoveryActionSvc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "issue_graph_liveness",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "issue_graph_liveness",
|
||||
fingerprint: "graph-liveness:blocked-without-blocker",
|
||||
evidence: { latestIssueStatus: "in_progress" },
|
||||
nextAction: "Choose a disposition with a live continuation path.",
|
||||
wakePolicy: { type: "manual" },
|
||||
});
|
||||
const app = createApp();
|
||||
|
||||
const rejected = await request(app)
|
||||
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
|
||||
.send({
|
||||
actionId: action.id,
|
||||
outcome: "blocked",
|
||||
sourceIssueStatus: "blocked",
|
||||
})
|
||||
.expect(422);
|
||||
|
||||
expect(rejected.body.error).toContain("requires an unresolved first-class blocker");
|
||||
|
||||
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
|
||||
expect(sourceIssue?.status).toBe("in_progress");
|
||||
|
||||
const [actionRow] = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.id, action.id));
|
||||
expect(actionRow).toMatchObject({
|
||||
status: "active",
|
||||
outcome: null,
|
||||
resolvedAt: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("allows blocked recovery resolution when the source issue has an unresolved first-class blocker", async () => {
|
||||
const { companyId, managerId, sourceIssueId, prefix } = await seedCompany();
|
||||
const blockerIssueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: blockerIssueId,
|
||||
companyId,
|
||||
title: "Unblock recovery disposition",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
assigneeAgentId: managerId,
|
||||
issueNumber: 2,
|
||||
identifier: `${prefix}-2`,
|
||||
});
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId: blockerIssueId,
|
||||
relatedIssueId: sourceIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
const recoveryActionSvc = issueRecoveryActionService(db);
|
||||
const action = await recoveryActionSvc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "issue_graph_liveness",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "issue_graph_liveness",
|
||||
fingerprint: "graph-liveness:blocked-with-blocker",
|
||||
evidence: { latestIssueStatus: "in_progress" },
|
||||
nextAction: "Wait for the blocker before continuing.",
|
||||
wakePolicy: { type: "manual" },
|
||||
});
|
||||
const app = createApp();
|
||||
|
||||
const resolved = await request(app)
|
||||
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
|
||||
.send({
|
||||
actionId: action.id,
|
||||
outcome: "blocked",
|
||||
sourceIssueStatus: "blocked",
|
||||
resolutionNote: "The source issue is explicitly blocked by a follow-up.",
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
expect(resolved.body.issue).toMatchObject({
|
||||
id: sourceIssueId,
|
||||
status: "blocked",
|
||||
activeRecoveryAction: null,
|
||||
});
|
||||
expect(resolved.body.recoveryAction).toMatchObject({
|
||||
id: action.id,
|
||||
status: "resolved",
|
||||
outcome: "blocked",
|
||||
resolutionNote: "The source issue is explicitly blocked by a follow-up.",
|
||||
});
|
||||
expect(await recoveryActionSvc.getActiveForIssue(companyId, sourceIssueId)).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects false-positive recovery resolution without an explicit source issue status", async () => {
|
||||
const { companyId, managerId, sourceIssueId } = await seedCompany();
|
||||
const recoveryActionSvc = issueRecoveryActionService(db);
|
||||
const action = await recoveryActionSvc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "issue_graph_liveness",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "issue_graph_liveness",
|
||||
fingerprint: "graph-liveness:fingerprint",
|
||||
evidence: { latestIssueStatus: "in_progress" },
|
||||
nextAction: "Confirm whether the issue is actually stranded.",
|
||||
wakePolicy: { type: "manual" },
|
||||
});
|
||||
const app = createApp();
|
||||
|
||||
await request(app)
|
||||
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
|
||||
.send({
|
||||
actionId: action.id,
|
||||
outcome: "false_positive",
|
||||
resolutionNote: "The source issue still has a live execution path.",
|
||||
})
|
||||
.expect(400);
|
||||
|
||||
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
|
||||
expect(sourceIssue?.status).toBe("in_progress");
|
||||
|
||||
const [actionRow] = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.id, action.id));
|
||||
expect(actionRow).toMatchObject({
|
||||
status: "active",
|
||||
outcome: null,
|
||||
resolutionNote: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("allows false-positive recovery resolution to restore a blocked source issue in the same request", async () => {
|
||||
const { companyId, managerId, sourceIssueId } = await seedCompany();
|
||||
await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, sourceIssueId));
|
||||
const recoveryActionSvc = issueRecoveryActionService(db);
|
||||
const action = await recoveryActionSvc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "issue_graph_liveness",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "issue_graph_liveness",
|
||||
fingerprint: "graph-liveness:false-positive-unblock",
|
||||
evidence: { latestIssueStatus: "blocked" },
|
||||
nextAction: "Confirm whether the issue is actually stranded.",
|
||||
wakePolicy: { type: "manual" },
|
||||
});
|
||||
const app = createApp();
|
||||
|
||||
const resolved = await request(app)
|
||||
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
|
||||
.send({
|
||||
actionId: action.id,
|
||||
outcome: "false_positive",
|
||||
sourceIssueStatus: "in_review",
|
||||
resolutionNote: "Recovery signal was stale; return to review.",
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
expect(resolved.body.issue).toMatchObject({
|
||||
id: sourceIssueId,
|
||||
status: "in_review",
|
||||
activeRecoveryAction: null,
|
||||
});
|
||||
expect(resolved.body.recoveryAction).toMatchObject({
|
||||
id: action.id,
|
||||
status: "resolved",
|
||||
outcome: "false_positive",
|
||||
resolutionNote: "Recovery signal was stale; return to review.",
|
||||
});
|
||||
});
|
||||
|
||||
it("enforces company scope when resolving recovery actions", async () => {
|
||||
const { companyId, managerId, sourceIssueId } = await seedCompany();
|
||||
const recoveryActionSvc = issueRecoveryActionService(db);
|
||||
const action = await recoveryActionSvc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "missing_disposition",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "successful_run_missing_issue_disposition",
|
||||
fingerprint: "missing-disposition:fingerprint",
|
||||
evidence: { sourceRunId: "run-1" },
|
||||
nextAction: "Choose a valid issue disposition.",
|
||||
wakePolicy: { type: "wake_owner" },
|
||||
});
|
||||
const app = createApp({
|
||||
type: "agent",
|
||||
agentId: randomUUID(),
|
||||
companyId: randomUUID(),
|
||||
runId: randomUUID(),
|
||||
source: "agent_jwt",
|
||||
});
|
||||
|
||||
await request(app)
|
||||
.post(`/api/issues/${sourceIssueId}/recovery-actions/resolve`)
|
||||
.send({
|
||||
actionId: action.id,
|
||||
outcome: "restored",
|
||||
sourceIssueStatus: "done",
|
||||
})
|
||||
.expect(403);
|
||||
|
||||
const [actionRow] = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.id, action.id));
|
||||
expect(actionRow?.status).toBe("active");
|
||||
});
|
||||
});
|
||||
|
|
@ -58,6 +58,10 @@ function registerModuleMocks() {
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
logActivity: vi.fn(async () => undefined),
|
||||
projectService: () => ({}),
|
||||
|
|
|
|||
|
|
@ -86,6 +86,10 @@ function registerModuleMocks() {
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockInteractionService,
|
||||
logActivity: mockLogActivity,
|
||||
|
|
|
|||
|
|
@ -73,6 +73,10 @@ vi.mock("../services/index.js", () => ({
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockIssueThreadInteractionService,
|
||||
logActivity: vi.fn(async () => undefined),
|
||||
|
|
@ -131,6 +135,10 @@ function registerModuleMocks() {
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockIssueThreadInteractionService,
|
||||
logActivity: vi.fn(async () => undefined),
|
||||
|
|
|
|||
|
|
@ -115,6 +115,10 @@ function registerRouteMocks() {
|
|||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
logActivity: mockLogActivity,
|
||||
projectService: () => ({}),
|
||||
|
|
|
|||
|
|
@ -111,6 +111,10 @@ vi.mock("../services/index.js", () => ({
|
|||
heartbeatService: () => mockHeartbeatService,
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
issueApprovalService: () => ({}),
|
||||
issueRecoveryActionService: () => ({
|
||||
getActiveForIssue: vi.fn(async () => null),
|
||||
listActiveForIssues: vi.fn(async () => new Map()),
|
||||
}),
|
||||
issueReferenceService: () => mockIssueReferenceService,
|
||||
issueService: () => mockIssueService,
|
||||
logActivity: mockLogActivity,
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import {
|
|||
heartbeatService,
|
||||
ISSUE_LIST_DEFAULT_LIMIT,
|
||||
issueApprovalService,
|
||||
issueRecoveryActionService,
|
||||
issueService,
|
||||
logActivity,
|
||||
syncInstructionsBundleConfigFromFilePath,
|
||||
|
|
@ -1741,16 +1742,18 @@ export function agentRoutes(
|
|||
}
|
||||
|
||||
const issuesSvc = issueService(db);
|
||||
const recoveryActionsSvc = issueRecoveryActionService(db);
|
||||
const rows = await issuesSvc.list(req.actor.companyId, {
|
||||
assigneeAgentId: req.actor.agentId,
|
||||
status: "todo,in_progress,blocked",
|
||||
includeRoutineExecutions: true,
|
||||
limit: ISSUE_LIST_DEFAULT_LIMIT,
|
||||
});
|
||||
const dependencyReadiness = await issuesSvc.listDependencyReadiness(
|
||||
req.actor.companyId,
|
||||
rows.map((issue) => issue.id),
|
||||
);
|
||||
const issueIds = rows.map((issue) => issue.id);
|
||||
const [dependencyReadiness, recoveryActionByIssue] = await Promise.all([
|
||||
issuesSvc.listDependencyReadiness(req.actor.companyId, issueIds),
|
||||
recoveryActionsSvc.listActiveForIssues(req.actor.companyId, issueIds),
|
||||
]);
|
||||
|
||||
res.json(
|
||||
rows.map((issue) => ({
|
||||
|
|
@ -1764,6 +1767,7 @@ export function agentRoutes(
|
|||
parentId: issue.parentId,
|
||||
updatedAt: issue.updatedAt,
|
||||
activeRun: issue.activeRun,
|
||||
activeRecoveryAction: recoveryActionByIssue.get(issue.id) ?? null,
|
||||
dependencyReady: dependencyReadiness.get(issue.id)?.isDependencyReady ?? true,
|
||||
unresolvedBlockerCount: dependencyReadiness.get(issue.id)?.unresolvedBlockerCount ?? 0,
|
||||
unresolvedBlockerIssueIds: dependencyReadiness.get(issue.id)?.unresolvedBlockerIssueIds ?? [],
|
||||
|
|
|
|||
|
|
@ -2,9 +2,16 @@ import { randomUUID } from "node:crypto";
|
|||
import { Router, type Request, type Response } from "express";
|
||||
import multer from "multer";
|
||||
import { z } from "zod";
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import { and, desc, eq, inArray, notInArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { activityLog, executionWorkspaces, issueExecutionDecisions, projectWorkspaces } from "@paperclipai/db";
|
||||
import {
|
||||
activityLog,
|
||||
executionWorkspaces,
|
||||
issueExecutionDecisions,
|
||||
issueRelations,
|
||||
issues as issueRows,
|
||||
projectWorkspaces,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
addIssueCommentSchema,
|
||||
acceptIssueThreadInteractionSchema,
|
||||
|
|
@ -18,6 +25,7 @@ import {
|
|||
createChildIssueSchema,
|
||||
createIssueSchema,
|
||||
resolveCreateIssueStatusDefault,
|
||||
resolveIssueRecoveryActionSchema,
|
||||
feedbackTargetTypeSchema,
|
||||
feedbackTraceStatusSchema,
|
||||
feedbackVoteValueSchema,
|
||||
|
|
@ -37,6 +45,7 @@ import {
|
|||
type CompanySearchQuery,
|
||||
type CompanySearchResponse,
|
||||
type ExecutionWorkspace,
|
||||
type IssueRelationIssueSummary,
|
||||
type SuccessfulRunHandoffState,
|
||||
} from "@paperclipai/shared";
|
||||
import { trackAgentTaskCompleted } from "@paperclipai/shared/telemetry";
|
||||
|
|
@ -53,6 +62,7 @@ import {
|
|||
goalService,
|
||||
heartbeatService,
|
||||
issueApprovalService,
|
||||
issueRecoveryActionService,
|
||||
issueThreadInteractionService,
|
||||
ISSUE_LIST_DEFAULT_LIMIT,
|
||||
ISSUE_LIST_MAX_LIMIT,
|
||||
|
|
@ -405,6 +415,47 @@ async function listSuccessfulRunHandoffStates(
|
|||
return states;
|
||||
}
|
||||
|
||||
type RecoveryActionsLister = {
|
||||
listActiveForIssues: (
|
||||
companyId: string,
|
||||
sourceIssueIds: string[],
|
||||
) => Promise<Map<string, NonNullable<IssueRelationIssueSummary["activeRecoveryAction"]>>>;
|
||||
};
|
||||
|
||||
async function relationRecoveryActionMap(
|
||||
recoveryActionsSvc: RecoveryActionsLister,
|
||||
companyId: string,
|
||||
relations: { blockedBy: IssueRelationIssueSummary[]; blocks: IssueRelationIssueSummary[] },
|
||||
): Promise<Map<string, NonNullable<IssueRelationIssueSummary["activeRecoveryAction"]>>> {
|
||||
const candidates: IssueRelationIssueSummary[] = [];
|
||||
const visit = (summary: IssueRelationIssueSummary) => {
|
||||
candidates.push(summary);
|
||||
for (const terminal of summary.terminalBlockers ?? []) {
|
||||
visit(terminal);
|
||||
}
|
||||
};
|
||||
for (const blocker of relations.blockedBy) visit(blocker);
|
||||
for (const blocking of relations.blocks) visit(blocking);
|
||||
if (candidates.length === 0) return new Map();
|
||||
const ids = [...new Set(candidates.map((summary) => summary.id))];
|
||||
return recoveryActionsSvc.listActiveForIssues(companyId, ids);
|
||||
}
|
||||
|
||||
function withRecoveryActionsOnRelationSummaries(
|
||||
relations: { blockedBy: IssueRelationIssueSummary[]; blocks: IssueRelationIssueSummary[] },
|
||||
recoveryActionByIssueId: Map<string, NonNullable<IssueRelationIssueSummary["activeRecoveryAction"]>>,
|
||||
) {
|
||||
const augment = (summary: IssueRelationIssueSummary): IssueRelationIssueSummary => ({
|
||||
...summary,
|
||||
activeRecoveryAction: recoveryActionByIssueId.get(summary.id) ?? summary.activeRecoveryAction ?? null,
|
||||
terminalBlockers: summary.terminalBlockers?.map(augment),
|
||||
});
|
||||
return {
|
||||
blockedBy: relations.blockedBy.map(augment),
|
||||
blocks: relations.blocks.map(augment),
|
||||
};
|
||||
}
|
||||
|
||||
const ACTIVE_REVIEW_APPROVAL_STATUSES = new Set(["pending", "revision_requested"]);
|
||||
|
||||
const INVALID_AGENT_IN_REVIEW_DISPOSITION_MESSAGE =
|
||||
|
|
@ -787,6 +838,7 @@ export function issueRoutes(
|
|||
const projectsSvc = projectService(db);
|
||||
const goalsSvc = goalService(db);
|
||||
const issueApprovalsSvc = issueApprovalService(db);
|
||||
const recoveryActionsSvc = issueRecoveryActionService(db);
|
||||
const executionWorkspacesSvc = executionWorkspaceServiceDirect(db);
|
||||
const workProductsSvc = workProductService(db);
|
||||
const documentsSvc = documentService(db);
|
||||
|
|
@ -1447,14 +1499,15 @@ export function issueRoutes(
|
|||
limit,
|
||||
offset,
|
||||
});
|
||||
const handoffStates = await listSuccessfulRunHandoffStates(
|
||||
db,
|
||||
companyId,
|
||||
result.map((issue) => issue.id),
|
||||
);
|
||||
const issueIds = result.map((issue) => issue.id);
|
||||
const [handoffStates, recoveryActionByIssue] = await Promise.all([
|
||||
listSuccessfulRunHandoffStates(db, companyId, issueIds),
|
||||
recoveryActionsSvc.listActiveForIssues(companyId, issueIds),
|
||||
]);
|
||||
res.json(result.map((issue) => ({
|
||||
...issue,
|
||||
successfulRunHandoff: handoffStates.get(issue.id) ?? null,
|
||||
activeRecoveryAction: recoveryActionByIssue.get(issue.id) ?? null,
|
||||
})));
|
||||
});
|
||||
|
||||
|
|
@ -1541,6 +1594,7 @@ export function issueRoutes(
|
|||
attachments,
|
||||
continuationSummary,
|
||||
currentExecutionWorkspace,
|
||||
activeRecoveryAction,
|
||||
] =
|
||||
await Promise.all([
|
||||
resolveIssueProjectAndGoal(issue),
|
||||
|
|
@ -1554,7 +1608,17 @@ export function issueRoutes(
|
|||
svc.listAttachments(issue.id),
|
||||
documentsSvc.getIssueDocumentByKey(issue.id, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY),
|
||||
currentExecutionWorkspacePromise,
|
||||
recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id),
|
||||
]);
|
||||
const recoveryActionsByRelationIssue = await relationRecoveryActionMap(
|
||||
recoveryActionsSvc,
|
||||
issue.companyId,
|
||||
relations,
|
||||
);
|
||||
const relationsWithRecoveryActions = withRecoveryActionsOnRelationSummaries(
|
||||
relations,
|
||||
recoveryActionsByRelationIssue,
|
||||
);
|
||||
|
||||
res.json({
|
||||
issue: {
|
||||
|
|
@ -1567,12 +1631,13 @@ export function issueRoutes(
|
|||
...(blockerAttention ? { blockerAttention } : {}),
|
||||
productivityReview,
|
||||
scheduledRetry,
|
||||
activeRecoveryAction,
|
||||
priority: issue.priority,
|
||||
projectId: issue.projectId,
|
||||
goalId: goal?.id ?? issue.goalId,
|
||||
parentId: issue.parentId,
|
||||
blockedBy: relations.blockedBy,
|
||||
blocks: relations.blocks,
|
||||
blockedBy: relationsWithRecoveryActions.blockedBy,
|
||||
blocks: relationsWithRecoveryActions.blocks,
|
||||
assigneeAgentId: issue.assigneeAgentId,
|
||||
assigneeUserId: issue.assigneeUserId,
|
||||
originKind: issue.originKind,
|
||||
|
|
@ -1649,6 +1714,7 @@ export function issueRoutes(
|
|||
referenceSummary,
|
||||
successfulRunHandoffStates,
|
||||
scheduledRetry,
|
||||
activeRecoveryAction,
|
||||
] = await Promise.all([
|
||||
resolveIssueProjectAndGoal(issue),
|
||||
svc.getAncestors(issue.id),
|
||||
|
|
@ -1660,7 +1726,17 @@ export function issueRoutes(
|
|||
issueReferencesSvc.listIssueReferenceSummary(issue.id),
|
||||
listSuccessfulRunHandoffStates(db, issue.companyId, [issue.id]),
|
||||
svc.getCurrentScheduledRetry(issue.id),
|
||||
recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id),
|
||||
]);
|
||||
const recoveryActionsByRelationIssue = await relationRecoveryActionMap(
|
||||
recoveryActionsSvc,
|
||||
issue.companyId,
|
||||
relations,
|
||||
);
|
||||
const relationsWithRecoveryActions = withRecoveryActionsOnRelationSummaries(
|
||||
relations,
|
||||
recoveryActionsByRelationIssue,
|
||||
);
|
||||
const mentionedProjects = mentionedProjectIds.length > 0
|
||||
? await projectsSvc.listByIds(issue.companyId, mentionedProjectIds)
|
||||
: [];
|
||||
|
|
@ -1676,8 +1752,9 @@ export function issueRoutes(
|
|||
productivityReview,
|
||||
successfulRunHandoff: successfulRunHandoffStates.get(issue.id) ?? null,
|
||||
scheduledRetry,
|
||||
blockedBy: relations.blockedBy,
|
||||
blocks: relations.blocks,
|
||||
activeRecoveryAction,
|
||||
blockedBy: relationsWithRecoveryActions.blockedBy,
|
||||
blocks: relationsWithRecoveryActions.blocks,
|
||||
relatedWork: referenceSummary,
|
||||
referencedIssueIdentifiers: referenceSummary.outbound.map((item) => item.issue.identifier ?? item.issue.id),
|
||||
...documentPayload,
|
||||
|
|
@ -1689,6 +1766,148 @@ export function issueRoutes(
|
|||
});
|
||||
});
|
||||
|
||||
router.get("/issues/:id/recovery-actions", async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const issue = await svc.getById(id);
|
||||
if (!issue) {
|
||||
res.status(404).json({ error: "Issue not found" });
|
||||
return;
|
||||
}
|
||||
assertCompanyAccess(req, issue.companyId);
|
||||
const active = await recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id);
|
||||
res.json({
|
||||
active,
|
||||
actions: active ? [active] : [],
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/issues/:id/recovery-actions/resolve", validate(resolveIssueRecoveryActionSchema), async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const existing = await svc.getById(id);
|
||||
if (!existing) {
|
||||
res.status(404).json({ error: "Issue not found" });
|
||||
return;
|
||||
}
|
||||
assertCompanyAccess(req, existing.companyId);
|
||||
if (!(await assertAgentIssueMutationAllowed(req, res, existing))) return;
|
||||
|
||||
const { actionId, outcome, sourceIssueStatus, resolutionNote } = req.body;
|
||||
if (outcome === "false_positive" || outcome === "cancelled") {
|
||||
assertBoard(req);
|
||||
}
|
||||
|
||||
const actor = getActorInfo(req);
|
||||
const updateFields = sourceIssueStatus ? { status: sourceIssueStatus } : {};
|
||||
await assertAgentInReviewReviewPath({
|
||||
existing,
|
||||
updateFields,
|
||||
actorType: req.actor.type,
|
||||
});
|
||||
|
||||
const actionStatus = outcome === "cancelled" ? "cancelled" : "resolved";
|
||||
const result = await db.transaction(async (tx) => {
|
||||
let issue = existing;
|
||||
if (outcome === "blocked") {
|
||||
const unresolvedBlockers = await tx
|
||||
.select({ id: issueRows.id })
|
||||
.from(issueRelations)
|
||||
.innerJoin(issueRows, eq(issueRelations.issueId, issueRows.id))
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, existing.companyId),
|
||||
eq(issueRelations.relatedIssueId, existing.id),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
notInArray(issueRows.status, ["done", "cancelled"]),
|
||||
),
|
||||
)
|
||||
.limit(1);
|
||||
if (unresolvedBlockers.length === 0) {
|
||||
throw unprocessable("Blocked recovery resolution requires an unresolved first-class blocker on the source issue");
|
||||
}
|
||||
}
|
||||
|
||||
if (sourceIssueStatus) {
|
||||
const updatedIssue = await svc.update(
|
||||
id,
|
||||
{
|
||||
status: sourceIssueStatus,
|
||||
actorAgentId: actor.agentId ?? null,
|
||||
actorUserId: actor.actorType === "user" ? actor.actorId : null,
|
||||
},
|
||||
tx,
|
||||
);
|
||||
if (!updatedIssue) throw notFound("Issue not found");
|
||||
issue = updatedIssue;
|
||||
}
|
||||
|
||||
const recoveryAction = await recoveryActionsSvc.resolveActiveForIssue(
|
||||
{
|
||||
companyId: existing.companyId,
|
||||
sourceIssueId: existing.id,
|
||||
actionId: actionId ?? null,
|
||||
status: actionStatus,
|
||||
outcome,
|
||||
resolutionNote: resolutionNote ?? null,
|
||||
},
|
||||
tx,
|
||||
);
|
||||
if (!recoveryAction) throw notFound("Active recovery action not found");
|
||||
|
||||
return { issue, recoveryAction };
|
||||
});
|
||||
|
||||
await routinesSvc.syncRunStatusForIssue(result.issue.id);
|
||||
|
||||
if (sourceIssueStatus && existing.status !== result.issue.status) {
|
||||
await logActivity(db, {
|
||||
companyId: result.issue.companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
action: "issue.updated",
|
||||
entityType: "issue",
|
||||
entityId: result.issue.id,
|
||||
details: {
|
||||
identifier: result.issue.identifier,
|
||||
status: result.issue.status,
|
||||
source: "recovery_action_resolution",
|
||||
recoveryActionId: result.recoveryAction.id,
|
||||
_previous: {
|
||||
status: existing.status,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
await logActivity(db, {
|
||||
companyId: result.issue.companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
action: "issue.recovery_action_resolved",
|
||||
entityType: "issue",
|
||||
entityId: result.issue.id,
|
||||
details: {
|
||||
identifier: result.issue.identifier,
|
||||
recoveryActionId: result.recoveryAction.id,
|
||||
recoveryActionStatus: result.recoveryAction.status,
|
||||
outcome: result.recoveryAction.outcome,
|
||||
sourceIssueStatus: sourceIssueStatus ?? null,
|
||||
resolutionNote: result.recoveryAction.resolutionNote,
|
||||
},
|
||||
});
|
||||
|
||||
res.json({
|
||||
issue: {
|
||||
...result.issue,
|
||||
activeRecoveryAction: null,
|
||||
},
|
||||
recoveryAction: result.recoveryAction,
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/issues/:id/work-products", async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const issue = await svc.getById(id);
|
||||
|
|
|
|||
|
|
@ -1691,6 +1691,7 @@ function shouldAutoCheckoutIssueForWake(input: {
|
|||
const wakeReason = readNonEmptyString(input.contextSnapshot?.wakeReason);
|
||||
if (!wakeReason) return false;
|
||||
if (wakeReason === "issue_comment_mentioned") return false;
|
||||
if (wakeReason === "source_scoped_recovery_action") return false;
|
||||
if (wakeReason.startsWith("execution_")) return false;
|
||||
|
||||
return true;
|
||||
|
|
@ -5883,8 +5884,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
|||
|
||||
// Fix A (lazy locking): stamp executionRunId now that the run is actually running,
|
||||
// not at queue time. Guard is idempotent — safe if called more than once.
|
||||
const claimedIssueId = readNonEmptyString(parseObject(claimed.contextSnapshot).issueId);
|
||||
if (claimedIssueId) {
|
||||
const claimedContext = parseObject(claimed.contextSnapshot);
|
||||
const claimedIssueId = readNonEmptyString(claimedContext.issueId);
|
||||
const claimedWakeReason = readNonEmptyString(claimedContext.wakeReason);
|
||||
if (claimedIssueId && claimedWakeReason !== "source_scoped_recovery_action") {
|
||||
const claimedAgent = await getAgent(claimed.agentId);
|
||||
await db
|
||||
.update(issues)
|
||||
|
|
@ -7873,7 +7876,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
|||
});
|
||||
const livenessRun = finalizedRun;
|
||||
await refreshContinuationSummaryForRun(livenessRun, agent);
|
||||
if (issueId && outcome === "succeeded") {
|
||||
const skipRunIssueComment = parseObject(livenessRun.contextSnapshot).skipIssueComment === true;
|
||||
if (issueId && outcome === "succeeded" && !skipRunIssueComment) {
|
||||
try {
|
||||
const existingRunComment = await findRunIssueComment(livenessRun.id, livenessRun.companyId, issueId);
|
||||
if (!existingRunComment) {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ export { issueThreadInteractionService } from "./issue-thread-interactions.js";
|
|||
export { issueTreeControlService } from "./issue-tree-control.js";
|
||||
export { issueApprovalService } from "./issue-approvals.js";
|
||||
export { issueReferenceService } from "./issue-references.js";
|
||||
export { issueRecoveryActionService } from "./issue-recovery-actions.js";
|
||||
export { goalService } from "./goals.js";
|
||||
export { activityService, type ActivityFilters } from "./activity.js";
|
||||
export { approvalService } from "./approvals.js";
|
||||
|
|
|
|||
295
server/src/services/issue-recovery-actions.ts
Normal file
295
server/src/services/issue-recovery-actions.ts
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { issueRecoveryActions } from "@paperclipai/db";
|
||||
import type {
|
||||
IssueRecoveryAction,
|
||||
IssueRecoveryActionKind,
|
||||
IssueRecoveryActionOwnerType,
|
||||
IssueRecoveryActionOutcome,
|
||||
IssueRecoveryActionStatus,
|
||||
} from "@paperclipai/shared";
|
||||
|
||||
const ACTIVE_RECOVERY_ACTION_STATUSES = ["active", "escalated"] as const satisfies readonly IssueRecoveryActionStatus[];
|
||||
const MAX_UPSERT_RETRIES = 3;
|
||||
|
||||
type IssueRecoveryActionRow = typeof issueRecoveryActions.$inferSelect;
|
||||
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
type DbOrTransaction = Db | DbTransaction;
|
||||
|
||||
export type UpsertIssueRecoveryActionInput = {
|
||||
companyId: string;
|
||||
sourceIssueId: string;
|
||||
recoveryIssueId?: string | null;
|
||||
kind: IssueRecoveryActionKind;
|
||||
ownerType?: IssueRecoveryActionOwnerType;
|
||||
ownerAgentId?: string | null;
|
||||
ownerUserId?: string | null;
|
||||
previousOwnerAgentId?: string | null;
|
||||
returnOwnerAgentId?: string | null;
|
||||
cause: string;
|
||||
fingerprint: string;
|
||||
evidence?: Record<string, unknown>;
|
||||
nextAction: string;
|
||||
wakePolicy?: Record<string, unknown> | null;
|
||||
monitorPolicy?: Record<string, unknown> | null;
|
||||
maxAttempts?: number | null;
|
||||
timeoutAt?: Date | null;
|
||||
lastAttemptAt?: Date | null;
|
||||
};
|
||||
|
||||
export type ResolveIssueRecoveryActionInput = {
|
||||
companyId: string;
|
||||
sourceIssueId: string;
|
||||
actionId?: string | null;
|
||||
status: Extract<IssueRecoveryActionStatus, "resolved" | "cancelled">;
|
||||
outcome: IssueRecoveryActionOutcome;
|
||||
resolutionNote?: string | null;
|
||||
};
|
||||
|
||||
function toReadModel(row: IssueRecoveryActionRow): IssueRecoveryAction {
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
sourceIssueId: row.sourceIssueId,
|
||||
recoveryIssueId: row.recoveryIssueId,
|
||||
kind: row.kind as IssueRecoveryAction["kind"],
|
||||
status: row.status as IssueRecoveryAction["status"],
|
||||
ownerType: row.ownerType as IssueRecoveryAction["ownerType"],
|
||||
ownerAgentId: row.ownerAgentId,
|
||||
ownerUserId: row.ownerUserId,
|
||||
previousOwnerAgentId: row.previousOwnerAgentId,
|
||||
returnOwnerAgentId: row.returnOwnerAgentId,
|
||||
cause: row.cause,
|
||||
fingerprint: row.fingerprint,
|
||||
evidence: row.evidence,
|
||||
nextAction: row.nextAction,
|
||||
wakePolicy: row.wakePolicy,
|
||||
monitorPolicy: row.monitorPolicy,
|
||||
attemptCount: row.attemptCount,
|
||||
maxAttempts: row.maxAttempts,
|
||||
timeoutAt: row.timeoutAt,
|
||||
lastAttemptAt: row.lastAttemptAt,
|
||||
outcome: row.outcome as IssueRecoveryAction["outcome"],
|
||||
resolutionNote: row.resolutionNote,
|
||||
resolvedAt: row.resolvedAt,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function isUniqueRecoveryActionConflict(error: unknown) {
|
||||
const maybe = error as { code?: string; constraint?: string; message?: string } | null;
|
||||
return Boolean(
|
||||
maybe &&
|
||||
maybe.code === "23505" &&
|
||||
(
|
||||
maybe.constraint === "issue_recovery_actions_active_source_uq" ||
|
||||
maybe.constraint === "issue_recovery_actions_active_fingerprint_uq" ||
|
||||
typeof maybe.message === "string" && (
|
||||
maybe.message.includes("issue_recovery_actions_active_source_uq") ||
|
||||
maybe.message.includes("issue_recovery_actions_active_fingerprint_uq")
|
||||
)
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function issueRecoveryActionService(db: Db) {
|
||||
const upsertQueues = new Map<string, Promise<void>>();
|
||||
|
||||
async function runExclusiveUpsert<T>(
|
||||
input: UpsertIssueRecoveryActionInput,
|
||||
task: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const key = `${input.companyId}:${input.sourceIssueId}`;
|
||||
const previous = upsertQueues.get(key) ?? Promise.resolve();
|
||||
let release: () => void = () => {};
|
||||
const current = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const next = previous.catch(() => undefined).then(() => current);
|
||||
upsertQueues.set(key, next);
|
||||
|
||||
await previous.catch(() => undefined);
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
release();
|
||||
if (upsertQueues.get(key) === next) {
|
||||
upsertQueues.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function getActiveForIssue(companyId: string, sourceIssueId: string): Promise<IssueRecoveryAction | null> {
|
||||
const row = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, companyId),
|
||||
eq(issueRecoveryActions.sourceIssueId, sourceIssueId),
|
||||
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(issueRecoveryActions.updatedAt))
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row ? toReadModel(row) : null;
|
||||
}
|
||||
|
||||
async function listActiveForIssues(companyId: string, sourceIssueIds: string[]) {
|
||||
if (sourceIssueIds.length === 0) return new Map<string, IssueRecoveryAction>();
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, companyId),
|
||||
inArray(issueRecoveryActions.sourceIssueId, [...new Set(sourceIssueIds)]),
|
||||
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(issueRecoveryActions.updatedAt));
|
||||
const result = new Map<string, IssueRecoveryAction>();
|
||||
for (const row of rows) {
|
||||
if (!result.has(row.sourceIssueId)) result.set(row.sourceIssueId, toReadModel(row));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function retryUpsertSourceScoped(
|
||||
input: UpsertIssueRecoveryActionInput,
|
||||
retryCount: number,
|
||||
error?: unknown,
|
||||
): Promise<IssueRecoveryAction> {
|
||||
if (retryCount >= MAX_UPSERT_RETRIES) {
|
||||
if (error) throw error;
|
||||
throw new Error(
|
||||
`Failed to upsert active recovery action for issue ${input.sourceIssueId} after ${MAX_UPSERT_RETRIES} retries`,
|
||||
);
|
||||
}
|
||||
return upsertSourceScopedUnlocked(input, retryCount + 1);
|
||||
}
|
||||
|
||||
async function upsertSourceScopedUnlocked(
|
||||
input: UpsertIssueRecoveryActionInput,
|
||||
retryCount = 0,
|
||||
): Promise<IssueRecoveryAction> {
|
||||
const existing = await getActiveForIssue(input.companyId, input.sourceIssueId);
|
||||
const now = new Date();
|
||||
const ownerType = input.ownerType ?? (input.ownerAgentId ? "agent" : "board");
|
||||
if (existing) {
|
||||
const [updated] = await db
|
||||
.update(issueRecoveryActions)
|
||||
.set({
|
||||
recoveryIssueId: input.recoveryIssueId ?? null,
|
||||
kind: input.kind,
|
||||
status: "active",
|
||||
ownerType,
|
||||
ownerAgentId: input.ownerAgentId ?? null,
|
||||
ownerUserId: input.ownerUserId ?? null,
|
||||
previousOwnerAgentId: input.previousOwnerAgentId ?? existing.previousOwnerAgentId,
|
||||
returnOwnerAgentId: input.returnOwnerAgentId ?? existing.returnOwnerAgentId,
|
||||
cause: input.cause,
|
||||
fingerprint: input.fingerprint,
|
||||
evidence: input.evidence ?? existing.evidence,
|
||||
nextAction: input.nextAction,
|
||||
wakePolicy: input.wakePolicy ?? null,
|
||||
monitorPolicy: input.monitorPolicy ?? null,
|
||||
attemptCount: existing.attemptCount + 1,
|
||||
maxAttempts: input.maxAttempts ?? null,
|
||||
timeoutAt: input.timeoutAt ?? null,
|
||||
lastAttemptAt: input.lastAttemptAt ?? now,
|
||||
outcome: null,
|
||||
resolutionNote: null,
|
||||
resolvedAt: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.id, existing.id),
|
||||
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
if (!updated) {
|
||||
return retryUpsertSourceScoped(input, retryCount);
|
||||
}
|
||||
return toReadModel(updated!);
|
||||
}
|
||||
|
||||
try {
|
||||
const [created] = await db
|
||||
.insert(issueRecoveryActions)
|
||||
.values({
|
||||
companyId: input.companyId,
|
||||
sourceIssueId: input.sourceIssueId,
|
||||
recoveryIssueId: input.recoveryIssueId ?? null,
|
||||
kind: input.kind,
|
||||
status: "active",
|
||||
ownerType,
|
||||
ownerAgentId: input.ownerAgentId ?? null,
|
||||
ownerUserId: input.ownerUserId ?? null,
|
||||
previousOwnerAgentId: input.previousOwnerAgentId ?? null,
|
||||
returnOwnerAgentId: input.returnOwnerAgentId ?? null,
|
||||
cause: input.cause,
|
||||
fingerprint: input.fingerprint,
|
||||
evidence: input.evidence ?? {},
|
||||
nextAction: input.nextAction,
|
||||
wakePolicy: input.wakePolicy ?? null,
|
||||
monitorPolicy: input.monitorPolicy ?? null,
|
||||
attemptCount: 1,
|
||||
maxAttempts: input.maxAttempts ?? null,
|
||||
timeoutAt: input.timeoutAt ?? null,
|
||||
lastAttemptAt: input.lastAttemptAt ?? now,
|
||||
})
|
||||
.returning();
|
||||
return toReadModel(created!);
|
||||
} catch (error) {
|
||||
if (!isUniqueRecoveryActionConflict(error)) throw error;
|
||||
return retryUpsertSourceScoped(input, retryCount, error);
|
||||
}
|
||||
}
|
||||
|
||||
async function upsertSourceScoped(
|
||||
input: UpsertIssueRecoveryActionInput,
|
||||
): Promise<IssueRecoveryAction> {
|
||||
return runExclusiveUpsert(input, () => upsertSourceScopedUnlocked(input));
|
||||
}
|
||||
|
||||
async function resolveActiveForIssue(
|
||||
input: ResolveIssueRecoveryActionInput,
|
||||
dbOrTx: DbOrTransaction = db,
|
||||
): Promise<IssueRecoveryAction | null> {
|
||||
const now = new Date();
|
||||
const predicates = [
|
||||
eq(issueRecoveryActions.companyId, input.companyId),
|
||||
eq(issueRecoveryActions.sourceIssueId, input.sourceIssueId),
|
||||
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
|
||||
];
|
||||
if (input.actionId) {
|
||||
predicates.push(eq(issueRecoveryActions.id, input.actionId));
|
||||
}
|
||||
|
||||
const [updated] = await dbOrTx
|
||||
.update(issueRecoveryActions)
|
||||
.set({
|
||||
status: input.status,
|
||||
outcome: input.outcome,
|
||||
resolutionNote: input.resolutionNote ?? null,
|
||||
resolvedAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(...predicates))
|
||||
.returning();
|
||||
|
||||
return updated ? toReadModel(updated) : null;
|
||||
}
|
||||
|
||||
return {
|
||||
getActiveForIssue,
|
||||
listActiveForIssues,
|
||||
resolveActiveForIssue,
|
||||
upsertSourceScoped,
|
||||
};
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@ import {
|
|||
issueAttachments,
|
||||
issueInboxArchives,
|
||||
issueLabels,
|
||||
issueRecoveryActions,
|
||||
issueRelations,
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
|
|
@ -1355,6 +1356,18 @@ async function listIssueBlockerAttentionMap(
|
|||
explicitWaitingIssueIds.add(parsed.issueId);
|
||||
explicitWaitingIssueIds.add(parsed.leafIssueId);
|
||||
}
|
||||
|
||||
const recoveryActionRows: Array<{ sourceIssueId: string }> = await dbOrTx
|
||||
.select({ sourceIssueId: issueRecoveryActions.sourceIssueId })
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, companyId),
|
||||
inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds),
|
||||
),
|
||||
);
|
||||
for (const row of recoveryActionRows) explicitWaitingIssueIds.add(row.sourceIssueId);
|
||||
}
|
||||
|
||||
const agentRows: IssueBlockerAttentionAgentRow[] = agentIds.size > 0
|
||||
|
|
|
|||
|
|
@ -12,10 +12,12 @@ import {
|
|||
agentWakeupRequests,
|
||||
approvals,
|
||||
companies,
|
||||
issueComments,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRunWatchdogDecisions,
|
||||
heartbeatRuns,
|
||||
issueApprovals,
|
||||
issueRecoveryActions,
|
||||
issueRelations,
|
||||
issueThreadInteractions,
|
||||
issues,
|
||||
|
|
@ -29,6 +31,7 @@ import { redactSensitiveText } from "../../redaction.js";
|
|||
import { logActivity } from "../activity-log.js";
|
||||
import { budgetService } from "../budgets.js";
|
||||
import { instanceSettingsService } from "../instance-settings.js";
|
||||
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
||||
import { issueTreeControlService } from "../issue-tree-control.js";
|
||||
import { issueService } from "../issues.js";
|
||||
import { getRunLogStore } from "../run-log-store.js";
|
||||
|
|
@ -37,6 +40,7 @@ import {
|
|||
FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,
|
||||
SUCCESSFUL_RUN_MISSING_STATE_REASON,
|
||||
buildSuccessfulRunHandoffExhaustedNotice,
|
||||
noticeMetadataReferencesRecoveryAction,
|
||||
type SuccessfulRunHandoffNotice,
|
||||
} from "./successful-run-handoff.js";
|
||||
import {
|
||||
|
|
@ -386,6 +390,7 @@ function buildLivenessOriginalIssueComment(finding: IssueLivenessFinding, escala
|
|||
|
||||
export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) {
|
||||
const issuesSvc = issueService(db);
|
||||
const recoveryActionsSvc = issueRecoveryActionService(db);
|
||||
const treeControlSvc = issueTreeControlService(db);
|
||||
const budgets = budgetService(db);
|
||||
const instanceSettings = instanceSettingsService(db);
|
||||
|
|
@ -1566,6 +1571,136 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
return recovery;
|
||||
}
|
||||
|
||||
function strandedRecoveryActionKind(cause: StrandedRecoveryCause) {
|
||||
return cause === SUCCESSFUL_RUN_MISSING_STATE_REASON
|
||||
? "missing_disposition" as const
|
||||
: "stranded_assigned_issue" as const;
|
||||
}
|
||||
|
||||
function strandedRecoveryActionFingerprint(input: {
|
||||
issue: typeof issues.$inferSelect;
|
||||
recoveryCause: StrandedRecoveryCause;
|
||||
}) {
|
||||
return [
|
||||
"source_scoped_recovery",
|
||||
input.issue.companyId,
|
||||
input.issue.id,
|
||||
input.recoveryCause,
|
||||
].join(":");
|
||||
}
|
||||
|
||||
function buildStrandedRecoveryActionEvidence(input: {
|
||||
issue: typeof issues.$inferSelect;
|
||||
latestRun: LatestIssueRun;
|
||||
previousStatus: "todo" | "in_progress";
|
||||
recoveryCause: StrandedRecoveryCause;
|
||||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||||
}) {
|
||||
const context = parseObject(input.latestRun?.contextSnapshot);
|
||||
return {
|
||||
sourceIssueId: input.issue.id,
|
||||
sourceIdentifier: input.issue.identifier,
|
||||
previousStatus: input.previousStatus,
|
||||
latestIssueStatus: input.issue.status,
|
||||
latestRunId: input.latestRun?.id ?? null,
|
||||
latestRunStatus: input.latestRun?.status ?? null,
|
||||
latestRunErrorCode: input.latestRun?.errorCode ?? null,
|
||||
retryReason: readNonEmptyString(context.retryReason) ?? null,
|
||||
recoveryCause: input.recoveryCause,
|
||||
sourceRunId: input.successfulRunHandoffEvidence?.sourceRunId ?? null,
|
||||
correctiveRunId: input.successfulRunHandoffEvidence?.correctiveRunId ?? null,
|
||||
missingDisposition: input.successfulRunHandoffEvidence?.missingDisposition ?? null,
|
||||
handoffAttempt: input.successfulRunHandoffEvidence?.handoffAttempt ?? null,
|
||||
maxHandoffAttempts: input.successfulRunHandoffEvidence?.maxHandoffAttempts ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureSourceScopedStrandedRecoveryAction(input: {
|
||||
issue: typeof issues.$inferSelect;
|
||||
latestRun: LatestIssueRun;
|
||||
previousStatus: "todo" | "in_progress";
|
||||
recoveryCause?: StrandedRecoveryCause;
|
||||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||||
}) {
|
||||
const recoveryCause = input.recoveryCause ?? "stranded_assigned_issue";
|
||||
const ownerAgentId = await resolveStrandedIssueRecoveryOwnerAgentId(input.issue);
|
||||
const now = new Date();
|
||||
const action = await recoveryActionsSvc.upsertSourceScoped({
|
||||
companyId: input.issue.companyId,
|
||||
sourceIssueId: input.issue.id,
|
||||
kind: strandedRecoveryActionKind(recoveryCause),
|
||||
ownerType: ownerAgentId ? "agent" : "board",
|
||||
ownerAgentId,
|
||||
previousOwnerAgentId: input.issue.assigneeAgentId,
|
||||
returnOwnerAgentId: input.issue.assigneeAgentId,
|
||||
cause: recoveryCause,
|
||||
fingerprint: strandedRecoveryActionFingerprint({
|
||||
issue: input.issue,
|
||||
recoveryCause,
|
||||
}),
|
||||
evidence: buildStrandedRecoveryActionEvidence({
|
||||
issue: input.issue,
|
||||
latestRun: input.latestRun,
|
||||
previousStatus: input.previousStatus,
|
||||
recoveryCause,
|
||||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||||
}),
|
||||
nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON
|
||||
? "Choose and record a valid issue disposition without copying transcript content."
|
||||
: "Restore a live execution path, fix the runtime/adapter failure, or record an intentional manual resolution.",
|
||||
wakePolicy: ownerAgentId
|
||||
? {
|
||||
type: "wake_owner",
|
||||
reason: "source_scoped_recovery_action",
|
||||
ownerAgentId,
|
||||
}
|
||||
: {
|
||||
type: "board_escalation",
|
||||
reason: "no_invokable_recovery_owner",
|
||||
},
|
||||
monitorPolicy: null,
|
||||
maxAttempts: null,
|
||||
lastAttemptAt: now,
|
||||
});
|
||||
|
||||
return action;
|
||||
}
|
||||
|
||||
async function enqueueSourceScopedStrandedRecoveryWake(input: {
|
||||
action: Awaited<ReturnType<typeof recoveryActionsSvc.upsertSourceScoped>>;
|
||||
issue: typeof issues.$inferSelect;
|
||||
latestRun: LatestIssueRun;
|
||||
recoveryCause: StrandedRecoveryCause;
|
||||
}) {
|
||||
if (!input.action.ownerAgentId) return;
|
||||
await deps.enqueueWakeup(input.action.ownerAgentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "source_scoped_recovery_action",
|
||||
idempotencyKey: `source_scoped_recovery_action:${input.action.id}:${input.action.attemptCount}`,
|
||||
payload: withRecoveryModelProfileHint({
|
||||
issueId: input.issue.id,
|
||||
sourceIssueId: input.issue.id,
|
||||
recoveryActionId: input.action.id,
|
||||
strandedRunId: input.latestRun?.id ?? null,
|
||||
recoveryCause: input.recoveryCause,
|
||||
}),
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
contextSnapshot: withRecoveryModelProfileHint({
|
||||
issueId: input.issue.id,
|
||||
taskId: input.issue.id,
|
||||
wakeReason: "source_scoped_recovery_action",
|
||||
skipIssueComment: true,
|
||||
source: "issue_recovery_action",
|
||||
recoveryActionId: input.action.id,
|
||||
sourceIssueId: input.issue.id,
|
||||
strandedRunId: input.latestRun?.id ?? null,
|
||||
recoveryCause: input.recoveryCause,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function buildRecoveryIssueInPlaceEscalationComment(input: {
|
||||
issue: typeof issues.$inferSelect;
|
||||
previousStatus: "todo" | "in_progress";
|
||||
|
|
@ -1682,29 +1817,32 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
recoveryCause?: StrandedRecoveryCause;
|
||||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||||
}) {
|
||||
const nestedRecoverySuppressed = isStrandedIssueRecoveryIssue(input.issue);
|
||||
let recoveryIssue: typeof issues.$inferSelect | null = null;
|
||||
if (!nestedRecoverySuppressed) {
|
||||
recoveryIssue = await ensureStrandedIssueRecoveryIssue({
|
||||
if (isStrandedIssueRecoveryIssue(input.issue)) {
|
||||
return escalateStrandedRecoveryIssueInPlace({
|
||||
issue: input.issue,
|
||||
previousStatus: input.previousStatus,
|
||||
latestRun: input.latestRun,
|
||||
recoveryCause: input.recoveryCause,
|
||||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||||
});
|
||||
}
|
||||
|
||||
const recoveryCause = input.recoveryCause ?? "stranded_assigned_issue";
|
||||
const recoveryAction = await ensureSourceScopedStrandedRecoveryAction({
|
||||
issue: input.issue,
|
||||
previousStatus: input.previousStatus,
|
||||
latestRun: input.latestRun,
|
||||
recoveryCause,
|
||||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||||
});
|
||||
const blockerIds = await existingUnresolvedBlockerIssueIds(input.issue.companyId, input.issue.id);
|
||||
const nextBlockerIds = recoveryIssue
|
||||
? [...new Set([...blockerIds, recoveryIssue.id])]
|
||||
: blockerIds;
|
||||
const updated = await issuesSvc.update(input.issue.id, {
|
||||
status: "blocked",
|
||||
blockedByIssueIds: nextBlockerIds,
|
||||
blockedByIssueIds: blockerIds,
|
||||
assigneeAgentId: recoveryAction.ownerAgentId ?? input.issue.assigneeAgentId,
|
||||
});
|
||||
if (!updated) return null;
|
||||
|
||||
const prefix = await getCompanyIssuePrefix(input.issue.companyId);
|
||||
const recoveryOwner = recoveryIssue?.assigneeAgentId ? await getAgent(recoveryIssue.assigneeAgentId) : null;
|
||||
const recoveryOwner = recoveryAction.ownerAgentId ? await getAgent(recoveryAction.ownerAgentId) : null;
|
||||
const sourceAssignee = input.issue.assigneeAgentId ? await getAgent(input.issue.assigneeAgentId) : null;
|
||||
let notice: SuccessfulRunHandoffNotice | null = null;
|
||||
if (input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON && input.successfulRunHandoffEvidence) {
|
||||
|
|
@ -1715,39 +1853,60 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
: null,
|
||||
correctiveRun: input.latestRun ? { id: input.latestRun.id, status: input.latestRun.status } : null,
|
||||
sourceAssignee,
|
||||
recoveryIssue,
|
||||
recoveryIssue: null,
|
||||
recoveryActionId: recoveryAction.id,
|
||||
recoveryOwner,
|
||||
latestIssueStatus: input.issue.status,
|
||||
latestHandoffRunStatus: input.latestRun?.status ?? "unknown",
|
||||
missingDisposition: input.successfulRunHandoffEvidence.missingDisposition,
|
||||
});
|
||||
}
|
||||
let recoveryLine: string;
|
||||
if (nestedRecoverySuppressed) {
|
||||
recoveryLine = await buildNestedStrandedRecoveryLine(input.issue, prefix);
|
||||
} else if (recoveryIssue) {
|
||||
recoveryLine = [
|
||||
const recoveryLine = recoveryAction.ownerAgentId
|
||||
? [
|
||||
"",
|
||||
`- Recovery issue: ${issueUiLink({ identifier: recoveryIssue.identifier, id: recoveryIssue.id }, prefix)}`,
|
||||
`- Recovery action: \`${recoveryAction.id}\``,
|
||||
`- Recovery owner: ${agentUiLink(recoveryOwner, prefix)}`,
|
||||
"- Next action: the recovery owner should either restore a live execution path or record the manual resolution, then mark the recovery issue done.",
|
||||
].join("\n");
|
||||
} else {
|
||||
recoveryLine = [
|
||||
"- Next action: the recovery owner should either restore a live execution path or record the manual resolution on the source issue.",
|
||||
].join("\n")
|
||||
: [
|
||||
"",
|
||||
"- Recovery issue: none created because Paperclip could not find an invokable manager, creator, or executive owner with budget available.",
|
||||
`- Recovery action: \`${recoveryAction.id}\``,
|
||||
"- Recovery owner: board escalation, because Paperclip could not find an invokable manager, creator, or executive owner with budget available.",
|
||||
"- Next action: a board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (notice) {
|
||||
await issuesSvc.addComment(input.issue.id, notice.body, {}, {
|
||||
authorType: "system",
|
||||
presentation: notice.presentation,
|
||||
metadata: notice.metadata,
|
||||
});
|
||||
} else {
|
||||
await issuesSvc.addComment(input.issue.id, `${input.comment ?? ""}${recoveryLine}`, {});
|
||||
if (recoveryAction.attemptCount === 1) {
|
||||
const escalationCommentMarker = `Recovery action: \`${recoveryAction.id}\``;
|
||||
|
||||
const hasEscalationComment = await db
|
||||
.select({ id: issueComments.id, body: issueComments.body, metadata: issueComments.metadata })
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.issueId, input.issue.id),
|
||||
eq(issueComments.authorType, "system"),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(issueComments.createdAt))
|
||||
.limit(50)
|
||||
.then((rows) => rows.some((row) =>
|
||||
(row.body ?? "").includes(escalationCommentMarker) ||
|
||||
noticeMetadataReferencesRecoveryAction(row.metadata, recoveryAction.id),
|
||||
));
|
||||
|
||||
if (!hasEscalationComment) {
|
||||
if (notice) {
|
||||
await issuesSvc.addComment(input.issue.id, notice.body, {}, {
|
||||
authorType: "system",
|
||||
presentation: notice.presentation,
|
||||
metadata: notice.metadata,
|
||||
});
|
||||
} else {
|
||||
await issuesSvc.addComment(input.issue.id, `${input.comment ?? ""}${recoveryLine}`, {}, {
|
||||
authorType: "system",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await logActivity(db, {
|
||||
|
|
@ -1772,12 +1931,44 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
latestRunId: input.latestRun?.id ?? null,
|
||||
latestRunStatus: input.latestRun?.status ?? null,
|
||||
latestRunErrorCode: input.latestRun?.errorCode ?? null,
|
||||
recoveryIssueId: recoveryIssue?.id ?? null,
|
||||
nestedRecoverySuppressed,
|
||||
blockerIssueIds: nextBlockerIds,
|
||||
recoveryActionId: recoveryAction.id,
|
||||
recoveryOwnerAgentId: recoveryAction.ownerAgentId,
|
||||
previousOwnerAgentId: recoveryAction.previousOwnerAgentId,
|
||||
returnOwnerAgentId: recoveryAction.returnOwnerAgentId,
|
||||
blockerIssueIds: blockerIds,
|
||||
},
|
||||
});
|
||||
|
||||
await enqueueSourceScopedStrandedRecoveryWake({
|
||||
action: recoveryAction,
|
||||
issue: input.issue,
|
||||
latestRun: input.latestRun,
|
||||
recoveryCause,
|
||||
});
|
||||
|
||||
if (recoveryAction.ownerAgentId && recoveryAction.ownerAgentId === input.issue.assigneeAgentId) {
|
||||
const [currentIssue] = await db
|
||||
.select({
|
||||
status: issues.status,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, input.issue.id))
|
||||
.limit(1);
|
||||
if (
|
||||
currentIssue &&
|
||||
(currentIssue.status !== "blocked" ||
|
||||
currentIssue.assigneeAgentId !== recoveryAction.ownerAgentId)
|
||||
) {
|
||||
const reblocked = await issuesSvc.update(input.issue.id, {
|
||||
status: "blocked",
|
||||
blockedByIssueIds: blockerIds,
|
||||
assigneeAgentId: recoveryAction.ownerAgentId,
|
||||
});
|
||||
if (reblocked) return reblocked;
|
||||
}
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
|
|
@ -2038,6 +2229,33 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
}
|
||||
|
||||
async function collectIssueGraphLivenessFindings() {
|
||||
const issueRowsPromise = Promise.resolve(db
|
||||
.select({
|
||||
id: issues.id,
|
||||
companyId: issues.companyId,
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
projectId: issues.projectId,
|
||||
goalId: issues.goalId,
|
||||
parentId: issues.parentId,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
createdByAgentId: issues.createdByAgentId,
|
||||
createdByUserId: issues.createdByUserId,
|
||||
executionPolicy: issues.executionPolicy,
|
||||
executionState: issues.executionState,
|
||||
monitorNextCheckAt: issues.monitorNextCheckAt,
|
||||
monitorAttemptCount: issues.monitorAttemptCount,
|
||||
})
|
||||
.from(issues)
|
||||
.where(
|
||||
and(
|
||||
isNull(issues.hiddenAt),
|
||||
notInArray(issues.originKind, [RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation]),
|
||||
),
|
||||
));
|
||||
|
||||
const [
|
||||
issueRows,
|
||||
relationRows,
|
||||
|
|
@ -2048,33 +2266,9 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
interactionRows,
|
||||
approvalRows,
|
||||
recoveryIssueRows,
|
||||
recoveryActionRows,
|
||||
] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
id: issues.id,
|
||||
companyId: issues.companyId,
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
projectId: issues.projectId,
|
||||
goalId: issues.goalId,
|
||||
parentId: issues.parentId,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
createdByAgentId: issues.createdByAgentId,
|
||||
createdByUserId: issues.createdByUserId,
|
||||
executionPolicy: issues.executionPolicy,
|
||||
executionState: issues.executionState,
|
||||
monitorNextCheckAt: issues.monitorNextCheckAt,
|
||||
monitorAttemptCount: issues.monitorAttemptCount,
|
||||
})
|
||||
.from(issues)
|
||||
.where(
|
||||
and(
|
||||
isNull(issues.hiddenAt),
|
||||
notInArray(issues.originKind, [RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation]),
|
||||
),
|
||||
),
|
||||
issueRowsPromise,
|
||||
db
|
||||
.select({
|
||||
companyId: issueRelations.companyId,
|
||||
|
|
@ -2164,6 +2358,24 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
notInArray(issues.status, ["done", "cancelled"]),
|
||||
),
|
||||
),
|
||||
issueRowsPromise.then((rows) => {
|
||||
const issueIdsUnderAnalysis = rows.map((row) => row.id);
|
||||
return issueIdsUnderAnalysis.length === 0
|
||||
? []
|
||||
: db
|
||||
.select({
|
||||
companyId: issueRecoveryActions.companyId,
|
||||
issueId: issueRecoveryActions.sourceIssueId,
|
||||
status: issueRecoveryActions.status,
|
||||
})
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
inArray(issueRecoveryActions.sourceIssueId, issueIdsUnderAnalysis),
|
||||
),
|
||||
);
|
||||
}),
|
||||
]);
|
||||
|
||||
const openRecoveryIssues = recoveryIssueRows.flatMap((row) => {
|
||||
|
|
@ -2217,7 +2429,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
})),
|
||||
pendingInteractions: interactionRows,
|
||||
pendingApprovals: approvalRows,
|
||||
openRecoveryIssues,
|
||||
openRecoveryIssues: openRecoveryIssues.concat(recoveryActionRows),
|
||||
now: new Date(),
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
decideSuccessfulRunHandoff,
|
||||
isIdempotentFinishSuccessfulRunHandoffWakeStatus,
|
||||
isSuccessfulRunHandoffRequiredNoticeBody,
|
||||
noticeMetadataReferencesRecoveryAction,
|
||||
} from "./successful-run-handoff.js";
|
||||
|
||||
const run = {
|
||||
|
|
@ -256,6 +257,7 @@ describe("successful run handoff decision", () => {
|
|||
title: "Recover missing next step PAP-1",
|
||||
status: "todo",
|
||||
} as any,
|
||||
recoveryActionId: "77777777-7777-4777-8777-777777777777",
|
||||
recoveryOwner: { id: "66666666-6666-4666-8666-666666666666", name: "CTO" } as any,
|
||||
latestIssueStatus: "in_progress",
|
||||
latestHandoffRunStatus: "failed",
|
||||
|
|
@ -273,7 +275,7 @@ describe("successful run handoff decision", () => {
|
|||
expect.objectContaining({
|
||||
title: "Recovery owner",
|
||||
rows: expect.arrayContaining([
|
||||
expect.objectContaining({ type: "issue_link", identifier: "PAP-2" }),
|
||||
expect.objectContaining({ type: "key_value", label: "Recovery action", value: "77777777-7777-4777-8777-777777777777" }),
|
||||
expect.objectContaining({ type: "agent_link", label: "Recovery owner", name: "CTO" }),
|
||||
]),
|
||||
}),
|
||||
|
|
@ -286,6 +288,8 @@ describe("successful run handoff decision", () => {
|
|||
]),
|
||||
}),
|
||||
]));
|
||||
expect(noticeMetadataReferencesRecoveryAction(notice.metadata, "77777777-7777-4777-8777-777777777777")).toBe(true);
|
||||
expect(noticeMetadataReferencesRecoveryAction(notice.metadata, "88888888-8888-4888-8888-888888888888")).toBe(false);
|
||||
});
|
||||
|
||||
it("recognizes new notices and legacy markdown headings for fallback deduplication", () => {
|
||||
|
|
|
|||
|
|
@ -61,6 +61,19 @@ export type SuccessfulRunHandoffNotice = {
|
|||
metadata: IssueCommentMetadata;
|
||||
};
|
||||
|
||||
export function noticeMetadataReferencesRecoveryAction(
|
||||
metadata: IssueCommentMetadata | null | undefined,
|
||||
recoveryActionId: string,
|
||||
) {
|
||||
return (metadata?.sections ?? []).some((section) =>
|
||||
section.rows.some((row) =>
|
||||
row.type === "key_value" &&
|
||||
row.label === "Recovery action" &&
|
||||
row.value === recoveryActionId,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export type SuccessfulRunHandoffDecision =
|
||||
| {
|
||||
kind: "enqueue";
|
||||
|
|
@ -181,6 +194,7 @@ export function buildSuccessfulRunHandoffExhaustedNotice(input: {
|
|||
correctiveRun: NullableNoticeRun;
|
||||
sourceAssignee: NullableNoticeAgent;
|
||||
recoveryIssue: NullableNoticeIssue;
|
||||
recoveryActionId?: string | null;
|
||||
recoveryOwner: NullableNoticeAgent;
|
||||
latestIssueStatus: string;
|
||||
latestHandoffRunStatus: string;
|
||||
|
|
@ -200,7 +214,9 @@ export function buildSuccessfulRunHandoffExhaustedNotice(input: {
|
|||
title: "Recovery owner",
|
||||
rows: [
|
||||
issueLinkRow("Source issue", input.issue),
|
||||
issueLinkRow("Recovery issue", input.recoveryIssue),
|
||||
input.recoveryActionId
|
||||
? keyValueRow("Recovery action", input.recoveryActionId)
|
||||
: issueLinkRow("Recovery issue", input.recoveryIssue),
|
||||
agentLinkRow("Recovery owner", input.recoveryOwner),
|
||||
agentLinkRow("Source assignee", input.sourceAssignee),
|
||||
keyValueRow("Suggested action", "choose and record a valid issue disposition without copying transcript content"),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue