mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-16 10:50:38 +09:00
Cancel stale queued heartbeats when issue graph changes (PAP-2314) (#4534)
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
868d08903e
commit
82e257c7ba
21 changed files with 1991 additions and 238 deletions
|
|
@ -4,7 +4,9 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
|
|||
import {
|
||||
activityLog,
|
||||
agents,
|
||||
budgetPolicies,
|
||||
companies,
|
||||
costEvents,
|
||||
createDb,
|
||||
executionWorkspaces,
|
||||
heartbeatRuns,
|
||||
|
|
@ -191,7 +193,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
type: "blocks",
|
||||
});
|
||||
|
||||
return { companyId, managerId, blockedIssueId, blockerIssueId };
|
||||
return { companyId, managerId, coderId, blockedIssueId, blockerIssueId };
|
||||
}
|
||||
|
||||
it("keeps liveness findings advisory when auto recovery is disabled", async () => {
|
||||
|
|
@ -342,6 +344,71 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
expect(events.some((event) => event.action === "issue.blockers.updated")).toBe(true);
|
||||
});
|
||||
|
||||
it("skips budget-blocked direct owners and assigns recovery to the manager fallback", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, managerId, coderId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
updatedAt: issueTimestamp,
|
||||
})
|
||||
.where(eq(issues.id, blockerIssueId));
|
||||
await db.insert(budgetPolicies).values({
|
||||
companyId,
|
||||
scopeType: "agent",
|
||||
scopeId: coderId,
|
||||
metric: "billed_cents",
|
||||
windowKind: "calendar_month_utc",
|
||||
amount: 1,
|
||||
hardStopEnabled: true,
|
||||
isActive: true,
|
||||
});
|
||||
await db.insert(costEvents).values({
|
||||
companyId,
|
||||
agentId: coderId,
|
||||
issueId: blockerIssueId,
|
||||
provider: "test",
|
||||
biller: "test",
|
||||
billingType: "tokens",
|
||||
model: "test-model",
|
||||
costCents: 1,
|
||||
occurredAt: new Date(),
|
||||
});
|
||||
|
||||
const result = await heartbeatService(db).reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.escalationsCreated).toBe(1);
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(1);
|
||||
expect(escalations[0]).toMatchObject({
|
||||
parentId: blockerIssueId,
|
||||
assigneeAgentId: managerId,
|
||||
originId: [
|
||||
"harness_liveness",
|
||||
companyId,
|
||||
blockedIssueId,
|
||||
"in_review_without_action_path",
|
||||
blockerIssueId,
|
||||
].join(":"),
|
||||
});
|
||||
|
||||
const events = await db.select().from(activityLog).where(eq(activityLog.companyId, companyId));
|
||||
const createdEvent = events.find((event) => event.action === "issue.harness_liveness_escalation_created");
|
||||
expect(createdEvent?.details).toMatchObject({
|
||||
ownerSelection: {
|
||||
selectedAgentId: managerId,
|
||||
selectedReason: "assignee_reporting_chain",
|
||||
budgetBlockedCandidateAgentIds: [coderId],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("parents recovery under the leaf blocker without inheriting dependent or blocker execution state for manager-owned recovery", async () => {
|
||||
await enableAutoRecovery();
|
||||
await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
|
||||
|
|
|
|||
545
server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
Normal file
545
server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
Normal file
|
|
@ -0,0 +1,545 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
activityLog,
|
||||
agents,
|
||||
agentRuntimeState,
|
||||
agentWakeupRequests,
|
||||
companies,
|
||||
companySkills,
|
||||
createDb,
|
||||
documentRevisions,
|
||||
documents,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
issueRelations,
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
import { runningProcesses } from "../adapters/index.ts";
|
||||
|
||||
const mockAdapterExecute = vi.hoisted(() =>
|
||||
vi.fn(async () => ({
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorMessage: null,
|
||||
summary: "Stale-queue invalidation test run.",
|
||||
provider: "test",
|
||||
model: "test-model",
|
||||
})),
|
||||
);
|
||||
|
||||
vi.mock("../adapters/index.ts", async () => {
|
||||
const actual = await vi.importActual<typeof import("../adapters/index.ts")>("../adapters/index.ts");
|
||||
return {
|
||||
...actual,
|
||||
getServerAdapter: vi.fn(() => ({
|
||||
supportsLocalAgentJwt: false,
|
||||
execute: mockAdapterExecute,
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres heartbeat stale-queue invalidation tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function ensureIssueRelationsTable(db: ReturnType<typeof createDb>) {
|
||||
await db.execute(sql.raw(`
|
||||
CREATE TABLE IF NOT EXISTS "issue_relations" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
"company_id" uuid NOT NULL,
|
||||
"issue_id" uuid NOT NULL,
|
||||
"related_issue_id" uuid NOT NULL,
|
||||
"type" text NOT NULL,
|
||||
"created_by_agent_id" uuid,
|
||||
"created_by_user_id" text,
|
||||
"created_at" timestamptz NOT NULL DEFAULT now(),
|
||||
"updated_at" timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
`));
|
||||
}
|
||||
|
||||
async function waitForCondition(fn: () => Promise<boolean>, timeoutMs = 3_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (await fn()) return true;
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
return fn();
|
||||
}
|
||||
|
||||
type SeedOptions = {
|
||||
agentName?: string;
|
||||
agentRole?: string;
|
||||
maxConcurrentRuns?: number;
|
||||
};
|
||||
|
||||
type SeedResult = {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
};
|
||||
|
||||
describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let heartbeat!: ReturnType<typeof heartbeatService>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-stale-queue-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
heartbeat = heartbeatService(db);
|
||||
await ensureIssueRelationsTable(db);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
mockAdapterExecute.mockReset();
|
||||
mockAdapterExecute.mockImplementation(async () => ({
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorMessage: null,
|
||||
summary: "Stale-queue invalidation test run.",
|
||||
provider: "test",
|
||||
model: "test-model",
|
||||
}));
|
||||
runningProcesses.clear();
|
||||
let idlePolls = 0;
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
const runs = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns);
|
||||
const hasActiveRun = runs.some((run) => run.status === "queued" || run.status === "running");
|
||||
if (!hasActiveRun) {
|
||||
idlePolls += 1;
|
||||
if (idlePolls >= 3) break;
|
||||
} else {
|
||||
idlePolls = 0;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
await db.delete(companySkills);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issueDocuments);
|
||||
await db.delete(documentRevisions);
|
||||
await db.delete(documents);
|
||||
await db.delete(issueRelations);
|
||||
await db.delete(issueTreeHolds);
|
||||
await db.delete(issues);
|
||||
await db.delete(heartbeatRunEvents);
|
||||
await db.delete(activityLog);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(agentRuntimeState);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedCompanyAndAgent(opts: SeedOptions = {}): Promise<SeedResult> {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: opts.agentName ?? "ClaudeCoder",
|
||||
role: opts.agentRole ?? "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
wakeOnDemand: true,
|
||||
maxConcurrentRuns: opts.maxConcurrentRuns ?? 1,
|
||||
},
|
||||
},
|
||||
permissions: {},
|
||||
});
|
||||
return { companyId, agentId };
|
||||
}
|
||||
|
||||
async function seedQueuedRun(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
issueId: string;
|
||||
wakeReason: string;
|
||||
contextExtras?: Record<string, unknown>;
|
||||
invocationSource?: "assignment" | "automation";
|
||||
}) {
|
||||
const wakeupRequestId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
await db.insert(agentWakeupRequests).values({
|
||||
id: wakeupRequestId,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
source: input.invocationSource ?? "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: input.wakeReason,
|
||||
payload: { issueId: input.issueId },
|
||||
status: "queued",
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
invocationSource: input.invocationSource ?? "assignment",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: input.issueId,
|
||||
wakeReason: input.wakeReason,
|
||||
...(input.contextExtras ?? {}),
|
||||
},
|
||||
});
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId })
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId));
|
||||
return { runId, wakeupRequestId };
|
||||
}
|
||||
|
||||
it("cancels queued runs when the issue assignee changes before the run starts", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent({ agentName: "OriginalCoder" });
|
||||
const replacementAgentId = randomUUID();
|
||||
await db.insert(agents).values({
|
||||
id: replacementAgentId,
|
||||
companyId,
|
||||
name: "ReplacementCoder",
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
wakeOnDemand: true,
|
||||
maxConcurrentRuns: 1,
|
||||
},
|
||||
},
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Reassigned task",
|
||||
status: "in_progress",
|
||||
priority: "high",
|
||||
assigneeAgentId: replacementAgentId,
|
||||
});
|
||||
|
||||
const { runId, wakeupRequestId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
|
||||
const [run, wakeup] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
status: heartbeatRuns.status,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
resultJson: heartbeatRuns.resultJson,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
expect(run?.status).toBe("cancelled");
|
||||
expect(run?.errorCode).toBe("issue_assignee_changed");
|
||||
expect(run?.resultJson).toMatchObject({ stopReason: "issue_assignee_changed" });
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(wakeup?.error).toContain("assignee changed");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels queued runs when the issue reaches a terminal status before the run starts", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Already-completed task",
|
||||
status: "done",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
|
||||
const { runId, wakeupRequestId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
|
||||
const [run, wakeup] = await Promise.all([
|
||||
db
|
||||
.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({ status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
expect(run?.status).toBe("cancelled");
|
||||
expect(run?.errorCode).toBe("issue_terminal_status");
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels queued in_review runs when the current participant changes before the run starts", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const otherAgentId = randomUUID();
|
||||
await db.insert(agents).values({
|
||||
id: otherAgentId,
|
||||
companyId,
|
||||
name: "ReviewerAgent",
|
||||
role: "qa",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } },
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "In-review task now owned by reviewer",
|
||||
status: "in_review",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
executionState: {
|
||||
status: "pending",
|
||||
currentStageId: randomUUID(),
|
||||
currentStageIndex: 0,
|
||||
currentStageType: "review",
|
||||
currentParticipant: { type: "agent", agentId: otherAgentId, userId: null },
|
||||
returnAssignee: { type: "agent", agentId, userId: null },
|
||||
reviewRequest: null,
|
||||
completedStageIds: [],
|
||||
lastDecisionId: null,
|
||||
lastDecisionOutcome: null,
|
||||
},
|
||||
});
|
||||
|
||||
const { runId, wakeupRequestId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
|
||||
const [run, wakeup] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
status: heartbeatRuns.status,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
resultJson: heartbeatRuns.resultJson,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
expect(run?.status).toBe("cancelled");
|
||||
expect(run?.errorCode).toBe("issue_review_participant_changed");
|
||||
expect(run?.resultJson).toMatchObject({ stopReason: "issue_review_participant_changed" });
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(wakeup?.error).toContain("in-review participant changed");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still runs comment-driven wakes on in_review issues even when the agent is no longer the current participant", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const otherAgentId = randomUUID();
|
||||
await db.insert(agents).values({
|
||||
id: otherAgentId,
|
||||
companyId,
|
||||
name: "ReviewerAgent",
|
||||
role: "qa",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } },
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
const issueId = randomUUID();
|
||||
const commentId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "In-review task with comment feedback",
|
||||
status: "in_review",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
executionState: {
|
||||
status: "pending",
|
||||
currentStageId: randomUUID(),
|
||||
currentStageIndex: 0,
|
||||
currentStageType: "review",
|
||||
currentParticipant: { type: "agent", agentId: otherAgentId, userId: null },
|
||||
returnAssignee: { type: "agent", agentId, userId: null },
|
||||
reviewRequest: null,
|
||||
completedStageIds: [],
|
||||
lastDecisionId: null,
|
||||
lastDecisionOutcome: null,
|
||||
},
|
||||
});
|
||||
await db.insert(issueComments).values({
|
||||
id: commentId,
|
||||
companyId,
|
||||
issueId,
|
||||
authorAgentId: otherAgentId,
|
||||
body: "Review feedback comment",
|
||||
});
|
||||
|
||||
const { runId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_commented",
|
||||
invocationSource: "automation",
|
||||
contextExtras: {
|
||||
commentId,
|
||||
wakeCommentId: commentId,
|
||||
source: "issue.comment",
|
||||
},
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
});
|
||||
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(run?.status).toBe("succeeded");
|
||||
expect(run?.errorCode).toBeNull();
|
||||
});
|
||||
|
||||
it("baseline: runs queued runs when the issue is in_progress with the same assignee", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Still actionable",
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
|
||||
const { runId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
});
|
||||
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(run?.status).toBe("succeeded");
|
||||
expect(run?.errorCode).toBeNull();
|
||||
expect(mockAdapterExecute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -253,6 +253,109 @@ describeEmbeddedPostgres("issue blocker attention", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("flags a chain whose leaf is in_review without an action path as stalled", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBV");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBV-1", title: "Parent", status: "blocked" });
|
||||
const reviewLeafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBV-2",
|
||||
title: "Stalled review leaf",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: reviewLeafId, blockedIssueId: parentId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "stalled",
|
||||
reason: "stalled_review",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 0,
|
||||
stalledBlockerCount: 1,
|
||||
attentionBlockerCount: 0,
|
||||
sampleBlockerIdentifier: "PBV-2",
|
||||
sampleStalledBlockerIdentifier: "PBV-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not flag an in_review leaf as stalled when an active run is still progressing it", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBW");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBW-1", title: "Parent", status: "blocked" });
|
||||
const reviewLeafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBW-2",
|
||||
title: "Active review leaf",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: reviewLeafId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: reviewLeafId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "covered",
|
||||
stalledBlockerCount: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("flags a deep chain whose leaf is stalled in_review through multiple layers", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBZ");
|
||||
const rootId = await insertIssue({ companyId, identifier: "PBZ-1", title: "Root", status: "blocked" });
|
||||
const midId = await insertIssue({ companyId, identifier: "PBZ-2", title: "Mid blocker", status: "blocked" });
|
||||
const leafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBZ-3",
|
||||
title: "Stalled leaf",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: midId, blockedIssueId: rootId });
|
||||
await block({ companyId, blockerIssueId: leafId, blockedIssueId: midId });
|
||||
|
||||
const root = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === rootId);
|
||||
|
||||
expect(root?.blockerAttention).toMatchObject({
|
||||
state: "stalled",
|
||||
reason: "stalled_review",
|
||||
stalledBlockerCount: 1,
|
||||
sampleStalledBlockerIdentifier: "PBZ-3",
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers needs_attention over stalled when the chain also has a hard attention case", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBQ");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBQ-1", title: "Parent", status: "blocked" });
|
||||
const reviewLeafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBQ-2",
|
||||
title: "Stalled review leaf",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
const cancelledLeafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBQ-3",
|
||||
title: "Cancelled blocker",
|
||||
status: "cancelled",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: reviewLeafId, blockedIssueId: parentId });
|
||||
await block({ companyId, blockerIssueId: cancelledLeafId, blockedIssueId: parentId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
coveredBlockerCount: 0,
|
||||
stalledBlockerCount: 1,
|
||||
attentionBlockerCount: 1,
|
||||
sampleStalledBlockerIdentifier: "PBQ-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not treat a scheduled retry as actively covered work", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBY");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBY-1", title: "Parent", status: "blocked" });
|
||||
|
|
|
|||
|
|
@ -234,4 +234,191 @@ describe("issue graph liveness classifier", () => {
|
|||
incidentKey: `harness_liveness:${companyId}:${blockedId}:invalid_review_participant:missing-agent`,
|
||||
});
|
||||
});
|
||||
|
||||
it("detects the PAP-2239-style blocked chain at the first stalled in_review leaf without duplicate findings", () => {
|
||||
const phaseIssueId = "phase-issue-1";
|
||||
const reviewLeafId = "review-leaf-1";
|
||||
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
issue({
|
||||
id: "pap-2239",
|
||||
identifier: "PAP-2239",
|
||||
title: "External object reference project",
|
||||
status: "blocked",
|
||||
}),
|
||||
issue({
|
||||
id: phaseIssueId,
|
||||
identifier: "PAP-2276",
|
||||
title: "UX acceptance review phase",
|
||||
status: "blocked",
|
||||
assigneeAgentId: coderId,
|
||||
}),
|
||||
issue({
|
||||
id: reviewLeafId,
|
||||
identifier: "PAP-2279",
|
||||
title: "Screenshot acceptance review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
executionState: null,
|
||||
}),
|
||||
],
|
||||
relations: [
|
||||
{ companyId, blockerIssueId: phaseIssueId, blockedIssueId: "pap-2239" },
|
||||
{ companyId, blockerIssueId: reviewLeafId, blockedIssueId: phaseIssueId },
|
||||
],
|
||||
agents: [agent(), manager],
|
||||
});
|
||||
|
||||
expect(findings).toHaveLength(1);
|
||||
expect(findings[0]).toMatchObject({
|
||||
issueId: "pap-2239",
|
||||
identifier: "PAP-2239",
|
||||
state: "in_review_without_action_path",
|
||||
recoveryIssueId: reviewLeafId,
|
||||
recommendedOwnerAgentId: coderId,
|
||||
dependencyPath: [
|
||||
expect.objectContaining({ issueId: "pap-2239" }),
|
||||
expect.objectContaining({ issueId: phaseIssueId }),
|
||||
expect.objectContaining({ issueId: reviewLeafId }),
|
||||
],
|
||||
incidentKey: `harness_liveness:${companyId}:pap-2239:in_review_without_action_path:${reviewLeafId}`,
|
||||
});
|
||||
});
|
||||
|
||||
it("skips paused stalled review assignees when choosing recovery owner candidates", () => {
|
||||
const reviewIssueId = "review-1";
|
||||
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
issue({
|
||||
id: reviewIssueId,
|
||||
identifier: "PAP-2279",
|
||||
title: "Screenshot acceptance review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
executionState: null,
|
||||
}),
|
||||
],
|
||||
relations: [],
|
||||
agents: [agent({ status: "paused" }), manager],
|
||||
});
|
||||
|
||||
expect(findings).toHaveLength(1);
|
||||
expect(findings[0]).toMatchObject({
|
||||
state: "in_review_without_action_path",
|
||||
recommendedOwnerAgentId: managerId,
|
||||
});
|
||||
expect(findings[0]?.recommendedOwnerCandidates).toEqual([
|
||||
{
|
||||
agentId: managerId,
|
||||
reason: "assignee_reporting_chain",
|
||||
sourceIssueId: reviewIssueId,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not flag healthy in_review issues with an explicit action path", () => {
|
||||
const reviewIssueId = "review-1";
|
||||
const baseReviewIssue = issue({
|
||||
id: reviewIssueId,
|
||||
identifier: "PAP-2279",
|
||||
title: "Screenshot acceptance review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
executionState: null,
|
||||
});
|
||||
|
||||
const cases = [
|
||||
{
|
||||
name: "typed agent participant",
|
||||
issue: {
|
||||
...baseReviewIssue,
|
||||
executionState: {
|
||||
currentParticipant: { type: "agent", agentId: coderId },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "typed user participant",
|
||||
issue: {
|
||||
...baseReviewIssue,
|
||||
executionState: {
|
||||
currentParticipant: { type: "user", userId: "board-user-1" },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "user owner",
|
||||
issue: { ...baseReviewIssue, assigneeAgentId: null, assigneeUserId: "board-user-1" },
|
||||
},
|
||||
{
|
||||
name: "active run",
|
||||
issue: baseReviewIssue,
|
||||
activeRuns: [{ companyId, issueId: reviewIssueId, agentId: coderId, status: "running" }],
|
||||
},
|
||||
{
|
||||
name: "queued wake",
|
||||
issue: baseReviewIssue,
|
||||
queuedWakeRequests: [{ companyId, issueId: reviewIssueId, agentId: coderId, status: "queued" }],
|
||||
},
|
||||
{
|
||||
name: "pending interaction",
|
||||
issue: baseReviewIssue,
|
||||
pendingInteractions: [{ companyId, issueId: reviewIssueId, status: "pending" }],
|
||||
},
|
||||
{
|
||||
name: "pending approval",
|
||||
issue: baseReviewIssue,
|
||||
pendingApprovals: [{ companyId, issueId: reviewIssueId, status: "pending" }],
|
||||
},
|
||||
{
|
||||
name: "open recovery issue",
|
||||
issue: baseReviewIssue,
|
||||
openRecoveryIssues: [{ companyId, issueId: reviewIssueId, status: "todo" }],
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [testCase.issue],
|
||||
relations: [],
|
||||
agents: [agent(), manager],
|
||||
activeRuns: testCase.activeRuns,
|
||||
queuedWakeRequests: testCase.queuedWakeRequests,
|
||||
pendingInteractions: testCase.pendingInteractions,
|
||||
pendingApprovals: testCase.pendingApprovals,
|
||||
openRecoveryIssues: testCase.openRecoveryIssues,
|
||||
});
|
||||
|
||||
expect(findings, testCase.name).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores cross-company waiting paths for stalled in_review issues", () => {
|
||||
const reviewIssueId = "review-1";
|
||||
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
issue({
|
||||
id: reviewIssueId,
|
||||
identifier: "PAP-2279",
|
||||
title: "Screenshot acceptance review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
executionState: null,
|
||||
}),
|
||||
],
|
||||
relations: [],
|
||||
agents: [agent(), manager],
|
||||
pendingInteractions: [{ companyId: "other-company", issueId: reviewIssueId, status: "pending" }],
|
||||
openRecoveryIssues: [{ companyId: "other-company", issueId: reviewIssueId, status: "todo" }],
|
||||
});
|
||||
|
||||
expect(findings).toHaveLength(1);
|
||||
expect(findings[0]).toMatchObject({
|
||||
state: "in_review_without_action_path",
|
||||
recoveryIssueId: reviewIssueId,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ import {
|
|||
sanitizeRuntimeServiceBaseEnv,
|
||||
} from "./workspace-runtime.js";
|
||||
import { issueService } from "./issues.js";
|
||||
import { parseIssueExecutionState } from "./issue-execution-policy.js";
|
||||
import {
|
||||
ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS,
|
||||
isVerifiedIssueTreeControlInteractionWake,
|
||||
|
|
@ -3792,6 +3793,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
|||
logger.info({ runId: run.id, issueId, unresolvedBlockerCount }, "claimQueuedRun: cancelled blocked queued run");
|
||||
return null;
|
||||
}
|
||||
|
||||
const staleness = await evaluateQueuedRunStaleness(run, issueId, context);
|
||||
if (staleness.stale) {
|
||||
await cancelQueuedRunForStaleIssue(run, issueId, staleness);
|
||||
logger.info(
|
||||
{ runId: run.id, issueId, errorCode: staleness.errorCode },
|
||||
"claimQueuedRun: cancelled stale queued run",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const claimedAt = new Date();
|
||||
|
|
@ -3912,6 +3923,151 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
|||
return cancelled;
|
||||
}
|
||||
|
||||
type QueuedRunStaleness =
|
||||
| { stale: false }
|
||||
| {
|
||||
stale: true;
|
||||
reason: string;
|
||||
errorCode:
|
||||
| "issue_not_found"
|
||||
| "issue_assignee_changed"
|
||||
| "issue_terminal_status"
|
||||
| "issue_review_participant_changed";
|
||||
details: Record<string, unknown>;
|
||||
};
|
||||
|
||||
async function evaluateQueuedRunStaleness(
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
issueId: string,
|
||||
context: Record<string, unknown>,
|
||||
): Promise<QueuedRunStaleness> {
|
||||
const issue = await db
|
||||
.select({
|
||||
id: issues.id,
|
||||
status: issues.status,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionState: issues.executionState,
|
||||
})
|
||||
.from(issues)
|
||||
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
if (!issue) {
|
||||
return {
|
||||
stale: true,
|
||||
errorCode: "issue_not_found",
|
||||
reason: "Cancelled because the target issue no longer exists",
|
||||
details: { issueId },
|
||||
};
|
||||
}
|
||||
|
||||
const wakeCommentId = deriveCommentId(context, null);
|
||||
const isInteractionWake = allowsIssueInteractionWake(context);
|
||||
const resumeIntent = context.resumeIntent === true || context.followUpRequested === true;
|
||||
|
||||
if (issue.assigneeAgentId !== run.agentId && !isInteractionWake) {
|
||||
return {
|
||||
stale: true,
|
||||
errorCode: "issue_assignee_changed",
|
||||
reason:
|
||||
"Cancelled because issue assignee changed before the queued run could start; the new owner will be woken instead",
|
||||
details: {
|
||||
issueId,
|
||||
previousAssigneeAgentId: run.agentId,
|
||||
currentAssigneeAgentId: issue.assigneeAgentId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (issue.status === "done" || issue.status === "cancelled") {
|
||||
if (!resumeIntent && !wakeCommentId) {
|
||||
return {
|
||||
stale: true,
|
||||
errorCode: "issue_terminal_status",
|
||||
reason: `Cancelled because issue reached terminal status (${issue.status}) before the queued run could start`,
|
||||
details: { issueId, currentStatus: issue.status },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (issue.status === "in_review") {
|
||||
const executionState = parseIssueExecutionState(issue.executionState);
|
||||
const currentParticipant = executionState?.currentParticipant ?? null;
|
||||
if (currentParticipant) {
|
||||
const participantMatches =
|
||||
currentParticipant.type === "agent" && currentParticipant.agentId === run.agentId;
|
||||
if (!participantMatches && !wakeCommentId) {
|
||||
return {
|
||||
stale: true,
|
||||
errorCode: "issue_review_participant_changed",
|
||||
reason:
|
||||
"Cancelled because the in-review participant changed before the queued run could start; the current participant will be woken instead",
|
||||
details: {
|
||||
issueId,
|
||||
currentStageType: executionState?.currentStageType ?? null,
|
||||
currentParticipant,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { stale: false };
|
||||
}
|
||||
|
||||
async function cancelQueuedRunForStaleIssue(
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
issueId: string,
|
||||
staleness: Extract<QueuedRunStaleness, { stale: true }>,
|
||||
) {
|
||||
const now = new Date();
|
||||
const cancelled = await setRunStatus(run.id, "cancelled", {
|
||||
finishedAt: now,
|
||||
error: staleness.reason,
|
||||
errorCode: staleness.errorCode,
|
||||
resultJson: {
|
||||
...parseObject(run.resultJson),
|
||||
stopReason: staleness.errorCode,
|
||||
effectiveTimeoutSec: 0,
|
||||
timeoutConfigured: false,
|
||||
timeoutSource: "stale_queued_run_gate",
|
||||
timeoutFired: false,
|
||||
},
|
||||
});
|
||||
if (!cancelled) return null;
|
||||
|
||||
await setWakeupStatus(run.wakeupRequestId, "skipped", {
|
||||
finishedAt: now,
|
||||
error: staleness.reason,
|
||||
});
|
||||
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(issues.companyId, run.companyId),
|
||||
eq(issues.id, issueId),
|
||||
eq(issues.executionRunId, run.id),
|
||||
),
|
||||
);
|
||||
|
||||
await appendRunEvent(cancelled, await nextRunEventSeq(cancelled.id), {
|
||||
eventType: "lifecycle",
|
||||
stream: "system",
|
||||
level: "warn",
|
||||
message: staleness.reason,
|
||||
payload: staleness.details,
|
||||
});
|
||||
|
||||
return cancelled;
|
||||
}
|
||||
|
||||
async function finalizeAgentStatus(
|
||||
agentId: string,
|
||||
outcome: "succeeded" | "failed" | "cancelled" | "timed_out",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import { Buffer } from "node:buffer";
|
||||
import { and, asc, desc, eq, gt, inArray, isNull, lt, ne, or, sql } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, gt, inArray, isNull, lt, ne, notInArray, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
activityLog,
|
||||
agentWakeupRequests,
|
||||
agents,
|
||||
approvals,
|
||||
assets,
|
||||
companies,
|
||||
companyMemberships,
|
||||
|
|
@ -12,6 +13,7 @@ import {
|
|||
goals,
|
||||
heartbeatRuns,
|
||||
executionWorkspaces,
|
||||
issueApprovals,
|
||||
issueAttachments,
|
||||
issueInboxArchives,
|
||||
issueLabels,
|
||||
|
|
@ -19,6 +21,7 @@ import {
|
|||
issueComments,
|
||||
issueDocuments,
|
||||
issueReadStates,
|
||||
issueThreadInteractions,
|
||||
issues,
|
||||
labels,
|
||||
projectWorkspaces,
|
||||
|
|
@ -660,6 +663,10 @@ async function withIssueLabels(dbOrTx: any, rows: IssueRow[]): Promise<IssueWith
|
|||
const ACTIVE_RUN_STATUSES = ["queued", "running"];
|
||||
const BLOCKER_ATTENTION_ACTIVE_RUN_STATUSES = ["queued", "running"];
|
||||
const BLOCKER_ATTENTION_ACTIVE_WAKE_STATUSES = ["queued", "deferred_issue_execution"];
|
||||
const BLOCKER_ATTENTION_PENDING_INTERACTION_STATUSES = ["pending"];
|
||||
const BLOCKER_ATTENTION_PENDING_APPROVAL_STATUSES = ["pending", "revision_requested"];
|
||||
const BLOCKER_ATTENTION_OPEN_RECOVERY_ORIGIN_KIND = "harness_liveness_escalation";
|
||||
const BLOCKER_ATTENTION_OPEN_RECOVERY_TERMINAL_STATUSES = ["done", "cancelled"];
|
||||
const BLOCKER_ATTENTION_MAX_DEPTH = 8;
|
||||
const BLOCKER_ATTENTION_MAX_NODES = 2000;
|
||||
const BLOCKER_ATTENTION_INVOKABLE_AGENT_STATUSES = new Set(["active", "idle", "running", "error"]);
|
||||
|
|
@ -742,8 +749,10 @@ function createIssueBlockerAttention(input: Partial<IssueBlockerAttention> = {})
|
|||
reason: input.reason ?? null,
|
||||
unresolvedBlockerCount: input.unresolvedBlockerCount ?? 0,
|
||||
coveredBlockerCount: input.coveredBlockerCount ?? 0,
|
||||
stalledBlockerCount: input.stalledBlockerCount ?? 0,
|
||||
attentionBlockerCount: input.attentionBlockerCount ?? 0,
|
||||
sampleBlockerIdentifier: input.sampleBlockerIdentifier ?? null,
|
||||
sampleStalledBlockerIdentifier: input.sampleStalledBlockerIdentifier ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -1026,6 +1035,55 @@ async function listIssueBlockerAttentionMap(
|
|||
}
|
||||
}
|
||||
|
||||
const reviewNodeIds = [...nodesById.values()]
|
||||
.filter((node) => node.status === "in_review")
|
||||
.map((node) => node.id);
|
||||
const explicitWaitingIssueIds = new Set<string>();
|
||||
if (reviewNodeIds.length > 0) {
|
||||
for (const chunk of chunkList(reviewNodeIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
|
||||
const interactionRows: Array<{ issueId: string }> = await dbOrTx
|
||||
.select({ issueId: issueThreadInteractions.issueId })
|
||||
.from(issueThreadInteractions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueThreadInteractions.companyId, companyId),
|
||||
inArray(issueThreadInteractions.status, BLOCKER_ATTENTION_PENDING_INTERACTION_STATUSES),
|
||||
inArray(issueThreadInteractions.issueId, chunk),
|
||||
),
|
||||
);
|
||||
for (const row of interactionRows) explicitWaitingIssueIds.add(row.issueId);
|
||||
|
||||
const approvalRows: Array<{ issueId: string }> = await dbOrTx
|
||||
.select({ issueId: issueApprovals.issueId })
|
||||
.from(issueApprovals)
|
||||
.innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))
|
||||
.where(
|
||||
and(
|
||||
eq(issueApprovals.companyId, companyId),
|
||||
inArray(approvals.status, BLOCKER_ATTENTION_PENDING_APPROVAL_STATUSES),
|
||||
inArray(issueApprovals.issueId, chunk),
|
||||
),
|
||||
);
|
||||
for (const row of approvalRows) explicitWaitingIssueIds.add(row.issueId);
|
||||
|
||||
const recoveryRows: Array<{ originId: string | null }> = await dbOrTx
|
||||
.select({ originId: issues.originId })
|
||||
.from(issues)
|
||||
.where(
|
||||
and(
|
||||
eq(issues.companyId, companyId),
|
||||
eq(issues.originKind, BLOCKER_ATTENTION_OPEN_RECOVERY_ORIGIN_KIND),
|
||||
isNull(issues.hiddenAt),
|
||||
inArray(issues.originId, chunk),
|
||||
notInArray(issues.status, BLOCKER_ATTENTION_OPEN_RECOVERY_TERMINAL_STATUSES),
|
||||
),
|
||||
);
|
||||
for (const row of recoveryRows) {
|
||||
if (row.originId) explicitWaitingIssueIds.add(row.originId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const agentRows: IssueBlockerAttentionAgentRow[] = agentIds.size > 0
|
||||
? await dbOrTx
|
||||
.select({
|
||||
|
|
@ -1038,39 +1096,83 @@ async function listIssueBlockerAttentionMap(
|
|||
: [];
|
||||
const agentsById = new Map(agentRows.map((agent) => [agent.id, agent]));
|
||||
|
||||
type PathClassification = { covered: boolean; sampleBlockerIdentifier: string | null };
|
||||
type PathClassification = {
|
||||
covered: boolean;
|
||||
stalled: boolean;
|
||||
sampleBlockerIdentifier: string | null;
|
||||
sampleStalledBlockerIdentifier: string | null;
|
||||
};
|
||||
const classifyPath = (
|
||||
nodeId: string,
|
||||
seen: Set<string>,
|
||||
): PathClassification => {
|
||||
if (truncated || seen.has(nodeId)) return { covered: false, sampleBlockerIdentifier: blockerSampleIdentifier(nodesById.get(nodeId)) };
|
||||
const sample = blockerSampleIdentifier(nodesById.get(nodeId));
|
||||
if (truncated || seen.has(nodeId)) {
|
||||
return { covered: false, stalled: false, sampleBlockerIdentifier: sample, sampleStalledBlockerIdentifier: null };
|
||||
}
|
||||
const node = nodesById.get(nodeId);
|
||||
if (!node || node.companyId !== companyId) return { covered: false, sampleBlockerIdentifier: nodeId };
|
||||
if (node.status === "done") return { covered: true, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
if (activeIssueIds.has(node.id)) return { covered: true, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
if (node.status === "cancelled") return { covered: false, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
if (!node || node.companyId !== companyId) {
|
||||
return { covered: false, stalled: false, sampleBlockerIdentifier: nodeId, sampleStalledBlockerIdentifier: null };
|
||||
}
|
||||
const nodeSample = blockerSampleIdentifier(node);
|
||||
if (node.status === "done") {
|
||||
return { covered: true, stalled: false, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: null };
|
||||
}
|
||||
if (node.status === "in_review") {
|
||||
const hasWaitingPath = activeIssueIds.has(node.id) || Boolean(node.assigneeUserId) || explicitWaitingIssueIds.has(node.id);
|
||||
if (hasWaitingPath) {
|
||||
return { covered: true, stalled: false, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: null };
|
||||
}
|
||||
return { covered: false, stalled: true, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: nodeSample };
|
||||
}
|
||||
if (activeIssueIds.has(node.id)) {
|
||||
return { covered: true, stalled: false, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: null };
|
||||
}
|
||||
if (node.status === "cancelled") {
|
||||
return { covered: false, stalled: false, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: null };
|
||||
}
|
||||
|
||||
const downstream = (edgesByIssueId.get(node.id) ?? []).filter((edge) => nodesById.get(edge.blockerIssueId)?.status !== "done");
|
||||
if (downstream.length > 0) {
|
||||
const nextSeen = new Set(seen);
|
||||
nextSeen.add(nodeId);
|
||||
const classified = downstream.map((edge) => classifyPath(edge.blockerIssueId, nextSeen));
|
||||
const attention = classified.find((result) => !result.covered);
|
||||
if (attention) return attention;
|
||||
const stalledChild = classified.find((result) => result.stalled || result.sampleStalledBlockerIdentifier);
|
||||
const sampleStalled = stalledChild?.sampleStalledBlockerIdentifier ?? null;
|
||||
const hardAttention = classified.find((result) => !result.covered && !result.stalled);
|
||||
if (hardAttention) {
|
||||
return {
|
||||
covered: false,
|
||||
stalled: false,
|
||||
sampleBlockerIdentifier: hardAttention.sampleBlockerIdentifier,
|
||||
sampleStalledBlockerIdentifier: sampleStalled,
|
||||
};
|
||||
}
|
||||
const stalledEntry = classified.find((result) => result.stalled);
|
||||
if (stalledEntry) {
|
||||
return {
|
||||
covered: false,
|
||||
stalled: true,
|
||||
sampleBlockerIdentifier: stalledEntry.sampleBlockerIdentifier,
|
||||
sampleStalledBlockerIdentifier: sampleStalled,
|
||||
};
|
||||
}
|
||||
return {
|
||||
covered: true,
|
||||
sampleBlockerIdentifier: classified[0]?.sampleBlockerIdentifier ?? blockerSampleIdentifier(node),
|
||||
stalled: false,
|
||||
sampleBlockerIdentifier: classified[0]?.sampleBlockerIdentifier ?? nodeSample,
|
||||
sampleStalledBlockerIdentifier: null,
|
||||
};
|
||||
}
|
||||
|
||||
if (node.assigneeAgentId) {
|
||||
const assignee = agentsById.get(node.assigneeAgentId);
|
||||
if (!assignee || assignee.companyId !== companyId || !BLOCKER_ATTENTION_INVOKABLE_AGENT_STATUSES.has(assignee.status)) {
|
||||
return { covered: false, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
return { covered: false, stalled: false, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: null };
|
||||
}
|
||||
}
|
||||
|
||||
return { covered: false, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
return { covered: false, stalled: false, sampleBlockerIdentifier: nodeSample, sampleStalledBlockerIdentifier: null };
|
||||
};
|
||||
|
||||
for (const root of roots) {
|
||||
|
|
@ -1088,22 +1190,41 @@ async function listIssueBlockerAttentionMap(
|
|||
result: classifyPath(edge.blockerIssueId, new Set([root.id])),
|
||||
}));
|
||||
const coveredBlockerCount = classified.filter((entry) => entry.result.covered).length;
|
||||
const attentionBlockerCount = classified.length - coveredBlockerCount;
|
||||
const attentionEntry = classified.find((entry) => !entry.result.covered);
|
||||
const sampleEntry = attentionEntry ?? classified[0] ?? null;
|
||||
const stalledBlockerCount = classified.filter((entry) => entry.result.stalled).length;
|
||||
const attentionBlockerCount = classified.length - coveredBlockerCount - stalledBlockerCount;
|
||||
const hardAttentionEntry = classified.find((entry) => !entry.result.covered && !entry.result.stalled);
|
||||
const stalledEntry = classified.find((entry) => entry.result.stalled);
|
||||
const sampleEntry = hardAttentionEntry ?? stalledEntry ?? classified[0] ?? null;
|
||||
const sampleNode = sampleEntry ? nodesById.get(sampleEntry.edge.blockerIssueId) : null;
|
||||
const sampleStalledFromChain = classified
|
||||
.map((entry) => entry.result.sampleStalledBlockerIdentifier)
|
||||
.find((value) => value);
|
||||
|
||||
let state: IssueBlockerAttention["state"];
|
||||
let reason: IssueBlockerAttention["reason"];
|
||||
if (attentionBlockerCount > 0) {
|
||||
state = "needs_attention";
|
||||
reason = "attention_required";
|
||||
} else if (stalledBlockerCount > 0) {
|
||||
state = "stalled";
|
||||
reason = "stalled_review";
|
||||
} else {
|
||||
state = "covered";
|
||||
reason = topLevelEdges.every((edge) => nodesById.get(edge.blockerIssueId)?.parentId === root.id)
|
||||
? "active_child"
|
||||
: "active_dependency";
|
||||
}
|
||||
|
||||
attentionMap.set(root.id, createIssueBlockerAttention({
|
||||
state: attentionBlockerCount === 0 ? "covered" : "needs_attention",
|
||||
reason: attentionBlockerCount === 0
|
||||
? topLevelEdges.every((edge) => nodesById.get(edge.blockerIssueId)?.parentId === root.id)
|
||||
? "active_child"
|
||||
: "active_dependency"
|
||||
: "attention_required",
|
||||
state,
|
||||
reason,
|
||||
unresolvedBlockerCount: topLevelEdges.length,
|
||||
coveredBlockerCount,
|
||||
stalledBlockerCount,
|
||||
attentionBlockerCount,
|
||||
sampleBlockerIdentifier: sampleEntry?.result.sampleBlockerIdentifier ?? blockerSampleIdentifier(sampleNode),
|
||||
sampleStalledBlockerIdentifier:
|
||||
stalledEntry?.result.sampleStalledBlockerIdentifier ?? sampleStalledFromChain ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@ export type IssueLivenessState =
|
|||
| "blocked_by_unassigned_issue"
|
||||
| "blocked_by_uninvokable_assignee"
|
||||
| "blocked_by_cancelled_issue"
|
||||
| "invalid_review_participant";
|
||||
| "invalid_review_participant"
|
||||
| "in_review_without_action_path";
|
||||
|
||||
export interface IssueLivenessIssueInput {
|
||||
id: string;
|
||||
|
|
@ -47,6 +48,12 @@ export interface IssueLivenessExecutionPathInput {
|
|||
status: string;
|
||||
}
|
||||
|
||||
export interface IssueLivenessWaitingPathInput {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface IssueLivenessDependencyPathEntry {
|
||||
issueId: string;
|
||||
identifier: string | null;
|
||||
|
|
@ -89,6 +96,9 @@ export interface IssueGraphLivenessInput {
|
|||
agents: IssueLivenessAgentInput[];
|
||||
activeRuns?: IssueLivenessExecutionPathInput[];
|
||||
queuedWakeRequests?: IssueLivenessExecutionPathInput[];
|
||||
pendingInteractions?: IssueLivenessWaitingPathInput[];
|
||||
pendingApprovals?: IssueLivenessWaitingPathInput[];
|
||||
openRecoveryIssues?: IssueLivenessWaitingPathInput[];
|
||||
}
|
||||
|
||||
const INVOKABLE_AGENT_STATUSES = new Set(["active", "idle", "running", "error"]);
|
||||
|
|
@ -122,6 +132,14 @@ function hasActiveExecutionPath(
|
|||
);
|
||||
}
|
||||
|
||||
function hasWaitingPath(
|
||||
companyId: string,
|
||||
issueId: string,
|
||||
waitingPaths: IssueLivenessWaitingPathInput[],
|
||||
) {
|
||||
return waitingPaths.some((entry) => entry.companyId === companyId && entry.issueId === issueId);
|
||||
}
|
||||
|
||||
function readPrincipalAgentId(principal: unknown): string | null {
|
||||
if (!principal || typeof principal !== "object") return null;
|
||||
const value = principal as Record<string, unknown>;
|
||||
|
|
@ -293,120 +311,225 @@ export function classifyIssueGraphLiveness(input: IssueGraphLivenessInput): Issu
|
|||
const issuesById = new Map(input.issues.map((issue) => [issue.id, issue]));
|
||||
const agentsById = new Map(input.agents.map((agent) => [agent.id, agent]));
|
||||
const blockersByBlockedIssueId = new Map<string, IssueLivenessRelationInput[]>();
|
||||
const unresolvedBlockers = new Set<string>();
|
||||
const findings: IssueLivenessFinding[] = [];
|
||||
const activeRuns = input.activeRuns ?? [];
|
||||
const queuedWakeRequests = input.queuedWakeRequests ?? [];
|
||||
const pendingInteractions = input.pendingInteractions ?? [];
|
||||
const pendingApprovals = input.pendingApprovals ?? [];
|
||||
const openRecoveryIssues = input.openRecoveryIssues ?? [];
|
||||
|
||||
for (const relation of input.relations) {
|
||||
const list = blockersByBlockedIssueId.get(relation.blockedIssueId) ?? [];
|
||||
list.push(relation);
|
||||
blockersByBlockedIssueId.set(relation.blockedIssueId, list);
|
||||
|
||||
const blocker = issuesById.get(relation.blockerIssueId);
|
||||
const blocked = issuesById.get(relation.blockedIssueId);
|
||||
if (
|
||||
blocker &&
|
||||
blocked &&
|
||||
blocker.companyId === relation.companyId &&
|
||||
blocked.companyId === relation.companyId &&
|
||||
blocker.status !== "done" &&
|
||||
blocker.status !== "cancelled" &&
|
||||
blocked.status === "blocked"
|
||||
) {
|
||||
unresolvedBlockers.add(blocker.id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const relations of blockersByBlockedIssueId.values()) {
|
||||
relations.sort((left, right) => {
|
||||
const leftIssue = issuesById.get(left.blockerIssueId);
|
||||
const rightIssue = issuesById.get(right.blockerIssueId);
|
||||
const leftLabel = leftIssue ? issueLabel(leftIssue) : left.blockerIssueId;
|
||||
const rightLabel = rightIssue ? issueLabel(rightIssue) : right.blockerIssueId;
|
||||
return leftLabel.localeCompare(rightLabel);
|
||||
});
|
||||
}
|
||||
|
||||
function hasExplicitWaitingPath(issue: IssueLivenessIssueInput) {
|
||||
return Boolean(issue.assigneeUserId) ||
|
||||
hasActiveExecutionPath(issue.companyId, issue.id, activeRuns, queuedWakeRequests) ||
|
||||
hasWaitingPath(issue.companyId, issue.id, pendingInteractions) ||
|
||||
hasWaitingPath(issue.companyId, issue.id, pendingApprovals) ||
|
||||
hasWaitingPath(issue.companyId, issue.id, openRecoveryIssues);
|
||||
}
|
||||
|
||||
function reviewFinding(
|
||||
source: IssueLivenessIssueInput,
|
||||
reviewIssue: IssueLivenessIssueInput,
|
||||
dependencyPath: IssueLivenessIssueInput[],
|
||||
): IssueLivenessFinding | null {
|
||||
if (reviewIssue.status !== "in_review") return null;
|
||||
if (hasExplicitWaitingPath(reviewIssue)) return null;
|
||||
|
||||
const ownerCandidates = ownerCandidatesForRecoveryIssue(reviewIssue, input.agents, agentsById, {
|
||||
includeStalledAssignee: true,
|
||||
});
|
||||
|
||||
const participant = reviewIssue.executionState?.currentParticipant;
|
||||
const participantAgentId = readPrincipalAgentId(participant);
|
||||
if (participantAgentId) {
|
||||
const participantAgent = agentsById.get(participantAgentId);
|
||||
if (isInvokableAgent(participantAgent) && participantAgent?.companyId === reviewIssue.companyId) return null;
|
||||
|
||||
return finding({
|
||||
issue: source,
|
||||
state: "invalid_review_participant",
|
||||
reason: participantAgent
|
||||
? `${issueLabel(reviewIssue)} is in review, but current participant agent is ${participantAgent.status}.`
|
||||
: `${issueLabel(reviewIssue)} is in review, but current participant agent cannot be resolved.`,
|
||||
dependencyPath,
|
||||
recoveryIssue: reviewIssue,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Repair ${issueLabel(reviewIssue)}'s review participant or return the issue to an active assignee with a clear change request.`,
|
||||
participantAgentId,
|
||||
});
|
||||
}
|
||||
|
||||
if (principalIsResolvableUser(participant)) return null;
|
||||
|
||||
if (reviewIssue.executionState) {
|
||||
return finding({
|
||||
issue: source,
|
||||
state: "invalid_review_participant",
|
||||
reason: `${issueLabel(reviewIssue)} is in review, but its current participant cannot be resolved.`,
|
||||
dependencyPath,
|
||||
recoveryIssue: reviewIssue,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Repair ${issueLabel(reviewIssue)}'s review participant or return the issue to an active assignee with a clear change request.`,
|
||||
});
|
||||
}
|
||||
|
||||
if (!reviewIssue.assigneeAgentId || reviewIssue.assigneeUserId) return null;
|
||||
|
||||
return finding({
|
||||
issue: source,
|
||||
state: "in_review_without_action_path",
|
||||
reason: `${issueLabel(reviewIssue)} is in review with an agent assignee but no participant, interaction, approval, user owner, wake, active run, or recovery issue owning the next action.`,
|
||||
dependencyPath,
|
||||
recoveryIssue: reviewIssue,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Review ${issueLabel(reviewIssue)} and make the next action explicit: add a reviewer/interaction, return it to active work with a change request, mark it done if accepted, or open a bounded recovery issue.`,
|
||||
blockerIssueId: reviewIssue.id,
|
||||
});
|
||||
}
|
||||
|
||||
function blockedFindingForLeaf(
|
||||
source: IssueLivenessIssueInput,
|
||||
blocker: IssueLivenessIssueInput,
|
||||
dependencyPath: IssueLivenessIssueInput[],
|
||||
): IssueLivenessFinding | null {
|
||||
const ownerCandidates = ownerCandidatesForRecoveryIssue(blocker, input.agents, agentsById, {
|
||||
includeStalledAssignee: true,
|
||||
});
|
||||
|
||||
if (blocker.status === "cancelled") {
|
||||
return finding({
|
||||
issue: source,
|
||||
state: "blocked_by_cancelled_issue",
|
||||
reason: `${issueLabel(source)} is still blocked by cancelled issue ${issueLabel(blocker)}.`,
|
||||
dependencyPath,
|
||||
recoveryIssue: blocker,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Inspect ${issueLabel(blocker)} and either remove it from ${issueLabel(source)}'s blockers or replace it with an actionable unblock issue.`,
|
||||
blockerIssueId: blocker.id,
|
||||
});
|
||||
}
|
||||
|
||||
if (hasExplicitWaitingPath(blocker)) return null;
|
||||
|
||||
if (blocker.status === "in_review") {
|
||||
return reviewFinding(source, blocker, dependencyPath);
|
||||
}
|
||||
|
||||
if (!blocker.assigneeAgentId && !blocker.assigneeUserId) {
|
||||
return finding({
|
||||
issue: source,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
reason: `${issueLabel(source)} is blocked by unassigned issue ${issueLabel(blocker)} with no user owner.`,
|
||||
dependencyPath,
|
||||
recoveryIssue: blocker,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Assign ${issueLabel(blocker)} to an owner who can complete it, or remove it from ${issueLabel(source)}'s blockers if it is no longer required.`,
|
||||
blockerIssueId: blocker.id,
|
||||
});
|
||||
}
|
||||
|
||||
if (!blocker.assigneeAgentId) return null;
|
||||
|
||||
const blockerAgent = agentsById.get(blocker.assigneeAgentId);
|
||||
if (!blockerAgent || blockerAgent.companyId !== source.companyId || BLOCKING_AGENT_STATUSES.has(blockerAgent.status)) {
|
||||
return finding({
|
||||
issue: source,
|
||||
state: "blocked_by_uninvokable_assignee",
|
||||
reason: blockerAgent
|
||||
? `${issueLabel(source)} is blocked by ${issueLabel(blocker)}, but its assignee is ${blockerAgent.status}.`
|
||||
: `${issueLabel(source)} is blocked by ${issueLabel(blocker)}, but its assignee no longer exists.`,
|
||||
dependencyPath,
|
||||
recoveryIssue: blocker,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Review ${issueLabel(blocker)} and assign it to an active owner or replace the blocker with an actionable issue.`,
|
||||
blockerIssueId: blocker.id,
|
||||
});
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function firstBlockedChainFinding(
|
||||
source: IssueLivenessIssueInput,
|
||||
current: IssueLivenessIssueInput,
|
||||
dependencyPath: IssueLivenessIssueInput[],
|
||||
seen: Set<string>,
|
||||
): IssueLivenessFinding | null {
|
||||
if (seen.has(current.id)) return null;
|
||||
seen.add(current.id);
|
||||
|
||||
const relations = blockersByBlockedIssueId.get(current.id) ?? [];
|
||||
for (const relation of relations) {
|
||||
if (relation.companyId !== current.companyId || relation.companyId !== source.companyId) continue;
|
||||
const blocker = issuesById.get(relation.blockerIssueId);
|
||||
if (!blocker || blocker.companyId !== source.companyId || blocker.status === "done") continue;
|
||||
const path = [...dependencyPath, blocker];
|
||||
|
||||
if (blocker.status === "blocked") {
|
||||
const nested = firstBlockedChainFinding(source, blocker, path, new Set(seen));
|
||||
if (nested) return nested;
|
||||
if (hasExplicitWaitingPath(blocker)) continue;
|
||||
}
|
||||
|
||||
const leafFinding = blockedFindingForLeaf(source, blocker, path);
|
||||
if (leafFinding) return leafFinding;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
for (const issue of input.issues) {
|
||||
if (issue.status === "blocked") {
|
||||
const relations = blockersByBlockedIssueId.get(issue.id) ?? [];
|
||||
for (const relation of relations) {
|
||||
if (relation.companyId !== issue.companyId) continue;
|
||||
const blocker = issuesById.get(relation.blockerIssueId);
|
||||
if (!blocker || blocker.companyId !== issue.companyId || blocker.status === "done") continue;
|
||||
const ownerCandidates = ownerCandidatesForRecoveryIssue(blocker, input.agents, agentsById, {
|
||||
includeStalledAssignee: true,
|
||||
});
|
||||
|
||||
if (blocker.status === "cancelled") {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "blocked_by_cancelled_issue",
|
||||
reason: `${issueLabel(issue)} is still blocked by cancelled issue ${issueLabel(blocker)}.`,
|
||||
dependencyPath: [issue, blocker],
|
||||
recoveryIssue: blocker,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Inspect ${issueLabel(blocker)} and either remove it from ${issueLabel(issue)}'s blockers or replace it with an actionable unblock issue.`,
|
||||
blockerIssueId: blocker.id,
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!blocker.assigneeAgentId && !blocker.assigneeUserId) {
|
||||
if (hasActiveExecutionPath(issue.companyId, blocker.id, activeRuns, queuedWakeRequests)) continue;
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
reason: `${issueLabel(issue)} is blocked by unassigned issue ${issueLabel(blocker)} with no user owner.`,
|
||||
dependencyPath: [issue, blocker],
|
||||
recoveryIssue: blocker,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Assign ${issueLabel(blocker)} to an owner who can complete it, or remove it from ${issueLabel(issue)}'s blockers if it is no longer required.`,
|
||||
blockerIssueId: blocker.id,
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!blocker.assigneeAgentId) continue;
|
||||
if (hasActiveExecutionPath(issue.companyId, blocker.id, activeRuns, queuedWakeRequests)) continue;
|
||||
|
||||
const blockerAgent = agentsById.get(blocker.assigneeAgentId);
|
||||
if (!blockerAgent || blockerAgent.companyId !== issue.companyId || BLOCKING_AGENT_STATUSES.has(blockerAgent.status)) {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "blocked_by_uninvokable_assignee",
|
||||
reason: blockerAgent
|
||||
? `${issueLabel(issue)} is blocked by ${issueLabel(blocker)}, but its assignee is ${blockerAgent.status}.`
|
||||
: `${issueLabel(issue)} is blocked by ${issueLabel(blocker)}, but its assignee no longer exists.`,
|
||||
dependencyPath: [issue, blocker],
|
||||
recoveryIssue: blocker,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Review ${issueLabel(blocker)} and assign it to an active owner or replace the blocker with an actionable issue.`,
|
||||
blockerIssueId: blocker.id,
|
||||
}));
|
||||
}
|
||||
}
|
||||
if (unresolvedBlockers.has(issue.id)) continue;
|
||||
const chainFinding = firstBlockedChainFinding(issue, issue, [issue], new Set());
|
||||
if (chainFinding) findings.push(chainFinding);
|
||||
}
|
||||
|
||||
if (issue.status !== "in_review" || !issue.executionState) continue;
|
||||
const ownerCandidates = ownerCandidatesForRecoveryIssue(issue, input.agents, agentsById);
|
||||
const participant = issue.executionState.currentParticipant;
|
||||
const participantAgentId = readPrincipalAgentId(participant);
|
||||
if (participantAgentId) {
|
||||
const participantAgent = agentsById.get(participantAgentId);
|
||||
if (!isInvokableAgent(participantAgent) || participantAgent?.companyId !== issue.companyId) {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "invalid_review_participant",
|
||||
reason: participantAgent
|
||||
? `${issueLabel(issue)} is in review, but current participant agent is ${participantAgent.status}.`
|
||||
: `${issueLabel(issue)} is in review, but current participant agent cannot be resolved.`,
|
||||
dependencyPath: [issue],
|
||||
recoveryIssue: issue,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Repair ${issueLabel(issue)}'s review participant or return the issue to an active assignee with a clear change request.`,
|
||||
participantAgentId,
|
||||
}));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!principalIsResolvableUser(participant)) {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "invalid_review_participant",
|
||||
reason: `${issueLabel(issue)} is in review, but its current participant cannot be resolved.`,
|
||||
dependencyPath: [issue],
|
||||
recoveryIssue: issue,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Repair ${issueLabel(issue)}'s review participant or return the issue to an active assignee with a clear change request.`,
|
||||
}));
|
||||
if (issue.status === "in_review" && !unresolvedBlockers.has(issue.id)) {
|
||||
const review = reviewFinding(issue, issue, [issue]);
|
||||
if (review) findings.push(review);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,11 +3,14 @@ import type { Db } from "@paperclipai/db";
|
|||
import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
approvals,
|
||||
companies,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRunWatchdogDecisions,
|
||||
heartbeatRuns,
|
||||
issueApprovals,
|
||||
issueRelations,
|
||||
issueThreadInteractions,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import { parseObject, asBoolean, asNumber } from "../../adapters/utils.js";
|
||||
|
|
@ -1540,7 +1543,17 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
}
|
||||
|
||||
async function collectIssueGraphLivenessFindings() {
|
||||
const [issueRows, relationRows, agentRows, activeRunRows, activeIssueRunRows, wakeRows] = await Promise.all([
|
||||
const [
|
||||
issueRows,
|
||||
relationRows,
|
||||
agentRows,
|
||||
activeRunRows,
|
||||
activeIssueRunRows,
|
||||
wakeRows,
|
||||
interactionRows,
|
||||
approvalRows,
|
||||
recoveryIssueRows,
|
||||
] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
id: issues.id,
|
||||
|
|
@ -1617,8 +1630,50 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
})
|
||||
.from(agentWakeupRequests)
|
||||
.where(inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"])),
|
||||
db
|
||||
.select({
|
||||
companyId: issueThreadInteractions.companyId,
|
||||
issueId: issueThreadInteractions.issueId,
|
||||
status: issueThreadInteractions.status,
|
||||
})
|
||||
.from(issueThreadInteractions)
|
||||
.where(eq(issueThreadInteractions.status, "pending")),
|
||||
db
|
||||
.select({
|
||||
companyId: issueApprovals.companyId,
|
||||
issueId: issueApprovals.issueId,
|
||||
status: approvals.status,
|
||||
})
|
||||
.from(issueApprovals)
|
||||
.innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))
|
||||
.where(inArray(approvals.status, ["pending", "revision_requested"])),
|
||||
db
|
||||
.select({
|
||||
companyId: issues.companyId,
|
||||
id: issues.id,
|
||||
status: issues.status,
|
||||
originId: issues.originId,
|
||||
})
|
||||
.from(issues)
|
||||
.where(
|
||||
and(
|
||||
isNull(issues.hiddenAt),
|
||||
eq(issues.originKind, STRANDED_ISSUE_RECOVERY_ORIGIN_KIND),
|
||||
notInArray(issues.status, ["done", "cancelled"]),
|
||||
),
|
||||
),
|
||||
]);
|
||||
|
||||
const openRecoveryIssues = recoveryIssueRows.flatMap((row) => {
|
||||
const issueId = readNonEmptyString(row.originId);
|
||||
if (!issueId) return [];
|
||||
return [{
|
||||
companyId: row.companyId,
|
||||
issueId,
|
||||
status: row.status,
|
||||
}];
|
||||
});
|
||||
|
||||
return classifyIssueGraphLiveness({
|
||||
issues: issueRows,
|
||||
relations: relationRows,
|
||||
|
|
@ -1640,6 +1695,9 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
|||
status: row.status,
|
||||
issueId: issueIdFromWakePayload(row.payload),
|
||||
})),
|
||||
pendingInteractions: interactionRows,
|
||||
pendingApprovals: approvalRows,
|
||||
openRecoveryIssues,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue