mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-16 19:00:38 +09:00
Cancel stale queued heartbeats when issue graph changes (PAP-2314) (#4534)
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
parent
868d08903e
commit
82e257c7ba
21 changed files with 1991 additions and 238 deletions
|
|
@ -4,7 +4,9 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
|
|||
import {
|
||||
activityLog,
|
||||
agents,
|
||||
budgetPolicies,
|
||||
companies,
|
||||
costEvents,
|
||||
createDb,
|
||||
executionWorkspaces,
|
||||
heartbeatRuns,
|
||||
|
|
@ -191,7 +193,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
type: "blocks",
|
||||
});
|
||||
|
||||
return { companyId, managerId, blockedIssueId, blockerIssueId };
|
||||
return { companyId, managerId, coderId, blockedIssueId, blockerIssueId };
|
||||
}
|
||||
|
||||
it("keeps liveness findings advisory when auto recovery is disabled", async () => {
|
||||
|
|
@ -342,6 +344,71 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
expect(events.some((event) => event.action === "issue.blockers.updated")).toBe(true);
|
||||
});
|
||||
|
||||
it("skips budget-blocked direct owners and assigns recovery to the manager fallback", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, managerId, coderId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
updatedAt: issueTimestamp,
|
||||
})
|
||||
.where(eq(issues.id, blockerIssueId));
|
||||
await db.insert(budgetPolicies).values({
|
||||
companyId,
|
||||
scopeType: "agent",
|
||||
scopeId: coderId,
|
||||
metric: "billed_cents",
|
||||
windowKind: "calendar_month_utc",
|
||||
amount: 1,
|
||||
hardStopEnabled: true,
|
||||
isActive: true,
|
||||
});
|
||||
await db.insert(costEvents).values({
|
||||
companyId,
|
||||
agentId: coderId,
|
||||
issueId: blockerIssueId,
|
||||
provider: "test",
|
||||
biller: "test",
|
||||
billingType: "tokens",
|
||||
model: "test-model",
|
||||
costCents: 1,
|
||||
occurredAt: new Date(),
|
||||
});
|
||||
|
||||
const result = await heartbeatService(db).reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.escalationsCreated).toBe(1);
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(1);
|
||||
expect(escalations[0]).toMatchObject({
|
||||
parentId: blockerIssueId,
|
||||
assigneeAgentId: managerId,
|
||||
originId: [
|
||||
"harness_liveness",
|
||||
companyId,
|
||||
blockedIssueId,
|
||||
"in_review_without_action_path",
|
||||
blockerIssueId,
|
||||
].join(":"),
|
||||
});
|
||||
|
||||
const events = await db.select().from(activityLog).where(eq(activityLog.companyId, companyId));
|
||||
const createdEvent = events.find((event) => event.action === "issue.harness_liveness_escalation_created");
|
||||
expect(createdEvent?.details).toMatchObject({
|
||||
ownerSelection: {
|
||||
selectedAgentId: managerId,
|
||||
selectedReason: "assignee_reporting_chain",
|
||||
budgetBlockedCandidateAgentIds: [coderId],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("parents recovery under the leaf blocker without inheriting dependent or blocker execution state for manager-owned recovery", async () => {
|
||||
await enableAutoRecovery();
|
||||
await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
|
||||
|
|
|
|||
545
server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
Normal file
545
server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
Normal file
|
|
@ -0,0 +1,545 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
activityLog,
|
||||
agents,
|
||||
agentRuntimeState,
|
||||
agentWakeupRequests,
|
||||
companies,
|
||||
companySkills,
|
||||
createDb,
|
||||
documentRevisions,
|
||||
documents,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
issueRelations,
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
import { runningProcesses } from "../adapters/index.ts";
|
||||
|
||||
const mockAdapterExecute = vi.hoisted(() =>
|
||||
vi.fn(async () => ({
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorMessage: null,
|
||||
summary: "Stale-queue invalidation test run.",
|
||||
provider: "test",
|
||||
model: "test-model",
|
||||
})),
|
||||
);
|
||||
|
||||
vi.mock("../adapters/index.ts", async () => {
|
||||
const actual = await vi.importActual<typeof import("../adapters/index.ts")>("../adapters/index.ts");
|
||||
return {
|
||||
...actual,
|
||||
getServerAdapter: vi.fn(() => ({
|
||||
supportsLocalAgentJwt: false,
|
||||
execute: mockAdapterExecute,
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres heartbeat stale-queue invalidation tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function ensureIssueRelationsTable(db: ReturnType<typeof createDb>) {
|
||||
await db.execute(sql.raw(`
|
||||
CREATE TABLE IF NOT EXISTS "issue_relations" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
"company_id" uuid NOT NULL,
|
||||
"issue_id" uuid NOT NULL,
|
||||
"related_issue_id" uuid NOT NULL,
|
||||
"type" text NOT NULL,
|
||||
"created_by_agent_id" uuid,
|
||||
"created_by_user_id" text,
|
||||
"created_at" timestamptz NOT NULL DEFAULT now(),
|
||||
"updated_at" timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
`));
|
||||
}
|
||||
|
||||
async function waitForCondition(fn: () => Promise<boolean>, timeoutMs = 3_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (await fn()) return true;
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
return fn();
|
||||
}
|
||||
|
||||
type SeedOptions = {
|
||||
agentName?: string;
|
||||
agentRole?: string;
|
||||
maxConcurrentRuns?: number;
|
||||
};
|
||||
|
||||
type SeedResult = {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
};
|
||||
|
||||
describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let heartbeat!: ReturnType<typeof heartbeatService>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-stale-queue-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
heartbeat = heartbeatService(db);
|
||||
await ensureIssueRelationsTable(db);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
mockAdapterExecute.mockReset();
|
||||
mockAdapterExecute.mockImplementation(async () => ({
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorMessage: null,
|
||||
summary: "Stale-queue invalidation test run.",
|
||||
provider: "test",
|
||||
model: "test-model",
|
||||
}));
|
||||
runningProcesses.clear();
|
||||
let idlePolls = 0;
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
const runs = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns);
|
||||
const hasActiveRun = runs.some((run) => run.status === "queued" || run.status === "running");
|
||||
if (!hasActiveRun) {
|
||||
idlePolls += 1;
|
||||
if (idlePolls >= 3) break;
|
||||
} else {
|
||||
idlePolls = 0;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
await db.delete(companySkills);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issueDocuments);
|
||||
await db.delete(documentRevisions);
|
||||
await db.delete(documents);
|
||||
await db.delete(issueRelations);
|
||||
await db.delete(issueTreeHolds);
|
||||
await db.delete(issues);
|
||||
await db.delete(heartbeatRunEvents);
|
||||
await db.delete(activityLog);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(agentRuntimeState);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedCompanyAndAgent(opts: SeedOptions = {}): Promise<SeedResult> {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: opts.agentName ?? "ClaudeCoder",
|
||||
role: opts.agentRole ?? "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
wakeOnDemand: true,
|
||||
maxConcurrentRuns: opts.maxConcurrentRuns ?? 1,
|
||||
},
|
||||
},
|
||||
permissions: {},
|
||||
});
|
||||
return { companyId, agentId };
|
||||
}
|
||||
|
||||
async function seedQueuedRun(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
issueId: string;
|
||||
wakeReason: string;
|
||||
contextExtras?: Record<string, unknown>;
|
||||
invocationSource?: "assignment" | "automation";
|
||||
}) {
|
||||
const wakeupRequestId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
await db.insert(agentWakeupRequests).values({
|
||||
id: wakeupRequestId,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
source: input.invocationSource ?? "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: input.wakeReason,
|
||||
payload: { issueId: input.issueId },
|
||||
status: "queued",
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
invocationSource: input.invocationSource ?? "assignment",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: input.issueId,
|
||||
wakeReason: input.wakeReason,
|
||||
...(input.contextExtras ?? {}),
|
||||
},
|
||||
});
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId })
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId));
|
||||
return { runId, wakeupRequestId };
|
||||
}
|
||||
|
||||
it("cancels queued runs when the issue assignee changes before the run starts", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent({ agentName: "OriginalCoder" });
|
||||
const replacementAgentId = randomUUID();
|
||||
await db.insert(agents).values({
|
||||
id: replacementAgentId,
|
||||
companyId,
|
||||
name: "ReplacementCoder",
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
wakeOnDemand: true,
|
||||
maxConcurrentRuns: 1,
|
||||
},
|
||||
},
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Reassigned task",
|
||||
status: "in_progress",
|
||||
priority: "high",
|
||||
assigneeAgentId: replacementAgentId,
|
||||
});
|
||||
|
||||
const { runId, wakeupRequestId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
|
||||
const [run, wakeup] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
status: heartbeatRuns.status,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
resultJson: heartbeatRuns.resultJson,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
expect(run?.status).toBe("cancelled");
|
||||
expect(run?.errorCode).toBe("issue_assignee_changed");
|
||||
expect(run?.resultJson).toMatchObject({ stopReason: "issue_assignee_changed" });
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(wakeup?.error).toContain("assignee changed");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels queued runs when the issue reaches a terminal status before the run starts", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Already-completed task",
|
||||
status: "done",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
|
||||
const { runId, wakeupRequestId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
|
||||
const [run, wakeup] = await Promise.all([
|
||||
db
|
||||
.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({ status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
expect(run?.status).toBe("cancelled");
|
||||
expect(run?.errorCode).toBe("issue_terminal_status");
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels queued in_review runs when the current participant changes before the run starts", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const otherAgentId = randomUUID();
|
||||
await db.insert(agents).values({
|
||||
id: otherAgentId,
|
||||
companyId,
|
||||
name: "ReviewerAgent",
|
||||
role: "qa",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } },
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "In-review task now owned by reviewer",
|
||||
status: "in_review",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
executionState: {
|
||||
status: "pending",
|
||||
currentStageId: randomUUID(),
|
||||
currentStageIndex: 0,
|
||||
currentStageType: "review",
|
||||
currentParticipant: { type: "agent", agentId: otherAgentId, userId: null },
|
||||
returnAssignee: { type: "agent", agentId, userId: null },
|
||||
reviewRequest: null,
|
||||
completedStageIds: [],
|
||||
lastDecisionId: null,
|
||||
lastDecisionOutcome: null,
|
||||
},
|
||||
});
|
||||
|
||||
const { runId, wakeupRequestId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
|
||||
const [run, wakeup] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
status: heartbeatRuns.status,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
resultJson: heartbeatRuns.resultJson,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
expect(run?.status).toBe("cancelled");
|
||||
expect(run?.errorCode).toBe("issue_review_participant_changed");
|
||||
expect(run?.resultJson).toMatchObject({ stopReason: "issue_review_participant_changed" });
|
||||
expect(wakeup?.status).toBe("skipped");
|
||||
expect(wakeup?.error).toContain("in-review participant changed");
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still runs comment-driven wakes on in_review issues even when the agent is no longer the current participant", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const otherAgentId = randomUUID();
|
||||
await db.insert(agents).values({
|
||||
id: otherAgentId,
|
||||
companyId,
|
||||
name: "ReviewerAgent",
|
||||
role: "qa",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } },
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
const issueId = randomUUID();
|
||||
const commentId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "In-review task with comment feedback",
|
||||
status: "in_review",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
executionState: {
|
||||
status: "pending",
|
||||
currentStageId: randomUUID(),
|
||||
currentStageIndex: 0,
|
||||
currentStageType: "review",
|
||||
currentParticipant: { type: "agent", agentId: otherAgentId, userId: null },
|
||||
returnAssignee: { type: "agent", agentId, userId: null },
|
||||
reviewRequest: null,
|
||||
completedStageIds: [],
|
||||
lastDecisionId: null,
|
||||
lastDecisionOutcome: null,
|
||||
},
|
||||
});
|
||||
await db.insert(issueComments).values({
|
||||
id: commentId,
|
||||
companyId,
|
||||
issueId,
|
||||
authorAgentId: otherAgentId,
|
||||
body: "Review feedback comment",
|
||||
});
|
||||
|
||||
const { runId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_commented",
|
||||
invocationSource: "automation",
|
||||
contextExtras: {
|
||||
commentId,
|
||||
wakeCommentId: commentId,
|
||||
source: "issue.comment",
|
||||
},
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
});
|
||||
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(run?.status).toBe("succeeded");
|
||||
expect(run?.errorCode).toBeNull();
|
||||
});
|
||||
|
||||
it("baseline: runs queued runs when the issue is in_progress with the same assignee", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Still actionable",
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
|
||||
const { runId } = await seedQueuedRun({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
});
|
||||
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(run?.status).toBe("succeeded");
|
||||
expect(run?.errorCode).toBeNull();
|
||||
expect(mockAdapterExecute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -253,6 +253,109 @@ describeEmbeddedPostgres("issue blocker attention", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("flags a chain whose leaf is in_review without an action path as stalled", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBV");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBV-1", title: "Parent", status: "blocked" });
|
||||
const reviewLeafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBV-2",
|
||||
title: "Stalled review leaf",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: reviewLeafId, blockedIssueId: parentId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "stalled",
|
||||
reason: "stalled_review",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 0,
|
||||
stalledBlockerCount: 1,
|
||||
attentionBlockerCount: 0,
|
||||
sampleBlockerIdentifier: "PBV-2",
|
||||
sampleStalledBlockerIdentifier: "PBV-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not flag an in_review leaf as stalled when an active run is still progressing it", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBW");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBW-1", title: "Parent", status: "blocked" });
|
||||
const reviewLeafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBW-2",
|
||||
title: "Active review leaf",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: reviewLeafId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: reviewLeafId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "covered",
|
||||
stalledBlockerCount: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("flags a deep chain whose leaf is stalled in_review through multiple layers", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBZ");
|
||||
const rootId = await insertIssue({ companyId, identifier: "PBZ-1", title: "Root", status: "blocked" });
|
||||
const midId = await insertIssue({ companyId, identifier: "PBZ-2", title: "Mid blocker", status: "blocked" });
|
||||
const leafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBZ-3",
|
||||
title: "Stalled leaf",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: midId, blockedIssueId: rootId });
|
||||
await block({ companyId, blockerIssueId: leafId, blockedIssueId: midId });
|
||||
|
||||
const root = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === rootId);
|
||||
|
||||
expect(root?.blockerAttention).toMatchObject({
|
||||
state: "stalled",
|
||||
reason: "stalled_review",
|
||||
stalledBlockerCount: 1,
|
||||
sampleStalledBlockerIdentifier: "PBZ-3",
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers needs_attention over stalled when the chain also has a hard attention case", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBQ");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBQ-1", title: "Parent", status: "blocked" });
|
||||
const reviewLeafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBQ-2",
|
||||
title: "Stalled review leaf",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
const cancelledLeafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBQ-3",
|
||||
title: "Cancelled blocker",
|
||||
status: "cancelled",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: reviewLeafId, blockedIssueId: parentId });
|
||||
await block({ companyId, blockerIssueId: cancelledLeafId, blockedIssueId: parentId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
coveredBlockerCount: 0,
|
||||
stalledBlockerCount: 1,
|
||||
attentionBlockerCount: 1,
|
||||
sampleStalledBlockerIdentifier: "PBQ-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not treat a scheduled retry as actively covered work", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBY");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBY-1", title: "Parent", status: "blocked" });
|
||||
|
|
|
|||
|
|
@ -234,4 +234,191 @@ describe("issue graph liveness classifier", () => {
|
|||
incidentKey: `harness_liveness:${companyId}:${blockedId}:invalid_review_participant:missing-agent`,
|
||||
});
|
||||
});
|
||||
|
||||
it("detects the PAP-2239-style blocked chain at the first stalled in_review leaf without duplicate findings", () => {
|
||||
const phaseIssueId = "phase-issue-1";
|
||||
const reviewLeafId = "review-leaf-1";
|
||||
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
issue({
|
||||
id: "pap-2239",
|
||||
identifier: "PAP-2239",
|
||||
title: "External object reference project",
|
||||
status: "blocked",
|
||||
}),
|
||||
issue({
|
||||
id: phaseIssueId,
|
||||
identifier: "PAP-2276",
|
||||
title: "UX acceptance review phase",
|
||||
status: "blocked",
|
||||
assigneeAgentId: coderId,
|
||||
}),
|
||||
issue({
|
||||
id: reviewLeafId,
|
||||
identifier: "PAP-2279",
|
||||
title: "Screenshot acceptance review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
executionState: null,
|
||||
}),
|
||||
],
|
||||
relations: [
|
||||
{ companyId, blockerIssueId: phaseIssueId, blockedIssueId: "pap-2239" },
|
||||
{ companyId, blockerIssueId: reviewLeafId, blockedIssueId: phaseIssueId },
|
||||
],
|
||||
agents: [agent(), manager],
|
||||
});
|
||||
|
||||
expect(findings).toHaveLength(1);
|
||||
expect(findings[0]).toMatchObject({
|
||||
issueId: "pap-2239",
|
||||
identifier: "PAP-2239",
|
||||
state: "in_review_without_action_path",
|
||||
recoveryIssueId: reviewLeafId,
|
||||
recommendedOwnerAgentId: coderId,
|
||||
dependencyPath: [
|
||||
expect.objectContaining({ issueId: "pap-2239" }),
|
||||
expect.objectContaining({ issueId: phaseIssueId }),
|
||||
expect.objectContaining({ issueId: reviewLeafId }),
|
||||
],
|
||||
incidentKey: `harness_liveness:${companyId}:pap-2239:in_review_without_action_path:${reviewLeafId}`,
|
||||
});
|
||||
});
|
||||
|
||||
it("skips paused stalled review assignees when choosing recovery owner candidates", () => {
|
||||
const reviewIssueId = "review-1";
|
||||
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
issue({
|
||||
id: reviewIssueId,
|
||||
identifier: "PAP-2279",
|
||||
title: "Screenshot acceptance review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
executionState: null,
|
||||
}),
|
||||
],
|
||||
relations: [],
|
||||
agents: [agent({ status: "paused" }), manager],
|
||||
});
|
||||
|
||||
expect(findings).toHaveLength(1);
|
||||
expect(findings[0]).toMatchObject({
|
||||
state: "in_review_without_action_path",
|
||||
recommendedOwnerAgentId: managerId,
|
||||
});
|
||||
expect(findings[0]?.recommendedOwnerCandidates).toEqual([
|
||||
{
|
||||
agentId: managerId,
|
||||
reason: "assignee_reporting_chain",
|
||||
sourceIssueId: reviewIssueId,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not flag healthy in_review issues with an explicit action path", () => {
|
||||
const reviewIssueId = "review-1";
|
||||
const baseReviewIssue = issue({
|
||||
id: reviewIssueId,
|
||||
identifier: "PAP-2279",
|
||||
title: "Screenshot acceptance review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
executionState: null,
|
||||
});
|
||||
|
||||
const cases = [
|
||||
{
|
||||
name: "typed agent participant",
|
||||
issue: {
|
||||
...baseReviewIssue,
|
||||
executionState: {
|
||||
currentParticipant: { type: "agent", agentId: coderId },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "typed user participant",
|
||||
issue: {
|
||||
...baseReviewIssue,
|
||||
executionState: {
|
||||
currentParticipant: { type: "user", userId: "board-user-1" },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "user owner",
|
||||
issue: { ...baseReviewIssue, assigneeAgentId: null, assigneeUserId: "board-user-1" },
|
||||
},
|
||||
{
|
||||
name: "active run",
|
||||
issue: baseReviewIssue,
|
||||
activeRuns: [{ companyId, issueId: reviewIssueId, agentId: coderId, status: "running" }],
|
||||
},
|
||||
{
|
||||
name: "queued wake",
|
||||
issue: baseReviewIssue,
|
||||
queuedWakeRequests: [{ companyId, issueId: reviewIssueId, agentId: coderId, status: "queued" }],
|
||||
},
|
||||
{
|
||||
name: "pending interaction",
|
||||
issue: baseReviewIssue,
|
||||
pendingInteractions: [{ companyId, issueId: reviewIssueId, status: "pending" }],
|
||||
},
|
||||
{
|
||||
name: "pending approval",
|
||||
issue: baseReviewIssue,
|
||||
pendingApprovals: [{ companyId, issueId: reviewIssueId, status: "pending" }],
|
||||
},
|
||||
{
|
||||
name: "open recovery issue",
|
||||
issue: baseReviewIssue,
|
||||
openRecoveryIssues: [{ companyId, issueId: reviewIssueId, status: "todo" }],
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [testCase.issue],
|
||||
relations: [],
|
||||
agents: [agent(), manager],
|
||||
activeRuns: testCase.activeRuns,
|
||||
queuedWakeRequests: testCase.queuedWakeRequests,
|
||||
pendingInteractions: testCase.pendingInteractions,
|
||||
pendingApprovals: testCase.pendingApprovals,
|
||||
openRecoveryIssues: testCase.openRecoveryIssues,
|
||||
});
|
||||
|
||||
expect(findings, testCase.name).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores cross-company waiting paths for stalled in_review issues", () => {
|
||||
const reviewIssueId = "review-1";
|
||||
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
issue({
|
||||
id: reviewIssueId,
|
||||
identifier: "PAP-2279",
|
||||
title: "Screenshot acceptance review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderId,
|
||||
executionState: null,
|
||||
}),
|
||||
],
|
||||
relations: [],
|
||||
agents: [agent(), manager],
|
||||
pendingInteractions: [{ companyId: "other-company", issueId: reviewIssueId, status: "pending" }],
|
||||
openRecoveryIssues: [{ companyId: "other-company", issueId: reviewIssueId, status: "todo" }],
|
||||
});
|
||||
|
||||
expect(findings).toHaveLength(1);
|
||||
expect(findings[0]).toMatchObject({
|
||||
state: "in_review_without_action_path",
|
||||
recoveryIssueId: reviewIssueId,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue