mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-15 18:30:39 +09:00
[codex] Add runtime lifecycle recovery and live issue visibility (#4419)
This commit is contained in:
Dotta
GitHub
parent
9a8d219949
commit
5a0c1979cf
121 changed files with 9625 additions and 2044 deletions
10
server/src/__tests__/README.md
Normal file
10
server/src/__tests__/README.md
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# Server Tests
|
||||
|
||||
Server tests that need a real PostgreSQL process must use
|
||||
`./helpers/embedded-postgres.ts` instead of constructing `embedded-postgres`
|
||||
directly.
|
||||
|
||||
The shared helper creates a throwaway data directory and a reserved-safe
|
||||
loopback port for each test database. This protects the live Paperclip
|
||||
control-plane Postgres from server vitest runs; see PAP-2033 for the incident
|
||||
that introduced this guard.
|
||||
|
|
@ -424,7 +424,7 @@ describeEmbeddedPostgres("activity service", () => {
|
|||
expect(backfilledRun).toMatchObject({
|
||||
runId,
|
||||
livenessState: "plan_only",
|
||||
livenessReason: "Run described future work without concrete action evidence",
|
||||
livenessReason: "Run described runnable future work without concrete action evidence",
|
||||
lastUsefulActionAt: null,
|
||||
});
|
||||
});
|
||||
|
|
@ -530,7 +530,7 @@ describeEmbeddedPostgres("activity service", () => {
|
|||
expect(backfilledRun).toMatchObject({
|
||||
runId,
|
||||
livenessState: "plan_only",
|
||||
livenessReason: "Run described future work without concrete action evidence",
|
||||
livenessReason: "Run described runnable future work without concrete action evidence",
|
||||
lastUsefulActionAt: null,
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ const mockHeartbeatService = vi.hoisted(() => ({
|
|||
buildRunOutputSilence: vi.fn(),
|
||||
getRunIssueSummary: vi.fn(),
|
||||
getActiveRunIssueSummaryForAgent: vi.fn(),
|
||||
buildRunOutputSilence: vi.fn(),
|
||||
getRunLogAccess: vi.fn(),
|
||||
readLog: vi.fn(),
|
||||
}));
|
||||
|
|
@ -173,6 +174,7 @@ describe("agent live run routes", () => {
|
|||
issueId: "issue-1",
|
||||
});
|
||||
mockHeartbeatService.getActiveRunIssueSummaryForAgent.mockResolvedValue(null);
|
||||
mockHeartbeatService.buildRunOutputSilence.mockResolvedValue(null);
|
||||
mockHeartbeatService.getRunLogAccess.mockResolvedValue({
|
||||
id: "run-1",
|
||||
companyId: "company-1",
|
||||
|
|
@ -209,6 +211,7 @@ describe("agent live run routes", () => {
|
|||
issueId: "issue-1",
|
||||
agentName: "Builder",
|
||||
adapterType: "codex_local",
|
||||
outputSilence: null,
|
||||
});
|
||||
expect(res.body).not.toHaveProperty("resultJson");
|
||||
expect(res.body).not.toHaveProperty("contextSnapshot");
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import net from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { eq } from "drizzle-orm";
|
||||
|
|
@ -8,14 +7,12 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
|
|||
import { writePaperclipSkillSyncPreference } from "@paperclipai/adapter-utils/server-utils";
|
||||
import {
|
||||
agents,
|
||||
applyPendingMigrations,
|
||||
companies,
|
||||
companySkills,
|
||||
costEvents,
|
||||
createDb,
|
||||
documents,
|
||||
documentRevisions,
|
||||
ensurePostgresDatabase,
|
||||
feedbackExports,
|
||||
feedbackVotes,
|
||||
heartbeatRuns,
|
||||
|
|
@ -25,72 +22,7 @@ import {
|
|||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import { feedbackService } from "../services/feedback.ts";
|
||||
|
||||
type EmbeddedPostgresInstance = {
|
||||
initialise(): Promise<void>;
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
type EmbeddedPostgresCtor = new (opts: {
|
||||
databaseDir: string;
|
||||
user: string;
|
||||
password: string;
|
||||
port: number;
|
||||
persistent: boolean;
|
||||
initdbFlags?: string[];
|
||||
onLog?: (message: unknown) => void;
|
||||
onError?: (message: unknown) => void;
|
||||
}) => EmbeddedPostgresInstance;
|
||||
|
||||
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
|
||||
const mod = await import("embedded-postgres");
|
||||
return mod.default as EmbeddedPostgresCtor;
|
||||
}
|
||||
|
||||
async function getAvailablePort(): Promise<number> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const server = net.createServer();
|
||||
server.unref();
|
||||
server.on("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
server.close(() => reject(new Error("Failed to allocate test port")));
|
||||
return;
|
||||
}
|
||||
const { port } = address;
|
||||
server.close((error) => {
|
||||
if (error) reject(error);
|
||||
else resolve(port);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function startTempDatabase() {
|
||||
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-feedback-service-"));
|
||||
const port = await getAvailablePort();
|
||||
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
|
||||
const instance = new EmbeddedPostgres({
|
||||
databaseDir: dataDir,
|
||||
user: "paperclip",
|
||||
password: "paperclip",
|
||||
port,
|
||||
persistent: true,
|
||||
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
|
||||
onLog: () => {},
|
||||
onError: () => {},
|
||||
});
|
||||
await instance.initialise();
|
||||
await instance.start();
|
||||
|
||||
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
|
||||
await ensurePostgresDatabase(adminConnectionString, "paperclip");
|
||||
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
|
||||
await applyPendingMigrations(connectionString);
|
||||
return { connectionString, dataDir, instance };
|
||||
}
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.ts";
|
||||
|
||||
async function closeDbClient(db: ReturnType<typeof createDb> | undefined) {
|
||||
await db?.$client?.end?.({ timeout: 0 });
|
||||
|
|
@ -99,17 +31,15 @@ async function closeDbClient(db: ReturnType<typeof createDb> | undefined) {
|
|||
describe("feedbackService.saveIssueVote", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let svc!: ReturnType<typeof feedbackService>;
|
||||
let instance: EmbeddedPostgresInstance | null = null;
|
||||
let dataDir = "";
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
let tempDirs: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
const started = await startTempDatabase();
|
||||
const started = await startEmbeddedPostgresTestDatabase("paperclip-feedback-service-");
|
||||
db = createDb(started.connectionString);
|
||||
svc = feedbackService(db);
|
||||
instance = started.instance;
|
||||
dataDir = started.dataDir;
|
||||
}, 20_000);
|
||||
tempDb = started;
|
||||
}, 120_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(feedbackExports);
|
||||
|
|
@ -134,10 +64,7 @@ describe("feedbackService.saveIssueVote", () => {
|
|||
|
||||
afterAll(async () => {
|
||||
await closeDbClient(db);
|
||||
await instance?.stop();
|
||||
if (dataDir) {
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedIssueWithAgentComment() {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,549 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRunWatchdogDecisions,
|
||||
heartbeatRuns,
|
||||
issueRelations,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import {
|
||||
ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS,
|
||||
ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS,
|
||||
ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS,
|
||||
heartbeatService,
|
||||
} from "../services/heartbeat.ts";
|
||||
import { recoveryService } from "../services/recovery/service.ts";
|
||||
import { getRunLogStore } from "../services/run-log-store.ts";
|
||||
|
||||
const mockAdapterExecute = vi.hoisted(() =>
|
||||
vi.fn(async () => ({
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorMessage: null,
|
||||
summary: "Acknowledged stale-run evaluation.",
|
||||
provider: "test",
|
||||
model: "test-model",
|
||||
})),
|
||||
);
|
||||
|
||||
vi.mock("../telemetry.ts", () => ({
|
||||
getTelemetryClient: () => ({ track: vi.fn() }),
|
||||
}));
|
||||
|
||||
vi.mock("@paperclipai/shared/telemetry", async () => {
|
||||
const actual = await vi.importActual<typeof import("@paperclipai/shared/telemetry")>(
|
||||
"@paperclipai/shared/telemetry",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
trackAgentFirstHeartbeat: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../adapters/index.ts", async () => {
|
||||
const actual = await vi.importActual<typeof import("../adapters/index.ts")>("../adapters/index.ts");
|
||||
return {
|
||||
...actual,
|
||||
getServerAdapter: vi.fn(() => ({
|
||||
supportsLocalAgentJwt: false,
|
||||
execute: mockAdapterExecute,
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres active-run output watchdog tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("active-run output watchdog", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-active-run-output-watchdog-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 30_000);
|
||||
|
||||
afterEach(async () => {
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
const activeRuns = await db
|
||||
.select({ id: heartbeatRuns.id })
|
||||
.from(heartbeatRuns)
|
||||
.where(sql`${heartbeatRuns.status} in ('queued', 'running')`);
|
||||
if (activeRuns.length === 0) break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
}
|
||||
await db.execute(sql.raw(`TRUNCATE TABLE "companies" CASCADE`));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedRunningRun(opts: { now: Date; ageMs: number; withOutput?: boolean; logChunk?: string }) {
|
||||
const companyId = randomUUID();
|
||||
const managerId = randomUUID();
|
||||
const coderId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const issuePrefix = `W${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const startedAt = new Date(opts.now.getTime() - opts.ageMs);
|
||||
const lastOutputAt = opts.withOutput ? new Date(opts.now.getTime() - 5 * 60 * 1000) : null;
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Watchdog Co",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values([
|
||||
{
|
||||
id: managerId,
|
||||
companyId,
|
||||
name: "CTO",
|
||||
role: "cto",
|
||||
status: "idle",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
id: coderId,
|
||||
companyId,
|
||||
name: "Coder",
|
||||
role: "engineer",
|
||||
status: "running",
|
||||
reportsTo: managerId,
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Long running implementation",
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: coderId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
updatedAt: startedAt,
|
||||
createdAt: startedAt,
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId,
|
||||
agentId: coderId,
|
||||
status: "running",
|
||||
invocationSource: "assignment",
|
||||
triggerDetail: "system",
|
||||
startedAt,
|
||||
processStartedAt: startedAt,
|
||||
lastOutputAt,
|
||||
lastOutputSeq: opts.withOutput ? 3 : 0,
|
||||
lastOutputStream: opts.withOutput ? "stdout" : null,
|
||||
contextSnapshot: { issueId },
|
||||
stdoutExcerpt: "OPENAI_API_KEY=sk-test-secret-value should not leak",
|
||||
logBytes: 0,
|
||||
});
|
||||
if (opts.logChunk) {
|
||||
const store = getRunLogStore();
|
||||
const handle = await store.begin({ companyId, agentId: coderId, runId });
|
||||
const logBytes = await store.append(handle, {
|
||||
stream: "stdout",
|
||||
chunk: opts.logChunk,
|
||||
ts: startedAt.toISOString(),
|
||||
});
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
logStore: handle.store,
|
||||
logRef: handle.logRef,
|
||||
logBytes,
|
||||
})
|
||||
.where(eq(heartbeatRuns.id, runId));
|
||||
}
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
return { companyId, managerId, coderId, issueId, runId, issuePrefix };
|
||||
}
|
||||
|
||||
it("creates one medium-priority evaluation issue for a suspicious silent run", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, runId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const first = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const second = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
|
||||
expect(first.created).toBe(1);
|
||||
expect(second.created).toBe(0);
|
||||
expect(second.existing).toBe(1);
|
||||
|
||||
const evaluations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stale_active_run_evaluation")));
|
||||
expect(evaluations).toHaveLength(1);
|
||||
expect(["todo", "in_progress"]).toContain(evaluations[0]?.status);
|
||||
expect(evaluations[0]).toMatchObject({
|
||||
priority: "medium",
|
||||
assigneeAgentId: managerId,
|
||||
originId: runId,
|
||||
originFingerprint: `stale_active_run:${companyId}:${runId}`,
|
||||
});
|
||||
expect(evaluations[0]?.description).toContain("Decision Checklist");
|
||||
expect(evaluations[0]?.description).not.toContain("sk-test-secret-value");
|
||||
});
|
||||
|
||||
it("redacts sensitive values from actual run-log evidence", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const leakedJwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
|
||||
const leakedGithubToken = "ghp_1234567890abcdefghijklmnopqrstuvwxyz";
|
||||
const { companyId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
logChunk: [
|
||||
"Authorization: Bearer live-bearer-token-value",
|
||||
`POST payload {"apiKey":"json-secret-value","token":"${leakedJwt}"}`,
|
||||
`GITHUB_TOKEN=${leakedGithubToken}`,
|
||||
].join("\n"),
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
|
||||
const [evaluation] = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stale_active_run_evaluation")));
|
||||
expect(evaluation?.description).toContain("***REDACTED***");
|
||||
expect(evaluation?.description).not.toContain("live-bearer-token-value");
|
||||
expect(evaluation?.description).not.toContain("json-secret-value");
|
||||
expect(evaluation?.description).not.toContain(leakedJwt);
|
||||
expect(evaluation?.description).not.toContain(leakedGithubToken);
|
||||
});
|
||||
|
||||
it("raises critical stale-run evaluations and blocks the source issue", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, issueId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
|
||||
expect(result.created).toBe(1);
|
||||
const [evaluation] = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stale_active_run_evaluation")));
|
||||
expect(evaluation?.priority).toBe("high");
|
||||
|
||||
const [blocker] = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(and(eq(issueRelations.companyId, companyId), eq(issueRelations.relatedIssueId, issueId)));
|
||||
expect(blocker?.issueId).toBe(evaluation?.id);
|
||||
|
||||
const [source] = await db.select().from(issues).where(eq(issues.id, issueId));
|
||||
expect(source?.status).toBe("blocked");
|
||||
});
|
||||
|
||||
it("skips snoozed runs and healthy noisy runs", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const stale = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const noisy = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
|
||||
withOutput: true,
|
||||
});
|
||||
await db.insert(heartbeatRunWatchdogDecisions).values({
|
||||
companyId: stale.companyId,
|
||||
runId: stale.runId,
|
||||
decision: "snooze",
|
||||
snoozedUntil: new Date(now.getTime() + 60 * 60 * 1000),
|
||||
reason: "Intentional quiet run",
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const staleResult = await heartbeat.scanSilentActiveRuns({ now, companyId: stale.companyId });
|
||||
const noisyResult = await heartbeat.scanSilentActiveRuns({ now, companyId: noisy.companyId });
|
||||
|
||||
expect(staleResult).toMatchObject({ created: 0, snoozed: 1 });
|
||||
expect(noisyResult).toMatchObject({ scanned: 0, created: 0 });
|
||||
});
|
||||
|
||||
it("records watchdog decisions through recovery owner authorization", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, runId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn() });
|
||||
|
||||
const scan = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const evaluationIssueId = scan.evaluationIssueIds[0];
|
||||
expect(evaluationIssueId).toBeTruthy();
|
||||
|
||||
await expect(
|
||||
recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: randomUUID() },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "not my recovery issue",
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 403 });
|
||||
|
||||
const snoozedUntil = new Date(now.getTime() + 60 * 60 * 1000);
|
||||
const decision = await recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
decision: "snooze",
|
||||
evaluationIssueId,
|
||||
reason: "Long compile with no output",
|
||||
snoozedUntil,
|
||||
});
|
||||
|
||||
expect(decision).toMatchObject({
|
||||
runId,
|
||||
evaluationIssueId,
|
||||
decision: "snooze",
|
||||
createdByAgentId: managerId,
|
||||
});
|
||||
await expect(recovery.buildRunOutputSilence({
|
||||
id: runId,
|
||||
companyId,
|
||||
status: "running",
|
||||
lastOutputAt: null,
|
||||
lastOutputSeq: 0,
|
||||
lastOutputStream: null,
|
||||
processStartedAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 60_000),
|
||||
startedAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 60_000),
|
||||
createdAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 60_000),
|
||||
}, now)).resolves.toMatchObject({
|
||||
level: "snoozed",
|
||||
snoozedUntil,
|
||||
evaluationIssueId,
|
||||
});
|
||||
});
|
||||
|
||||
it("re-arms continue decisions after the default quiet window", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, runId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn() });
|
||||
|
||||
const scan = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const evaluationIssueId = scan.evaluationIssueIds[0];
|
||||
expect(evaluationIssueId).toBeTruthy();
|
||||
|
||||
const decision = await recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "Current evidence is acceptable; keep watching.",
|
||||
now,
|
||||
});
|
||||
const rearmAt = new Date(now.getTime() + ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS);
|
||||
expect(decision).toMatchObject({
|
||||
runId,
|
||||
evaluationIssueId,
|
||||
decision: "continue",
|
||||
createdByAgentId: managerId,
|
||||
});
|
||||
expect(decision.snoozedUntil?.toISOString()).toBe(rearmAt.toISOString());
|
||||
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, evaluationIssueId));
|
||||
|
||||
const beforeRearm = await heartbeat.scanSilentActiveRuns({
|
||||
now: new Date(rearmAt.getTime() - 60_000),
|
||||
companyId,
|
||||
});
|
||||
expect(beforeRearm).toMatchObject({ created: 0, snoozed: 1 });
|
||||
|
||||
const afterRearm = await heartbeat.scanSilentActiveRuns({
|
||||
now: new Date(rearmAt.getTime() + 60_000),
|
||||
companyId,
|
||||
});
|
||||
expect(afterRearm.created).toBe(1);
|
||||
expect(afterRearm.evaluationIssueIds[0]).not.toBe(evaluationIssueId);
|
||||
|
||||
const evaluations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stale_active_run_evaluation")));
|
||||
expect(evaluations.filter((issue) => !["done", "cancelled"].includes(issue.status))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("rejects agent watchdog decisions using issues not bound to the target run", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, coderId, runId, issuePrefix } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn() });
|
||||
|
||||
const scan = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const evaluationIssueId = scan.evaluationIssueIds[0];
|
||||
expect(evaluationIssueId).toBeTruthy();
|
||||
|
||||
const unrelatedIssueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: unrelatedIssueId,
|
||||
companyId,
|
||||
title: "Assigned but unrelated",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
assigneeAgentId: managerId,
|
||||
issueNumber: 20,
|
||||
identifier: `${issuePrefix}-20`,
|
||||
});
|
||||
|
||||
const otherRunId = randomUUID();
|
||||
const otherEvaluationIssueId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: otherRunId,
|
||||
companyId,
|
||||
agentId: coderId,
|
||||
status: "running",
|
||||
invocationSource: "assignment",
|
||||
triggerDetail: "system",
|
||||
startedAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 120_000),
|
||||
processStartedAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 120_000),
|
||||
lastOutputAt: null,
|
||||
lastOutputSeq: 0,
|
||||
lastOutputStream: null,
|
||||
contextSnapshot: {},
|
||||
logBytes: 0,
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: otherEvaluationIssueId,
|
||||
companyId,
|
||||
title: "Other run evaluation",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
assigneeAgentId: managerId,
|
||||
issueNumber: 21,
|
||||
identifier: `${issuePrefix}-21`,
|
||||
originKind: "stale_active_run_evaluation",
|
||||
originId: otherRunId,
|
||||
originFingerprint: `stale_active_run:${companyId}:${otherRunId}`,
|
||||
});
|
||||
|
||||
const attempts = [
|
||||
{ decision: "continue" as const, evaluationIssueId: unrelatedIssueId },
|
||||
{ decision: "dismissed_false_positive" as const, evaluationIssueId: unrelatedIssueId },
|
||||
{
|
||||
decision: "snooze" as const,
|
||||
evaluationIssueId: unrelatedIssueId,
|
||||
snoozedUntil: new Date(now.getTime() + 60 * 60 * 1000),
|
||||
},
|
||||
{ decision: "continue" as const, evaluationIssueId: otherEvaluationIssueId },
|
||||
];
|
||||
|
||||
for (const attempt of attempts) {
|
||||
await expect(
|
||||
recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
reason: "malicious or stale binding",
|
||||
...attempt,
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 403 });
|
||||
}
|
||||
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, evaluationIssueId));
|
||||
await expect(
|
||||
recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "closed evaluation should not authorize",
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 403 });
|
||||
});
|
||||
|
||||
it("validates createdByRunId before storing watchdog decisions", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, runId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn() });
|
||||
|
||||
const scan = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const evaluationIssueId = scan.evaluationIssueIds[0];
|
||||
expect(evaluationIssueId).toBeTruthy();
|
||||
|
||||
await expect(
|
||||
recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "client supplied another agent run",
|
||||
createdByRunId: runId,
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 403 });
|
||||
|
||||
const managerRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: managerRunId,
|
||||
companyId,
|
||||
agentId: managerId,
|
||||
status: "running",
|
||||
invocationSource: "assignment",
|
||||
triggerDetail: "system",
|
||||
startedAt: now,
|
||||
processStartedAt: now,
|
||||
lastOutputAt: now,
|
||||
lastOutputSeq: 1,
|
||||
lastOutputStream: "stdout",
|
||||
contextSnapshot: {},
|
||||
logBytes: 0,
|
||||
});
|
||||
|
||||
const decision = await recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId, runId: managerRunId },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "valid current actor run",
|
||||
createdByRunId: randomUUID(),
|
||||
});
|
||||
expect(decision.createdByRunId).toBe(managerRunId);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,8 +1,4 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import net from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createServer } from "node:http";
|
||||
import { and, asc, eq } from "drizzle-orm";
|
||||
import { WebSocketServer } from "ws";
|
||||
|
|
@ -10,81 +6,14 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|||
import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
applyPendingMigrations,
|
||||
companies,
|
||||
createDb,
|
||||
ensurePostgresDatabase,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
|
||||
type EmbeddedPostgresInstance = {
|
||||
initialise(): Promise<void>;
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
type EmbeddedPostgresCtor = new (opts: {
|
||||
databaseDir: string;
|
||||
user: string;
|
||||
password: string;
|
||||
port: number;
|
||||
persistent: boolean;
|
||||
initdbFlags?: string[];
|
||||
onLog?: (message: unknown) => void;
|
||||
onError?: (message: unknown) => void;
|
||||
}) => EmbeddedPostgresInstance;
|
||||
|
||||
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
|
||||
const mod = await import("embedded-postgres");
|
||||
return mod.default as EmbeddedPostgresCtor;
|
||||
}
|
||||
|
||||
async function getAvailablePort(): Promise<number> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const server = net.createServer();
|
||||
server.unref();
|
||||
server.on("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
server.close(() => reject(new Error("Failed to allocate test port")));
|
||||
return;
|
||||
}
|
||||
const { port } = address;
|
||||
server.close((error) => {
|
||||
if (error) reject(error);
|
||||
else resolve(port);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function startTempDatabase() {
|
||||
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-heartbeat-comment-wake-"));
|
||||
const port = await getAvailablePort();
|
||||
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
|
||||
const instance = new EmbeddedPostgres({
|
||||
databaseDir: dataDir,
|
||||
user: "paperclip",
|
||||
password: "paperclip",
|
||||
port,
|
||||
persistent: true,
|
||||
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
|
||||
onLog: () => {},
|
||||
onError: () => {},
|
||||
});
|
||||
await instance.initialise();
|
||||
await instance.start();
|
||||
|
||||
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
|
||||
await ensurePostgresDatabase(adminConnectionString, "paperclip");
|
||||
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
|
||||
await applyPendingMigrations(connectionString);
|
||||
return { connectionString, instance, dataDir };
|
||||
}
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.ts";
|
||||
|
||||
async function waitFor(condition: () => boolean | Promise<boolean>, timeoutMs = 10_000, intervalMs = 50) {
|
||||
const startedAt = Date.now();
|
||||
|
|
@ -218,22 +147,17 @@ async function createControlledGatewayServer() {
|
|||
|
||||
describe("heartbeat comment wake batching", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let instance: EmbeddedPostgresInstance | null = null;
|
||||
let dataDir = "";
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
const started = await startTempDatabase();
|
||||
const started = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-comment-wake-");
|
||||
db = createDb(started.connectionString);
|
||||
instance = started.instance;
|
||||
dataDir = started.dataDir;
|
||||
}, 45_000);
|
||||
tempDb = started;
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await closeDbClient(db);
|
||||
await instance?.stop();
|
||||
if (dataDir) {
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
it("defers approval-approved wakes for a running issue so the assignee resumes after the run", async () => {
|
||||
|
|
@ -862,6 +786,206 @@ describe("heartbeat comment wake batching", () => {
|
|||
}
|
||||
}, 120_000);
|
||||
|
||||
it("does not reopen a finished issue when the deferred comment wake came from another agent", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
const assigneeAgentId = randomUUID();
|
||||
const mentionedAgentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
try {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
await db.insert(agents).values([
|
||||
{
|
||||
id: assigneeAgentId,
|
||||
companyId,
|
||||
name: "Primary Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
id: mentionedAgentId,
|
||||
companyId,
|
||||
name: "Mentioned Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Do not reopen from agent mention",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
assigneeAgentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
});
|
||||
|
||||
const firstRun = await heartbeat.wakeup(assigneeAgentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
},
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
});
|
||||
|
||||
expect(firstRun).not.toBeNull();
|
||||
await waitFor(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, firstRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "running";
|
||||
});
|
||||
|
||||
const comment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorAgentId: assigneeAgentId,
|
||||
createdByRunId: firstRun?.id ?? null,
|
||||
body: "@Mentioned Agent please review after I finish",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const deferredRun = await heartbeat.wakeup(mentionedAgentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_comment_mentioned",
|
||||
payload: { issueId, commentId: comment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: comment.id,
|
||||
wakeCommentId: comment.id,
|
||||
wakeReason: "issue_comment_mentioned",
|
||||
source: "comment.mention",
|
||||
},
|
||||
requestedByActorType: "agent",
|
||||
requestedByActorId: assigneeAgentId,
|
||||
});
|
||||
|
||||
expect(deferredRun).toBeNull();
|
||||
|
||||
await waitFor(async () => {
|
||||
const deferred = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, mentionedAgentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return Boolean(deferred);
|
||||
});
|
||||
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
status: "done",
|
||||
completedAt: new Date(),
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, issueId));
|
||||
|
||||
gateway.releaseFirstWait();
|
||||
|
||||
await waitFor(() => gateway.getAgentPayloads().length === 2, 90_000);
|
||||
await waitFor(async () => {
|
||||
const runs = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.companyId, companyId));
|
||||
return runs.length === 2 && runs.every((run) => run.status === "succeeded");
|
||||
}, 90_000);
|
||||
|
||||
const issueAfterPromotion = await db
|
||||
.select({
|
||||
status: issues.status,
|
||||
completedAt: issues.completedAt,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(issueAfterPromotion).toMatchObject({
|
||||
status: "done",
|
||||
});
|
||||
expect(issueAfterPromotion?.completedAt).not.toBeNull();
|
||||
|
||||
const secondPayload = gateway.getAgentPayloads()[1] ?? {};
|
||||
expect(secondPayload.paperclip).toMatchObject({
|
||||
wake: {
|
||||
reason: "issue_comment_mentioned",
|
||||
commentIds: [comment.id],
|
||||
latestCommentId: comment.id,
|
||||
issue: {
|
||||
id: issueId,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
title: "Do not reopen from agent mention",
|
||||
status: "done",
|
||||
priority: "medium",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(String(secondPayload.message ?? "")).toContain("please review after I finish");
|
||||
} finally {
|
||||
gateway.releaseFirstWait();
|
||||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("queues exactly one follow-up run when an issue-bound run exits without a comment", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
|
|
@ -1172,6 +1296,20 @@ describe("heartbeat comment wake batching", () => {
|
|||
wakeReason: "issue_comment_mentioned",
|
||||
});
|
||||
|
||||
const issueAfterMention = await db
|
||||
.select({
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(issueAfterMention?.assigneeAgentId).toBe(primaryAgentId);
|
||||
expect(issueAfterMention?.executionRunId).not.toBe(mentionedRuns[0]?.id);
|
||||
expect(issueAfterMention?.executionAgentNameKey).not.toBe("mentioned agent");
|
||||
|
||||
const primaryRuns = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
|
|
@ -1198,6 +1336,155 @@ describe("heartbeat comment wake batching", () => {
|
|||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("does not mark a direct mentioned-agent run as the issue execution owner", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
const primaryAgentId = randomUUID();
|
||||
const mentionedAgentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
try {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
await db.insert(agents).values([
|
||||
{
|
||||
id: primaryAgentId,
|
||||
companyId,
|
||||
name: "Primary Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
id: mentionedAgentId,
|
||||
companyId,
|
||||
name: "Mentioned Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Mention should not steal execution ownership",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
assigneeAgentId: primaryAgentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
});
|
||||
|
||||
const mentionComment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorUserId: "user-1",
|
||||
body: "@Mentioned Agent please inspect this.",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const mentionRun = await heartbeat.wakeup(mentionedAgentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_comment_mentioned",
|
||||
payload: { issueId, commentId: mentionComment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: mentionComment.id,
|
||||
wakeCommentId: mentionComment.id,
|
||||
wakeReason: "issue_comment_mentioned",
|
||||
source: "comment.mention",
|
||||
},
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "user-1",
|
||||
});
|
||||
|
||||
expect(mentionRun).not.toBeNull();
|
||||
await waitFor(() => gateway.getAgentPayloads().length === 1);
|
||||
|
||||
const issueDuringMention = await db
|
||||
.select({
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(issueDuringMention).toMatchObject({
|
||||
assigneeAgentId: primaryAgentId,
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
});
|
||||
|
||||
gateway.releaseFirstWait();
|
||||
await waitFor(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, mentionRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
}, 90_000);
|
||||
|
||||
const issueAfterMention = await db
|
||||
.select({
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(issueAfterMention).toMatchObject({
|
||||
assigneeAgentId: primaryAgentId,
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
});
|
||||
} finally {
|
||||
gateway.releaseFirstWait();
|
||||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
it("treats the automatic run summary as fallback-only when the run already posted a comment", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
|
|
|
|||
|
|
@ -347,6 +347,198 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
|
|||
expect(blockedWakeRequestCount).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("cancels stale queued runs when issue blockers are still unresolved", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const blockerId = randomUUID();
|
||||
const blockedIssueId = randomUUID();
|
||||
const readyIssueId = randomUUID();
|
||||
const blockedWakeupRequestId = randomUUID();
|
||||
const readyWakeupRequestId = randomUUID();
|
||||
const blockedRunId = randomUUID();
|
||||
const readyRunId = randomUUID();
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "QAChecker",
|
||||
role: "qa",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
wakeOnDemand: true,
|
||||
maxConcurrentRuns: 2,
|
||||
},
|
||||
},
|
||||
permissions: {},
|
||||
});
|
||||
await db.insert(issues).values([
|
||||
{
|
||||
id: blockerId,
|
||||
companyId,
|
||||
title: "Security review",
|
||||
status: "blocked",
|
||||
priority: "high",
|
||||
},
|
||||
{
|
||||
id: blockedIssueId,
|
||||
companyId,
|
||||
title: "QA validation",
|
||||
status: "blocked",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
},
|
||||
{
|
||||
id: readyIssueId,
|
||||
companyId,
|
||||
title: "Ready QA task",
|
||||
status: "todo",
|
||||
priority: "low",
|
||||
assigneeAgentId: agentId,
|
||||
},
|
||||
]);
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId: blockerId,
|
||||
relatedIssueId: blockedIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
await db.insert(agentWakeupRequests).values([
|
||||
{
|
||||
id: blockedWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "transient_failure_retry",
|
||||
payload: { issueId: blockedIssueId },
|
||||
status: "queued",
|
||||
},
|
||||
{
|
||||
id: readyWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId: readyIssueId },
|
||||
status: "queued",
|
||||
},
|
||||
]);
|
||||
await db.insert(heartbeatRuns).values([
|
||||
{
|
||||
id: blockedRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId: blockedWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: blockedIssueId,
|
||||
wakeReason: "transient_failure_retry",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: readyRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
invocationSource: "assignment",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId: readyWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: readyIssueId,
|
||||
wakeReason: "issue_assigned",
|
||||
},
|
||||
},
|
||||
]);
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: blockedRunId })
|
||||
.where(eq(agentWakeupRequests.id, blockedWakeupRequestId));
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: readyRunId })
|
||||
.where(eq(agentWakeupRequests.id, readyWakeupRequestId));
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: blockedRunId,
|
||||
executionAgentNameKey: "qa-checker",
|
||||
executionLockedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, blockedIssueId));
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, readyRunId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
});
|
||||
|
||||
const [blockedRun, blockedWakeup, blockedIssue, readyRun] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
status: heartbeatRuns.status,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
finishedAt: heartbeatRuns.finishedAt,
|
||||
resultJson: heartbeatRuns.resultJson,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, blockedRunId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({
|
||||
status: agentWakeupRequests.status,
|
||||
error: agentWakeupRequests.error,
|
||||
})
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, blockedWakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
executionLockedAt: issues.executionLockedAt,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, blockedIssueId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, readyRunId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
expect(blockedRun?.status).toBe("cancelled");
|
||||
expect(blockedRun?.errorCode).toBe("issue_dependencies_blocked");
|
||||
expect(blockedRun?.finishedAt).toBeTruthy();
|
||||
expect(blockedRun?.resultJson).toMatchObject({ stopReason: "issue_dependencies_blocked" });
|
||||
expect(blockedWakeup?.status).toBe("skipped");
|
||||
expect(blockedWakeup?.error).toContain("dependencies are still blocked");
|
||||
expect(blockedIssue).toMatchObject({
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
});
|
||||
expect(readyRun?.status).toBe("succeeded");
|
||||
expect(mockAdapterExecute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("suppresses normal wakeups while allowing comment interaction wakes under a pause hold", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
|
|
@ -425,12 +617,39 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
|
|||
.then((rows) => rows[0] ?? null);
|
||||
expect(skippedWake).toMatchObject({ status: "skipped", reason: "issue_tree_hold_active" });
|
||||
|
||||
const childCommentId = randomUUID();
|
||||
await db.insert(issueComments).values({
|
||||
id: childCommentId,
|
||||
companyId,
|
||||
issueId: childIssueId,
|
||||
authorUserId: "board-user",
|
||||
body: "Please respond while this hold is active.",
|
||||
});
|
||||
|
||||
const forgedChildCommentWake = await heartbeat.wakeup(agentId, {
|
||||
source: "on_demand",
|
||||
triggerDetail: "manual",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: childIssueId, commentId: childCommentId },
|
||||
requestedByActorType: "agent",
|
||||
requestedByActorId: agentId,
|
||||
});
|
||||
expect(forgedChildCommentWake).toBeNull();
|
||||
|
||||
const childCommentWake = await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: childIssueId, commentId: randomUUID() },
|
||||
contextSnapshot: { issueId: childIssueId, wakeReason: "issue_commented" },
|
||||
payload: { issueId: childIssueId, commentId: childCommentId },
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "board-user",
|
||||
contextSnapshot: {
|
||||
issueId: childIssueId,
|
||||
commentId: childCommentId,
|
||||
wakeCommentId: childCommentId,
|
||||
wakeReason: "issue_commented",
|
||||
source: "issue.comment",
|
||||
},
|
||||
});
|
||||
|
||||
expect(childCommentWake).not.toBeNull();
|
||||
|
|
@ -494,12 +713,29 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
|
|||
releasePolicy: { strategy: "manual", note: "full_pause" },
|
||||
});
|
||||
|
||||
const rootCommentId = randomUUID();
|
||||
await db.insert(issueComments).values({
|
||||
id: rootCommentId,
|
||||
companyId,
|
||||
issueId: rootIssueId,
|
||||
authorUserId: "board-user",
|
||||
body: "Please respond while this hold is active.",
|
||||
});
|
||||
|
||||
const rootCommentWake = await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: rootIssueId, commentId: randomUUID() },
|
||||
contextSnapshot: { issueId: rootIssueId, wakeReason: "issue_commented" },
|
||||
payload: { issueId: rootIssueId, commentId: rootCommentId },
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "board-user",
|
||||
contextSnapshot: {
|
||||
issueId: rootIssueId,
|
||||
commentId: rootCommentId,
|
||||
wakeCommentId: rootCommentId,
|
||||
wakeReason: "issue_commented",
|
||||
source: "issue.comment",
|
||||
},
|
||||
});
|
||||
|
||||
expect(rootCommentWake).not.toBeNull();
|
||||
|
|
|
|||
|
|
@ -4,13 +4,16 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
|
|||
import {
|
||||
activityLog,
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
companies,
|
||||
createDb,
|
||||
executionWorkspaces,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueRelations,
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
projects,
|
||||
projectWorkspaces,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
|
|
@ -55,6 +58,7 @@ vi.mock("../adapters/index.ts", async () => {
|
|||
});
|
||||
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
import { instanceSettingsService } from "../services/instance-settings.ts";
|
||||
import { runningProcesses } from "../adapters/index.ts";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
|
|
@ -94,13 +98,23 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
await db.execute(sql.raw(`TRUNCATE TABLE "companies" CASCADE`));
|
||||
await instanceSettingsService(db).updateExperimental({
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedBlockedChain() {
|
||||
async function enableAutoRecovery() {
|
||||
await instanceSettingsService(db).updateExperimental({
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
});
|
||||
}
|
||||
|
||||
async function seedBlockedChain(opts: { stale?: boolean } = {}) {
|
||||
const companyId = randomUUID();
|
||||
const managerId = randomUUID();
|
||||
const coderId = randomUUID();
|
||||
|
|
@ -124,7 +138,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
status: "idle",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: false } },
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
|
|
@ -136,11 +150,14 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
reportsTo: managerId,
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: false } },
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
|
||||
const issueTimestamp = opts.stale === false
|
||||
? new Date()
|
||||
: new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
await db.insert(issues).values([
|
||||
{
|
||||
id: blockedIssueId,
|
||||
|
|
@ -151,6 +168,8 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
assigneeAgentId: coderId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
},
|
||||
{
|
||||
id: blockerIssueId,
|
||||
|
|
@ -160,6 +179,8 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
priority: "medium",
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
},
|
||||
]);
|
||||
|
||||
|
|
@ -173,7 +194,91 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
return { companyId, managerId, blockedIssueId, blockerIssueId };
|
||||
}
|
||||
|
||||
it("creates one manager escalation, preserves blockers, and wakes the assignee", async () => {
|
||||
it("keeps liveness findings advisory when auto recovery is disabled", async () => {
|
||||
const { companyId } = await seedBlockedChain();
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(1);
|
||||
expect(result.autoRecoveryEnabled).toBe(false);
|
||||
expect(result.escalationsCreated).toBe(0);
|
||||
expect(result.skippedAutoRecoveryDisabled).toBe(1);
|
||||
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not create recovery issues until the dependency path is stale for 24 hours", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId } = await seedBlockedChain({ stale: false });
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(1);
|
||||
expect(result.escalationsCreated).toBe(0);
|
||||
expect(result.skippedAutoRecoveryTooYoung).toBe(1);
|
||||
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("suppresses liveness escalation when the source issue is under an active pause hold", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, blockedIssueId } = await seedBlockedChain();
|
||||
|
||||
await db.insert(issueTreeHolds).values({
|
||||
companyId,
|
||||
rootIssueId: blockedIssueId,
|
||||
mode: "pause",
|
||||
status: "active",
|
||||
reason: "pause liveness recovery subtree",
|
||||
releasePolicy: { strategy: "manual" },
|
||||
});
|
||||
|
||||
const result = await heartbeatService(db).reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(1);
|
||||
expect(result.escalationsCreated).toBe(0);
|
||||
expect(result.existingEscalations).toBe(0);
|
||||
expect(result.skipped).toBe(1);
|
||||
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("treats an active executionRunId on the leaf blocker as a live execution path", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, managerId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId,
|
||||
agentId: managerId,
|
||||
status: "running",
|
||||
contextSnapshot: { issueId: blockedIssueId },
|
||||
});
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, blockerIssueId));
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(0);
|
||||
expect(result.escalationsCreated).toBe(0);
|
||||
});
|
||||
|
||||
it("creates one manager escalation, preserves blockers, and records owner selection", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, managerId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
|
|
@ -182,7 +287,6 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
|
||||
expect(first.escalationsCreated).toBe(1);
|
||||
expect(second.escalationsCreated).toBe(0);
|
||||
expect(second.existingEscalations).toBe(1);
|
||||
|
||||
const escalations = await db
|
||||
.select()
|
||||
|
|
@ -195,9 +299,15 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
);
|
||||
expect(escalations).toHaveLength(1);
|
||||
expect(escalations[0]).toMatchObject({
|
||||
parentId: blockedIssueId,
|
||||
parentId: blockerIssueId,
|
||||
assigneeAgentId: managerId,
|
||||
status: expect.stringMatching(/^(todo|in_progress|done)$/),
|
||||
originFingerprint: [
|
||||
"harness_liveness_leaf",
|
||||
companyId,
|
||||
"blocked_by_unassigned_issue",
|
||||
blockerIssueId,
|
||||
].join(":"),
|
||||
});
|
||||
|
||||
const blockers = await db
|
||||
|
|
@ -213,15 +323,217 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
expect(comments[0]?.body).toContain("harness-level liveness incident");
|
||||
expect(comments[0]?.body).toContain(escalations[0]?.identifier ?? escalations[0]!.id);
|
||||
|
||||
const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, managerId));
|
||||
expect(wakes.some((wake) => wake.reason === "issue_assigned")).toBe(true);
|
||||
|
||||
const events = await db.select().from(activityLog).where(eq(activityLog.companyId, companyId));
|
||||
expect(events.some((event) => event.action === "issue.harness_liveness_escalation_created")).toBe(true);
|
||||
const createdEvent = events.find((event) => event.action === "issue.harness_liveness_escalation_created");
|
||||
expect(createdEvent).toBeTruthy();
|
||||
expect(createdEvent?.details).toMatchObject({
|
||||
recoveryIssueId: blockerIssueId,
|
||||
ownerSelection: {
|
||||
selectedAgentId: managerId,
|
||||
selectedReason: "root_agent",
|
||||
selectedSourceIssueId: blockerIssueId,
|
||||
},
|
||||
workspaceSelection: {
|
||||
reuseRecoveryExecutionWorkspace: false,
|
||||
inheritedExecutionWorkspaceFromIssueId: null,
|
||||
projectWorkspaceSourceIssueId: blockerIssueId,
|
||||
},
|
||||
});
|
||||
expect(events.some((event) => event.action === "issue.blockers.updated")).toBe(true);
|
||||
});
|
||||
|
||||
it("parents recovery under the leaf blocker without inheriting dependent or blocker execution state for manager-owned recovery", async () => {
|
||||
await enableAutoRecovery();
|
||||
await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
|
||||
|
||||
const companyId = randomUUID();
|
||||
const managerId = randomUUID();
|
||||
const blockedIssueId = randomUUID();
|
||||
const blockerIssueId = randomUUID();
|
||||
const dependentProjectId = randomUUID();
|
||||
const blockerProjectId = randomUUID();
|
||||
const dependentProjectWorkspaceId = randomUUID();
|
||||
const blockerProjectWorkspaceId = randomUUID();
|
||||
const dependentExecutionWorkspaceId = randomUUID();
|
||||
const blockerExecutionWorkspaceId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: managerId,
|
||||
companyId,
|
||||
name: "Root Operator",
|
||||
role: "operator",
|
||||
status: "idle",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: false } },
|
||||
permissions: {},
|
||||
});
|
||||
await db.insert(projects).values([
|
||||
{
|
||||
id: dependentProjectId,
|
||||
companyId,
|
||||
name: "Dependent workspace project",
|
||||
status: "in_progress",
|
||||
},
|
||||
{
|
||||
id: blockerProjectId,
|
||||
companyId,
|
||||
name: "Blocker workspace project",
|
||||
status: "in_progress",
|
||||
},
|
||||
]);
|
||||
await db.insert(projectWorkspaces).values([
|
||||
{
|
||||
id: dependentProjectWorkspaceId,
|
||||
companyId,
|
||||
projectId: dependentProjectId,
|
||||
name: "Dependent primary",
|
||||
},
|
||||
{
|
||||
id: blockerProjectWorkspaceId,
|
||||
companyId,
|
||||
projectId: blockerProjectId,
|
||||
name: "Blocker primary",
|
||||
},
|
||||
]);
|
||||
await db.insert(executionWorkspaces).values([
|
||||
{
|
||||
id: dependentExecutionWorkspaceId,
|
||||
companyId,
|
||||
projectId: dependentProjectId,
|
||||
projectWorkspaceId: dependentProjectWorkspaceId,
|
||||
mode: "operator_branch",
|
||||
strategyType: "git_worktree",
|
||||
name: "Dependent branch",
|
||||
status: "active",
|
||||
providerType: "git_worktree",
|
||||
},
|
||||
{
|
||||
id: blockerExecutionWorkspaceId,
|
||||
companyId,
|
||||
projectId: blockerProjectId,
|
||||
projectWorkspaceId: blockerProjectWorkspaceId,
|
||||
mode: "operator_branch",
|
||||
strategyType: "git_worktree",
|
||||
name: "Blocker branch",
|
||||
status: "active",
|
||||
providerType: "git_worktree",
|
||||
},
|
||||
]);
|
||||
await db.insert(issues).values([
|
||||
{
|
||||
id: blockedIssueId,
|
||||
companyId,
|
||||
projectId: dependentProjectId,
|
||||
projectWorkspaceId: dependentProjectWorkspaceId,
|
||||
executionWorkspaceId: dependentExecutionWorkspaceId,
|
||||
executionWorkspacePreference: "reuse_existing",
|
||||
executionWorkspaceSettings: { mode: "operator_branch" },
|
||||
title: "Blocked dependent",
|
||||
status: "blocked",
|
||||
priority: "medium",
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
},
|
||||
{
|
||||
id: blockerIssueId,
|
||||
companyId,
|
||||
projectId: blockerProjectId,
|
||||
projectWorkspaceId: blockerProjectWorkspaceId,
|
||||
executionWorkspaceId: blockerExecutionWorkspaceId,
|
||||
executionWorkspacePreference: "reuse_existing",
|
||||
executionWorkspaceSettings: { mode: "operator_branch" },
|
||||
title: "Unassigned leaf blocker",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
},
|
||||
]);
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId: blockerIssueId,
|
||||
relatedIssueId: blockedIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
|
||||
const result = await heartbeatService(db).reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.escalationsCreated).toBe(1);
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(1);
|
||||
expect(escalations[0]).toMatchObject({
|
||||
parentId: blockerIssueId,
|
||||
projectId: blockerProjectId,
|
||||
projectWorkspaceId: blockerProjectWorkspaceId,
|
||||
executionWorkspaceId: null,
|
||||
executionWorkspacePreference: null,
|
||||
assigneeAgentId: managerId,
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses one open recovery issue for multiple dependents with the same leaf blocker", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const secondBlockedIssueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
await db.insert(issues).values({
|
||||
id: secondBlockedIssueId,
|
||||
companyId,
|
||||
title: "Second blocked parent",
|
||||
status: "blocked",
|
||||
priority: "medium",
|
||||
issueNumber: 3,
|
||||
identifier: `${issuePrefix}-3`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
});
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId: blockerIssueId,
|
||||
relatedIssueId: secondBlockedIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(2);
|
||||
expect(result.escalationsCreated).toBe(1);
|
||||
expect(result.existingEscalations).toBe(1);
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(1);
|
||||
|
||||
const blockers = await db
|
||||
.select({ blockedIssueId: issueRelations.relatedIssueId })
|
||||
.from(issueRelations)
|
||||
.where(and(eq(issueRelations.companyId, companyId), eq(issueRelations.issueId, escalations[0]!.id)));
|
||||
expect(blockers.map((row) => row.blockedIssueId).sort()).toEqual(
|
||||
[blockedIssueId, secondBlockedIssueId].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("creates a fresh escalation when the previous matching escalation is terminal", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, managerId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const heartbeat = heartbeatService(db);
|
||||
const incidentKey = [
|
||||
|
|
@ -265,7 +577,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
expect(openEscalations).toHaveLength(2);
|
||||
const freshEscalation = openEscalations.find((issue) => issue.status !== "done");
|
||||
expect(freshEscalation).toMatchObject({
|
||||
parentId: blockedIssueId,
|
||||
parentId: blockerIssueId,
|
||||
assigneeAgentId: managerId,
|
||||
status: expect.stringMatching(/^(todo|in_progress|done)$/),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { eq, or, inArray } from "drizzle-orm";
|
||||
import { and, eq, or, inArray } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
activityLog,
|
||||
|
|
@ -17,6 +17,8 @@ import {
|
|||
issueComments,
|
||||
issueDocuments,
|
||||
issueRelations,
|
||||
issueTreeHoldMembers,
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
|
|
@ -309,6 +311,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
await db.delete(documentRevisions);
|
||||
await db.delete(documents);
|
||||
await db.delete(issueRelations);
|
||||
await db.delete(issueTreeHoldMembers);
|
||||
await db.delete(issueTreeHolds);
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issueDocuments);
|
||||
|
|
@ -454,11 +458,13 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
runStatus: "failed" | "timed_out" | "cancelled" | "succeeded";
|
||||
retryReason?: "assignment_recovery" | "issue_continuation_needed" | null;
|
||||
assignToUser?: boolean;
|
||||
activePauseHold?: boolean;
|
||||
}) {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const wakeupRequestId = randomUUID();
|
||||
const rootIssueId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const now = new Date("2026-03-19T00:00:00.000Z");
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
|
|
@ -520,22 +526,128 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
error: input.runStatus === "succeeded" ? null : "run failed before issue advanced",
|
||||
});
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Recover stranded assigned work",
|
||||
status: input.status,
|
||||
await db.insert(issues).values([
|
||||
...(input.activePauseHold
|
||||
? [{
|
||||
id: rootIssueId,
|
||||
companyId,
|
||||
title: "Paused recovery root",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
}]
|
||||
: []),
|
||||
{
|
||||
id: issueId,
|
||||
companyId,
|
||||
parentId: input.activePauseHold ? rootIssueId : null,
|
||||
title: "Recover stranded assigned work",
|
||||
status: input.status,
|
||||
priority: "medium",
|
||||
assigneeAgentId: input.assignToUser ? null : agentId,
|
||||
assigneeUserId: input.assignToUser ? "user-1" : null,
|
||||
checkoutRunId: input.status === "in_progress" ? runId : null,
|
||||
executionRunId: null,
|
||||
issueNumber: input.activePauseHold ? 2 : 1,
|
||||
identifier: `${issuePrefix}-${input.activePauseHold ? 2 : 1}`,
|
||||
startedAt: input.status === "in_progress" ? now : null,
|
||||
},
|
||||
]);
|
||||
|
||||
if (input.activePauseHold) {
|
||||
await db.insert(issueTreeHolds).values({
|
||||
companyId,
|
||||
rootIssueId,
|
||||
mode: "pause",
|
||||
status: "active",
|
||||
reason: "pause recovery subtree",
|
||||
releasePolicy: { strategy: "manual" },
|
||||
});
|
||||
}
|
||||
|
||||
return { companyId, agentId, runId, wakeupRequestId, issueId, rootIssueId };
|
||||
}
|
||||
|
||||
async function expectStrandedRecoveryArtifacts(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
issueId: string;
|
||||
runId: string;
|
||||
previousStatus: "todo" | "in_progress";
|
||||
retryReason: "assignment_recovery" | "issue_continuation_needed";
|
||||
}) {
|
||||
const recovery = await waitForValue(async () =>
|
||||
db.select().from(issues).where(
|
||||
and(
|
||||
eq(issues.companyId, input.companyId),
|
||||
eq(issues.originKind, "stranded_issue_recovery"),
|
||||
eq(issues.originId, input.issueId),
|
||||
),
|
||||
).then((rows) => rows[0] ?? null),
|
||||
);
|
||||
if (!recovery) throw new Error("Expected stranded issue recovery issue to be created");
|
||||
|
||||
expect(recovery).toMatchObject({
|
||||
companyId: input.companyId,
|
||||
parentId: input.issueId,
|
||||
assigneeAgentId: input.agentId,
|
||||
originKind: "stranded_issue_recovery",
|
||||
originId: input.issueId,
|
||||
originRunId: input.runId,
|
||||
priority: "medium",
|
||||
assigneeAgentId: input.assignToUser ? null : agentId,
|
||||
assigneeUserId: input.assignToUser ? "user-1" : null,
|
||||
checkoutRunId: input.status === "in_progress" ? runId : null,
|
||||
executionRunId: null,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
startedAt: input.status === "in_progress" ? now : null,
|
||||
});
|
||||
expect(recovery.title).toContain("Recover stalled issue");
|
||||
expect(recovery.description).toContain(`Previous source status: \`${input.previousStatus}\``);
|
||||
expect(recovery.description).toContain(`Retry reason: \`${input.retryReason}\``);
|
||||
expect(recovery.description).toContain("Fix the runtime/adapter problem");
|
||||
|
||||
const relation = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, input.companyId),
|
||||
eq(issueRelations.issueId, recovery.id),
|
||||
eq(issueRelations.relatedIssueId, input.issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(relation).toBeTruthy();
|
||||
|
||||
const wakeups = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.agentId, input.agentId));
|
||||
const recoveryWakeup = wakeups.find((wakeup) => {
|
||||
const payload = wakeup.payload as Record<string, unknown> | null;
|
||||
return payload?.issueId === recovery.id &&
|
||||
payload?.sourceIssueId === input.issueId &&
|
||||
payload?.strandedRunId === input.runId;
|
||||
});
|
||||
expect(recoveryWakeup).toMatchObject({
|
||||
companyId: input.companyId,
|
||||
reason: "issue_assigned",
|
||||
source: "assignment",
|
||||
});
|
||||
|
||||
return { companyId, agentId, runId, wakeupRequestId, issueId };
|
||||
const recoveryRun = recoveryWakeup?.runId
|
||||
? await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, recoveryWakeup.runId))
|
||||
.then((rows) => rows[0] ?? null)
|
||||
: null;
|
||||
expect(recoveryRun?.contextSnapshot).toMatchObject({
|
||||
issueId: recovery.id,
|
||||
taskId: recovery.id,
|
||||
source: "stranded_issue_recovery",
|
||||
sourceIssueId: input.issueId,
|
||||
strandedRunId: input.runId,
|
||||
});
|
||||
|
||||
return recovery;
|
||||
}
|
||||
|
||||
async function seedQueuedIssueRunFixture() {
|
||||
|
|
@ -728,11 +840,28 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
it("blocks the issue when process-loss retry is exhausted and the immediate continuation recovery also fails", async () => {
|
||||
mockAdapterExecute.mockRejectedValueOnce(new Error("continuation recovery failed"));
|
||||
|
||||
const { agentId, runId, issueId } = await seedRunFixture({
|
||||
const { companyId, agentId, runId, issueId } = await seedRunFixture({
|
||||
agentStatus: "idle",
|
||||
processPid: 999_999_999,
|
||||
processLossRetryCount: 1,
|
||||
});
|
||||
const resolvedBlockerId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
await db.insert(issues).values({
|
||||
id: resolvedBlockerId,
|
||||
companyId,
|
||||
title: "Already completed prerequisite",
|
||||
status: "done",
|
||||
priority: "medium",
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
});
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId: resolvedBlockerId,
|
||||
relatedIssueId: issueId,
|
||||
type: "blocks",
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reapOrphanedRuns();
|
||||
|
|
@ -759,7 +888,29 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
);
|
||||
expect(blockedIssue?.status).toBe("blocked");
|
||||
expect(blockedIssue?.executionRunId).toBeNull();
|
||||
expect(blockedIssue?.checkoutRunId).toBe(continuationRun?.id ?? null);
|
||||
expect(blockedIssue?.checkoutRunId).toBeNull();
|
||||
if (!continuationRun?.id) throw new Error("Expected continuation recovery run to exist");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
runId: continuationRun.id,
|
||||
previousStatus: "in_progress",
|
||||
retryReason: "issue_continuation_needed",
|
||||
});
|
||||
|
||||
const blockerRelations = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.relatedIssueId, issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
);
|
||||
expect(blockerRelations.map((relation) => relation.issueId)).toEqual([recovery.id]);
|
||||
|
||||
const comments = await waitForValue(async () => {
|
||||
const rows = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
|
|
@ -767,6 +918,49 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
});
|
||||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("retried continuation");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
});
|
||||
|
||||
it("does not block paused-tree work when immediate continuation recovery is suppressed by the hold", async () => {
|
||||
const { companyId, agentId, runId, issueId } = await seedRunFixture({
|
||||
agentStatus: "idle",
|
||||
processPid: 999_999_999,
|
||||
processLossRetryCount: 1,
|
||||
});
|
||||
await db.insert(issueTreeHolds).values({
|
||||
companyId,
|
||||
rootIssueId: issueId,
|
||||
mode: "pause",
|
||||
status: "active",
|
||||
reason: "pause immediate recovery subtree",
|
||||
releasePolicy: { strategy: "manual" },
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reapOrphanedRuns();
|
||||
expect(result.reaped).toBe(1);
|
||||
expect(result.runIds).toEqual([runId]);
|
||||
|
||||
const runs = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.agentId, agentId));
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(runs[0]?.status).toBe("failed");
|
||||
|
||||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("in_progress");
|
||||
expect(issue?.executionRunId).toBeNull();
|
||||
expect(issue?.checkoutRunId).toBe(runId);
|
||||
|
||||
const recoveryIssues = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
|
||||
expect(recoveryIssues).toHaveLength(0);
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("schedules a bounded retry for codex transient upstream failures instead of blocking the issue immediately", async () => {
|
||||
|
|
@ -901,7 +1095,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
});
|
||||
|
||||
it("blocks assigned todo work after the one automatic dispatch recovery was already used", async () => {
|
||||
const { issueId } = await seedStrandedIssueFixture({
|
||||
const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({
|
||||
status: "todo",
|
||||
runStatus: "failed",
|
||||
retryReason: "assignment_recovery",
|
||||
|
|
@ -916,10 +1110,20 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("blocked");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
runId,
|
||||
previousStatus: "todo",
|
||||
retryReason: "assignment_recovery",
|
||||
});
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("retried dispatch");
|
||||
expect(comments[0]?.body).toContain("Latest retry failure: `process_lost` - run failed before issue advanced.");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
});
|
||||
|
||||
it("assigns open unassigned blockers back to their creator agent", async () => {
|
||||
|
|
@ -1206,7 +1410,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
expect(wakes.some((row) => row.reason === "run_liveness_continuation")).toBe(false);
|
||||
});
|
||||
it("blocks stranded in-progress work after the continuation retry was already used", async () => {
|
||||
const { issueId } = await seedStrandedIssueFixture({
|
||||
const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({
|
||||
status: "in_progress",
|
||||
runStatus: "failed",
|
||||
retryReason: "issue_continuation_needed",
|
||||
|
|
@ -1221,10 +1425,65 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("blocked");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
runId,
|
||||
previousStatus: "in_progress",
|
||||
retryReason: "issue_continuation_needed",
|
||||
});
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("retried continuation");
|
||||
expect(comments[0]?.body).toContain("Latest retry failure: `process_lost` - run failed before issue advanced.");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
});
|
||||
|
||||
it("does not escalate paused-tree recovery when the automatic continuation retry was cancelled by the hold", async () => {
|
||||
const { companyId, agentId, issueId } = await seedStrandedIssueFixture({
|
||||
status: "in_progress",
|
||||
runStatus: "cancelled",
|
||||
retryReason: "issue_continuation_needed",
|
||||
activePauseHold: true,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileStrandedAssignedIssues();
|
||||
expect(result.dispatchRequeued).toBe(0);
|
||||
expect(result.continuationRequeued).toBe(0);
|
||||
expect(result.escalated).toBe(0);
|
||||
expect(result.skipped).toBe(1);
|
||||
expect(result.issueIds).toEqual([]);
|
||||
|
||||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("in_progress");
|
||||
expect(issue?.checkoutRunId).toBeTruthy();
|
||||
|
||||
const recoveryIssues = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
|
||||
expect(recoveryIssues).toHaveLength(0);
|
||||
|
||||
const blockerRelations = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.relatedIssueId, issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
);
|
||||
expect(blockerRelations).toHaveLength(0);
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(0);
|
||||
|
||||
const wakeups = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId));
|
||||
expect(wakeups).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("re-enqueues continuation when the latest automatic continuation succeeded without closing the issue", async () => {
|
||||
|
|
|
|||
30
server/src/__tests__/heartbeat-start-lock.test.ts
Normal file
30
server/src/__tests__/heartbeat-start-lock.test.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { withAgentStartLock } from "../services/agent-start-lock.ts";
|
||||
|
||||
describe("heartbeat agent start lock", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("does not let a stale start lock freeze later queued-run starts", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const agentId = randomUUID();
|
||||
const firstStart = vi.fn(() => new Promise<void>(() => undefined));
|
||||
const secondStart = vi.fn(async () => "started");
|
||||
|
||||
void withAgentStartLock(agentId, firstStart);
|
||||
await Promise.resolve();
|
||||
expect(firstStart).toHaveBeenCalledTimes(1);
|
||||
|
||||
const secondStartResult = withAgentStartLock(agentId, secondStart);
|
||||
await Promise.resolve();
|
||||
expect(secondStart).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
|
||||
await expect(secondStartResult).resolves.toBe("started");
|
||||
expect(secondStart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -58,6 +58,7 @@ describe("instance settings routes", () => {
|
|||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
});
|
||||
mockInstanceSettingsService.updateGeneral.mockResolvedValue({
|
||||
id: "instance-settings-1",
|
||||
|
|
@ -73,6 +74,7 @@ describe("instance settings routes", () => {
|
|||
enableEnvironments: true,
|
||||
enableIsolatedWorkspaces: true,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
},
|
||||
});
|
||||
mockInstanceSettingsService.listCompanyIds.mockResolvedValue(["company-1", "company-2"]);
|
||||
|
|
@ -92,6 +94,7 @@ describe("instance settings routes", () => {
|
|||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
});
|
||||
|
||||
const patchRes = await request(app)
|
||||
|
|
@ -103,7 +106,7 @@ describe("instance settings routes", () => {
|
|||
enableIsolatedWorkspaces: true,
|
||||
});
|
||||
expect(mockLogActivity).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
}, 10_000);
|
||||
|
||||
it("allows local board users to update guarded dev-server auto-restart", async () => {
|
||||
const app = await createApp({
|
||||
|
|
@ -118,8 +121,28 @@ describe("instance settings routes", () => {
|
|||
.send({ autoRestartDevServerWhenIdle: true })
|
||||
.expect(200);
|
||||
|
||||
expect(
|
||||
mockInstanceSettingsService.updateExperimental.mock.calls.some(
|
||||
([patch]) => patch?.autoRestartDevServerWhenIdle === true,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("allows local board users to update issue graph liveness auto-recovery", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
userId: "local-board",
|
||||
source: "local_implicit",
|
||||
isInstanceAdmin: true,
|
||||
});
|
||||
|
||||
await request(app)
|
||||
.patch("/api/instance/settings/experimental")
|
||||
.send({ enableIssueGraphLivenessAutoRecovery: true })
|
||||
.expect(200);
|
||||
|
||||
expect(mockInstanceSettingsService.updateExperimental).toHaveBeenCalledWith({
|
||||
autoRestartDevServerWhenIdle: true,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
function createSelectChain(rows: unknown[]) {
|
||||
const query = {
|
||||
|
|
@ -44,8 +44,6 @@ function createInvite(overrides: Record<string, unknown> = {}) {
|
|||
};
|
||||
}
|
||||
|
||||
let currentAccessModule: Awaited<ReturnType<typeof vi.importActual<typeof import("../routes/access.js")>>> | null = null;
|
||||
|
||||
async function createApp(
|
||||
db: Record<string, unknown>,
|
||||
network: {
|
||||
|
|
@ -54,11 +52,9 @@ async function createApp(
|
|||
},
|
||||
) {
|
||||
const [access, middleware] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/access.js")>("../routes/access.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
import("../routes/access.js"),
|
||||
import("../middleware/index.js"),
|
||||
]);
|
||||
currentAccessModule = access;
|
||||
access.setInviteResolutionNetworkForTest(network);
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = { type: "anon" };
|
||||
|
|
@ -71,6 +67,7 @@ async function createApp(
|
|||
deploymentExposure: "private",
|
||||
bindHost: "127.0.0.1",
|
||||
allowedHostnames: [],
|
||||
inviteResolutionNetwork: network,
|
||||
}),
|
||||
);
|
||||
app.use(middleware.errorHandler);
|
||||
|
|
@ -79,43 +76,43 @@ async function createApp(
|
|||
|
||||
describe.sequential("GET /invites/:token/test-resolution", () => {
|
||||
beforeEach(() => {
|
||||
currentAccessModule = null;
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
currentAccessModule?.setInviteResolutionNetworkForTest(null);
|
||||
});
|
||||
it("rejects private, local, multicast, and reserved targets before probing", async () => {
|
||||
const cases = [
|
||||
["localhost", "http://localhost:3100/api/health", "127.0.0.1"],
|
||||
["IPv4 loopback", "http://127.0.0.1:3100/api/health", "127.0.0.1"],
|
||||
["IPv6 loopback", "http://[::1]:3100/api/health", "::1"],
|
||||
["IPv4-mapped IPv6 loopback hex", "http://[::ffff:7f00:1]/api/health", "::ffff:7f00:1"],
|
||||
["IPv4-mapped IPv6 RFC1918 hex", "http://[::ffff:c0a8:101]/api/health", "::ffff:c0a8:101"],
|
||||
["RFC1918 10/8", "http://10.0.0.5/api/health", "10.0.0.5"],
|
||||
["RFC1918 172.16/12", "http://172.16.10.5/api/health", "172.16.10.5"],
|
||||
["RFC1918 192.168/16", "http://192.168.1.10/api/health", "192.168.1.10"],
|
||||
["link-local metadata", "http://169.254.169.254/latest/meta-data", "169.254.169.254"],
|
||||
["multicast", "http://224.0.0.1/probe", "224.0.0.1"],
|
||||
["NAT64 well-known prefix", "https://gateway.example.test/health", "64:ff9b::0a00:0001"],
|
||||
["NAT64 local-use prefix", "https://gateway.example.test/health", "64:ff9b:1::0a00:0001"],
|
||||
] as const;
|
||||
|
||||
it.each([
|
||||
["localhost", "http://localhost:3100/api/health", "127.0.0.1"],
|
||||
["IPv4 loopback", "http://127.0.0.1:3100/api/health", "127.0.0.1"],
|
||||
["IPv6 loopback", "http://[::1]:3100/api/health", "::1"],
|
||||
["IPv4-mapped IPv6 loopback hex", "http://[::ffff:7f00:1]/api/health", "::ffff:7f00:1"],
|
||||
["IPv4-mapped IPv6 RFC1918 hex", "http://[::ffff:c0a8:101]/api/health", "::ffff:c0a8:101"],
|
||||
["RFC1918 10/8", "http://10.0.0.5/api/health", "10.0.0.5"],
|
||||
["RFC1918 172.16/12", "http://172.16.10.5/api/health", "172.16.10.5"],
|
||||
["RFC1918 192.168/16", "http://192.168.1.10/api/health", "192.168.1.10"],
|
||||
["link-local metadata", "http://169.254.169.254/latest/meta-data", "169.254.169.254"],
|
||||
["multicast", "http://224.0.0.1/probe", "224.0.0.1"],
|
||||
["NAT64 well-known prefix", "https://gateway.example.test/health", "64:ff9b::0a00:0001"],
|
||||
["NAT64 local-use prefix", "https://gateway.example.test/health", "64:ff9b:1::0a00:0001"],
|
||||
])("rejects %s targets before probing", async (_label, url, address) => {
|
||||
const lookup = vi.fn().mockResolvedValue([{ address, family: address.includes(":") ? 6 : 4 }]);
|
||||
const requestHead = vi.fn();
|
||||
const app = await createApp(createDbStub([createInvite()]), { lookup, requestHead });
|
||||
for (const [label, url, address] of cases) {
|
||||
const lookup = vi.fn().mockResolvedValue([{ address, family: address.includes(":") ? 6 : 4 }]);
|
||||
const requestHead = vi.fn();
|
||||
const app = await createApp(createDbStub([createInvite()]), { lookup, requestHead });
|
||||
|
||||
const res = await request(app)
|
||||
.get("/api/invites/pcp_invite_test/test-resolution")
|
||||
.query({ url });
|
||||
const res = await request(app)
|
||||
.get("/api/invites/pcp_invite_test/test-resolution")
|
||||
.query({ url });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe(
|
||||
"url resolves to a private, local, multicast, or reserved address",
|
||||
);
|
||||
expect(requestHead).not.toHaveBeenCalled();
|
||||
}, 15_000);
|
||||
expect(res.status, label).toBe(400);
|
||||
expect(res.body.error).toBe(
|
||||
"url resolves to a private, local, multicast, or reserved address",
|
||||
);
|
||||
expect(requestHead).not.toHaveBeenCalled();
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("rejects hostnames that resolve to private addresses", async () => {
|
||||
it.sequential("rejects hostnames that resolve to private addresses", async () => {
|
||||
const lookup = vi.fn().mockResolvedValue([{ address: "10.1.2.3", family: 4 }]);
|
||||
const requestHead = vi.fn();
|
||||
const app = await createApp(createDbStub([createInvite()]), { lookup, requestHead });
|
||||
|
|
@ -132,7 +129,7 @@ describe.sequential("GET /invites/:token/test-resolution", () => {
|
|||
expect(requestHead).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects hostnames when any resolved address is private", async () => {
|
||||
it.sequential("rejects hostnames when any resolved address is private", async () => {
|
||||
const lookup = vi.fn().mockResolvedValue([
|
||||
{ address: "127.0.0.1", family: 4 },
|
||||
{ address: "93.184.216.34", family: 4 },
|
||||
|
|
@ -148,7 +145,7 @@ describe.sequential("GET /invites/:token/test-resolution", () => {
|
|||
expect(requestHead).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows public HTTPS targets through the resolved and pinned probe path", async () => {
|
||||
it.sequential("allows public HTTPS targets through the resolved and pinned probe path", async () => {
|
||||
const lookup = vi.fn().mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
|
||||
const requestHead = vi.fn().mockResolvedValue({ httpStatus: 204 });
|
||||
const app = await createApp(createDbStub([createInvite()]), { lookup, requestHead });
|
||||
|
|
@ -177,7 +174,7 @@ describe.sequential("GET /invites/:token/test-resolution", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
it.sequential.each([
|
||||
["missing invite", []],
|
||||
["revoked invite", [createInvite({ revokedAt: new Date("2026-03-07T00:05:00.000Z") })]],
|
||||
["expired invite", [createInvite({ expiresAt: new Date("2020-03-07T00:10:00.000Z") })]],
|
||||
|
|
|
|||
280
server/src/__tests__/issue-blocker-attention.test.ts
Normal file
280
server/src/__tests__/issue-blocker-attention.test.ts
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issueRelations,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres issue blocker attention tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("issue blocker attention", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let svc!: ReturnType<typeof issueService>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
svc = issueService(db);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(issueRelations);
|
||||
await db.delete(issues);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function createCompany(prefix = "PBA") {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: `Company ${prefix}`,
|
||||
issuePrefix: prefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: `${prefix} Agent`,
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
});
|
||||
return { companyId, agentId };
|
||||
}
|
||||
|
||||
async function insertIssue(input: {
|
||||
companyId: string;
|
||||
id?: string;
|
||||
identifier: string;
|
||||
title: string;
|
||||
status: string;
|
||||
parentId?: string | null;
|
||||
assigneeAgentId?: string | null;
|
||||
}) {
|
||||
const id = input.id ?? randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id,
|
||||
companyId: input.companyId,
|
||||
identifier: input.identifier,
|
||||
title: input.title,
|
||||
status: input.status,
|
||||
priority: "medium",
|
||||
parentId: input.parentId ?? null,
|
||||
assigneeAgentId: input.assigneeAgentId ?? null,
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
async function block(input: { companyId: string; blockerIssueId: string; blockedIssueId: string }) {
|
||||
await db.insert(issueRelations).values({
|
||||
companyId: input.companyId,
|
||||
issueId: input.blockerIssueId,
|
||||
relatedIssueId: input.blockedIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
}
|
||||
|
||||
async function activeRun(input: { companyId: string; agentId: string; issueId: string; status?: string; current?: boolean }) {
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
status: input.status ?? "running",
|
||||
contextSnapshot: { issueId: input.issueId },
|
||||
});
|
||||
if (input.current !== false) {
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, input.issueId));
|
||||
}
|
||||
return runId;
|
||||
}
|
||||
|
||||
it("classifies a blocked parent as covered when its child has a running execution path", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBC");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBC-1", title: "Parent", status: "blocked" });
|
||||
const childId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBC-2",
|
||||
title: "Running child",
|
||||
status: "todo",
|
||||
parentId,
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: childId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: childId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "covered",
|
||||
reason: "active_child",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 1,
|
||||
attentionBlockerCount: 0,
|
||||
sampleBlockerIdentifier: "PBC-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps mixed blockers attention-required when any path lacks active work", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBM");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBM-1", title: "Parent", status: "blocked" });
|
||||
const activeChildId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBM-2",
|
||||
title: "Running child",
|
||||
status: "todo",
|
||||
parentId,
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
const idleBlockerId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBM-3",
|
||||
title: "Idle blocker",
|
||||
status: "todo",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: activeChildId, blockedIssueId: parentId });
|
||||
await block({ companyId, blockerIssueId: idleBlockerId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: activeChildId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
unresolvedBlockerCount: 2,
|
||||
coveredBlockerCount: 1,
|
||||
attentionBlockerCount: 1,
|
||||
sampleBlockerIdentifier: "PBM-3",
|
||||
});
|
||||
});
|
||||
|
||||
it("covers recursive blocker chains when the downstream leaf has active work", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBR");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBR-1", title: "Parent", status: "blocked" });
|
||||
const blockerId = await insertIssue({ companyId, identifier: "PBR-2", title: "Blocked dependency", status: "blocked" });
|
||||
const leafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBR-3",
|
||||
title: "Running leaf",
|
||||
status: "todo",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
|
||||
await block({ companyId, blockerIssueId: leafId, blockedIssueId: blockerId });
|
||||
await activeRun({ companyId, agentId, issueId: leafId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "covered",
|
||||
reason: "active_dependency",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 1,
|
||||
attentionBlockerCount: 0,
|
||||
sampleBlockerIdentifier: "PBR-3",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let another company's active run cover the blocker", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBS");
|
||||
const other = await createCompany("PBT");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBS-1", title: "Parent", status: "blocked" });
|
||||
const blockerId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBS-2",
|
||||
title: "Same-company blocker",
|
||||
status: "todo",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId: other.companyId, agentId: other.agentId, issueId: blockerId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 0,
|
||||
attentionBlockerCount: 1,
|
||||
sampleBlockerIdentifier: "PBS-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not cover a blocker from a stale run the issue no longer owns", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBX");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBX-1", title: "Parent", status: "blocked" });
|
||||
const blockerId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBX-2",
|
||||
title: "Previously running blocker",
|
||||
status: "blocked",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: blockerId, current: false });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 0,
|
||||
attentionBlockerCount: 1,
|
||||
sampleBlockerIdentifier: "PBX-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not treat a scheduled retry as actively covered work", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBY");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBY-1", title: "Parent", status: "blocked" });
|
||||
const blockerId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBY-2",
|
||||
title: "Retrying blocker",
|
||||
status: "blocked",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: blockerId, status: "scheduled_retry" });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 0,
|
||||
attentionBlockerCount: 1,
|
||||
sampleBlockerIdentifier: "PBY-2",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -28,6 +28,7 @@ const mockHeartbeatService = vi.hoisted(() => ({
|
|||
|
||||
const mockAgentService = vi.hoisted(() => ({
|
||||
getById: vi.fn(),
|
||||
list: vi.fn(),
|
||||
resolveByReference: vi.fn(),
|
||||
}));
|
||||
|
||||
|
|
@ -61,80 +62,82 @@ const mockIssueThreadInteractionService = vi.hoisted(() => ({
|
|||
expireRequestConfirmationsSupersededByComment: vi.fn(async () => []),
|
||||
expireStaleRequestConfirmationsForIssueDocument: vi.fn(async () => []),
|
||||
}));
|
||||
const mockIssueTreeControlService = vi.hoisted(() => ({
|
||||
getActivePauseHoldGate: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
function registerModuleMocks() {
|
||||
vi.doMock("@paperclipai/shared/telemetry", () => ({
|
||||
trackAgentTaskCompleted: vi.fn(),
|
||||
trackErrorHandlerCrash: vi.fn(),
|
||||
}));
|
||||
vi.mock("@paperclipai/shared/telemetry", () => ({
|
||||
trackAgentTaskCompleted: vi.fn(),
|
||||
trackErrorHandlerCrash: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.doMock("../telemetry.js", () => ({
|
||||
getTelemetryClient: vi.fn(() => ({ track: vi.fn() })),
|
||||
}));
|
||||
vi.mock("../telemetry.js", () => ({
|
||||
getTelemetryClient: vi.fn(() => ({ track: vi.fn() })),
|
||||
}));
|
||||
|
||||
vi.doMock("../services/access.js", () => ({
|
||||
accessService: () => mockAccessService,
|
||||
}));
|
||||
vi.mock("../services/access.js", () => ({
|
||||
accessService: () => mockAccessService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/activity-log.js", () => ({
|
||||
logActivity: mockLogActivity,
|
||||
}));
|
||||
vi.mock("../services/activity-log.js", () => ({
|
||||
logActivity: mockLogActivity,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/agents.js", () => ({
|
||||
agentService: () => mockAgentService,
|
||||
}));
|
||||
vi.mock("../services/agents.js", () => ({
|
||||
agentService: () => mockAgentService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/feedback.js", () => ({
|
||||
feedbackService: () => mockFeedbackService,
|
||||
}));
|
||||
vi.mock("../services/feedback.js", () => ({
|
||||
feedbackService: () => mockFeedbackService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/heartbeat.js", () => ({
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
}));
|
||||
vi.mock("../services/heartbeat.js", () => ({
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/instance-settings.js", () => ({
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
}));
|
||||
vi.mock("../services/instance-settings.js", () => ({
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/issues.js", () => ({
|
||||
issueService: () => mockIssueService,
|
||||
}));
|
||||
vi.mock("../services/issues.js", () => ({
|
||||
issueService: () => mockIssueService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/routines.js", () => ({
|
||||
routineService: () => mockRoutineService,
|
||||
}));
|
||||
vi.mock("../services/routines.js", () => ({
|
||||
routineService: () => mockRoutineService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/index.js", () => ({
|
||||
accessService: () => mockAccessService,
|
||||
agentService: () => mockAgentService,
|
||||
documentService: () => ({}),
|
||||
executionWorkspaceService: () => ({}),
|
||||
feedbackService: () => mockFeedbackService,
|
||||
goalService: () => ({}),
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
issueApprovalService: () => ({}),
|
||||
issueReferenceService: () => ({
|
||||
deleteDocumentSource: async () => undefined,
|
||||
diffIssueReferenceSummary: () => ({
|
||||
addedReferencedIssues: [],
|
||||
removedReferencedIssues: [],
|
||||
currentReferencedIssues: [],
|
||||
}),
|
||||
emptySummary: () => ({ outbound: [], inbound: [] }),
|
||||
listIssueReferenceSummary: async () => ({ outbound: [], inbound: [] }),
|
||||
syncComment: async () => undefined,
|
||||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
vi.mock("../services/index.js", () => ({
|
||||
accessService: () => mockAccessService,
|
||||
agentService: () => mockAgentService,
|
||||
documentService: () => ({}),
|
||||
executionWorkspaceService: () => ({}),
|
||||
feedbackService: () => mockFeedbackService,
|
||||
goalService: () => ({}),
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
issueApprovalService: () => ({}),
|
||||
issueReferenceService: () => ({
|
||||
deleteDocumentSource: async () => undefined,
|
||||
diffIssueReferenceSummary: () => ({
|
||||
addedReferencedIssues: [],
|
||||
removedReferencedIssues: [],
|
||||
currentReferencedIssues: [],
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockIssueThreadInteractionService,
|
||||
logActivity: mockLogActivity,
|
||||
projectService: () => ({}),
|
||||
routineService: () => mockRoutineService,
|
||||
workProductService: () => ({}),
|
||||
}));
|
||||
}
|
||||
emptySummary: () => ({ outbound: [], inbound: [] }),
|
||||
listIssueReferenceSummary: async () => ({ outbound: [], inbound: [] }),
|
||||
syncComment: async () => undefined,
|
||||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockIssueThreadInteractionService,
|
||||
issueTreeControlService: () => mockIssueTreeControlService,
|
||||
logActivity: mockLogActivity,
|
||||
projectService: () => ({}),
|
||||
routineService: () => mockRoutineService,
|
||||
workProductService: () => ({}),
|
||||
}));
|
||||
|
||||
function createApp() {
|
||||
const app = express();
|
||||
|
|
@ -144,8 +147,8 @@ function createApp() {
|
|||
|
||||
async function installActor(app: express.Express, actor?: Record<string, unknown>) {
|
||||
const [{ issueRoutes }, { errorHandler }] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/issues.js")>("../routes/issues.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
import("../routes/issues.js"),
|
||||
import("../middleware/index.js"),
|
||||
]);
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = actor ?? {
|
||||
|
|
@ -173,7 +176,7 @@ async function normalizePolicy(input: {
|
|||
return normalizeIssueExecutionPolicy(input);
|
||||
}
|
||||
|
||||
function makeIssue(status: "todo" | "done" | "blocked") {
|
||||
function makeIssue(status: "todo" | "done" | "blocked" | "cancelled" | "in_progress") {
|
||||
return {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
companyId: "company-1",
|
||||
|
|
@ -186,25 +189,23 @@ function makeIssue(status: "todo" | "done" | "blocked") {
|
|||
};
|
||||
}
|
||||
|
||||
describe("issue comment reopen routes", () => {
|
||||
function agentActor(agentId = "22222222-2222-4222-8222-222222222222") {
|
||||
return {
|
||||
type: "agent",
|
||||
agentId,
|
||||
companyId: "company-1",
|
||||
source: "agent_key",
|
||||
runId: "run-1",
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForWakeup(assertion: () => void) {
|
||||
await vi.waitFor(assertion);
|
||||
}
|
||||
|
||||
describe.sequential("issue comment reopen routes", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("@paperclipai/shared/telemetry");
|
||||
vi.doUnmock("../telemetry.js");
|
||||
vi.doUnmock("../services/access.js");
|
||||
vi.doUnmock("../services/activity-log.js");
|
||||
vi.doUnmock("../services/agents.js");
|
||||
vi.doUnmock("../services/feedback.js");
|
||||
vi.doUnmock("../services/heartbeat.js");
|
||||
vi.doUnmock("../services/index.js");
|
||||
vi.doUnmock("../services/instance-settings.js");
|
||||
vi.doUnmock("../services/issues.js");
|
||||
vi.doUnmock("../services/routines.js");
|
||||
vi.doUnmock("../routes/issues.js");
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
registerModuleMocks();
|
||||
vi.resetAllMocks();
|
||||
vi.clearAllMocks();
|
||||
mockIssueService.getById.mockReset();
|
||||
mockIssueService.assertCheckoutOwner.mockReset();
|
||||
mockIssueService.update.mockReset();
|
||||
|
|
@ -221,6 +222,7 @@ describe("issue comment reopen routes", () => {
|
|||
mockHeartbeatService.getActiveRunForAgent.mockReset();
|
||||
mockHeartbeatService.cancelRun.mockReset();
|
||||
mockAgentService.getById.mockReset();
|
||||
mockAgentService.list.mockReset();
|
||||
mockAgentService.resolveByReference.mockReset();
|
||||
mockLogActivity.mockReset();
|
||||
mockFeedbackService.listIssueVotesForUser.mockReset();
|
||||
|
|
@ -228,6 +230,7 @@ describe("issue comment reopen routes", () => {
|
|||
mockInstanceSettingsService.get.mockReset();
|
||||
mockInstanceSettingsService.listCompanyIds.mockReset();
|
||||
mockRoutineService.syncRunStatusForIssue.mockReset();
|
||||
mockIssueTreeControlService.getActivePauseHoldGate.mockReset();
|
||||
mockTxInsertValues.mockReset();
|
||||
mockTxInsert.mockReset();
|
||||
mockDb.transaction.mockReset();
|
||||
|
|
@ -255,6 +258,7 @@ describe("issue comment reopen routes", () => {
|
|||
});
|
||||
mockInstanceSettingsService.listCompanyIds.mockResolvedValue(["company-1"]);
|
||||
mockRoutineService.syncRunStatusForIssue.mockResolvedValue(undefined);
|
||||
mockIssueTreeControlService.getActivePauseHoldGate.mockResolvedValue(null);
|
||||
mockIssueService.addComment.mockResolvedValue({
|
||||
id: "comment-1",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
|
|
@ -280,12 +284,36 @@ describe("issue comment reopen routes", () => {
|
|||
mockAccessService.canUser.mockResolvedValue(false);
|
||||
mockAccessService.hasPermission.mockResolvedValue(false);
|
||||
mockAgentService.getById.mockResolvedValue(null);
|
||||
mockAgentService.resolveByReference.mockImplementation(async (_companyId: string, reference: string) => ({
|
||||
ambiguous: false,
|
||||
agent: {
|
||||
id: reference,
|
||||
mockAgentService.list.mockResolvedValue([
|
||||
{
|
||||
id: "22222222-2222-4222-8222-222222222222",
|
||||
reportsTo: null,
|
||||
permissions: { canCreateAgents: false },
|
||||
},
|
||||
}));
|
||||
{
|
||||
id: "44444444-4444-4444-8444-444444444444",
|
||||
reportsTo: null,
|
||||
permissions: { canCreateAgents: false },
|
||||
},
|
||||
]);
|
||||
mockAgentService.resolveByReference.mockImplementation(async (_companyId: string, reference: string) => {
|
||||
if (reference === "ambiguous-codex") {
|
||||
return { ambiguous: true, agent: null };
|
||||
}
|
||||
if (reference === "missing-codex") {
|
||||
return { ambiguous: false, agent: null };
|
||||
}
|
||||
if (reference === "codexcoder") {
|
||||
return {
|
||||
ambiguous: false,
|
||||
agent: { id: "33333333-3333-4333-8333-333333333333" },
|
||||
};
|
||||
}
|
||||
return {
|
||||
ambiguous: false,
|
||||
agent: { id: reference },
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
it("treats reopen=true as a no-op when the issue is already open", async () => {
|
||||
|
|
@ -350,10 +378,6 @@ describe("issue comment reopen routes", () => {
|
|||
...makeIssue("todo"),
|
||||
...patch,
|
||||
}));
|
||||
mockAgentService.resolveByReference.mockResolvedValue({
|
||||
ambiguous: false,
|
||||
agent: { id: "33333333-3333-4333-8333-333333333333" },
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
|
|
@ -371,14 +395,10 @@ describe("issue comment reopen routes", () => {
|
|||
|
||||
it("rejects ambiguous assignee shortnames", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("todo"));
|
||||
mockAgentService.resolveByReference.mockResolvedValue({
|
||||
ambiguous: true,
|
||||
agent: null,
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ assigneeAgentId: "codexcoder" });
|
||||
.send({ assigneeAgentId: "ambiguous-codex" });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.error).toContain("ambiguous");
|
||||
|
|
@ -387,14 +407,10 @@ describe("issue comment reopen routes", () => {
|
|||
|
||||
it("rejects missing assignee shortnames", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("todo"));
|
||||
mockAgentService.resolveByReference.mockResolvedValue({
|
||||
ambiguous: false,
|
||||
agent: null,
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ assigneeAgentId: "codexcoder" });
|
||||
.send({ assigneeAgentId: "missing-codex" });
|
||||
|
||||
expect(res.status).toBe(404);
|
||||
expect(res.body.error).toBe("Agent not found");
|
||||
|
|
@ -450,7 +466,7 @@ describe("issue comment reopen routes", () => {
|
|||
"11111111-1111-4111-8111-111111111111",
|
||||
{ status: "todo" },
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
|
|
@ -458,7 +474,38 @@ describe("issue comment reopen routes", () => {
|
|||
reopenedFrom: "done",
|
||||
}),
|
||||
}),
|
||||
));
|
||||
});
|
||||
|
||||
it("does not implicitly reopen closed issues via POST comments for agent-authored comments", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueService.addComment.mockResolvedValue({
|
||||
id: "comment-1",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
companyId: "company-1",
|
||||
body: "hello",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
authorAgentId: "33333333-3333-4333-8333-333333333333",
|
||||
authorUserId: null,
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp(), {
|
||||
type: "agent",
|
||||
agentId: "33333333-3333-4333-8333-333333333333",
|
||||
companyId: "company-1",
|
||||
source: "agent_key",
|
||||
runId: "77777777-7777-4777-8777-777777777777",
|
||||
}))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "hello" });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockIssueService.update).not.toHaveBeenCalledWith(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
{ status: "todo" },
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves assigned blocked issues back to todo via POST comments", async () => {
|
||||
|
|
@ -477,7 +524,7 @@ describe("issue comment reopen routes", () => {
|
|||
"11111111-1111-4111-8111-111111111111",
|
||||
{ status: "todo" },
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
|
|
@ -493,7 +540,7 @@ describe("issue comment reopen routes", () => {
|
|||
reopenedFrom: "blocked",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
|
||||
it("does not move dependency-blocked issues to todo via POST comments", async () => {
|
||||
|
|
@ -513,7 +560,7 @@ describe("issue comment reopen routes", () => {
|
|||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_commented",
|
||||
|
|
@ -527,7 +574,7 @@ describe("issue comment reopen routes", () => {
|
|||
wakeReason: "issue_commented",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
|
||||
it("does not implicitly reopen closed issues via POST comments when no agent is assigned", async () => {
|
||||
|
|
@ -565,7 +612,7 @@ describe("issue comment reopen routes", () => {
|
|||
actorUserId: "local-board",
|
||||
}),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
|
|
@ -575,7 +622,42 @@ describe("issue comment reopen routes", () => {
|
|||
mutation: "comment",
|
||||
}),
|
||||
}),
|
||||
));
|
||||
});
|
||||
|
||||
it("does not implicitly reopen closed issues via the PATCH comment path for agent-authored comments", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueService.addComment.mockResolvedValue({
|
||||
id: "comment-1",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
companyId: "company-1",
|
||||
body: "hello",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
authorAgentId: "33333333-3333-4333-8333-333333333333",
|
||||
authorUserId: null,
|
||||
});
|
||||
mockIssueService.update.mockImplementation(async (_id: string, patch: Record<string, unknown>) => ({
|
||||
...makeIssue("done"),
|
||||
...patch,
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp(), {
|
||||
type: "agent",
|
||||
agentId: "33333333-3333-4333-8333-333333333333",
|
||||
companyId: "company-1",
|
||||
source: "agent_key",
|
||||
runId: "88888888-8888-4888-8888-888888888888",
|
||||
}))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ comment: "hello" });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockIssueService.update).not.toHaveBeenCalledWith(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
expect.objectContaining({ status: "todo" }),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not move dependency-blocked issues to todo via the PATCH comment path", async () => {
|
||||
|
|
@ -609,7 +691,7 @@ describe("issue comment reopen routes", () => {
|
|||
"11111111-1111-4111-8111-111111111111",
|
||||
expect.objectContaining({ status: "todo" }),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_commented",
|
||||
|
|
@ -618,7 +700,7 @@ describe("issue comment reopen routes", () => {
|
|||
mutation: "comment",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
|
||||
it("wakes the assignee when an assigned blocked issue moves back to todo", async () => {
|
||||
|
|
@ -630,6 +712,34 @@ describe("issue comment reopen routes", () => {
|
|||
updatedAt: new Date(),
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ status: "todo" });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_status_changed",
|
||||
payload: expect.objectContaining({
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
mutation: "update",
|
||||
}),
|
||||
}),
|
||||
));
|
||||
});
|
||||
|
||||
it("wakes the assignee when an assigned done issue moves back to todo", async () => {
|
||||
const issue = makeIssue("done");
|
||||
mockIssueService.getById.mockResolvedValue(issue);
|
||||
mockIssueService.update.mockImplementation(async (_id: string, patch: Record<string, unknown>) => ({
|
||||
...issue,
|
||||
...patch,
|
||||
updatedAt: new Date(),
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ status: "todo" });
|
||||
|
|
@ -645,9 +755,166 @@ describe("issue comment reopen routes", () => {
|
|||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
mutation: "update",
|
||||
}),
|
||||
contextSnapshot: expect.objectContaining({
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
source: "issue.status_change",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("explicit same-agent resume works through the PATCH comment path", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueService.update.mockImplementation(async (_id: string, patch: Record<string, unknown>) => ({
|
||||
...makeIssue("done"),
|
||||
...patch,
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ comment: "please validate the follow-up", resume: true });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockIssueService.update).toHaveBeenCalledWith(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
expect.objectContaining({
|
||||
status: "todo",
|
||||
actorAgentId: "22222222-2222-4222-8222-222222222222",
|
||||
actorUserId: null,
|
||||
}),
|
||||
);
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
action: "issue.comment_added",
|
||||
details: expect.objectContaining({
|
||||
commentId: "comment-1",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
payload: expect.objectContaining({
|
||||
commentId: "comment-1",
|
||||
reopenedFrom: "done",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps generic same-agent comments on closed issues inert", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "follow-up note without intent" });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("explicit same-agent resume comments reopen closed issues and mark the wake payload", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueService.update.mockImplementation(async (_id: string, patch: Record<string, unknown>) => ({
|
||||
...makeIssue("done"),
|
||||
...patch,
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "please validate the follow-up", resume: true });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockIssueService.update).toHaveBeenCalledWith(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
{ status: "todo" },
|
||||
);
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
action: "issue.comment_added",
|
||||
details: expect.objectContaining({
|
||||
commentId: "comment-1",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
payload: expect.objectContaining({
|
||||
commentId: "comment-1",
|
||||
reopenedFrom: "done",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
contextSnapshot: expect.objectContaining({
|
||||
wakeReason: "issue_reopened_via_comment",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects explicit agent resume intent from a non-assignee", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor("44444444-4444-4444-8444-444444444444")))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "restart someone else's work", resume: true });
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.error).toBe("Agent cannot request follow-up for another agent's issue");
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockIssueService.addComment).not.toHaveBeenCalled();
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects explicit resume intent under an active pause hold", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueTreeControlService.getActivePauseHoldGate.mockResolvedValue({
|
||||
holdId: "hold-1",
|
||||
rootIssueId: "root-1",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
isRoot: false,
|
||||
mode: "pause",
|
||||
reason: "reviewing",
|
||||
releasePolicy: null,
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "please resume", resume: true });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.error).toBe("Issue follow-up blocked by active subtree pause hold");
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockIssueService.addComment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects explicit resume intent on cancelled issues", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("cancelled"));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "please resume", resume: true });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.error).toBe("Cancelled issues must be restored through the dedicated restore flow");
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockIssueService.addComment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("interrupts an active run before a combined comment update", async () => {
|
||||
const issue = {
|
||||
...makeIssue("todo"),
|
||||
|
|
@ -818,7 +1085,7 @@ describe("issue comment reopen routes", () => {
|
|||
instructions: "Please verify the fix against the reproduction steps and note any residual risk.",
|
||||
},
|
||||
});
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"33333333-3333-4333-8333-333333333333",
|
||||
expect.objectContaining({
|
||||
reason: "execution_review_requested",
|
||||
|
|
@ -834,7 +1101,7 @@ describe("issue comment reopen routes", () => {
|
|||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
|
||||
it("wakes the return assignee with execution_changes_requested", async () => {
|
||||
|
|
@ -886,7 +1153,7 @@ describe("issue comment reopen routes", () => {
|
|||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "execution_changes_requested",
|
||||
|
|
@ -900,6 +1167,6 @@ describe("issue comment reopen routes", () => {
|
|||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ describe("issue graph liveness classifier", () => {
|
|||
issueId: blockedId,
|
||||
identifier: "PAP-1703",
|
||||
state: "blocked_by_unassigned_issue",
|
||||
recoveryIssueId: blockerId,
|
||||
recommendedOwnerAgentId: managerId,
|
||||
dependencyPath: [
|
||||
expect.objectContaining({ issueId: blockedId }),
|
||||
|
|
@ -76,6 +77,57 @@ describe("issue graph liveness classifier", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("does not use free-form executive role or name matching for recovery ownership", () => {
|
||||
const rootAgentId = "root-agent";
|
||||
const spoofedExecutiveId = "spoofed-executive";
|
||||
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
issue({
|
||||
assigneeAgentId: null,
|
||||
createdByAgentId: null,
|
||||
}),
|
||||
issue({
|
||||
id: blockerId,
|
||||
identifier: "PAP-1704",
|
||||
title: "Missing unblock work",
|
||||
status: "todo",
|
||||
assigneeAgentId: null,
|
||||
createdByAgentId: null,
|
||||
}),
|
||||
],
|
||||
relations: blocks,
|
||||
agents: [
|
||||
agent({
|
||||
id: spoofedExecutiveId,
|
||||
name: "Chief Executive Recovery",
|
||||
role: "cto",
|
||||
title: "CEO",
|
||||
reportsTo: rootAgentId,
|
||||
}),
|
||||
agent({
|
||||
id: rootAgentId,
|
||||
name: "Root Operator",
|
||||
role: "operator",
|
||||
title: null,
|
||||
reportsTo: null,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(findings).toHaveLength(1);
|
||||
expect(findings[0]?.recommendedOwnerAgentId).toBe(rootAgentId);
|
||||
expect(findings[0]?.recommendedOwnerCandidates[0]).toMatchObject({
|
||||
agentId: rootAgentId,
|
||||
reason: "root_agent",
|
||||
sourceIssueId: blockerId,
|
||||
});
|
||||
expect(findings[0]?.recommendedOwnerCandidateAgentIds).toEqual([
|
||||
rootAgentId,
|
||||
spoofedExecutiveId,
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not flag a live blocked chain with an active assignee and wake path", () => {
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
|
|
|
|||
|
|
@ -195,6 +195,61 @@ describe("issue tree control routes", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("still marks affected issues cancelled when run interruption fails", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
userId: "user-1",
|
||||
companyIds: ["company-2"],
|
||||
source: "session",
|
||||
isInstanceAdmin: false,
|
||||
});
|
||||
mockTreeControlService.createHold.mockResolvedValue({
|
||||
hold: {
|
||||
id: "33333333-3333-4333-8333-333333333333",
|
||||
mode: "cancel",
|
||||
reason: "cancel subtree",
|
||||
},
|
||||
preview: {
|
||||
mode: "cancel",
|
||||
totals: { affectedIssues: 1 },
|
||||
warnings: [],
|
||||
activeRuns: [
|
||||
{
|
||||
id: "44444444-4444-4444-8444-444444444444",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
mockTreeControlService.cancelIssueStatusesForHold.mockResolvedValue({
|
||||
updatedIssueIds: ["11111111-1111-4111-8111-111111111111"],
|
||||
updatedIssues: [],
|
||||
});
|
||||
mockHeartbeatService.cancelRun.mockRejectedValue(new Error("adapter process did not exit"));
|
||||
|
||||
const res = await request(app)
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/tree-holds")
|
||||
.send({ mode: "cancel", reason: "cancel subtree" });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockHeartbeatService.cancelRun).toHaveBeenCalledWith("44444444-4444-4444-8444-444444444444");
|
||||
expect(mockTreeControlService.cancelIssueStatusesForHold).toHaveBeenCalledWith(
|
||||
"company-2",
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"33333333-3333-4333-8333-333333333333",
|
||||
);
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
action: "issue.tree_hold_run_interrupt_failed",
|
||||
entityId: "44444444-4444-4444-8444-444444444444",
|
||||
details: expect.objectContaining({
|
||||
error: "adapter process did not exit",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("restores affected issues and can request explicit wakeups", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@ import { eq, inArray } from "drizzle-orm";
|
|||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueTreeHoldMembers,
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
|
|
@ -38,8 +40,10 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
afterEach(async () => {
|
||||
await db.delete(issueTreeHoldMembers);
|
||||
await db.delete(issueTreeHolds);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issues);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
|
@ -340,6 +344,12 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
const childIssueId = randomUUID();
|
||||
const rootRunId = randomUUID();
|
||||
const childRunId = randomUUID();
|
||||
const forgedRunId = randomUUID();
|
||||
const rootWakeupRequestId = randomUUID();
|
||||
const childWakeupRequestId = randomUUID();
|
||||
const forgedWakeupRequestId = randomUUID();
|
||||
const rootCommentId = randomUUID();
|
||||
const childCommentId = randomUUID();
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
|
|
@ -377,6 +387,63 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
assigneeAgentId: agentId,
|
||||
},
|
||||
]);
|
||||
await db.insert(issueComments).values([
|
||||
{
|
||||
id: rootCommentId,
|
||||
companyId,
|
||||
issueId: rootIssueId,
|
||||
authorUserId: "board-user",
|
||||
body: "Please answer this root issue question.",
|
||||
},
|
||||
{
|
||||
id: childCommentId,
|
||||
companyId,
|
||||
issueId: childIssueId,
|
||||
authorUserId: "board-user",
|
||||
body: "Please answer this child issue question.",
|
||||
},
|
||||
]);
|
||||
await db.insert(agentWakeupRequests).values([
|
||||
{
|
||||
id: rootWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: rootIssueId, commentId: rootCommentId },
|
||||
status: "queued",
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "board-user",
|
||||
runId: rootRunId,
|
||||
},
|
||||
{
|
||||
id: forgedWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "on_demand",
|
||||
triggerDetail: "manual",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: childIssueId, commentId: childCommentId },
|
||||
status: "queued",
|
||||
requestedByActorType: "agent",
|
||||
requestedByActorId: agentId,
|
||||
runId: forgedRunId,
|
||||
},
|
||||
{
|
||||
id: childWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: childIssueId, commentId: childCommentId },
|
||||
status: "queued",
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "board-user",
|
||||
runId: childRunId,
|
||||
},
|
||||
]);
|
||||
await db.insert(heartbeatRuns).values([
|
||||
{
|
||||
id: rootRunId,
|
||||
|
|
@ -385,7 +452,29 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
contextSnapshot: { issueId: rootIssueId, wakeReason: "issue_commented", commentId: randomUUID() },
|
||||
wakeupRequestId: rootWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: rootIssueId,
|
||||
wakeReason: "issue_commented",
|
||||
commentId: rootCommentId,
|
||||
wakeCommentId: rootCommentId,
|
||||
source: "issue.comment",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: forgedRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
invocationSource: "on_demand",
|
||||
triggerDetail: "manual",
|
||||
status: "queued",
|
||||
wakeupRequestId: forgedWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: childIssueId,
|
||||
wakeReason: "issue_commented",
|
||||
commentId: childCommentId,
|
||||
wakeCommentId: childCommentId,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: childRunId,
|
||||
|
|
@ -394,7 +483,14 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
contextSnapshot: { issueId: childIssueId, wakeReason: "issue_commented", commentId: randomUUID() },
|
||||
wakeupRequestId: childWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: childIssueId,
|
||||
wakeReason: "issue_commented",
|
||||
commentId: childCommentId,
|
||||
wakeCommentId: childCommentId,
|
||||
source: "issue.comment",
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
|
|
@ -413,6 +509,13 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
mode: "pause",
|
||||
}),
|
||||
});
|
||||
await expect(issueSvc.checkout(childIssueId, agentId, ["todo"], forgedRunId)).rejects.toMatchObject({
|
||||
status: 409,
|
||||
details: expect.objectContaining({
|
||||
rootIssueId,
|
||||
mode: "pause",
|
||||
}),
|
||||
});
|
||||
|
||||
const checkedOutChild = await issueSvc.checkout(childIssueId, agentId, ["todo"], childRunId);
|
||||
expect(checkedOutChild.status).toBe("in_progress");
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ const mockIssueService = vi.hoisted(() => ({
|
|||
findMentionedProjectIds: vi.fn(),
|
||||
getCommentCursor: vi.fn(),
|
||||
getComment: vi.fn(),
|
||||
listBlockerAttention: vi.fn(),
|
||||
listAttachments: vi.fn(),
|
||||
}));
|
||||
|
||||
|
|
@ -166,6 +167,7 @@ describe("issue goal context routes", () => {
|
|||
latestCommentAt: null,
|
||||
});
|
||||
mockIssueService.getComment.mockResolvedValue(null);
|
||||
mockIssueService.listBlockerAttention.mockResolvedValue(new Map());
|
||||
mockIssueService.listAttachments.mockResolvedValue([]);
|
||||
mockDocumentsService.getIssueDocumentPayload.mockResolvedValue({});
|
||||
mockDocumentsService.getIssueDocumentByKey.mockResolvedValue(null);
|
||||
|
|
|
|||
|
|
@ -1401,6 +1401,49 @@ describeEmbeddedPostgres("issueService blockers and dependency wake readiness",
|
|||
expect(blockedRelations.blockedBy.map((relation) => relation.id)).toEqual([blockerId]);
|
||||
});
|
||||
|
||||
it("adds terminal blockers to immediate blocked-by summaries", async () => {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
const issueA = randomUUID();
|
||||
const issueB = randomUUID();
|
||||
const issueC = randomUUID();
|
||||
const issueD = randomUUID();
|
||||
await db.insert(issues).values([
|
||||
{ id: issueA, companyId, identifier: "PAP-1", title: "Issue A", status: "blocked", priority: "medium" },
|
||||
{ id: issueB, companyId, identifier: "PAP-2", title: "Issue B", status: "blocked", priority: "medium" },
|
||||
{ id: issueC, companyId, identifier: "PAP-3", title: "Issue C", status: "blocked", priority: "medium" },
|
||||
{ id: issueD, companyId, identifier: "PAP-4", title: "Issue D", status: "todo", priority: "high" },
|
||||
]);
|
||||
|
||||
await svc.update(issueC, { blockedByIssueIds: [issueD] });
|
||||
await svc.update(issueB, { blockedByIssueIds: [issueC] });
|
||||
await svc.update(issueA, { blockedByIssueIds: [issueB] });
|
||||
|
||||
const relations = await svc.getRelationSummaries(issueA);
|
||||
|
||||
expect(relations.blockedBy).toHaveLength(1);
|
||||
expect(relations.blockedBy[0]).toMatchObject({
|
||||
id: issueB,
|
||||
identifier: "PAP-2",
|
||||
title: "Issue B",
|
||||
terminalBlockers: [
|
||||
expect.objectContaining({
|
||||
id: issueD,
|
||||
identifier: "PAP-4",
|
||||
title: "Issue D",
|
||||
status: "todo",
|
||||
priority: "high",
|
||||
}),
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects blocking cycles", async () => {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
|
|
|
|||
146
server/src/__tests__/recovery-classifiers.test.ts
Normal file
146
server/src/__tests__/recovery-classifiers.test.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { classifyIssueGraphLiveness as classifyIssueGraphLivenessCompat } from "../services/issue-liveness.ts";
|
||||
import { decideRunLivenessContinuation as decideRunLivenessContinuationCompat } from "../services/run-continuations.ts";
|
||||
import {
|
||||
RECOVERY_KEY_PREFIXES,
|
||||
RECOVERY_ORIGIN_KINDS,
|
||||
RECOVERY_REASON_KINDS,
|
||||
buildIssueGraphLivenessIncidentKey,
|
||||
buildIssueGraphLivenessLeafKey,
|
||||
buildRunLivenessContinuationIdempotencyKey,
|
||||
classifyIssueGraphLiveness,
|
||||
decideRunLivenessContinuation,
|
||||
parseIssueGraphLivenessIncidentKey,
|
||||
} from "../services/recovery/index.ts";
|
||||
|
||||
const companyId = "company-1";
|
||||
const agentId = "agent-1";
|
||||
const managerId = "manager-1";
|
||||
const issueId = "issue-1";
|
||||
const blockerId = "blocker-1";
|
||||
const runId = "run-1";
|
||||
|
||||
describe("recovery classifier boundary", () => {
|
||||
it("keeps issue graph liveness classifier parity with the compatibility export", () => {
|
||||
const input = {
|
||||
issues: [
|
||||
{
|
||||
id: issueId,
|
||||
companyId,
|
||||
identifier: "PAP-2073",
|
||||
title: "Centralize recovery classifiers",
|
||||
status: "blocked",
|
||||
assigneeAgentId: agentId,
|
||||
assigneeUserId: null,
|
||||
createdByAgentId: null,
|
||||
createdByUserId: null,
|
||||
executionState: null,
|
||||
},
|
||||
{
|
||||
id: blockerId,
|
||||
companyId,
|
||||
identifier: "PAP-2074",
|
||||
title: "Move recovery side effects",
|
||||
status: "todo",
|
||||
assigneeAgentId: null,
|
||||
assigneeUserId: null,
|
||||
createdByAgentId: null,
|
||||
createdByUserId: null,
|
||||
executionState: null,
|
||||
},
|
||||
],
|
||||
relations: [{ companyId, blockerIssueId: blockerId, blockedIssueId: issueId }],
|
||||
agents: [
|
||||
{
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Coder",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
reportsTo: managerId,
|
||||
},
|
||||
{
|
||||
id: managerId,
|
||||
companyId,
|
||||
name: "CTO",
|
||||
role: "cto",
|
||||
status: "idle",
|
||||
reportsTo: null,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(classifyIssueGraphLiveness(input)).toEqual(classifyIssueGraphLivenessCompat(input));
|
||||
});
|
||||
|
||||
it("keeps run liveness continuation decision parity with the compatibility export", () => {
|
||||
const input = {
|
||||
run: {
|
||||
id: runId,
|
||||
companyId,
|
||||
agentId,
|
||||
continuationAttempt: 0,
|
||||
} as never,
|
||||
issue: {
|
||||
id: issueId,
|
||||
companyId,
|
||||
identifier: "PAP-2073",
|
||||
title: "Centralize recovery classifiers",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: agentId,
|
||||
executionState: null,
|
||||
projectId: null,
|
||||
} as never,
|
||||
agent: {
|
||||
id: agentId,
|
||||
companyId,
|
||||
status: "idle",
|
||||
} as never,
|
||||
livenessState: "plan_only" as const,
|
||||
livenessReason: "Planned without acting",
|
||||
nextAction: "Take the first concrete action.",
|
||||
budgetBlocked: false,
|
||||
idempotentWakeExists: false,
|
||||
};
|
||||
|
||||
expect(decideRunLivenessContinuation(input)).toEqual(decideRunLivenessContinuationCompat(input));
|
||||
});
|
||||
|
||||
it("keeps recovery origin and idempotency keys stable", () => {
|
||||
expect(RECOVERY_ORIGIN_KINDS).toMatchObject({
|
||||
issueGraphLivenessEscalation: "harness_liveness_escalation",
|
||||
strandedIssueRecovery: "stranded_issue_recovery",
|
||||
staleActiveRunEvaluation: "stale_active_run_evaluation",
|
||||
});
|
||||
expect(RECOVERY_REASON_KINDS.runLivenessContinuation).toBe("run_liveness_continuation");
|
||||
expect(RECOVERY_KEY_PREFIXES.issueGraphLivenessIncident).toBe("harness_liveness");
|
||||
expect(RECOVERY_KEY_PREFIXES.issueGraphLivenessLeaf).toBe("harness_liveness_leaf");
|
||||
|
||||
const incidentKey = buildIssueGraphLivenessIncidentKey({
|
||||
companyId,
|
||||
issueId,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
blockerIssueId: blockerId,
|
||||
});
|
||||
expect(incidentKey).toBe(
|
||||
"harness_liveness:company-1:issue-1:blocked_by_unassigned_issue:blocker-1",
|
||||
);
|
||||
expect(parseIssueGraphLivenessIncidentKey(incidentKey)).toEqual({
|
||||
companyId,
|
||||
issueId,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
leafIssueId: blockerId,
|
||||
});
|
||||
expect(buildIssueGraphLivenessLeafKey({
|
||||
companyId,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
leafIssueId: blockerId,
|
||||
})).toBe("harness_liveness_leaf:company-1:blocked_by_unassigned_issue:blocker-1");
|
||||
expect(buildRunLivenessContinuationIdempotencyKey({
|
||||
issueId,
|
||||
sourceRunId: runId,
|
||||
livenessState: "plan_only",
|
||||
nextAttempt: 1,
|
||||
})).toBe("run_liveness_continuation:issue-1:run-1:plan_only:1");
|
||||
});
|
||||
});
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { REDACTED_EVENT_VALUE, redactEventPayload, sanitizeRecord } from "../redaction.js";
|
||||
import { REDACTED_EVENT_VALUE, redactEventPayload, redactSensitiveText, sanitizeRecord } from "../redaction.js";
|
||||
|
||||
describe("redaction", () => {
|
||||
it("redacts sensitive keys and nested secret values", () => {
|
||||
|
|
@ -63,4 +63,25 @@ describe("redaction", () => {
|
|||
safe: "value",
|
||||
});
|
||||
});
|
||||
|
||||
it("redacts common secret shapes from unstructured text", () => {
|
||||
const jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
|
||||
const githubToken = "ghp_1234567890abcdefghijklmnopqrstuvwxyz";
|
||||
const input = [
|
||||
"Authorization: Bearer live-bearer-token-value",
|
||||
`payload {"apiKey":"json-secret-value"}`,
|
||||
`escaped {\\"apiKey\\":\\"escaped-json-secret\\"}`,
|
||||
`GITHUB_TOKEN=${githubToken}`,
|
||||
`session=${jwt}`,
|
||||
].join("\n");
|
||||
|
||||
const result = redactSensitiveText(input);
|
||||
|
||||
expect(result).toContain(REDACTED_EVENT_VALUE);
|
||||
expect(result).not.toContain("live-bearer-token-value");
|
||||
expect(result).not.toContain("json-secret-value");
|
||||
expect(result).not.toContain("escaped-json-secret");
|
||||
expect(result).not.toContain(githubToken);
|
||||
expect(result).not.toContain(jwt);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ describe("run liveness classifier", () => {
|
|||
});
|
||||
|
||||
expect(classification.livenessState).toBe("plan_only");
|
||||
expect(classification.actionability).toBe("runnable");
|
||||
expect(classification.nextAction).toContain("inspect the repo");
|
||||
});
|
||||
|
||||
|
|
@ -34,6 +35,7 @@ describe("run liveness classifier", () => {
|
|||
const classification = classifyRunLiveness(baseInput);
|
||||
|
||||
expect(classification.livenessState).toBe("empty_response");
|
||||
expect(classification.actionability).toBe("unknown");
|
||||
});
|
||||
|
||||
it("treats issue comments, documents, products, and actions as progress", () => {
|
||||
|
|
@ -128,5 +130,81 @@ describe("run liveness classifier", () => {
|
|||
});
|
||||
|
||||
expect(classification.livenessState).toBe("blocked");
|
||||
expect(classification.actionability).toBe("blocked_external");
|
||||
});
|
||||
|
||||
it("treats PAP-2000-style validation output as runnable follow-up, not an external blocker", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
resultJson: {
|
||||
summary: "PAP-1949 remains blocked until PAP-2000 is resolved.",
|
||||
},
|
||||
issueCommentBodies: [
|
||||
[
|
||||
"Validation is ready for the next pass.",
|
||||
"",
|
||||
"- Blocked chain context: PAP-1949 -> PAP-1999 -> PAP-2000",
|
||||
"- Next action: run npm test and report the row counts.",
|
||||
].join("\n"),
|
||||
],
|
||||
});
|
||||
|
||||
expect(classification.livenessState).toBe("plan_only");
|
||||
expect(classification.actionability).toBe("runnable");
|
||||
expect(classification.nextAction).toBe("run npm test and report the row counts.");
|
||||
});
|
||||
|
||||
it("prefers durable comments over raw transcript next-action noise", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
issueCommentBodies: ["Next action: run pnpm test -- --runInBand."],
|
||||
stdoutExcerpt: [
|
||||
"tool_call: write",
|
||||
"command: rm -rf production-data",
|
||||
"Next action: deploy to production",
|
||||
].join("\n"),
|
||||
});
|
||||
|
||||
expect(classification.actionability).toBe("runnable");
|
||||
expect(classification.nextAction).toBe("run pnpm test -- --runInBand.");
|
||||
});
|
||||
|
||||
it("keeps approval requests out of automatic continuation", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
resultJson: {
|
||||
summary: "Next action: wait for board approval before continuing.",
|
||||
},
|
||||
});
|
||||
|
||||
expect(classification.livenessState).toBe("blocked");
|
||||
expect(classification.actionability).toBe("approval_required");
|
||||
expect(classification.nextAction).toBe("wait for board approval before continuing.");
|
||||
});
|
||||
|
||||
it("routes production-sensitive next actions to manager review", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
resultJson: {
|
||||
summary: "Next action: deploy to production and verify live traffic.",
|
||||
},
|
||||
});
|
||||
|
||||
expect(classification.livenessState).toBe("needs_followup");
|
||||
expect(classification.actionability).toBe("manager_review");
|
||||
expect(classification.nextAction).toBe("deploy to production and verify live traffic.");
|
||||
});
|
||||
|
||||
it("marks unclear useful output as unknown actionability", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
resultJson: {
|
||||
summary: "Observed mixed output and left notes for a later pass.",
|
||||
},
|
||||
});
|
||||
|
||||
expect(classification.livenessState).toBe("needs_followup");
|
||||
expect(classification.actionability).toBe("unknown");
|
||||
expect(classification.nextAction).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue