mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-14 01:50:39 +09:00
[codex] Add runtime lifecycle recovery and live issue visibility (#4419)
This commit is contained in:
parent
9a8d219949
commit
5a0c1979cf
121 changed files with 9625 additions and 2044 deletions
10
server/src/__tests__/README.md
Normal file
10
server/src/__tests__/README.md
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# Server Tests
|
||||
|
||||
Server tests that need a real PostgreSQL process must use
|
||||
`./helpers/embedded-postgres.ts` instead of constructing `embedded-postgres`
|
||||
directly.
|
||||
|
||||
The shared helper creates a throwaway data directory and a reserved-safe
|
||||
loopback port for each test database. This protects the live Paperclip
|
||||
control-plane Postgres from server vitest runs; see PAP-2033 for the incident
|
||||
that introduced this guard.
|
||||
|
|
@ -424,7 +424,7 @@ describeEmbeddedPostgres("activity service", () => {
|
|||
expect(backfilledRun).toMatchObject({
|
||||
runId,
|
||||
livenessState: "plan_only",
|
||||
livenessReason: "Run described future work without concrete action evidence",
|
||||
livenessReason: "Run described runnable future work without concrete action evidence",
|
||||
lastUsefulActionAt: null,
|
||||
});
|
||||
});
|
||||
|
|
@ -530,7 +530,7 @@ describeEmbeddedPostgres("activity service", () => {
|
|||
expect(backfilledRun).toMatchObject({
|
||||
runId,
|
||||
livenessState: "plan_only",
|
||||
livenessReason: "Run described future work without concrete action evidence",
|
||||
livenessReason: "Run described runnable future work without concrete action evidence",
|
||||
lastUsefulActionAt: null,
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ const mockHeartbeatService = vi.hoisted(() => ({
|
|||
buildRunOutputSilence: vi.fn(),
|
||||
getRunIssueSummary: vi.fn(),
|
||||
getActiveRunIssueSummaryForAgent: vi.fn(),
|
||||
buildRunOutputSilence: vi.fn(),
|
||||
getRunLogAccess: vi.fn(),
|
||||
readLog: vi.fn(),
|
||||
}));
|
||||
|
|
@ -173,6 +174,7 @@ describe("agent live run routes", () => {
|
|||
issueId: "issue-1",
|
||||
});
|
||||
mockHeartbeatService.getActiveRunIssueSummaryForAgent.mockResolvedValue(null);
|
||||
mockHeartbeatService.buildRunOutputSilence.mockResolvedValue(null);
|
||||
mockHeartbeatService.getRunLogAccess.mockResolvedValue({
|
||||
id: "run-1",
|
||||
companyId: "company-1",
|
||||
|
|
@ -209,6 +211,7 @@ describe("agent live run routes", () => {
|
|||
issueId: "issue-1",
|
||||
agentName: "Builder",
|
||||
adapterType: "codex_local",
|
||||
outputSilence: null,
|
||||
});
|
||||
expect(res.body).not.toHaveProperty("resultJson");
|
||||
expect(res.body).not.toHaveProperty("contextSnapshot");
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import net from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { eq } from "drizzle-orm";
|
||||
|
|
@ -8,14 +7,12 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
|
|||
import { writePaperclipSkillSyncPreference } from "@paperclipai/adapter-utils/server-utils";
|
||||
import {
|
||||
agents,
|
||||
applyPendingMigrations,
|
||||
companies,
|
||||
companySkills,
|
||||
costEvents,
|
||||
createDb,
|
||||
documents,
|
||||
documentRevisions,
|
||||
ensurePostgresDatabase,
|
||||
feedbackExports,
|
||||
feedbackVotes,
|
||||
heartbeatRuns,
|
||||
|
|
@ -25,72 +22,7 @@ import {
|
|||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import { feedbackService } from "../services/feedback.ts";
|
||||
|
||||
type EmbeddedPostgresInstance = {
|
||||
initialise(): Promise<void>;
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
type EmbeddedPostgresCtor = new (opts: {
|
||||
databaseDir: string;
|
||||
user: string;
|
||||
password: string;
|
||||
port: number;
|
||||
persistent: boolean;
|
||||
initdbFlags?: string[];
|
||||
onLog?: (message: unknown) => void;
|
||||
onError?: (message: unknown) => void;
|
||||
}) => EmbeddedPostgresInstance;
|
||||
|
||||
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
|
||||
const mod = await import("embedded-postgres");
|
||||
return mod.default as EmbeddedPostgresCtor;
|
||||
}
|
||||
|
||||
async function getAvailablePort(): Promise<number> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const server = net.createServer();
|
||||
server.unref();
|
||||
server.on("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
server.close(() => reject(new Error("Failed to allocate test port")));
|
||||
return;
|
||||
}
|
||||
const { port } = address;
|
||||
server.close((error) => {
|
||||
if (error) reject(error);
|
||||
else resolve(port);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function startTempDatabase() {
|
||||
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-feedback-service-"));
|
||||
const port = await getAvailablePort();
|
||||
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
|
||||
const instance = new EmbeddedPostgres({
|
||||
databaseDir: dataDir,
|
||||
user: "paperclip",
|
||||
password: "paperclip",
|
||||
port,
|
||||
persistent: true,
|
||||
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
|
||||
onLog: () => {},
|
||||
onError: () => {},
|
||||
});
|
||||
await instance.initialise();
|
||||
await instance.start();
|
||||
|
||||
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
|
||||
await ensurePostgresDatabase(adminConnectionString, "paperclip");
|
||||
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
|
||||
await applyPendingMigrations(connectionString);
|
||||
return { connectionString, dataDir, instance };
|
||||
}
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.ts";
|
||||
|
||||
async function closeDbClient(db: ReturnType<typeof createDb> | undefined) {
|
||||
await db?.$client?.end?.({ timeout: 0 });
|
||||
|
|
@ -99,17 +31,15 @@ async function closeDbClient(db: ReturnType<typeof createDb> | undefined) {
|
|||
describe("feedbackService.saveIssueVote", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let svc!: ReturnType<typeof feedbackService>;
|
||||
let instance: EmbeddedPostgresInstance | null = null;
|
||||
let dataDir = "";
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
let tempDirs: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
const started = await startTempDatabase();
|
||||
const started = await startEmbeddedPostgresTestDatabase("paperclip-feedback-service-");
|
||||
db = createDb(started.connectionString);
|
||||
svc = feedbackService(db);
|
||||
instance = started.instance;
|
||||
dataDir = started.dataDir;
|
||||
}, 20_000);
|
||||
tempDb = started;
|
||||
}, 120_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(feedbackExports);
|
||||
|
|
@ -134,10 +64,7 @@ describe("feedbackService.saveIssueVote", () => {
|
|||
|
||||
afterAll(async () => {
|
||||
await closeDbClient(db);
|
||||
await instance?.stop();
|
||||
if (dataDir) {
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedIssueWithAgentComment() {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,549 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRunWatchdogDecisions,
|
||||
heartbeatRuns,
|
||||
issueRelations,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import {
|
||||
ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS,
|
||||
ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS,
|
||||
ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS,
|
||||
heartbeatService,
|
||||
} from "../services/heartbeat.ts";
|
||||
import { recoveryService } from "../services/recovery/service.ts";
|
||||
import { getRunLogStore } from "../services/run-log-store.ts";
|
||||
|
||||
const mockAdapterExecute = vi.hoisted(() =>
|
||||
vi.fn(async () => ({
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorMessage: null,
|
||||
summary: "Acknowledged stale-run evaluation.",
|
||||
provider: "test",
|
||||
model: "test-model",
|
||||
})),
|
||||
);
|
||||
|
||||
vi.mock("../telemetry.ts", () => ({
|
||||
getTelemetryClient: () => ({ track: vi.fn() }),
|
||||
}));
|
||||
|
||||
vi.mock("@paperclipai/shared/telemetry", async () => {
|
||||
const actual = await vi.importActual<typeof import("@paperclipai/shared/telemetry")>(
|
||||
"@paperclipai/shared/telemetry",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
trackAgentFirstHeartbeat: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../adapters/index.ts", async () => {
|
||||
const actual = await vi.importActual<typeof import("../adapters/index.ts")>("../adapters/index.ts");
|
||||
return {
|
||||
...actual,
|
||||
getServerAdapter: vi.fn(() => ({
|
||||
supportsLocalAgentJwt: false,
|
||||
execute: mockAdapterExecute,
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres active-run output watchdog tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("active-run output watchdog", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-active-run-output-watchdog-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 30_000);
|
||||
|
||||
afterEach(async () => {
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
const activeRuns = await db
|
||||
.select({ id: heartbeatRuns.id })
|
||||
.from(heartbeatRuns)
|
||||
.where(sql`${heartbeatRuns.status} in ('queued', 'running')`);
|
||||
if (activeRuns.length === 0) break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
}
|
||||
await db.execute(sql.raw(`TRUNCATE TABLE "companies" CASCADE`));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedRunningRun(opts: { now: Date; ageMs: number; withOutput?: boolean; logChunk?: string }) {
|
||||
const companyId = randomUUID();
|
||||
const managerId = randomUUID();
|
||||
const coderId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const issuePrefix = `W${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const startedAt = new Date(opts.now.getTime() - opts.ageMs);
|
||||
const lastOutputAt = opts.withOutput ? new Date(opts.now.getTime() - 5 * 60 * 1000) : null;
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Watchdog Co",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values([
|
||||
{
|
||||
id: managerId,
|
||||
companyId,
|
||||
name: "CTO",
|
||||
role: "cto",
|
||||
status: "idle",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
id: coderId,
|
||||
companyId,
|
||||
name: "Coder",
|
||||
role: "engineer",
|
||||
status: "running",
|
||||
reportsTo: managerId,
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Long running implementation",
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: coderId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
updatedAt: startedAt,
|
||||
createdAt: startedAt,
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId,
|
||||
agentId: coderId,
|
||||
status: "running",
|
||||
invocationSource: "assignment",
|
||||
triggerDetail: "system",
|
||||
startedAt,
|
||||
processStartedAt: startedAt,
|
||||
lastOutputAt,
|
||||
lastOutputSeq: opts.withOutput ? 3 : 0,
|
||||
lastOutputStream: opts.withOutput ? "stdout" : null,
|
||||
contextSnapshot: { issueId },
|
||||
stdoutExcerpt: "OPENAI_API_KEY=sk-test-secret-value should not leak",
|
||||
logBytes: 0,
|
||||
});
|
||||
if (opts.logChunk) {
|
||||
const store = getRunLogStore();
|
||||
const handle = await store.begin({ companyId, agentId: coderId, runId });
|
||||
const logBytes = await store.append(handle, {
|
||||
stream: "stdout",
|
||||
chunk: opts.logChunk,
|
||||
ts: startedAt.toISOString(),
|
||||
});
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
logStore: handle.store,
|
||||
logRef: handle.logRef,
|
||||
logBytes,
|
||||
})
|
||||
.where(eq(heartbeatRuns.id, runId));
|
||||
}
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
return { companyId, managerId, coderId, issueId, runId, issuePrefix };
|
||||
}
|
||||
|
||||
it("creates one medium-priority evaluation issue for a suspicious silent run", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, runId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const first = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const second = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
|
||||
expect(first.created).toBe(1);
|
||||
expect(second.created).toBe(0);
|
||||
expect(second.existing).toBe(1);
|
||||
|
||||
const evaluations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stale_active_run_evaluation")));
|
||||
expect(evaluations).toHaveLength(1);
|
||||
expect(["todo", "in_progress"]).toContain(evaluations[0]?.status);
|
||||
expect(evaluations[0]).toMatchObject({
|
||||
priority: "medium",
|
||||
assigneeAgentId: managerId,
|
||||
originId: runId,
|
||||
originFingerprint: `stale_active_run:${companyId}:${runId}`,
|
||||
});
|
||||
expect(evaluations[0]?.description).toContain("Decision Checklist");
|
||||
expect(evaluations[0]?.description).not.toContain("sk-test-secret-value");
|
||||
});
|
||||
|
||||
it("redacts sensitive values from actual run-log evidence", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const leakedJwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
|
||||
const leakedGithubToken = "ghp_1234567890abcdefghijklmnopqrstuvwxyz";
|
||||
const { companyId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
logChunk: [
|
||||
"Authorization: Bearer live-bearer-token-value",
|
||||
`POST payload {"apiKey":"json-secret-value","token":"${leakedJwt}"}`,
|
||||
`GITHUB_TOKEN=${leakedGithubToken}`,
|
||||
].join("\n"),
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
|
||||
const [evaluation] = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stale_active_run_evaluation")));
|
||||
expect(evaluation?.description).toContain("***REDACTED***");
|
||||
expect(evaluation?.description).not.toContain("live-bearer-token-value");
|
||||
expect(evaluation?.description).not.toContain("json-secret-value");
|
||||
expect(evaluation?.description).not.toContain(leakedJwt);
|
||||
expect(evaluation?.description).not.toContain(leakedGithubToken);
|
||||
});
|
||||
|
||||
it("raises critical stale-run evaluations and blocks the source issue", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, issueId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
|
||||
expect(result.created).toBe(1);
|
||||
const [evaluation] = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stale_active_run_evaluation")));
|
||||
expect(evaluation?.priority).toBe("high");
|
||||
|
||||
const [blocker] = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(and(eq(issueRelations.companyId, companyId), eq(issueRelations.relatedIssueId, issueId)));
|
||||
expect(blocker?.issueId).toBe(evaluation?.id);
|
||||
|
||||
const [source] = await db.select().from(issues).where(eq(issues.id, issueId));
|
||||
expect(source?.status).toBe("blocked");
|
||||
});
|
||||
|
||||
it("skips snoozed runs and healthy noisy runs", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const stale = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const noisy = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
|
||||
withOutput: true,
|
||||
});
|
||||
await db.insert(heartbeatRunWatchdogDecisions).values({
|
||||
companyId: stale.companyId,
|
||||
runId: stale.runId,
|
||||
decision: "snooze",
|
||||
snoozedUntil: new Date(now.getTime() + 60 * 60 * 1000),
|
||||
reason: "Intentional quiet run",
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const staleResult = await heartbeat.scanSilentActiveRuns({ now, companyId: stale.companyId });
|
||||
const noisyResult = await heartbeat.scanSilentActiveRuns({ now, companyId: noisy.companyId });
|
||||
|
||||
expect(staleResult).toMatchObject({ created: 0, snoozed: 1 });
|
||||
expect(noisyResult).toMatchObject({ scanned: 0, created: 0 });
|
||||
});
|
||||
|
||||
it("records watchdog decisions through recovery owner authorization", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, runId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn() });
|
||||
|
||||
const scan = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const evaluationIssueId = scan.evaluationIssueIds[0];
|
||||
expect(evaluationIssueId).toBeTruthy();
|
||||
|
||||
await expect(
|
||||
recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: randomUUID() },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "not my recovery issue",
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 403 });
|
||||
|
||||
const snoozedUntil = new Date(now.getTime() + 60 * 60 * 1000);
|
||||
const decision = await recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
decision: "snooze",
|
||||
evaluationIssueId,
|
||||
reason: "Long compile with no output",
|
||||
snoozedUntil,
|
||||
});
|
||||
|
||||
expect(decision).toMatchObject({
|
||||
runId,
|
||||
evaluationIssueId,
|
||||
decision: "snooze",
|
||||
createdByAgentId: managerId,
|
||||
});
|
||||
await expect(recovery.buildRunOutputSilence({
|
||||
id: runId,
|
||||
companyId,
|
||||
status: "running",
|
||||
lastOutputAt: null,
|
||||
lastOutputSeq: 0,
|
||||
lastOutputStream: null,
|
||||
processStartedAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 60_000),
|
||||
startedAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 60_000),
|
||||
createdAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 60_000),
|
||||
}, now)).resolves.toMatchObject({
|
||||
level: "snoozed",
|
||||
snoozedUntil,
|
||||
evaluationIssueId,
|
||||
});
|
||||
});
|
||||
|
||||
it("re-arms continue decisions after the default quiet window", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, runId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn() });
|
||||
|
||||
const scan = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const evaluationIssueId = scan.evaluationIssueIds[0];
|
||||
expect(evaluationIssueId).toBeTruthy();
|
||||
|
||||
const decision = await recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "Current evidence is acceptable; keep watching.",
|
||||
now,
|
||||
});
|
||||
const rearmAt = new Date(now.getTime() + ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS);
|
||||
expect(decision).toMatchObject({
|
||||
runId,
|
||||
evaluationIssueId,
|
||||
decision: "continue",
|
||||
createdByAgentId: managerId,
|
||||
});
|
||||
expect(decision.snoozedUntil?.toISOString()).toBe(rearmAt.toISOString());
|
||||
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, evaluationIssueId));
|
||||
|
||||
const beforeRearm = await heartbeat.scanSilentActiveRuns({
|
||||
now: new Date(rearmAt.getTime() - 60_000),
|
||||
companyId,
|
||||
});
|
||||
expect(beforeRearm).toMatchObject({ created: 0, snoozed: 1 });
|
||||
|
||||
const afterRearm = await heartbeat.scanSilentActiveRuns({
|
||||
now: new Date(rearmAt.getTime() + 60_000),
|
||||
companyId,
|
||||
});
|
||||
expect(afterRearm.created).toBe(1);
|
||||
expect(afterRearm.evaluationIssueIds[0]).not.toBe(evaluationIssueId);
|
||||
|
||||
const evaluations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stale_active_run_evaluation")));
|
||||
expect(evaluations.filter((issue) => !["done", "cancelled"].includes(issue.status))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("rejects agent watchdog decisions using issues not bound to the target run", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, coderId, runId, issuePrefix } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn() });
|
||||
|
||||
const scan = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const evaluationIssueId = scan.evaluationIssueIds[0];
|
||||
expect(evaluationIssueId).toBeTruthy();
|
||||
|
||||
const unrelatedIssueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: unrelatedIssueId,
|
||||
companyId,
|
||||
title: "Assigned but unrelated",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
assigneeAgentId: managerId,
|
||||
issueNumber: 20,
|
||||
identifier: `${issuePrefix}-20`,
|
||||
});
|
||||
|
||||
const otherRunId = randomUUID();
|
||||
const otherEvaluationIssueId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: otherRunId,
|
||||
companyId,
|
||||
agentId: coderId,
|
||||
status: "running",
|
||||
invocationSource: "assignment",
|
||||
triggerDetail: "system",
|
||||
startedAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 120_000),
|
||||
processStartedAt: new Date(now.getTime() - ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS - 120_000),
|
||||
lastOutputAt: null,
|
||||
lastOutputSeq: 0,
|
||||
lastOutputStream: null,
|
||||
contextSnapshot: {},
|
||||
logBytes: 0,
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: otherEvaluationIssueId,
|
||||
companyId,
|
||||
title: "Other run evaluation",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
assigneeAgentId: managerId,
|
||||
issueNumber: 21,
|
||||
identifier: `${issuePrefix}-21`,
|
||||
originKind: "stale_active_run_evaluation",
|
||||
originId: otherRunId,
|
||||
originFingerprint: `stale_active_run:${companyId}:${otherRunId}`,
|
||||
});
|
||||
|
||||
const attempts = [
|
||||
{ decision: "continue" as const, evaluationIssueId: unrelatedIssueId },
|
||||
{ decision: "dismissed_false_positive" as const, evaluationIssueId: unrelatedIssueId },
|
||||
{
|
||||
decision: "snooze" as const,
|
||||
evaluationIssueId: unrelatedIssueId,
|
||||
snoozedUntil: new Date(now.getTime() + 60 * 60 * 1000),
|
||||
},
|
||||
{ decision: "continue" as const, evaluationIssueId: otherEvaluationIssueId },
|
||||
];
|
||||
|
||||
for (const attempt of attempts) {
|
||||
await expect(
|
||||
recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
reason: "malicious or stale binding",
|
||||
...attempt,
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 403 });
|
||||
}
|
||||
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, evaluationIssueId));
|
||||
await expect(
|
||||
recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "closed evaluation should not authorize",
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 403 });
|
||||
});
|
||||
|
||||
it("validates createdByRunId before storing watchdog decisions", async () => {
|
||||
const now = new Date("2026-04-22T20:00:00.000Z");
|
||||
const { companyId, managerId, runId } = await seedRunningRun({
|
||||
now,
|
||||
ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn() });
|
||||
|
||||
const scan = await heartbeat.scanSilentActiveRuns({ now, companyId });
|
||||
const evaluationIssueId = scan.evaluationIssueIds[0];
|
||||
expect(evaluationIssueId).toBeTruthy();
|
||||
|
||||
await expect(
|
||||
recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "client supplied another agent run",
|
||||
createdByRunId: runId,
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 403 });
|
||||
|
||||
const managerRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: managerRunId,
|
||||
companyId,
|
||||
agentId: managerId,
|
||||
status: "running",
|
||||
invocationSource: "assignment",
|
||||
triggerDetail: "system",
|
||||
startedAt: now,
|
||||
processStartedAt: now,
|
||||
lastOutputAt: now,
|
||||
lastOutputSeq: 1,
|
||||
lastOutputStream: "stdout",
|
||||
contextSnapshot: {},
|
||||
logBytes: 0,
|
||||
});
|
||||
|
||||
const decision = await recovery.recordWatchdogDecision({
|
||||
runId,
|
||||
actor: { type: "agent", agentId: managerId, runId: managerRunId },
|
||||
decision: "continue",
|
||||
evaluationIssueId,
|
||||
reason: "valid current actor run",
|
||||
createdByRunId: randomUUID(),
|
||||
});
|
||||
expect(decision.createdByRunId).toBe(managerRunId);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,8 +1,4 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import net from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createServer } from "node:http";
|
||||
import { and, asc, eq } from "drizzle-orm";
|
||||
import { WebSocketServer } from "ws";
|
||||
|
|
@ -10,81 +6,14 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|||
import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
applyPendingMigrations,
|
||||
companies,
|
||||
createDb,
|
||||
ensurePostgresDatabase,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
|
||||
type EmbeddedPostgresInstance = {
|
||||
initialise(): Promise<void>;
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
type EmbeddedPostgresCtor = new (opts: {
|
||||
databaseDir: string;
|
||||
user: string;
|
||||
password: string;
|
||||
port: number;
|
||||
persistent: boolean;
|
||||
initdbFlags?: string[];
|
||||
onLog?: (message: unknown) => void;
|
||||
onError?: (message: unknown) => void;
|
||||
}) => EmbeddedPostgresInstance;
|
||||
|
||||
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
|
||||
const mod = await import("embedded-postgres");
|
||||
return mod.default as EmbeddedPostgresCtor;
|
||||
}
|
||||
|
||||
async function getAvailablePort(): Promise<number> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const server = net.createServer();
|
||||
server.unref();
|
||||
server.on("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
server.close(() => reject(new Error("Failed to allocate test port")));
|
||||
return;
|
||||
}
|
||||
const { port } = address;
|
||||
server.close((error) => {
|
||||
if (error) reject(error);
|
||||
else resolve(port);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function startTempDatabase() {
|
||||
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-heartbeat-comment-wake-"));
|
||||
const port = await getAvailablePort();
|
||||
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
|
||||
const instance = new EmbeddedPostgres({
|
||||
databaseDir: dataDir,
|
||||
user: "paperclip",
|
||||
password: "paperclip",
|
||||
port,
|
||||
persistent: true,
|
||||
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
|
||||
onLog: () => {},
|
||||
onError: () => {},
|
||||
});
|
||||
await instance.initialise();
|
||||
await instance.start();
|
||||
|
||||
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
|
||||
await ensurePostgresDatabase(adminConnectionString, "paperclip");
|
||||
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
|
||||
await applyPendingMigrations(connectionString);
|
||||
return { connectionString, instance, dataDir };
|
||||
}
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.ts";
|
||||
|
||||
async function waitFor(condition: () => boolean | Promise<boolean>, timeoutMs = 10_000, intervalMs = 50) {
|
||||
const startedAt = Date.now();
|
||||
|
|
@ -218,22 +147,17 @@ async function createControlledGatewayServer() {
|
|||
|
||||
describe("heartbeat comment wake batching", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let instance: EmbeddedPostgresInstance | null = null;
|
||||
let dataDir = "";
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
const started = await startTempDatabase();
|
||||
const started = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-comment-wake-");
|
||||
db = createDb(started.connectionString);
|
||||
instance = started.instance;
|
||||
dataDir = started.dataDir;
|
||||
}, 45_000);
|
||||
tempDb = started;
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await closeDbClient(db);
|
||||
await instance?.stop();
|
||||
if (dataDir) {
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
it("defers approval-approved wakes for a running issue so the assignee resumes after the run", async () => {
|
||||
|
|
@ -862,6 +786,206 @@ describe("heartbeat comment wake batching", () => {
|
|||
}
|
||||
}, 120_000);
|
||||
|
||||
it("does not reopen a finished issue when the deferred comment wake came from another agent", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
const assigneeAgentId = randomUUID();
|
||||
const mentionedAgentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
try {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
await db.insert(agents).values([
|
||||
{
|
||||
id: assigneeAgentId,
|
||||
companyId,
|
||||
name: "Primary Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
id: mentionedAgentId,
|
||||
companyId,
|
||||
name: "Mentioned Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Do not reopen from agent mention",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
assigneeAgentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
});
|
||||
|
||||
const firstRun = await heartbeat.wakeup(assigneeAgentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
},
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
});
|
||||
|
||||
expect(firstRun).not.toBeNull();
|
||||
await waitFor(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, firstRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "running";
|
||||
});
|
||||
|
||||
const comment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorAgentId: assigneeAgentId,
|
||||
createdByRunId: firstRun?.id ?? null,
|
||||
body: "@Mentioned Agent please review after I finish",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const deferredRun = await heartbeat.wakeup(mentionedAgentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_comment_mentioned",
|
||||
payload: { issueId, commentId: comment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: comment.id,
|
||||
wakeCommentId: comment.id,
|
||||
wakeReason: "issue_comment_mentioned",
|
||||
source: "comment.mention",
|
||||
},
|
||||
requestedByActorType: "agent",
|
||||
requestedByActorId: assigneeAgentId,
|
||||
});
|
||||
|
||||
expect(deferredRun).toBeNull();
|
||||
|
||||
await waitFor(async () => {
|
||||
const deferred = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, mentionedAgentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return Boolean(deferred);
|
||||
});
|
||||
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
status: "done",
|
||||
completedAt: new Date(),
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, issueId));
|
||||
|
||||
gateway.releaseFirstWait();
|
||||
|
||||
await waitFor(() => gateway.getAgentPayloads().length === 2, 90_000);
|
||||
await waitFor(async () => {
|
||||
const runs = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.companyId, companyId));
|
||||
return runs.length === 2 && runs.every((run) => run.status === "succeeded");
|
||||
}, 90_000);
|
||||
|
||||
const issueAfterPromotion = await db
|
||||
.select({
|
||||
status: issues.status,
|
||||
completedAt: issues.completedAt,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(issueAfterPromotion).toMatchObject({
|
||||
status: "done",
|
||||
});
|
||||
expect(issueAfterPromotion?.completedAt).not.toBeNull();
|
||||
|
||||
const secondPayload = gateway.getAgentPayloads()[1] ?? {};
|
||||
expect(secondPayload.paperclip).toMatchObject({
|
||||
wake: {
|
||||
reason: "issue_comment_mentioned",
|
||||
commentIds: [comment.id],
|
||||
latestCommentId: comment.id,
|
||||
issue: {
|
||||
id: issueId,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
title: "Do not reopen from agent mention",
|
||||
status: "done",
|
||||
priority: "medium",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(String(secondPayload.message ?? "")).toContain("please review after I finish");
|
||||
} finally {
|
||||
gateway.releaseFirstWait();
|
||||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("queues exactly one follow-up run when an issue-bound run exits without a comment", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
|
|
@ -1172,6 +1296,20 @@ describe("heartbeat comment wake batching", () => {
|
|||
wakeReason: "issue_comment_mentioned",
|
||||
});
|
||||
|
||||
const issueAfterMention = await db
|
||||
.select({
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(issueAfterMention?.assigneeAgentId).toBe(primaryAgentId);
|
||||
expect(issueAfterMention?.executionRunId).not.toBe(mentionedRuns[0]?.id);
|
||||
expect(issueAfterMention?.executionAgentNameKey).not.toBe("mentioned agent");
|
||||
|
||||
const primaryRuns = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
|
|
@ -1198,6 +1336,155 @@ describe("heartbeat comment wake batching", () => {
|
|||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("does not mark a direct mentioned-agent run as the issue execution owner", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
const primaryAgentId = randomUUID();
|
||||
const mentionedAgentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
try {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
await db.insert(agents).values([
|
||||
{
|
||||
id: primaryAgentId,
|
||||
companyId,
|
||||
name: "Primary Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
id: mentionedAgentId,
|
||||
companyId,
|
||||
name: "Mentioned Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Mention should not steal execution ownership",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
assigneeAgentId: primaryAgentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
});
|
||||
|
||||
const mentionComment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorUserId: "user-1",
|
||||
body: "@Mentioned Agent please inspect this.",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const mentionRun = await heartbeat.wakeup(mentionedAgentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_comment_mentioned",
|
||||
payload: { issueId, commentId: mentionComment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: mentionComment.id,
|
||||
wakeCommentId: mentionComment.id,
|
||||
wakeReason: "issue_comment_mentioned",
|
||||
source: "comment.mention",
|
||||
},
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "user-1",
|
||||
});
|
||||
|
||||
expect(mentionRun).not.toBeNull();
|
||||
await waitFor(() => gateway.getAgentPayloads().length === 1);
|
||||
|
||||
const issueDuringMention = await db
|
||||
.select({
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(issueDuringMention).toMatchObject({
|
||||
assigneeAgentId: primaryAgentId,
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
});
|
||||
|
||||
gateway.releaseFirstWait();
|
||||
await waitFor(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, mentionRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
}, 90_000);
|
||||
|
||||
const issueAfterMention = await db
|
||||
.select({
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(issueAfterMention).toMatchObject({
|
||||
assigneeAgentId: primaryAgentId,
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
});
|
||||
} finally {
|
||||
gateway.releaseFirstWait();
|
||||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
it("treats the automatic run summary as fallback-only when the run already posted a comment", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
|
|
|
|||
|
|
@ -347,6 +347,198 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
|
|||
expect(blockedWakeRequestCount).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("cancels stale queued runs when issue blockers are still unresolved", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const blockerId = randomUUID();
|
||||
const blockedIssueId = randomUUID();
|
||||
const readyIssueId = randomUUID();
|
||||
const blockedWakeupRequestId = randomUUID();
|
||||
const readyWakeupRequestId = randomUUID();
|
||||
const blockedRunId = randomUUID();
|
||||
const readyRunId = randomUUID();
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "QAChecker",
|
||||
role: "qa",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
wakeOnDemand: true,
|
||||
maxConcurrentRuns: 2,
|
||||
},
|
||||
},
|
||||
permissions: {},
|
||||
});
|
||||
await db.insert(issues).values([
|
||||
{
|
||||
id: blockerId,
|
||||
companyId,
|
||||
title: "Security review",
|
||||
status: "blocked",
|
||||
priority: "high",
|
||||
},
|
||||
{
|
||||
id: blockedIssueId,
|
||||
companyId,
|
||||
title: "QA validation",
|
||||
status: "blocked",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
},
|
||||
{
|
||||
id: readyIssueId,
|
||||
companyId,
|
||||
title: "Ready QA task",
|
||||
status: "todo",
|
||||
priority: "low",
|
||||
assigneeAgentId: agentId,
|
||||
},
|
||||
]);
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId: blockerId,
|
||||
relatedIssueId: blockedIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
await db.insert(agentWakeupRequests).values([
|
||||
{
|
||||
id: blockedWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "transient_failure_retry",
|
||||
payload: { issueId: blockedIssueId },
|
||||
status: "queued",
|
||||
},
|
||||
{
|
||||
id: readyWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId: readyIssueId },
|
||||
status: "queued",
|
||||
},
|
||||
]);
|
||||
await db.insert(heartbeatRuns).values([
|
||||
{
|
||||
id: blockedRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId: blockedWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: blockedIssueId,
|
||||
wakeReason: "transient_failure_retry",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: readyRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
invocationSource: "assignment",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId: readyWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: readyIssueId,
|
||||
wakeReason: "issue_assigned",
|
||||
},
|
||||
},
|
||||
]);
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: blockedRunId })
|
||||
.where(eq(agentWakeupRequests.id, blockedWakeupRequestId));
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: readyRunId })
|
||||
.where(eq(agentWakeupRequests.id, readyWakeupRequestId));
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: blockedRunId,
|
||||
executionAgentNameKey: "qa-checker",
|
||||
executionLockedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, blockedIssueId));
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, readyRunId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
});
|
||||
|
||||
const [blockedRun, blockedWakeup, blockedIssue, readyRun] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
status: heartbeatRuns.status,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
finishedAt: heartbeatRuns.finishedAt,
|
||||
resultJson: heartbeatRuns.resultJson,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, blockedRunId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({
|
||||
status: agentWakeupRequests.status,
|
||||
error: agentWakeupRequests.error,
|
||||
})
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, blockedWakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({
|
||||
executionRunId: issues.executionRunId,
|
||||
executionAgentNameKey: issues.executionAgentNameKey,
|
||||
executionLockedAt: issues.executionLockedAt,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, blockedIssueId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, readyRunId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
expect(blockedRun?.status).toBe("cancelled");
|
||||
expect(blockedRun?.errorCode).toBe("issue_dependencies_blocked");
|
||||
expect(blockedRun?.finishedAt).toBeTruthy();
|
||||
expect(blockedRun?.resultJson).toMatchObject({ stopReason: "issue_dependencies_blocked" });
|
||||
expect(blockedWakeup?.status).toBe("skipped");
|
||||
expect(blockedWakeup?.error).toContain("dependencies are still blocked");
|
||||
expect(blockedIssue).toMatchObject({
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
});
|
||||
expect(readyRun?.status).toBe("succeeded");
|
||||
expect(mockAdapterExecute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("suppresses normal wakeups while allowing comment interaction wakes under a pause hold", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
|
|
@ -425,12 +617,39 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
|
|||
.then((rows) => rows[0] ?? null);
|
||||
expect(skippedWake).toMatchObject({ status: "skipped", reason: "issue_tree_hold_active" });
|
||||
|
||||
const childCommentId = randomUUID();
|
||||
await db.insert(issueComments).values({
|
||||
id: childCommentId,
|
||||
companyId,
|
||||
issueId: childIssueId,
|
||||
authorUserId: "board-user",
|
||||
body: "Please respond while this hold is active.",
|
||||
});
|
||||
|
||||
const forgedChildCommentWake = await heartbeat.wakeup(agentId, {
|
||||
source: "on_demand",
|
||||
triggerDetail: "manual",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: childIssueId, commentId: childCommentId },
|
||||
requestedByActorType: "agent",
|
||||
requestedByActorId: agentId,
|
||||
});
|
||||
expect(forgedChildCommentWake).toBeNull();
|
||||
|
||||
const childCommentWake = await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: childIssueId, commentId: randomUUID() },
|
||||
contextSnapshot: { issueId: childIssueId, wakeReason: "issue_commented" },
|
||||
payload: { issueId: childIssueId, commentId: childCommentId },
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "board-user",
|
||||
contextSnapshot: {
|
||||
issueId: childIssueId,
|
||||
commentId: childCommentId,
|
||||
wakeCommentId: childCommentId,
|
||||
wakeReason: "issue_commented",
|
||||
source: "issue.comment",
|
||||
},
|
||||
});
|
||||
|
||||
expect(childCommentWake).not.toBeNull();
|
||||
|
|
@ -494,12 +713,29 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
|
|||
releasePolicy: { strategy: "manual", note: "full_pause" },
|
||||
});
|
||||
|
||||
const rootCommentId = randomUUID();
|
||||
await db.insert(issueComments).values({
|
||||
id: rootCommentId,
|
||||
companyId,
|
||||
issueId: rootIssueId,
|
||||
authorUserId: "board-user",
|
||||
body: "Please respond while this hold is active.",
|
||||
});
|
||||
|
||||
const rootCommentWake = await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: rootIssueId, commentId: randomUUID() },
|
||||
contextSnapshot: { issueId: rootIssueId, wakeReason: "issue_commented" },
|
||||
payload: { issueId: rootIssueId, commentId: rootCommentId },
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "board-user",
|
||||
contextSnapshot: {
|
||||
issueId: rootIssueId,
|
||||
commentId: rootCommentId,
|
||||
wakeCommentId: rootCommentId,
|
||||
wakeReason: "issue_commented",
|
||||
source: "issue.comment",
|
||||
},
|
||||
});
|
||||
|
||||
expect(rootCommentWake).not.toBeNull();
|
||||
|
|
|
|||
|
|
@ -4,13 +4,16 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
|
|||
import {
|
||||
activityLog,
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
companies,
|
||||
createDb,
|
||||
executionWorkspaces,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueRelations,
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
projects,
|
||||
projectWorkspaces,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
|
|
@ -55,6 +58,7 @@ vi.mock("../adapters/index.ts", async () => {
|
|||
});
|
||||
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
import { instanceSettingsService } from "../services/instance-settings.ts";
|
||||
import { runningProcesses } from "../adapters/index.ts";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
|
|
@ -94,13 +98,23 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
await db.execute(sql.raw(`TRUNCATE TABLE "companies" CASCADE`));
|
||||
await instanceSettingsService(db).updateExperimental({
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedBlockedChain() {
|
||||
async function enableAutoRecovery() {
|
||||
await instanceSettingsService(db).updateExperimental({
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
});
|
||||
}
|
||||
|
||||
async function seedBlockedChain(opts: { stale?: boolean } = {}) {
|
||||
const companyId = randomUUID();
|
||||
const managerId = randomUUID();
|
||||
const coderId = randomUUID();
|
||||
|
|
@ -124,7 +138,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
status: "idle",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: false } },
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
|
|
@ -136,11 +150,14 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
reportsTo: managerId,
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: false } },
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
|
||||
const issueTimestamp = opts.stale === false
|
||||
? new Date()
|
||||
: new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
await db.insert(issues).values([
|
||||
{
|
||||
id: blockedIssueId,
|
||||
|
|
@ -151,6 +168,8 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
assigneeAgentId: coderId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
},
|
||||
{
|
||||
id: blockerIssueId,
|
||||
|
|
@ -160,6 +179,8 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
priority: "medium",
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
},
|
||||
]);
|
||||
|
||||
|
|
@ -173,7 +194,91 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
return { companyId, managerId, blockedIssueId, blockerIssueId };
|
||||
}
|
||||
|
||||
it("creates one manager escalation, preserves blockers, and wakes the assignee", async () => {
|
||||
it("keeps liveness findings advisory when auto recovery is disabled", async () => {
|
||||
const { companyId } = await seedBlockedChain();
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(1);
|
||||
expect(result.autoRecoveryEnabled).toBe(false);
|
||||
expect(result.escalationsCreated).toBe(0);
|
||||
expect(result.skippedAutoRecoveryDisabled).toBe(1);
|
||||
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not create recovery issues until the dependency path is stale for 24 hours", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId } = await seedBlockedChain({ stale: false });
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(1);
|
||||
expect(result.escalationsCreated).toBe(0);
|
||||
expect(result.skippedAutoRecoveryTooYoung).toBe(1);
|
||||
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("suppresses liveness escalation when the source issue is under an active pause hold", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, blockedIssueId } = await seedBlockedChain();
|
||||
|
||||
await db.insert(issueTreeHolds).values({
|
||||
companyId,
|
||||
rootIssueId: blockedIssueId,
|
||||
mode: "pause",
|
||||
status: "active",
|
||||
reason: "pause liveness recovery subtree",
|
||||
releasePolicy: { strategy: "manual" },
|
||||
});
|
||||
|
||||
const result = await heartbeatService(db).reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(1);
|
||||
expect(result.escalationsCreated).toBe(0);
|
||||
expect(result.existingEscalations).toBe(0);
|
||||
expect(result.skipped).toBe(1);
|
||||
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("treats an active executionRunId on the leaf blocker as a live execution path", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, managerId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId,
|
||||
agentId: managerId,
|
||||
status: "running",
|
||||
contextSnapshot: { issueId: blockedIssueId },
|
||||
});
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, blockerIssueId));
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(0);
|
||||
expect(result.escalationsCreated).toBe(0);
|
||||
});
|
||||
|
||||
it("creates one manager escalation, preserves blockers, and records owner selection", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, managerId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
|
|
@ -182,7 +287,6 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
|
||||
expect(first.escalationsCreated).toBe(1);
|
||||
expect(second.escalationsCreated).toBe(0);
|
||||
expect(second.existingEscalations).toBe(1);
|
||||
|
||||
const escalations = await db
|
||||
.select()
|
||||
|
|
@ -195,9 +299,15 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
);
|
||||
expect(escalations).toHaveLength(1);
|
||||
expect(escalations[0]).toMatchObject({
|
||||
parentId: blockedIssueId,
|
||||
parentId: blockerIssueId,
|
||||
assigneeAgentId: managerId,
|
||||
status: expect.stringMatching(/^(todo|in_progress|done)$/),
|
||||
originFingerprint: [
|
||||
"harness_liveness_leaf",
|
||||
companyId,
|
||||
"blocked_by_unassigned_issue",
|
||||
blockerIssueId,
|
||||
].join(":"),
|
||||
});
|
||||
|
||||
const blockers = await db
|
||||
|
|
@ -213,15 +323,217 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
expect(comments[0]?.body).toContain("harness-level liveness incident");
|
||||
expect(comments[0]?.body).toContain(escalations[0]?.identifier ?? escalations[0]!.id);
|
||||
|
||||
const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, managerId));
|
||||
expect(wakes.some((wake) => wake.reason === "issue_assigned")).toBe(true);
|
||||
|
||||
const events = await db.select().from(activityLog).where(eq(activityLog.companyId, companyId));
|
||||
expect(events.some((event) => event.action === "issue.harness_liveness_escalation_created")).toBe(true);
|
||||
const createdEvent = events.find((event) => event.action === "issue.harness_liveness_escalation_created");
|
||||
expect(createdEvent).toBeTruthy();
|
||||
expect(createdEvent?.details).toMatchObject({
|
||||
recoveryIssueId: blockerIssueId,
|
||||
ownerSelection: {
|
||||
selectedAgentId: managerId,
|
||||
selectedReason: "root_agent",
|
||||
selectedSourceIssueId: blockerIssueId,
|
||||
},
|
||||
workspaceSelection: {
|
||||
reuseRecoveryExecutionWorkspace: false,
|
||||
inheritedExecutionWorkspaceFromIssueId: null,
|
||||
projectWorkspaceSourceIssueId: blockerIssueId,
|
||||
},
|
||||
});
|
||||
expect(events.some((event) => event.action === "issue.blockers.updated")).toBe(true);
|
||||
});
|
||||
|
||||
it("parents recovery under the leaf blocker without inheriting dependent or blocker execution state for manager-owned recovery", async () => {
|
||||
await enableAutoRecovery();
|
||||
await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
|
||||
|
||||
const companyId = randomUUID();
|
||||
const managerId = randomUUID();
|
||||
const blockedIssueId = randomUUID();
|
||||
const blockerIssueId = randomUUID();
|
||||
const dependentProjectId = randomUUID();
|
||||
const blockerProjectId = randomUUID();
|
||||
const dependentProjectWorkspaceId = randomUUID();
|
||||
const blockerProjectWorkspaceId = randomUUID();
|
||||
const dependentExecutionWorkspaceId = randomUUID();
|
||||
const blockerExecutionWorkspaceId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: managerId,
|
||||
companyId,
|
||||
name: "Root Operator",
|
||||
role: "operator",
|
||||
status: "idle",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: false } },
|
||||
permissions: {},
|
||||
});
|
||||
await db.insert(projects).values([
|
||||
{
|
||||
id: dependentProjectId,
|
||||
companyId,
|
||||
name: "Dependent workspace project",
|
||||
status: "in_progress",
|
||||
},
|
||||
{
|
||||
id: blockerProjectId,
|
||||
companyId,
|
||||
name: "Blocker workspace project",
|
||||
status: "in_progress",
|
||||
},
|
||||
]);
|
||||
await db.insert(projectWorkspaces).values([
|
||||
{
|
||||
id: dependentProjectWorkspaceId,
|
||||
companyId,
|
||||
projectId: dependentProjectId,
|
||||
name: "Dependent primary",
|
||||
},
|
||||
{
|
||||
id: blockerProjectWorkspaceId,
|
||||
companyId,
|
||||
projectId: blockerProjectId,
|
||||
name: "Blocker primary",
|
||||
},
|
||||
]);
|
||||
await db.insert(executionWorkspaces).values([
|
||||
{
|
||||
id: dependentExecutionWorkspaceId,
|
||||
companyId,
|
||||
projectId: dependentProjectId,
|
||||
projectWorkspaceId: dependentProjectWorkspaceId,
|
||||
mode: "operator_branch",
|
||||
strategyType: "git_worktree",
|
||||
name: "Dependent branch",
|
||||
status: "active",
|
||||
providerType: "git_worktree",
|
||||
},
|
||||
{
|
||||
id: blockerExecutionWorkspaceId,
|
||||
companyId,
|
||||
projectId: blockerProjectId,
|
||||
projectWorkspaceId: blockerProjectWorkspaceId,
|
||||
mode: "operator_branch",
|
||||
strategyType: "git_worktree",
|
||||
name: "Blocker branch",
|
||||
status: "active",
|
||||
providerType: "git_worktree",
|
||||
},
|
||||
]);
|
||||
await db.insert(issues).values([
|
||||
{
|
||||
id: blockedIssueId,
|
||||
companyId,
|
||||
projectId: dependentProjectId,
|
||||
projectWorkspaceId: dependentProjectWorkspaceId,
|
||||
executionWorkspaceId: dependentExecutionWorkspaceId,
|
||||
executionWorkspacePreference: "reuse_existing",
|
||||
executionWorkspaceSettings: { mode: "operator_branch" },
|
||||
title: "Blocked dependent",
|
||||
status: "blocked",
|
||||
priority: "medium",
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
},
|
||||
{
|
||||
id: blockerIssueId,
|
||||
companyId,
|
||||
projectId: blockerProjectId,
|
||||
projectWorkspaceId: blockerProjectWorkspaceId,
|
||||
executionWorkspaceId: blockerExecutionWorkspaceId,
|
||||
executionWorkspacePreference: "reuse_existing",
|
||||
executionWorkspaceSettings: { mode: "operator_branch" },
|
||||
title: "Unassigned leaf blocker",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
},
|
||||
]);
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId: blockerIssueId,
|
||||
relatedIssueId: blockedIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
|
||||
const result = await heartbeatService(db).reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.escalationsCreated).toBe(1);
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(1);
|
||||
expect(escalations[0]).toMatchObject({
|
||||
parentId: blockerIssueId,
|
||||
projectId: blockerProjectId,
|
||||
projectWorkspaceId: blockerProjectWorkspaceId,
|
||||
executionWorkspaceId: null,
|
||||
executionWorkspacePreference: null,
|
||||
assigneeAgentId: managerId,
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses one open recovery issue for multiple dependents with the same leaf blocker", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const secondBlockedIssueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
await db.insert(issues).values({
|
||||
id: secondBlockedIssueId,
|
||||
companyId,
|
||||
title: "Second blocked parent",
|
||||
status: "blocked",
|
||||
priority: "medium",
|
||||
issueNumber: 3,
|
||||
identifier: `${issuePrefix}-3`,
|
||||
createdAt: issueTimestamp,
|
||||
updatedAt: issueTimestamp,
|
||||
});
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId: blockerIssueId,
|
||||
relatedIssueId: secondBlockedIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(2);
|
||||
expect(result.escalationsCreated).toBe(1);
|
||||
expect(result.existingEscalations).toBe(1);
|
||||
const escalations = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
|
||||
expect(escalations).toHaveLength(1);
|
||||
|
||||
const blockers = await db
|
||||
.select({ blockedIssueId: issueRelations.relatedIssueId })
|
||||
.from(issueRelations)
|
||||
.where(and(eq(issueRelations.companyId, companyId), eq(issueRelations.issueId, escalations[0]!.id)));
|
||||
expect(blockers.map((row) => row.blockedIssueId).sort()).toEqual(
|
||||
[blockedIssueId, secondBlockedIssueId].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("creates a fresh escalation when the previous matching escalation is terminal", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId, managerId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const heartbeat = heartbeatService(db);
|
||||
const incidentKey = [
|
||||
|
|
@ -265,7 +577,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
|||
expect(openEscalations).toHaveLength(2);
|
||||
const freshEscalation = openEscalations.find((issue) => issue.status !== "done");
|
||||
expect(freshEscalation).toMatchObject({
|
||||
parentId: blockedIssueId,
|
||||
parentId: blockerIssueId,
|
||||
assigneeAgentId: managerId,
|
||||
status: expect.stringMatching(/^(todo|in_progress|done)$/),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { eq, or, inArray } from "drizzle-orm";
|
||||
import { and, eq, or, inArray } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
activityLog,
|
||||
|
|
@ -17,6 +17,8 @@ import {
|
|||
issueComments,
|
||||
issueDocuments,
|
||||
issueRelations,
|
||||
issueTreeHoldMembers,
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
|
|
@ -309,6 +311,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
await db.delete(documentRevisions);
|
||||
await db.delete(documents);
|
||||
await db.delete(issueRelations);
|
||||
await db.delete(issueTreeHoldMembers);
|
||||
await db.delete(issueTreeHolds);
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issueDocuments);
|
||||
|
|
@ -454,11 +458,13 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
runStatus: "failed" | "timed_out" | "cancelled" | "succeeded";
|
||||
retryReason?: "assignment_recovery" | "issue_continuation_needed" | null;
|
||||
assignToUser?: boolean;
|
||||
activePauseHold?: boolean;
|
||||
}) {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const wakeupRequestId = randomUUID();
|
||||
const rootIssueId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const now = new Date("2026-03-19T00:00:00.000Z");
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
|
|
@ -520,22 +526,128 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
error: input.runStatus === "succeeded" ? null : "run failed before issue advanced",
|
||||
});
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Recover stranded assigned work",
|
||||
status: input.status,
|
||||
await db.insert(issues).values([
|
||||
...(input.activePauseHold
|
||||
? [{
|
||||
id: rootIssueId,
|
||||
companyId,
|
||||
title: "Paused recovery root",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
}]
|
||||
: []),
|
||||
{
|
||||
id: issueId,
|
||||
companyId,
|
||||
parentId: input.activePauseHold ? rootIssueId : null,
|
||||
title: "Recover stranded assigned work",
|
||||
status: input.status,
|
||||
priority: "medium",
|
||||
assigneeAgentId: input.assignToUser ? null : agentId,
|
||||
assigneeUserId: input.assignToUser ? "user-1" : null,
|
||||
checkoutRunId: input.status === "in_progress" ? runId : null,
|
||||
executionRunId: null,
|
||||
issueNumber: input.activePauseHold ? 2 : 1,
|
||||
identifier: `${issuePrefix}-${input.activePauseHold ? 2 : 1}`,
|
||||
startedAt: input.status === "in_progress" ? now : null,
|
||||
},
|
||||
]);
|
||||
|
||||
if (input.activePauseHold) {
|
||||
await db.insert(issueTreeHolds).values({
|
||||
companyId,
|
||||
rootIssueId,
|
||||
mode: "pause",
|
||||
status: "active",
|
||||
reason: "pause recovery subtree",
|
||||
releasePolicy: { strategy: "manual" },
|
||||
});
|
||||
}
|
||||
|
||||
return { companyId, agentId, runId, wakeupRequestId, issueId, rootIssueId };
|
||||
}
|
||||
|
||||
async function expectStrandedRecoveryArtifacts(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
issueId: string;
|
||||
runId: string;
|
||||
previousStatus: "todo" | "in_progress";
|
||||
retryReason: "assignment_recovery" | "issue_continuation_needed";
|
||||
}) {
|
||||
const recovery = await waitForValue(async () =>
|
||||
db.select().from(issues).where(
|
||||
and(
|
||||
eq(issues.companyId, input.companyId),
|
||||
eq(issues.originKind, "stranded_issue_recovery"),
|
||||
eq(issues.originId, input.issueId),
|
||||
),
|
||||
).then((rows) => rows[0] ?? null),
|
||||
);
|
||||
if (!recovery) throw new Error("Expected stranded issue recovery issue to be created");
|
||||
|
||||
expect(recovery).toMatchObject({
|
||||
companyId: input.companyId,
|
||||
parentId: input.issueId,
|
||||
assigneeAgentId: input.agentId,
|
||||
originKind: "stranded_issue_recovery",
|
||||
originId: input.issueId,
|
||||
originRunId: input.runId,
|
||||
priority: "medium",
|
||||
assigneeAgentId: input.assignToUser ? null : agentId,
|
||||
assigneeUserId: input.assignToUser ? "user-1" : null,
|
||||
checkoutRunId: input.status === "in_progress" ? runId : null,
|
||||
executionRunId: null,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
startedAt: input.status === "in_progress" ? now : null,
|
||||
});
|
||||
expect(recovery.title).toContain("Recover stalled issue");
|
||||
expect(recovery.description).toContain(`Previous source status: \`${input.previousStatus}\``);
|
||||
expect(recovery.description).toContain(`Retry reason: \`${input.retryReason}\``);
|
||||
expect(recovery.description).toContain("Fix the runtime/adapter problem");
|
||||
|
||||
const relation = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, input.companyId),
|
||||
eq(issueRelations.issueId, recovery.id),
|
||||
eq(issueRelations.relatedIssueId, input.issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(relation).toBeTruthy();
|
||||
|
||||
const wakeups = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.agentId, input.agentId));
|
||||
const recoveryWakeup = wakeups.find((wakeup) => {
|
||||
const payload = wakeup.payload as Record<string, unknown> | null;
|
||||
return payload?.issueId === recovery.id &&
|
||||
payload?.sourceIssueId === input.issueId &&
|
||||
payload?.strandedRunId === input.runId;
|
||||
});
|
||||
expect(recoveryWakeup).toMatchObject({
|
||||
companyId: input.companyId,
|
||||
reason: "issue_assigned",
|
||||
source: "assignment",
|
||||
});
|
||||
|
||||
return { companyId, agentId, runId, wakeupRequestId, issueId };
|
||||
const recoveryRun = recoveryWakeup?.runId
|
||||
? await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, recoveryWakeup.runId))
|
||||
.then((rows) => rows[0] ?? null)
|
||||
: null;
|
||||
expect(recoveryRun?.contextSnapshot).toMatchObject({
|
||||
issueId: recovery.id,
|
||||
taskId: recovery.id,
|
||||
source: "stranded_issue_recovery",
|
||||
sourceIssueId: input.issueId,
|
||||
strandedRunId: input.runId,
|
||||
});
|
||||
|
||||
return recovery;
|
||||
}
|
||||
|
||||
async function seedQueuedIssueRunFixture() {
|
||||
|
|
@ -728,11 +840,28 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
it("blocks the issue when process-loss retry is exhausted and the immediate continuation recovery also fails", async () => {
|
||||
mockAdapterExecute.mockRejectedValueOnce(new Error("continuation recovery failed"));
|
||||
|
||||
const { agentId, runId, issueId } = await seedRunFixture({
|
||||
const { companyId, agentId, runId, issueId } = await seedRunFixture({
|
||||
agentStatus: "idle",
|
||||
processPid: 999_999_999,
|
||||
processLossRetryCount: 1,
|
||||
});
|
||||
const resolvedBlockerId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
await db.insert(issues).values({
|
||||
id: resolvedBlockerId,
|
||||
companyId,
|
||||
title: "Already completed prerequisite",
|
||||
status: "done",
|
||||
priority: "medium",
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
});
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId: resolvedBlockerId,
|
||||
relatedIssueId: issueId,
|
||||
type: "blocks",
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reapOrphanedRuns();
|
||||
|
|
@ -759,7 +888,29 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
);
|
||||
expect(blockedIssue?.status).toBe("blocked");
|
||||
expect(blockedIssue?.executionRunId).toBeNull();
|
||||
expect(blockedIssue?.checkoutRunId).toBe(continuationRun?.id ?? null);
|
||||
expect(blockedIssue?.checkoutRunId).toBeNull();
|
||||
if (!continuationRun?.id) throw new Error("Expected continuation recovery run to exist");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
runId: continuationRun.id,
|
||||
previousStatus: "in_progress",
|
||||
retryReason: "issue_continuation_needed",
|
||||
});
|
||||
|
||||
const blockerRelations = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.relatedIssueId, issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
);
|
||||
expect(blockerRelations.map((relation) => relation.issueId)).toEqual([recovery.id]);
|
||||
|
||||
const comments = await waitForValue(async () => {
|
||||
const rows = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
|
|
@ -767,6 +918,49 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
});
|
||||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("retried continuation");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
});
|
||||
|
||||
it("does not block paused-tree work when immediate continuation recovery is suppressed by the hold", async () => {
|
||||
const { companyId, agentId, runId, issueId } = await seedRunFixture({
|
||||
agentStatus: "idle",
|
||||
processPid: 999_999_999,
|
||||
processLossRetryCount: 1,
|
||||
});
|
||||
await db.insert(issueTreeHolds).values({
|
||||
companyId,
|
||||
rootIssueId: issueId,
|
||||
mode: "pause",
|
||||
status: "active",
|
||||
reason: "pause immediate recovery subtree",
|
||||
releasePolicy: { strategy: "manual" },
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reapOrphanedRuns();
|
||||
expect(result.reaped).toBe(1);
|
||||
expect(result.runIds).toEqual([runId]);
|
||||
|
||||
const runs = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.agentId, agentId));
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(runs[0]?.status).toBe("failed");
|
||||
|
||||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("in_progress");
|
||||
expect(issue?.executionRunId).toBeNull();
|
||||
expect(issue?.checkoutRunId).toBe(runId);
|
||||
|
||||
const recoveryIssues = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
|
||||
expect(recoveryIssues).toHaveLength(0);
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("schedules a bounded retry for codex transient upstream failures instead of blocking the issue immediately", async () => {
|
||||
|
|
@ -901,7 +1095,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
});
|
||||
|
||||
it("blocks assigned todo work after the one automatic dispatch recovery was already used", async () => {
|
||||
const { issueId } = await seedStrandedIssueFixture({
|
||||
const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({
|
||||
status: "todo",
|
||||
runStatus: "failed",
|
||||
retryReason: "assignment_recovery",
|
||||
|
|
@ -916,10 +1110,20 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("blocked");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
runId,
|
||||
previousStatus: "todo",
|
||||
retryReason: "assignment_recovery",
|
||||
});
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("retried dispatch");
|
||||
expect(comments[0]?.body).toContain("Latest retry failure: `process_lost` - run failed before issue advanced.");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
});
|
||||
|
||||
it("assigns open unassigned blockers back to their creator agent", async () => {
|
||||
|
|
@ -1206,7 +1410,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
expect(wakes.some((row) => row.reason === "run_liveness_continuation")).toBe(false);
|
||||
});
|
||||
it("blocks stranded in-progress work after the continuation retry was already used", async () => {
|
||||
const { issueId } = await seedStrandedIssueFixture({
|
||||
const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({
|
||||
status: "in_progress",
|
||||
runStatus: "failed",
|
||||
retryReason: "issue_continuation_needed",
|
||||
|
|
@ -1221,10 +1425,65 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("blocked");
|
||||
|
||||
const recovery = await expectStrandedRecoveryArtifacts({
|
||||
companyId,
|
||||
agentId,
|
||||
issueId,
|
||||
runId,
|
||||
previousStatus: "in_progress",
|
||||
retryReason: "issue_continuation_needed",
|
||||
});
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(1);
|
||||
expect(comments[0]?.body).toContain("retried continuation");
|
||||
expect(comments[0]?.body).toContain("Latest retry failure: `process_lost` - run failed before issue advanced.");
|
||||
expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
|
||||
});
|
||||
|
||||
it("does not escalate paused-tree recovery when the automatic continuation retry was cancelled by the hold", async () => {
|
||||
const { companyId, agentId, issueId } = await seedStrandedIssueFixture({
|
||||
status: "in_progress",
|
||||
runStatus: "cancelled",
|
||||
retryReason: "issue_continuation_needed",
|
||||
activePauseHold: true,
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileStrandedAssignedIssues();
|
||||
expect(result.dispatchRequeued).toBe(0);
|
||||
expect(result.continuationRequeued).toBe(0);
|
||||
expect(result.escalated).toBe(0);
|
||||
expect(result.skipped).toBe(1);
|
||||
expect(result.issueIds).toEqual([]);
|
||||
|
||||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
|
||||
expect(issue?.status).toBe("in_progress");
|
||||
expect(issue?.checkoutRunId).toBeTruthy();
|
||||
|
||||
const recoveryIssues = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
|
||||
expect(recoveryIssues).toHaveLength(0);
|
||||
|
||||
const blockerRelations = await db
|
||||
.select()
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.relatedIssueId, issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
);
|
||||
expect(blockerRelations).toHaveLength(0);
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(0);
|
||||
|
||||
const wakeups = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId));
|
||||
expect(wakeups).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("re-enqueues continuation when the latest automatic continuation succeeded without closing the issue", async () => {
|
||||
|
|
|
|||
30
server/src/__tests__/heartbeat-start-lock.test.ts
Normal file
30
server/src/__tests__/heartbeat-start-lock.test.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { withAgentStartLock } from "../services/agent-start-lock.ts";
|
||||
|
||||
describe("heartbeat agent start lock", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("does not let a stale start lock freeze later queued-run starts", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const agentId = randomUUID();
|
||||
const firstStart = vi.fn(() => new Promise<void>(() => undefined));
|
||||
const secondStart = vi.fn(async () => "started");
|
||||
|
||||
void withAgentStartLock(agentId, firstStart);
|
||||
await Promise.resolve();
|
||||
expect(firstStart).toHaveBeenCalledTimes(1);
|
||||
|
||||
const secondStartResult = withAgentStartLock(agentId, secondStart);
|
||||
await Promise.resolve();
|
||||
expect(secondStart).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
|
||||
await expect(secondStartResult).resolves.toBe("started");
|
||||
expect(secondStart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -58,6 +58,7 @@ describe("instance settings routes", () => {
|
|||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
});
|
||||
mockInstanceSettingsService.updateGeneral.mockResolvedValue({
|
||||
id: "instance-settings-1",
|
||||
|
|
@ -73,6 +74,7 @@ describe("instance settings routes", () => {
|
|||
enableEnvironments: true,
|
||||
enableIsolatedWorkspaces: true,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
},
|
||||
});
|
||||
mockInstanceSettingsService.listCompanyIds.mockResolvedValue(["company-1", "company-2"]);
|
||||
|
|
@ -92,6 +94,7 @@ describe("instance settings routes", () => {
|
|||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
});
|
||||
|
||||
const patchRes = await request(app)
|
||||
|
|
@ -103,7 +106,7 @@ describe("instance settings routes", () => {
|
|||
enableIsolatedWorkspaces: true,
|
||||
});
|
||||
expect(mockLogActivity).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
}, 10_000);
|
||||
|
||||
it("allows local board users to update guarded dev-server auto-restart", async () => {
|
||||
const app = await createApp({
|
||||
|
|
@ -118,8 +121,28 @@ describe("instance settings routes", () => {
|
|||
.send({ autoRestartDevServerWhenIdle: true })
|
||||
.expect(200);
|
||||
|
||||
expect(
|
||||
mockInstanceSettingsService.updateExperimental.mock.calls.some(
|
||||
([patch]) => patch?.autoRestartDevServerWhenIdle === true,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("allows local board users to update issue graph liveness auto-recovery", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
userId: "local-board",
|
||||
source: "local_implicit",
|
||||
isInstanceAdmin: true,
|
||||
});
|
||||
|
||||
await request(app)
|
||||
.patch("/api/instance/settings/experimental")
|
||||
.send({ enableIssueGraphLivenessAutoRecovery: true })
|
||||
.expect(200);
|
||||
|
||||
expect(mockInstanceSettingsService.updateExperimental).toHaveBeenCalledWith({
|
||||
autoRestartDevServerWhenIdle: true,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
function createSelectChain(rows: unknown[]) {
|
||||
const query = {
|
||||
|
|
@ -44,8 +44,6 @@ function createInvite(overrides: Record<string, unknown> = {}) {
|
|||
};
|
||||
}
|
||||
|
||||
let currentAccessModule: Awaited<ReturnType<typeof vi.importActual<typeof import("../routes/access.js")>>> | null = null;
|
||||
|
||||
async function createApp(
|
||||
db: Record<string, unknown>,
|
||||
network: {
|
||||
|
|
@ -54,11 +52,9 @@ async function createApp(
|
|||
},
|
||||
) {
|
||||
const [access, middleware] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/access.js")>("../routes/access.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
import("../routes/access.js"),
|
||||
import("../middleware/index.js"),
|
||||
]);
|
||||
currentAccessModule = access;
|
||||
access.setInviteResolutionNetworkForTest(network);
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = { type: "anon" };
|
||||
|
|
@ -71,6 +67,7 @@ async function createApp(
|
|||
deploymentExposure: "private",
|
||||
bindHost: "127.0.0.1",
|
||||
allowedHostnames: [],
|
||||
inviteResolutionNetwork: network,
|
||||
}),
|
||||
);
|
||||
app.use(middleware.errorHandler);
|
||||
|
|
@ -79,43 +76,43 @@ async function createApp(
|
|||
|
||||
describe.sequential("GET /invites/:token/test-resolution", () => {
|
||||
beforeEach(() => {
|
||||
currentAccessModule = null;
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
currentAccessModule?.setInviteResolutionNetworkForTest(null);
|
||||
});
|
||||
it("rejects private, local, multicast, and reserved targets before probing", async () => {
|
||||
const cases = [
|
||||
["localhost", "http://localhost:3100/api/health", "127.0.0.1"],
|
||||
["IPv4 loopback", "http://127.0.0.1:3100/api/health", "127.0.0.1"],
|
||||
["IPv6 loopback", "http://[::1]:3100/api/health", "::1"],
|
||||
["IPv4-mapped IPv6 loopback hex", "http://[::ffff:7f00:1]/api/health", "::ffff:7f00:1"],
|
||||
["IPv4-mapped IPv6 RFC1918 hex", "http://[::ffff:c0a8:101]/api/health", "::ffff:c0a8:101"],
|
||||
["RFC1918 10/8", "http://10.0.0.5/api/health", "10.0.0.5"],
|
||||
["RFC1918 172.16/12", "http://172.16.10.5/api/health", "172.16.10.5"],
|
||||
["RFC1918 192.168/16", "http://192.168.1.10/api/health", "192.168.1.10"],
|
||||
["link-local metadata", "http://169.254.169.254/latest/meta-data", "169.254.169.254"],
|
||||
["multicast", "http://224.0.0.1/probe", "224.0.0.1"],
|
||||
["NAT64 well-known prefix", "https://gateway.example.test/health", "64:ff9b::0a00:0001"],
|
||||
["NAT64 local-use prefix", "https://gateway.example.test/health", "64:ff9b:1::0a00:0001"],
|
||||
] as const;
|
||||
|
||||
it.each([
|
||||
["localhost", "http://localhost:3100/api/health", "127.0.0.1"],
|
||||
["IPv4 loopback", "http://127.0.0.1:3100/api/health", "127.0.0.1"],
|
||||
["IPv6 loopback", "http://[::1]:3100/api/health", "::1"],
|
||||
["IPv4-mapped IPv6 loopback hex", "http://[::ffff:7f00:1]/api/health", "::ffff:7f00:1"],
|
||||
["IPv4-mapped IPv6 RFC1918 hex", "http://[::ffff:c0a8:101]/api/health", "::ffff:c0a8:101"],
|
||||
["RFC1918 10/8", "http://10.0.0.5/api/health", "10.0.0.5"],
|
||||
["RFC1918 172.16/12", "http://172.16.10.5/api/health", "172.16.10.5"],
|
||||
["RFC1918 192.168/16", "http://192.168.1.10/api/health", "192.168.1.10"],
|
||||
["link-local metadata", "http://169.254.169.254/latest/meta-data", "169.254.169.254"],
|
||||
["multicast", "http://224.0.0.1/probe", "224.0.0.1"],
|
||||
["NAT64 well-known prefix", "https://gateway.example.test/health", "64:ff9b::0a00:0001"],
|
||||
["NAT64 local-use prefix", "https://gateway.example.test/health", "64:ff9b:1::0a00:0001"],
|
||||
])("rejects %s targets before probing", async (_label, url, address) => {
|
||||
const lookup = vi.fn().mockResolvedValue([{ address, family: address.includes(":") ? 6 : 4 }]);
|
||||
const requestHead = vi.fn();
|
||||
const app = await createApp(createDbStub([createInvite()]), { lookup, requestHead });
|
||||
for (const [label, url, address] of cases) {
|
||||
const lookup = vi.fn().mockResolvedValue([{ address, family: address.includes(":") ? 6 : 4 }]);
|
||||
const requestHead = vi.fn();
|
||||
const app = await createApp(createDbStub([createInvite()]), { lookup, requestHead });
|
||||
|
||||
const res = await request(app)
|
||||
.get("/api/invites/pcp_invite_test/test-resolution")
|
||||
.query({ url });
|
||||
const res = await request(app)
|
||||
.get("/api/invites/pcp_invite_test/test-resolution")
|
||||
.query({ url });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe(
|
||||
"url resolves to a private, local, multicast, or reserved address",
|
||||
);
|
||||
expect(requestHead).not.toHaveBeenCalled();
|
||||
}, 15_000);
|
||||
expect(res.status, label).toBe(400);
|
||||
expect(res.body.error).toBe(
|
||||
"url resolves to a private, local, multicast, or reserved address",
|
||||
);
|
||||
expect(requestHead).not.toHaveBeenCalled();
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("rejects hostnames that resolve to private addresses", async () => {
|
||||
it.sequential("rejects hostnames that resolve to private addresses", async () => {
|
||||
const lookup = vi.fn().mockResolvedValue([{ address: "10.1.2.3", family: 4 }]);
|
||||
const requestHead = vi.fn();
|
||||
const app = await createApp(createDbStub([createInvite()]), { lookup, requestHead });
|
||||
|
|
@ -132,7 +129,7 @@ describe.sequential("GET /invites/:token/test-resolution", () => {
|
|||
expect(requestHead).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects hostnames when any resolved address is private", async () => {
|
||||
it.sequential("rejects hostnames when any resolved address is private", async () => {
|
||||
const lookup = vi.fn().mockResolvedValue([
|
||||
{ address: "127.0.0.1", family: 4 },
|
||||
{ address: "93.184.216.34", family: 4 },
|
||||
|
|
@ -148,7 +145,7 @@ describe.sequential("GET /invites/:token/test-resolution", () => {
|
|||
expect(requestHead).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows public HTTPS targets through the resolved and pinned probe path", async () => {
|
||||
it.sequential("allows public HTTPS targets through the resolved and pinned probe path", async () => {
|
||||
const lookup = vi.fn().mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
|
||||
const requestHead = vi.fn().mockResolvedValue({ httpStatus: 204 });
|
||||
const app = await createApp(createDbStub([createInvite()]), { lookup, requestHead });
|
||||
|
|
@ -177,7 +174,7 @@ describe.sequential("GET /invites/:token/test-resolution", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
it.sequential.each([
|
||||
["missing invite", []],
|
||||
["revoked invite", [createInvite({ revokedAt: new Date("2026-03-07T00:05:00.000Z") })]],
|
||||
["expired invite", [createInvite({ expiresAt: new Date("2020-03-07T00:10:00.000Z") })]],
|
||||
|
|
|
|||
280
server/src/__tests__/issue-blocker-attention.test.ts
Normal file
280
server/src/__tests__/issue-blocker-attention.test.ts
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issueRelations,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres issue blocker attention tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("issue blocker attention", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let svc!: ReturnType<typeof issueService>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
svc = issueService(db);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(issueRelations);
|
||||
await db.delete(issues);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function createCompany(prefix = "PBA") {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: `Company ${prefix}`,
|
||||
issuePrefix: prefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: `${prefix} Agent`,
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
});
|
||||
return { companyId, agentId };
|
||||
}
|
||||
|
||||
async function insertIssue(input: {
|
||||
companyId: string;
|
||||
id?: string;
|
||||
identifier: string;
|
||||
title: string;
|
||||
status: string;
|
||||
parentId?: string | null;
|
||||
assigneeAgentId?: string | null;
|
||||
}) {
|
||||
const id = input.id ?? randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id,
|
||||
companyId: input.companyId,
|
||||
identifier: input.identifier,
|
||||
title: input.title,
|
||||
status: input.status,
|
||||
priority: "medium",
|
||||
parentId: input.parentId ?? null,
|
||||
assigneeAgentId: input.assigneeAgentId ?? null,
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
async function block(input: { companyId: string; blockerIssueId: string; blockedIssueId: string }) {
|
||||
await db.insert(issueRelations).values({
|
||||
companyId: input.companyId,
|
||||
issueId: input.blockerIssueId,
|
||||
relatedIssueId: input.blockedIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
}
|
||||
|
||||
async function activeRun(input: { companyId: string; agentId: string; issueId: string; status?: string; current?: boolean }) {
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
status: input.status ?? "running",
|
||||
contextSnapshot: { issueId: input.issueId },
|
||||
});
|
||||
if (input.current !== false) {
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, input.issueId));
|
||||
}
|
||||
return runId;
|
||||
}
|
||||
|
||||
it("classifies a blocked parent as covered when its child has a running execution path", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBC");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBC-1", title: "Parent", status: "blocked" });
|
||||
const childId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBC-2",
|
||||
title: "Running child",
|
||||
status: "todo",
|
||||
parentId,
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: childId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: childId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "covered",
|
||||
reason: "active_child",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 1,
|
||||
attentionBlockerCount: 0,
|
||||
sampleBlockerIdentifier: "PBC-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps mixed blockers attention-required when any path lacks active work", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBM");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBM-1", title: "Parent", status: "blocked" });
|
||||
const activeChildId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBM-2",
|
||||
title: "Running child",
|
||||
status: "todo",
|
||||
parentId,
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
const idleBlockerId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBM-3",
|
||||
title: "Idle blocker",
|
||||
status: "todo",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: activeChildId, blockedIssueId: parentId });
|
||||
await block({ companyId, blockerIssueId: idleBlockerId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: activeChildId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
unresolvedBlockerCount: 2,
|
||||
coveredBlockerCount: 1,
|
||||
attentionBlockerCount: 1,
|
||||
sampleBlockerIdentifier: "PBM-3",
|
||||
});
|
||||
});
|
||||
|
||||
it("covers recursive blocker chains when the downstream leaf has active work", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBR");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBR-1", title: "Parent", status: "blocked" });
|
||||
const blockerId = await insertIssue({ companyId, identifier: "PBR-2", title: "Blocked dependency", status: "blocked" });
|
||||
const leafId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBR-3",
|
||||
title: "Running leaf",
|
||||
status: "todo",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
|
||||
await block({ companyId, blockerIssueId: leafId, blockedIssueId: blockerId });
|
||||
await activeRun({ companyId, agentId, issueId: leafId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "covered",
|
||||
reason: "active_dependency",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 1,
|
||||
attentionBlockerCount: 0,
|
||||
sampleBlockerIdentifier: "PBR-3",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let another company's active run cover the blocker", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBS");
|
||||
const other = await createCompany("PBT");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBS-1", title: "Parent", status: "blocked" });
|
||||
const blockerId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBS-2",
|
||||
title: "Same-company blocker",
|
||||
status: "todo",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId: other.companyId, agentId: other.agentId, issueId: blockerId });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 0,
|
||||
attentionBlockerCount: 1,
|
||||
sampleBlockerIdentifier: "PBS-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not cover a blocker from a stale run the issue no longer owns", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBX");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBX-1", title: "Parent", status: "blocked" });
|
||||
const blockerId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBX-2",
|
||||
title: "Previously running blocker",
|
||||
status: "blocked",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: blockerId, current: false });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 0,
|
||||
attentionBlockerCount: 1,
|
||||
sampleBlockerIdentifier: "PBX-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not treat a scheduled retry as actively covered work", async () => {
|
||||
const { companyId, agentId } = await createCompany("PBY");
|
||||
const parentId = await insertIssue({ companyId, identifier: "PBY-1", title: "Parent", status: "blocked" });
|
||||
const blockerId = await insertIssue({
|
||||
companyId,
|
||||
identifier: "PBY-2",
|
||||
title: "Retrying blocker",
|
||||
status: "blocked",
|
||||
assigneeAgentId: agentId,
|
||||
});
|
||||
await block({ companyId, blockerIssueId: blockerId, blockedIssueId: parentId });
|
||||
await activeRun({ companyId, agentId, issueId: blockerId, status: "scheduled_retry" });
|
||||
|
||||
const parent = (await svc.list(companyId, { status: "blocked" })).find((issue) => issue.id === parentId);
|
||||
|
||||
expect(parent?.blockerAttention).toMatchObject({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
unresolvedBlockerCount: 1,
|
||||
coveredBlockerCount: 0,
|
||||
attentionBlockerCount: 1,
|
||||
sampleBlockerIdentifier: "PBY-2",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -28,6 +28,7 @@ const mockHeartbeatService = vi.hoisted(() => ({
|
|||
|
||||
const mockAgentService = vi.hoisted(() => ({
|
||||
getById: vi.fn(),
|
||||
list: vi.fn(),
|
||||
resolveByReference: vi.fn(),
|
||||
}));
|
||||
|
||||
|
|
@ -61,80 +62,82 @@ const mockIssueThreadInteractionService = vi.hoisted(() => ({
|
|||
expireRequestConfirmationsSupersededByComment: vi.fn(async () => []),
|
||||
expireStaleRequestConfirmationsForIssueDocument: vi.fn(async () => []),
|
||||
}));
|
||||
const mockIssueTreeControlService = vi.hoisted(() => ({
|
||||
getActivePauseHoldGate: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
function registerModuleMocks() {
|
||||
vi.doMock("@paperclipai/shared/telemetry", () => ({
|
||||
trackAgentTaskCompleted: vi.fn(),
|
||||
trackErrorHandlerCrash: vi.fn(),
|
||||
}));
|
||||
vi.mock("@paperclipai/shared/telemetry", () => ({
|
||||
trackAgentTaskCompleted: vi.fn(),
|
||||
trackErrorHandlerCrash: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.doMock("../telemetry.js", () => ({
|
||||
getTelemetryClient: vi.fn(() => ({ track: vi.fn() })),
|
||||
}));
|
||||
vi.mock("../telemetry.js", () => ({
|
||||
getTelemetryClient: vi.fn(() => ({ track: vi.fn() })),
|
||||
}));
|
||||
|
||||
vi.doMock("../services/access.js", () => ({
|
||||
accessService: () => mockAccessService,
|
||||
}));
|
||||
vi.mock("../services/access.js", () => ({
|
||||
accessService: () => mockAccessService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/activity-log.js", () => ({
|
||||
logActivity: mockLogActivity,
|
||||
}));
|
||||
vi.mock("../services/activity-log.js", () => ({
|
||||
logActivity: mockLogActivity,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/agents.js", () => ({
|
||||
agentService: () => mockAgentService,
|
||||
}));
|
||||
vi.mock("../services/agents.js", () => ({
|
||||
agentService: () => mockAgentService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/feedback.js", () => ({
|
||||
feedbackService: () => mockFeedbackService,
|
||||
}));
|
||||
vi.mock("../services/feedback.js", () => ({
|
||||
feedbackService: () => mockFeedbackService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/heartbeat.js", () => ({
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
}));
|
||||
vi.mock("../services/heartbeat.js", () => ({
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/instance-settings.js", () => ({
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
}));
|
||||
vi.mock("../services/instance-settings.js", () => ({
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/issues.js", () => ({
|
||||
issueService: () => mockIssueService,
|
||||
}));
|
||||
vi.mock("../services/issues.js", () => ({
|
||||
issueService: () => mockIssueService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/routines.js", () => ({
|
||||
routineService: () => mockRoutineService,
|
||||
}));
|
||||
vi.mock("../services/routines.js", () => ({
|
||||
routineService: () => mockRoutineService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/index.js", () => ({
|
||||
accessService: () => mockAccessService,
|
||||
agentService: () => mockAgentService,
|
||||
documentService: () => ({}),
|
||||
executionWorkspaceService: () => ({}),
|
||||
feedbackService: () => mockFeedbackService,
|
||||
goalService: () => ({}),
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
issueApprovalService: () => ({}),
|
||||
issueReferenceService: () => ({
|
||||
deleteDocumentSource: async () => undefined,
|
||||
diffIssueReferenceSummary: () => ({
|
||||
addedReferencedIssues: [],
|
||||
removedReferencedIssues: [],
|
||||
currentReferencedIssues: [],
|
||||
}),
|
||||
emptySummary: () => ({ outbound: [], inbound: [] }),
|
||||
listIssueReferenceSummary: async () => ({ outbound: [], inbound: [] }),
|
||||
syncComment: async () => undefined,
|
||||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
vi.mock("../services/index.js", () => ({
|
||||
accessService: () => mockAccessService,
|
||||
agentService: () => mockAgentService,
|
||||
documentService: () => ({}),
|
||||
executionWorkspaceService: () => ({}),
|
||||
feedbackService: () => mockFeedbackService,
|
||||
goalService: () => ({}),
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
issueApprovalService: () => ({}),
|
||||
issueReferenceService: () => ({
|
||||
deleteDocumentSource: async () => undefined,
|
||||
diffIssueReferenceSummary: () => ({
|
||||
addedReferencedIssues: [],
|
||||
removedReferencedIssues: [],
|
||||
currentReferencedIssues: [],
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockIssueThreadInteractionService,
|
||||
logActivity: mockLogActivity,
|
||||
projectService: () => ({}),
|
||||
routineService: () => mockRoutineService,
|
||||
workProductService: () => ({}),
|
||||
}));
|
||||
}
|
||||
emptySummary: () => ({ outbound: [], inbound: [] }),
|
||||
listIssueReferenceSummary: async () => ({ outbound: [], inbound: [] }),
|
||||
syncComment: async () => undefined,
|
||||
syncDocument: async () => undefined,
|
||||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => mockIssueThreadInteractionService,
|
||||
issueTreeControlService: () => mockIssueTreeControlService,
|
||||
logActivity: mockLogActivity,
|
||||
projectService: () => ({}),
|
||||
routineService: () => mockRoutineService,
|
||||
workProductService: () => ({}),
|
||||
}));
|
||||
|
||||
function createApp() {
|
||||
const app = express();
|
||||
|
|
@ -144,8 +147,8 @@ function createApp() {
|
|||
|
||||
async function installActor(app: express.Express, actor?: Record<string, unknown>) {
|
||||
const [{ issueRoutes }, { errorHandler }] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/issues.js")>("../routes/issues.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
import("../routes/issues.js"),
|
||||
import("../middleware/index.js"),
|
||||
]);
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = actor ?? {
|
||||
|
|
@ -173,7 +176,7 @@ async function normalizePolicy(input: {
|
|||
return normalizeIssueExecutionPolicy(input);
|
||||
}
|
||||
|
||||
function makeIssue(status: "todo" | "done" | "blocked") {
|
||||
function makeIssue(status: "todo" | "done" | "blocked" | "cancelled" | "in_progress") {
|
||||
return {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
companyId: "company-1",
|
||||
|
|
@ -186,25 +189,23 @@ function makeIssue(status: "todo" | "done" | "blocked") {
|
|||
};
|
||||
}
|
||||
|
||||
describe("issue comment reopen routes", () => {
|
||||
function agentActor(agentId = "22222222-2222-4222-8222-222222222222") {
|
||||
return {
|
||||
type: "agent",
|
||||
agentId,
|
||||
companyId: "company-1",
|
||||
source: "agent_key",
|
||||
runId: "run-1",
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForWakeup(assertion: () => void) {
|
||||
await vi.waitFor(assertion);
|
||||
}
|
||||
|
||||
describe.sequential("issue comment reopen routes", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("@paperclipai/shared/telemetry");
|
||||
vi.doUnmock("../telemetry.js");
|
||||
vi.doUnmock("../services/access.js");
|
||||
vi.doUnmock("../services/activity-log.js");
|
||||
vi.doUnmock("../services/agents.js");
|
||||
vi.doUnmock("../services/feedback.js");
|
||||
vi.doUnmock("../services/heartbeat.js");
|
||||
vi.doUnmock("../services/index.js");
|
||||
vi.doUnmock("../services/instance-settings.js");
|
||||
vi.doUnmock("../services/issues.js");
|
||||
vi.doUnmock("../services/routines.js");
|
||||
vi.doUnmock("../routes/issues.js");
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
registerModuleMocks();
|
||||
vi.resetAllMocks();
|
||||
vi.clearAllMocks();
|
||||
mockIssueService.getById.mockReset();
|
||||
mockIssueService.assertCheckoutOwner.mockReset();
|
||||
mockIssueService.update.mockReset();
|
||||
|
|
@ -221,6 +222,7 @@ describe("issue comment reopen routes", () => {
|
|||
mockHeartbeatService.getActiveRunForAgent.mockReset();
|
||||
mockHeartbeatService.cancelRun.mockReset();
|
||||
mockAgentService.getById.mockReset();
|
||||
mockAgentService.list.mockReset();
|
||||
mockAgentService.resolveByReference.mockReset();
|
||||
mockLogActivity.mockReset();
|
||||
mockFeedbackService.listIssueVotesForUser.mockReset();
|
||||
|
|
@ -228,6 +230,7 @@ describe("issue comment reopen routes", () => {
|
|||
mockInstanceSettingsService.get.mockReset();
|
||||
mockInstanceSettingsService.listCompanyIds.mockReset();
|
||||
mockRoutineService.syncRunStatusForIssue.mockReset();
|
||||
mockIssueTreeControlService.getActivePauseHoldGate.mockReset();
|
||||
mockTxInsertValues.mockReset();
|
||||
mockTxInsert.mockReset();
|
||||
mockDb.transaction.mockReset();
|
||||
|
|
@ -255,6 +258,7 @@ describe("issue comment reopen routes", () => {
|
|||
});
|
||||
mockInstanceSettingsService.listCompanyIds.mockResolvedValue(["company-1"]);
|
||||
mockRoutineService.syncRunStatusForIssue.mockResolvedValue(undefined);
|
||||
mockIssueTreeControlService.getActivePauseHoldGate.mockResolvedValue(null);
|
||||
mockIssueService.addComment.mockResolvedValue({
|
||||
id: "comment-1",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
|
|
@ -280,12 +284,36 @@ describe("issue comment reopen routes", () => {
|
|||
mockAccessService.canUser.mockResolvedValue(false);
|
||||
mockAccessService.hasPermission.mockResolvedValue(false);
|
||||
mockAgentService.getById.mockResolvedValue(null);
|
||||
mockAgentService.resolveByReference.mockImplementation(async (_companyId: string, reference: string) => ({
|
||||
ambiguous: false,
|
||||
agent: {
|
||||
id: reference,
|
||||
mockAgentService.list.mockResolvedValue([
|
||||
{
|
||||
id: "22222222-2222-4222-8222-222222222222",
|
||||
reportsTo: null,
|
||||
permissions: { canCreateAgents: false },
|
||||
},
|
||||
}));
|
||||
{
|
||||
id: "44444444-4444-4444-8444-444444444444",
|
||||
reportsTo: null,
|
||||
permissions: { canCreateAgents: false },
|
||||
},
|
||||
]);
|
||||
mockAgentService.resolveByReference.mockImplementation(async (_companyId: string, reference: string) => {
|
||||
if (reference === "ambiguous-codex") {
|
||||
return { ambiguous: true, agent: null };
|
||||
}
|
||||
if (reference === "missing-codex") {
|
||||
return { ambiguous: false, agent: null };
|
||||
}
|
||||
if (reference === "codexcoder") {
|
||||
return {
|
||||
ambiguous: false,
|
||||
agent: { id: "33333333-3333-4333-8333-333333333333" },
|
||||
};
|
||||
}
|
||||
return {
|
||||
ambiguous: false,
|
||||
agent: { id: reference },
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
it("treats reopen=true as a no-op when the issue is already open", async () => {
|
||||
|
|
@ -350,10 +378,6 @@ describe("issue comment reopen routes", () => {
|
|||
...makeIssue("todo"),
|
||||
...patch,
|
||||
}));
|
||||
mockAgentService.resolveByReference.mockResolvedValue({
|
||||
ambiguous: false,
|
||||
agent: { id: "33333333-3333-4333-8333-333333333333" },
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
|
|
@ -371,14 +395,10 @@ describe("issue comment reopen routes", () => {
|
|||
|
||||
it("rejects ambiguous assignee shortnames", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("todo"));
|
||||
mockAgentService.resolveByReference.mockResolvedValue({
|
||||
ambiguous: true,
|
||||
agent: null,
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ assigneeAgentId: "codexcoder" });
|
||||
.send({ assigneeAgentId: "ambiguous-codex" });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.error).toContain("ambiguous");
|
||||
|
|
@ -387,14 +407,10 @@ describe("issue comment reopen routes", () => {
|
|||
|
||||
it("rejects missing assignee shortnames", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("todo"));
|
||||
mockAgentService.resolveByReference.mockResolvedValue({
|
||||
ambiguous: false,
|
||||
agent: null,
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ assigneeAgentId: "codexcoder" });
|
||||
.send({ assigneeAgentId: "missing-codex" });
|
||||
|
||||
expect(res.status).toBe(404);
|
||||
expect(res.body.error).toBe("Agent not found");
|
||||
|
|
@ -450,7 +466,7 @@ describe("issue comment reopen routes", () => {
|
|||
"11111111-1111-4111-8111-111111111111",
|
||||
{ status: "todo" },
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
|
|
@ -458,7 +474,38 @@ describe("issue comment reopen routes", () => {
|
|||
reopenedFrom: "done",
|
||||
}),
|
||||
}),
|
||||
));
|
||||
});
|
||||
|
||||
it("does not implicitly reopen closed issues via POST comments for agent-authored comments", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueService.addComment.mockResolvedValue({
|
||||
id: "comment-1",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
companyId: "company-1",
|
||||
body: "hello",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
authorAgentId: "33333333-3333-4333-8333-333333333333",
|
||||
authorUserId: null,
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp(), {
|
||||
type: "agent",
|
||||
agentId: "33333333-3333-4333-8333-333333333333",
|
||||
companyId: "company-1",
|
||||
source: "agent_key",
|
||||
runId: "77777777-7777-4777-8777-777777777777",
|
||||
}))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "hello" });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockIssueService.update).not.toHaveBeenCalledWith(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
{ status: "todo" },
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves assigned blocked issues back to todo via POST comments", async () => {
|
||||
|
|
@ -477,7 +524,7 @@ describe("issue comment reopen routes", () => {
|
|||
"11111111-1111-4111-8111-111111111111",
|
||||
{ status: "todo" },
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
|
|
@ -493,7 +540,7 @@ describe("issue comment reopen routes", () => {
|
|||
reopenedFrom: "blocked",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
|
||||
it("does not move dependency-blocked issues to todo via POST comments", async () => {
|
||||
|
|
@ -513,7 +560,7 @@ describe("issue comment reopen routes", () => {
|
|||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_commented",
|
||||
|
|
@ -527,7 +574,7 @@ describe("issue comment reopen routes", () => {
|
|||
wakeReason: "issue_commented",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
|
||||
it("does not implicitly reopen closed issues via POST comments when no agent is assigned", async () => {
|
||||
|
|
@ -565,7 +612,7 @@ describe("issue comment reopen routes", () => {
|
|||
actorUserId: "local-board",
|
||||
}),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
|
|
@ -575,7 +622,42 @@ describe("issue comment reopen routes", () => {
|
|||
mutation: "comment",
|
||||
}),
|
||||
}),
|
||||
));
|
||||
});
|
||||
|
||||
it("does not implicitly reopen closed issues via the PATCH comment path for agent-authored comments", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueService.addComment.mockResolvedValue({
|
||||
id: "comment-1",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
companyId: "company-1",
|
||||
body: "hello",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
authorAgentId: "33333333-3333-4333-8333-333333333333",
|
||||
authorUserId: null,
|
||||
});
|
||||
mockIssueService.update.mockImplementation(async (_id: string, patch: Record<string, unknown>) => ({
|
||||
...makeIssue("done"),
|
||||
...patch,
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp(), {
|
||||
type: "agent",
|
||||
agentId: "33333333-3333-4333-8333-333333333333",
|
||||
companyId: "company-1",
|
||||
source: "agent_key",
|
||||
runId: "88888888-8888-4888-8888-888888888888",
|
||||
}))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ comment: "hello" });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockIssueService.update).not.toHaveBeenCalledWith(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
expect.objectContaining({ status: "todo" }),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not move dependency-blocked issues to todo via the PATCH comment path", async () => {
|
||||
|
|
@ -609,7 +691,7 @@ describe("issue comment reopen routes", () => {
|
|||
"11111111-1111-4111-8111-111111111111",
|
||||
expect.objectContaining({ status: "todo" }),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_commented",
|
||||
|
|
@ -618,7 +700,7 @@ describe("issue comment reopen routes", () => {
|
|||
mutation: "comment",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
|
||||
it("wakes the assignee when an assigned blocked issue moves back to todo", async () => {
|
||||
|
|
@ -630,6 +712,34 @@ describe("issue comment reopen routes", () => {
|
|||
updatedAt: new Date(),
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ status: "todo" });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_status_changed",
|
||||
payload: expect.objectContaining({
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
mutation: "update",
|
||||
}),
|
||||
}),
|
||||
));
|
||||
});
|
||||
|
||||
it("wakes the assignee when an assigned done issue moves back to todo", async () => {
|
||||
const issue = makeIssue("done");
|
||||
mockIssueService.getById.mockResolvedValue(issue);
|
||||
mockIssueService.update.mockImplementation(async (_id: string, patch: Record<string, unknown>) => ({
|
||||
...issue,
|
||||
...patch,
|
||||
updatedAt: new Date(),
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ status: "todo" });
|
||||
|
|
@ -645,9 +755,166 @@ describe("issue comment reopen routes", () => {
|
|||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
mutation: "update",
|
||||
}),
|
||||
contextSnapshot: expect.objectContaining({
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
source: "issue.status_change",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("explicit same-agent resume works through the PATCH comment path", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueService.update.mockImplementation(async (_id: string, patch: Record<string, unknown>) => ({
|
||||
...makeIssue("done"),
|
||||
...patch,
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.patch("/api/issues/11111111-1111-4111-8111-111111111111")
|
||||
.send({ comment: "please validate the follow-up", resume: true });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockIssueService.update).toHaveBeenCalledWith(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
expect.objectContaining({
|
||||
status: "todo",
|
||||
actorAgentId: "22222222-2222-4222-8222-222222222222",
|
||||
actorUserId: null,
|
||||
}),
|
||||
);
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
action: "issue.comment_added",
|
||||
details: expect.objectContaining({
|
||||
commentId: "comment-1",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
payload: expect.objectContaining({
|
||||
commentId: "comment-1",
|
||||
reopenedFrom: "done",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps generic same-agent comments on closed issues inert", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "follow-up note without intent" });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("explicit same-agent resume comments reopen closed issues and mark the wake payload", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueService.update.mockImplementation(async (_id: string, patch: Record<string, unknown>) => ({
|
||||
...makeIssue("done"),
|
||||
...patch,
|
||||
}));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "please validate the follow-up", resume: true });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockIssueService.update).toHaveBeenCalledWith(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
{ status: "todo" },
|
||||
);
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
action: "issue.comment_added",
|
||||
details: expect.objectContaining({
|
||||
commentId: "comment-1",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "issue_reopened_via_comment",
|
||||
payload: expect.objectContaining({
|
||||
commentId: "comment-1",
|
||||
reopenedFrom: "done",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
contextSnapshot: expect.objectContaining({
|
||||
wakeReason: "issue_reopened_via_comment",
|
||||
resumeIntent: true,
|
||||
followUpRequested: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects explicit agent resume intent from a non-assignee", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor("44444444-4444-4444-8444-444444444444")))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "restart someone else's work", resume: true });
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.error).toBe("Agent cannot request follow-up for another agent's issue");
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockIssueService.addComment).not.toHaveBeenCalled();
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects explicit resume intent under an active pause hold", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("done"));
|
||||
mockIssueTreeControlService.getActivePauseHoldGate.mockResolvedValue({
|
||||
holdId: "hold-1",
|
||||
rootIssueId: "root-1",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
isRoot: false,
|
||||
mode: "pause",
|
||||
reason: "reviewing",
|
||||
releasePolicy: null,
|
||||
});
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "please resume", resume: true });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.error).toBe("Issue follow-up blocked by active subtree pause hold");
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockIssueService.addComment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects explicit resume intent on cancelled issues", async () => {
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue("cancelled"));
|
||||
|
||||
const res = await request(await installActor(createApp(), agentActor()))
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/comments")
|
||||
.send({ body: "please resume", resume: true });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.error).toBe("Cancelled issues must be restored through the dedicated restore flow");
|
||||
expect(mockIssueService.update).not.toHaveBeenCalled();
|
||||
expect(mockIssueService.addComment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("interrupts an active run before a combined comment update", async () => {
|
||||
const issue = {
|
||||
...makeIssue("todo"),
|
||||
|
|
@ -818,7 +1085,7 @@ describe("issue comment reopen routes", () => {
|
|||
instructions: "Please verify the fix against the reproduction steps and note any residual risk.",
|
||||
},
|
||||
});
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"33333333-3333-4333-8333-333333333333",
|
||||
expect.objectContaining({
|
||||
reason: "execution_review_requested",
|
||||
|
|
@ -834,7 +1101,7 @@ describe("issue comment reopen routes", () => {
|
|||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
|
||||
it("wakes the return assignee with execution_changes_requested", async () => {
|
||||
|
|
@ -886,7 +1153,7 @@ describe("issue comment reopen routes", () => {
|
|||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
await waitForWakeup(() => expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith(
|
||||
"22222222-2222-4222-8222-222222222222",
|
||||
expect.objectContaining({
|
||||
reason: "execution_changes_requested",
|
||||
|
|
@ -900,6 +1167,6 @@ describe("issue comment reopen routes", () => {
|
|||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
));
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ describe("issue graph liveness classifier", () => {
|
|||
issueId: blockedId,
|
||||
identifier: "PAP-1703",
|
||||
state: "blocked_by_unassigned_issue",
|
||||
recoveryIssueId: blockerId,
|
||||
recommendedOwnerAgentId: managerId,
|
||||
dependencyPath: [
|
||||
expect.objectContaining({ issueId: blockedId }),
|
||||
|
|
@ -76,6 +77,57 @@ describe("issue graph liveness classifier", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("does not use free-form executive role or name matching for recovery ownership", () => {
|
||||
const rootAgentId = "root-agent";
|
||||
const spoofedExecutiveId = "spoofed-executive";
|
||||
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
issue({
|
||||
assigneeAgentId: null,
|
||||
createdByAgentId: null,
|
||||
}),
|
||||
issue({
|
||||
id: blockerId,
|
||||
identifier: "PAP-1704",
|
||||
title: "Missing unblock work",
|
||||
status: "todo",
|
||||
assigneeAgentId: null,
|
||||
createdByAgentId: null,
|
||||
}),
|
||||
],
|
||||
relations: blocks,
|
||||
agents: [
|
||||
agent({
|
||||
id: spoofedExecutiveId,
|
||||
name: "Chief Executive Recovery",
|
||||
role: "cto",
|
||||
title: "CEO",
|
||||
reportsTo: rootAgentId,
|
||||
}),
|
||||
agent({
|
||||
id: rootAgentId,
|
||||
name: "Root Operator",
|
||||
role: "operator",
|
||||
title: null,
|
||||
reportsTo: null,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(findings).toHaveLength(1);
|
||||
expect(findings[0]?.recommendedOwnerAgentId).toBe(rootAgentId);
|
||||
expect(findings[0]?.recommendedOwnerCandidates[0]).toMatchObject({
|
||||
agentId: rootAgentId,
|
||||
reason: "root_agent",
|
||||
sourceIssueId: blockerId,
|
||||
});
|
||||
expect(findings[0]?.recommendedOwnerCandidateAgentIds).toEqual([
|
||||
rootAgentId,
|
||||
spoofedExecutiveId,
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not flag a live blocked chain with an active assignee and wake path", () => {
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
issues: [
|
||||
|
|
|
|||
|
|
@ -195,6 +195,61 @@ describe("issue tree control routes", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("still marks affected issues cancelled when run interruption fails", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
userId: "user-1",
|
||||
companyIds: ["company-2"],
|
||||
source: "session",
|
||||
isInstanceAdmin: false,
|
||||
});
|
||||
mockTreeControlService.createHold.mockResolvedValue({
|
||||
hold: {
|
||||
id: "33333333-3333-4333-8333-333333333333",
|
||||
mode: "cancel",
|
||||
reason: "cancel subtree",
|
||||
},
|
||||
preview: {
|
||||
mode: "cancel",
|
||||
totals: { affectedIssues: 1 },
|
||||
warnings: [],
|
||||
activeRuns: [
|
||||
{
|
||||
id: "44444444-4444-4444-8444-444444444444",
|
||||
issueId: "11111111-1111-4111-8111-111111111111",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
mockTreeControlService.cancelIssueStatusesForHold.mockResolvedValue({
|
||||
updatedIssueIds: ["11111111-1111-4111-8111-111111111111"],
|
||||
updatedIssues: [],
|
||||
});
|
||||
mockHeartbeatService.cancelRun.mockRejectedValue(new Error("adapter process did not exit"));
|
||||
|
||||
const res = await request(app)
|
||||
.post("/api/issues/11111111-1111-4111-8111-111111111111/tree-holds")
|
||||
.send({ mode: "cancel", reason: "cancel subtree" });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockHeartbeatService.cancelRun).toHaveBeenCalledWith("44444444-4444-4444-8444-444444444444");
|
||||
expect(mockTreeControlService.cancelIssueStatusesForHold).toHaveBeenCalledWith(
|
||||
"company-2",
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"33333333-3333-4333-8333-333333333333",
|
||||
);
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
action: "issue.tree_hold_run_interrupt_failed",
|
||||
entityId: "44444444-4444-4444-8444-444444444444",
|
||||
details: expect.objectContaining({
|
||||
error: "adapter process did not exit",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("restores affected issues and can request explicit wakeups", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@ import { eq, inArray } from "drizzle-orm";
|
|||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueTreeHoldMembers,
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
|
|
@ -38,8 +40,10 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
afterEach(async () => {
|
||||
await db.delete(issueTreeHoldMembers);
|
||||
await db.delete(issueTreeHolds);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issues);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
|
@ -340,6 +344,12 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
const childIssueId = randomUUID();
|
||||
const rootRunId = randomUUID();
|
||||
const childRunId = randomUUID();
|
||||
const forgedRunId = randomUUID();
|
||||
const rootWakeupRequestId = randomUUID();
|
||||
const childWakeupRequestId = randomUUID();
|
||||
const forgedWakeupRequestId = randomUUID();
|
||||
const rootCommentId = randomUUID();
|
||||
const childCommentId = randomUUID();
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
|
|
@ -377,6 +387,63 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
assigneeAgentId: agentId,
|
||||
},
|
||||
]);
|
||||
await db.insert(issueComments).values([
|
||||
{
|
||||
id: rootCommentId,
|
||||
companyId,
|
||||
issueId: rootIssueId,
|
||||
authorUserId: "board-user",
|
||||
body: "Please answer this root issue question.",
|
||||
},
|
||||
{
|
||||
id: childCommentId,
|
||||
companyId,
|
||||
issueId: childIssueId,
|
||||
authorUserId: "board-user",
|
||||
body: "Please answer this child issue question.",
|
||||
},
|
||||
]);
|
||||
await db.insert(agentWakeupRequests).values([
|
||||
{
|
||||
id: rootWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: rootIssueId, commentId: rootCommentId },
|
||||
status: "queued",
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "board-user",
|
||||
runId: rootRunId,
|
||||
},
|
||||
{
|
||||
id: forgedWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "on_demand",
|
||||
triggerDetail: "manual",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: childIssueId, commentId: childCommentId },
|
||||
status: "queued",
|
||||
requestedByActorType: "agent",
|
||||
requestedByActorId: agentId,
|
||||
runId: forgedRunId,
|
||||
},
|
||||
{
|
||||
id: childWakeupRequestId,
|
||||
companyId,
|
||||
agentId,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId: childIssueId, commentId: childCommentId },
|
||||
status: "queued",
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "board-user",
|
||||
runId: childRunId,
|
||||
},
|
||||
]);
|
||||
await db.insert(heartbeatRuns).values([
|
||||
{
|
||||
id: rootRunId,
|
||||
|
|
@ -385,7 +452,29 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
contextSnapshot: { issueId: rootIssueId, wakeReason: "issue_commented", commentId: randomUUID() },
|
||||
wakeupRequestId: rootWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: rootIssueId,
|
||||
wakeReason: "issue_commented",
|
||||
commentId: rootCommentId,
|
||||
wakeCommentId: rootCommentId,
|
||||
source: "issue.comment",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: forgedRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
invocationSource: "on_demand",
|
||||
triggerDetail: "manual",
|
||||
status: "queued",
|
||||
wakeupRequestId: forgedWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: childIssueId,
|
||||
wakeReason: "issue_commented",
|
||||
commentId: childCommentId,
|
||||
wakeCommentId: childCommentId,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: childRunId,
|
||||
|
|
@ -394,7 +483,14 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
contextSnapshot: { issueId: childIssueId, wakeReason: "issue_commented", commentId: randomUUID() },
|
||||
wakeupRequestId: childWakeupRequestId,
|
||||
contextSnapshot: {
|
||||
issueId: childIssueId,
|
||||
wakeReason: "issue_commented",
|
||||
commentId: childCommentId,
|
||||
wakeCommentId: childCommentId,
|
||||
source: "issue.comment",
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
|
|
@ -413,6 +509,13 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
|||
mode: "pause",
|
||||
}),
|
||||
});
|
||||
await expect(issueSvc.checkout(childIssueId, agentId, ["todo"], forgedRunId)).rejects.toMatchObject({
|
||||
status: 409,
|
||||
details: expect.objectContaining({
|
||||
rootIssueId,
|
||||
mode: "pause",
|
||||
}),
|
||||
});
|
||||
|
||||
const checkedOutChild = await issueSvc.checkout(childIssueId, agentId, ["todo"], childRunId);
|
||||
expect(checkedOutChild.status).toBe("in_progress");
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ const mockIssueService = vi.hoisted(() => ({
|
|||
findMentionedProjectIds: vi.fn(),
|
||||
getCommentCursor: vi.fn(),
|
||||
getComment: vi.fn(),
|
||||
listBlockerAttention: vi.fn(),
|
||||
listAttachments: vi.fn(),
|
||||
}));
|
||||
|
||||
|
|
@ -166,6 +167,7 @@ describe("issue goal context routes", () => {
|
|||
latestCommentAt: null,
|
||||
});
|
||||
mockIssueService.getComment.mockResolvedValue(null);
|
||||
mockIssueService.listBlockerAttention.mockResolvedValue(new Map());
|
||||
mockIssueService.listAttachments.mockResolvedValue([]);
|
||||
mockDocumentsService.getIssueDocumentPayload.mockResolvedValue({});
|
||||
mockDocumentsService.getIssueDocumentByKey.mockResolvedValue(null);
|
||||
|
|
|
|||
|
|
@ -1401,6 +1401,49 @@ describeEmbeddedPostgres("issueService blockers and dependency wake readiness",
|
|||
expect(blockedRelations.blockedBy.map((relation) => relation.id)).toEqual([blockerId]);
|
||||
});
|
||||
|
||||
it("adds terminal blockers to immediate blocked-by summaries", async () => {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
const issueA = randomUUID();
|
||||
const issueB = randomUUID();
|
||||
const issueC = randomUUID();
|
||||
const issueD = randomUUID();
|
||||
await db.insert(issues).values([
|
||||
{ id: issueA, companyId, identifier: "PAP-1", title: "Issue A", status: "blocked", priority: "medium" },
|
||||
{ id: issueB, companyId, identifier: "PAP-2", title: "Issue B", status: "blocked", priority: "medium" },
|
||||
{ id: issueC, companyId, identifier: "PAP-3", title: "Issue C", status: "blocked", priority: "medium" },
|
||||
{ id: issueD, companyId, identifier: "PAP-4", title: "Issue D", status: "todo", priority: "high" },
|
||||
]);
|
||||
|
||||
await svc.update(issueC, { blockedByIssueIds: [issueD] });
|
||||
await svc.update(issueB, { blockedByIssueIds: [issueC] });
|
||||
await svc.update(issueA, { blockedByIssueIds: [issueB] });
|
||||
|
||||
const relations = await svc.getRelationSummaries(issueA);
|
||||
|
||||
expect(relations.blockedBy).toHaveLength(1);
|
||||
expect(relations.blockedBy[0]).toMatchObject({
|
||||
id: issueB,
|
||||
identifier: "PAP-2",
|
||||
title: "Issue B",
|
||||
terminalBlockers: [
|
||||
expect.objectContaining({
|
||||
id: issueD,
|
||||
identifier: "PAP-4",
|
||||
title: "Issue D",
|
||||
status: "todo",
|
||||
priority: "high",
|
||||
}),
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects blocking cycles", async () => {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
|
|
|
|||
146
server/src/__tests__/recovery-classifiers.test.ts
Normal file
146
server/src/__tests__/recovery-classifiers.test.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { classifyIssueGraphLiveness as classifyIssueGraphLivenessCompat } from "../services/issue-liveness.ts";
|
||||
import { decideRunLivenessContinuation as decideRunLivenessContinuationCompat } from "../services/run-continuations.ts";
|
||||
import {
|
||||
RECOVERY_KEY_PREFIXES,
|
||||
RECOVERY_ORIGIN_KINDS,
|
||||
RECOVERY_REASON_KINDS,
|
||||
buildIssueGraphLivenessIncidentKey,
|
||||
buildIssueGraphLivenessLeafKey,
|
||||
buildRunLivenessContinuationIdempotencyKey,
|
||||
classifyIssueGraphLiveness,
|
||||
decideRunLivenessContinuation,
|
||||
parseIssueGraphLivenessIncidentKey,
|
||||
} from "../services/recovery/index.ts";
|
||||
|
||||
const companyId = "company-1";
|
||||
const agentId = "agent-1";
|
||||
const managerId = "manager-1";
|
||||
const issueId = "issue-1";
|
||||
const blockerId = "blocker-1";
|
||||
const runId = "run-1";
|
||||
|
||||
describe("recovery classifier boundary", () => {
|
||||
it("keeps issue graph liveness classifier parity with the compatibility export", () => {
|
||||
const input = {
|
||||
issues: [
|
||||
{
|
||||
id: issueId,
|
||||
companyId,
|
||||
identifier: "PAP-2073",
|
||||
title: "Centralize recovery classifiers",
|
||||
status: "blocked",
|
||||
assigneeAgentId: agentId,
|
||||
assigneeUserId: null,
|
||||
createdByAgentId: null,
|
||||
createdByUserId: null,
|
||||
executionState: null,
|
||||
},
|
||||
{
|
||||
id: blockerId,
|
||||
companyId,
|
||||
identifier: "PAP-2074",
|
||||
title: "Move recovery side effects",
|
||||
status: "todo",
|
||||
assigneeAgentId: null,
|
||||
assigneeUserId: null,
|
||||
createdByAgentId: null,
|
||||
createdByUserId: null,
|
||||
executionState: null,
|
||||
},
|
||||
],
|
||||
relations: [{ companyId, blockerIssueId: blockerId, blockedIssueId: issueId }],
|
||||
agents: [
|
||||
{
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Coder",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
reportsTo: managerId,
|
||||
},
|
||||
{
|
||||
id: managerId,
|
||||
companyId,
|
||||
name: "CTO",
|
||||
role: "cto",
|
||||
status: "idle",
|
||||
reportsTo: null,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(classifyIssueGraphLiveness(input)).toEqual(classifyIssueGraphLivenessCompat(input));
|
||||
});
|
||||
|
||||
it("keeps run liveness continuation decision parity with the compatibility export", () => {
|
||||
const input = {
|
||||
run: {
|
||||
id: runId,
|
||||
companyId,
|
||||
agentId,
|
||||
continuationAttempt: 0,
|
||||
} as never,
|
||||
issue: {
|
||||
id: issueId,
|
||||
companyId,
|
||||
identifier: "PAP-2073",
|
||||
title: "Centralize recovery classifiers",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: agentId,
|
||||
executionState: null,
|
||||
projectId: null,
|
||||
} as never,
|
||||
agent: {
|
||||
id: agentId,
|
||||
companyId,
|
||||
status: "idle",
|
||||
} as never,
|
||||
livenessState: "plan_only" as const,
|
||||
livenessReason: "Planned without acting",
|
||||
nextAction: "Take the first concrete action.",
|
||||
budgetBlocked: false,
|
||||
idempotentWakeExists: false,
|
||||
};
|
||||
|
||||
expect(decideRunLivenessContinuation(input)).toEqual(decideRunLivenessContinuationCompat(input));
|
||||
});
|
||||
|
||||
it("keeps recovery origin and idempotency keys stable", () => {
|
||||
expect(RECOVERY_ORIGIN_KINDS).toMatchObject({
|
||||
issueGraphLivenessEscalation: "harness_liveness_escalation",
|
||||
strandedIssueRecovery: "stranded_issue_recovery",
|
||||
staleActiveRunEvaluation: "stale_active_run_evaluation",
|
||||
});
|
||||
expect(RECOVERY_REASON_KINDS.runLivenessContinuation).toBe("run_liveness_continuation");
|
||||
expect(RECOVERY_KEY_PREFIXES.issueGraphLivenessIncident).toBe("harness_liveness");
|
||||
expect(RECOVERY_KEY_PREFIXES.issueGraphLivenessLeaf).toBe("harness_liveness_leaf");
|
||||
|
||||
const incidentKey = buildIssueGraphLivenessIncidentKey({
|
||||
companyId,
|
||||
issueId,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
blockerIssueId: blockerId,
|
||||
});
|
||||
expect(incidentKey).toBe(
|
||||
"harness_liveness:company-1:issue-1:blocked_by_unassigned_issue:blocker-1",
|
||||
);
|
||||
expect(parseIssueGraphLivenessIncidentKey(incidentKey)).toEqual({
|
||||
companyId,
|
||||
issueId,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
leafIssueId: blockerId,
|
||||
});
|
||||
expect(buildIssueGraphLivenessLeafKey({
|
||||
companyId,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
leafIssueId: blockerId,
|
||||
})).toBe("harness_liveness_leaf:company-1:blocked_by_unassigned_issue:blocker-1");
|
||||
expect(buildRunLivenessContinuationIdempotencyKey({
|
||||
issueId,
|
||||
sourceRunId: runId,
|
||||
livenessState: "plan_only",
|
||||
nextAttempt: 1,
|
||||
})).toBe("run_liveness_continuation:issue-1:run-1:plan_only:1");
|
||||
});
|
||||
});
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { REDACTED_EVENT_VALUE, redactEventPayload, sanitizeRecord } from "../redaction.js";
|
||||
import { REDACTED_EVENT_VALUE, redactEventPayload, redactSensitiveText, sanitizeRecord } from "../redaction.js";
|
||||
|
||||
describe("redaction", () => {
|
||||
it("redacts sensitive keys and nested secret values", () => {
|
||||
|
|
@ -63,4 +63,25 @@ describe("redaction", () => {
|
|||
safe: "value",
|
||||
});
|
||||
});
|
||||
|
||||
it("redacts common secret shapes from unstructured text", () => {
|
||||
const jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
|
||||
const githubToken = "ghp_1234567890abcdefghijklmnopqrstuvwxyz";
|
||||
const input = [
|
||||
"Authorization: Bearer live-bearer-token-value",
|
||||
`payload {"apiKey":"json-secret-value"}`,
|
||||
`escaped {\\"apiKey\\":\\"escaped-json-secret\\"}`,
|
||||
`GITHUB_TOKEN=${githubToken}`,
|
||||
`session=${jwt}`,
|
||||
].join("\n");
|
||||
|
||||
const result = redactSensitiveText(input);
|
||||
|
||||
expect(result).toContain(REDACTED_EVENT_VALUE);
|
||||
expect(result).not.toContain("live-bearer-token-value");
|
||||
expect(result).not.toContain("json-secret-value");
|
||||
expect(result).not.toContain("escaped-json-secret");
|
||||
expect(result).not.toContain(githubToken);
|
||||
expect(result).not.toContain(jwt);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ describe("run liveness classifier", () => {
|
|||
});
|
||||
|
||||
expect(classification.livenessState).toBe("plan_only");
|
||||
expect(classification.actionability).toBe("runnable");
|
||||
expect(classification.nextAction).toContain("inspect the repo");
|
||||
});
|
||||
|
||||
|
|
@ -34,6 +35,7 @@ describe("run liveness classifier", () => {
|
|||
const classification = classifyRunLiveness(baseInput);
|
||||
|
||||
expect(classification.livenessState).toBe("empty_response");
|
||||
expect(classification.actionability).toBe("unknown");
|
||||
});
|
||||
|
||||
it("treats issue comments, documents, products, and actions as progress", () => {
|
||||
|
|
@ -128,5 +130,81 @@ describe("run liveness classifier", () => {
|
|||
});
|
||||
|
||||
expect(classification.livenessState).toBe("blocked");
|
||||
expect(classification.actionability).toBe("blocked_external");
|
||||
});
|
||||
|
||||
it("treats PAP-2000-style validation output as runnable follow-up, not an external blocker", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
resultJson: {
|
||||
summary: "PAP-1949 remains blocked until PAP-2000 is resolved.",
|
||||
},
|
||||
issueCommentBodies: [
|
||||
[
|
||||
"Validation is ready for the next pass.",
|
||||
"",
|
||||
"- Blocked chain context: PAP-1949 -> PAP-1999 -> PAP-2000",
|
||||
"- Next action: run npm test and report the row counts.",
|
||||
].join("\n"),
|
||||
],
|
||||
});
|
||||
|
||||
expect(classification.livenessState).toBe("plan_only");
|
||||
expect(classification.actionability).toBe("runnable");
|
||||
expect(classification.nextAction).toBe("run npm test and report the row counts.");
|
||||
});
|
||||
|
||||
it("prefers durable comments over raw transcript next-action noise", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
issueCommentBodies: ["Next action: run pnpm test -- --runInBand."],
|
||||
stdoutExcerpt: [
|
||||
"tool_call: write",
|
||||
"command: rm -rf production-data",
|
||||
"Next action: deploy to production",
|
||||
].join("\n"),
|
||||
});
|
||||
|
||||
expect(classification.actionability).toBe("runnable");
|
||||
expect(classification.nextAction).toBe("run pnpm test -- --runInBand.");
|
||||
});
|
||||
|
||||
it("keeps approval requests out of automatic continuation", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
resultJson: {
|
||||
summary: "Next action: wait for board approval before continuing.",
|
||||
},
|
||||
});
|
||||
|
||||
expect(classification.livenessState).toBe("blocked");
|
||||
expect(classification.actionability).toBe("approval_required");
|
||||
expect(classification.nextAction).toBe("wait for board approval before continuing.");
|
||||
});
|
||||
|
||||
it("routes production-sensitive next actions to manager review", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
resultJson: {
|
||||
summary: "Next action: deploy to production and verify live traffic.",
|
||||
},
|
||||
});
|
||||
|
||||
expect(classification.livenessState).toBe("needs_followup");
|
||||
expect(classification.actionability).toBe("manager_review");
|
||||
expect(classification.nextAction).toBe("deploy to production and verify live traffic.");
|
||||
});
|
||||
|
||||
it("marks unclear useful output as unknown actionability", () => {
|
||||
const classification = classifyRunLiveness({
|
||||
...baseInput,
|
||||
resultJson: {
|
||||
summary: "Observed mixed output and left notes for a later pass.",
|
||||
},
|
||||
});
|
||||
|
||||
expect(classification.livenessState).toBe("needs_followup");
|
||||
expect(classification.actionability).toBe("unknown");
|
||||
expect(classification.nextAction).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -698,6 +698,12 @@ export async function startServer(): Promise<StartedServer> {
|
|||
logger.warn({ ...reconciled }, "startup issue-graph liveness reconciliation created escalations");
|
||||
}
|
||||
})
|
||||
.then(async () => {
|
||||
const scanned = await heartbeat.scanSilentActiveRuns();
|
||||
if (scanned.created > 0 || scanned.escalated > 0) {
|
||||
logger.warn({ ...scanned }, "startup active-run output watchdog created review work");
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "startup heartbeat recovery failed");
|
||||
});
|
||||
|
|
@ -750,6 +756,12 @@ export async function startServer(): Promise<StartedServer> {
|
|||
logger.warn({ ...reconciled }, "periodic issue-graph liveness reconciliation created escalations");
|
||||
}
|
||||
})
|
||||
.then(async () => {
|
||||
const scanned = await heartbeat.scanSilentActiveRuns();
|
||||
if (scanned.created > 0 || scanned.escalated > 0) {
|
||||
logger.warn({ ...scanned }, "periodic active-run output watchdog created review work");
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "periodic heartbeat recovery failed");
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,16 @@
|
|||
const SECRET_PAYLOAD_KEY_RE =
|
||||
/(api[-_]?key|access[-_]?token|auth(?:_?token)?|authorization|bearer|secret|passwd|password|credential|jwt|private[-_]?key|cookie|connectionstring)/i;
|
||||
const JWT_VALUE_RE = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)?$/;
|
||||
const JWT_TEXT_RE = /\b[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}(?:\.[A-Za-z0-9_-]{8,})?\b/g;
|
||||
const OPENAI_KEY_TEXT_RE = /\bsk-[A-Za-z0-9_-]{12,}\b/g;
|
||||
const GITHUB_TOKEN_TEXT_RE = /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/g;
|
||||
const AUTHORIZATION_BEARER_TEXT_RE = /(\bAuthorization\s*:\s*Bearer\s+)[^\s"'`]+/gi;
|
||||
const ENV_SECRET_ASSIGNMENT_TEXT_RE =
|
||||
/(\b[A-Za-z0-9_]*(?:TOKEN|KEY|SECRET|PASSWORD|PASSWD|AUTHORIZATION|JWT)[A-Za-z0-9_]*\s*=\s*)[^\s"'`]+/gi;
|
||||
const JSON_SECRET_FIELD_TEXT_RE =
|
||||
/((?:"|')?(?:api[-_]?key|access[-_]?token|auth(?:_?token)?|authorization|bearer|secret|passwd|password|credential|jwt|private[-_]?key|cookie|connectionstring)(?:"|')?\s*:\s*(?:"|'))[^"'`\r\n]+((?:"|'))/gi;
|
||||
const ESCAPED_JSON_SECRET_FIELD_TEXT_RE =
|
||||
/((?:\\")?(?:api[-_]?key|access[-_]?token|auth(?:_?token)?|authorization|bearer|secret|passwd|password|credential|jwt|private[-_]?key|cookie|connectionstring)(?:\\")?\s*:\s*(?:\\"))[^\\\r\n]+((?:\\"))/gi;
|
||||
export const REDACTED_EVENT_VALUE = "***REDACTED***";
|
||||
|
||||
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
||||
|
|
@ -57,3 +67,14 @@ export function redactEventPayload(payload: Record<string, unknown> | null): Rec
|
|||
if (!isPlainObject(payload)) return payload;
|
||||
return sanitizeRecord(payload);
|
||||
}
|
||||
|
||||
export function redactSensitiveText(input: string): string {
|
||||
return input
|
||||
.replace(AUTHORIZATION_BEARER_TEXT_RE, `$1${REDACTED_EVENT_VALUE}`)
|
||||
.replace(JSON_SECRET_FIELD_TEXT_RE, `$1${REDACTED_EVENT_VALUE}$2`)
|
||||
.replace(ESCAPED_JSON_SECRET_FIELD_TEXT_RE, `$1${REDACTED_EVENT_VALUE}$2`)
|
||||
.replace(ENV_SECRET_ASSIGNMENT_TEXT_RE, `$1${REDACTED_EVENT_VALUE}`)
|
||||
.replace(OPENAI_KEY_TEXT_RE, REDACTED_EVENT_VALUE)
|
||||
.replace(GITHUB_TOKEN_TEXT_RE, REDACTED_EVENT_VALUE)
|
||||
.replace(JWT_TEXT_RE, REDACTED_EVENT_VALUE);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2270,11 +2270,14 @@ export function setInviteResolutionNetworkForTest(
|
|||
: defaultInviteResolutionNetwork;
|
||||
}
|
||||
|
||||
async function lookupInviteResolutionHostname(hostname: string) {
|
||||
async function lookupInviteResolutionHostname(
|
||||
hostname: string,
|
||||
network: InviteResolutionNetwork = inviteResolutionNetwork
|
||||
) {
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
try {
|
||||
return await Promise.race([
|
||||
inviteResolutionNetwork.lookup(hostname),
|
||||
network.lookup(hostname),
|
||||
new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(
|
||||
() =>
|
||||
|
|
@ -2296,7 +2299,8 @@ async function lookupInviteResolutionHostname(hostname: string) {
|
|||
}
|
||||
|
||||
async function resolveInviteResolutionTarget(
|
||||
url: URL
|
||||
url: URL,
|
||||
network: InviteResolutionNetwork = inviteResolutionNetwork
|
||||
): Promise<ResolvedInviteResolutionTarget> {
|
||||
const hostname = hostnameForResolution(url);
|
||||
if (parseIpv4Address(hostname)) {
|
||||
|
|
@ -2328,7 +2332,7 @@ async function resolveInviteResolutionTarget(
|
|||
tlsServername: undefined,
|
||||
};
|
||||
}
|
||||
const results = await lookupInviteResolutionHostname(hostname);
|
||||
const results = await lookupInviteResolutionHostname(hostname, network);
|
||||
if (results.length === 0) {
|
||||
throw badRequest("url hostname did not resolve to any addresses");
|
||||
}
|
||||
|
|
@ -2354,11 +2358,12 @@ async function resolveInviteResolutionTarget(
|
|||
|
||||
async function probeInviteResolutionTarget(
|
||||
target: ResolvedInviteResolutionTarget,
|
||||
timeoutMs: number
|
||||
timeoutMs: number,
|
||||
network: InviteResolutionNetwork = inviteResolutionNetwork
|
||||
): Promise<InviteResolutionProbe> {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const response = await inviteResolutionNetwork.requestHead(target, timeoutMs);
|
||||
const response = await network.requestHead(target, timeoutMs);
|
||||
const durationMs = Date.now() - startedAt;
|
||||
if (
|
||||
response.httpStatus !== null &&
|
||||
|
|
@ -2421,12 +2426,16 @@ export function accessRoutes(
|
|||
deploymentExposure: DeploymentExposure;
|
||||
bindHost: string;
|
||||
allowedHostnames: string[];
|
||||
inviteResolutionNetwork?: Partial<InviteResolutionNetwork>;
|
||||
}
|
||||
) {
|
||||
const router = Router();
|
||||
const access = accessService(db);
|
||||
const boardAuth = boardAuthService(db);
|
||||
const agents = agentService(db);
|
||||
const routeInviteResolutionNetwork = opts.inviteResolutionNetwork
|
||||
? { ...defaultInviteResolutionNetwork, ...opts.inviteResolutionNetwork }
|
||||
: inviteResolutionNetwork;
|
||||
|
||||
async function assertInstanceAdmin(req: Request) {
|
||||
if (req.actor.type !== "board") throw unauthorized();
|
||||
|
|
@ -3175,8 +3184,8 @@ export function accessRoutes(
|
|||
const timeoutMs = Number.isFinite(parsedTimeoutMs)
|
||||
? Math.max(1000, Math.min(15000, Math.floor(parsedTimeoutMs)))
|
||||
: 5000;
|
||||
const resolvedTarget = await resolveInviteResolutionTarget(target);
|
||||
const probe = await probeInviteResolutionTarget(resolvedTarget, timeoutMs);
|
||||
const resolvedTarget = await resolveInviteResolutionTarget(target, routeInviteResolutionNetwork);
|
||||
const probe = await probeInviteResolutionTarget(resolvedTarget, timeoutMs, routeInviteResolutionNetwork);
|
||||
res.json({
|
||||
inviteId: invite.id,
|
||||
testResolutionPath: `/api/invites/${token}/test-resolution`,
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ import {
|
|||
} from "../services/default-agent-instructions.js";
|
||||
import { getTelemetryClient } from "../telemetry.js";
|
||||
import { assertEnvironmentSelectionForCompany } from "./environment-selection.js";
|
||||
import { recoveryService } from "../services/recovery/service.js";
|
||||
|
||||
const RUN_LOG_DEFAULT_LIMIT_BYTES = 256_000;
|
||||
const RUN_LOG_MAX_LIMIT_BYTES = 1024 * 1024;
|
||||
|
|
@ -91,6 +92,12 @@ function readRunLogLimitBytes(value: unknown) {
|
|||
return Math.max(1, Math.min(RUN_LOG_MAX_LIMIT_BYTES, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
function readLiveRunsQueryInt(value: unknown, max: number, fallback = 0) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed)) return fallback;
|
||||
return Math.max(0, Math.min(max, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
export function agentRoutes(
|
||||
db: Db,
|
||||
options: { pluginWorkerManager?: PluginWorkerManager } = {},
|
||||
|
|
@ -142,6 +149,7 @@ export function agentRoutes(
|
|||
const heartbeat = heartbeatService(db, {
|
||||
pluginWorkerManager: options.pluginWorkerManager,
|
||||
});
|
||||
const recovery = recoveryService(db, { enqueueWakeup: heartbeat.wakeup });
|
||||
const issueApprovalsSvc = issueApprovalService(db);
|
||||
const secretsSvc = secretService(db);
|
||||
const instructions = agentInstructionsService();
|
||||
|
|
@ -2532,11 +2540,12 @@ export function agentRoutes(
|
|||
const companyId = req.params.companyId as string;
|
||||
assertCompanyAccess(req, companyId);
|
||||
|
||||
const minCountParam = req.query.minCount as string | undefined;
|
||||
const minCount = minCountParam ? Math.max(0, Math.min(20, parseInt(minCountParam, 10) || 0)) : 0;
|
||||
const minCount = readLiveRunsQueryInt(req.query.minCount, 50);
|
||||
const limit = readLiveRunsQueryInt(req.query.limit, 50);
|
||||
|
||||
const columns = {
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
status: heartbeatRuns.status,
|
||||
invocationSource: heartbeatRuns.invocationSource,
|
||||
triggerDetail: heartbeatRuns.triggerDetail,
|
||||
|
|
@ -2546,15 +2555,21 @@ export function agentRoutes(
|
|||
agentId: heartbeatRuns.agentId,
|
||||
agentName: agentsTable.name,
|
||||
adapterType: agentsTable.adapterType,
|
||||
logBytes: heartbeatRuns.logBytes,
|
||||
livenessState: heartbeatRuns.livenessState,
|
||||
livenessReason: heartbeatRuns.livenessReason,
|
||||
continuationAttempt: heartbeatRuns.continuationAttempt,
|
||||
lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt,
|
||||
nextAction: heartbeatRuns.nextAction,
|
||||
lastOutputAt: heartbeatRuns.lastOutputAt,
|
||||
lastOutputSeq: heartbeatRuns.lastOutputSeq,
|
||||
lastOutputStream: heartbeatRuns.lastOutputStream,
|
||||
lastOutputBytes: heartbeatRuns.lastOutputBytes,
|
||||
processStartedAt: heartbeatRuns.processStartedAt,
|
||||
issueId: sql<string | null>`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("issueId"),
|
||||
};
|
||||
|
||||
const liveRuns = await db
|
||||
const liveRunsQuery = db
|
||||
.select(columns)
|
||||
.from(heartbeatRuns)
|
||||
.innerJoin(agentsTable, eq(heartbeatRuns.agentId, agentsTable.id))
|
||||
|
|
@ -2566,7 +2581,10 @@ export function agentRoutes(
|
|||
)
|
||||
.orderBy(desc(heartbeatRuns.createdAt));
|
||||
|
||||
if (minCount > 0 && liveRuns.length < minCount) {
|
||||
const liveRuns = limit > 0 ? await liveRunsQuery.limit(limit) : await liveRunsQuery;
|
||||
const targetRunCount = limit > 0 ? Math.min(minCount, limit) : minCount;
|
||||
|
||||
if (targetRunCount > 0 && liveRuns.length < targetRunCount) {
|
||||
const activeIds = liveRuns.map((r) => r.id);
|
||||
const recentRuns = await db
|
||||
.select(columns)
|
||||
|
|
@ -2580,13 +2598,20 @@ export function agentRoutes(
|
|||
),
|
||||
)
|
||||
.orderBy(desc(heartbeatRuns.createdAt))
|
||||
.limit(minCount - liveRuns.length);
|
||||
.limit(targetRunCount - liveRuns.length);
|
||||
|
||||
res.json([...liveRuns, ...recentRuns]);
|
||||
const rows = [...liveRuns, ...recentRuns];
|
||||
res.json(await Promise.all(rows.map(async (run) => ({
|
||||
...run,
|
||||
outputSilence: await heartbeat.buildRunOutputSilence(run),
|
||||
}))));
|
||||
return;
|
||||
}
|
||||
|
||||
res.json(liveRuns);
|
||||
res.json(await Promise.all(liveRuns.map(async (run) => ({
|
||||
...run,
|
||||
outputSilence: await heartbeat.buildRunOutputSilence(run),
|
||||
}))));
|
||||
});
|
||||
|
||||
router.get("/heartbeat-runs/:runId", async (req, res) => {
|
||||
|
|
@ -2600,7 +2625,7 @@ export function agentRoutes(
|
|||
const retryExhaustedReason = await heartbeat.getRetryExhaustedReason(runId);
|
||||
res.json(
|
||||
redactCurrentUserValue(
|
||||
{ ...run, retryExhaustedReason },
|
||||
{ ...run, retryExhaustedReason, outputSilence: await heartbeat.buildRunOutputSilence(run) },
|
||||
await getCurrentUserRedactionOptions(),
|
||||
),
|
||||
);
|
||||
|
|
@ -2630,6 +2655,42 @@ export function agentRoutes(
|
|||
res.json(run);
|
||||
});
|
||||
|
||||
router.post("/heartbeat-runs/:runId/watchdog-decisions", async (req, res) => {
|
||||
const runId = req.params.runId as string;
|
||||
const existing = await heartbeat.getRun(runId);
|
||||
if (!existing) {
|
||||
res.status(404).json({ error: "Heartbeat run not found" });
|
||||
return;
|
||||
}
|
||||
assertCompanyAccess(req, existing.companyId);
|
||||
const decision = typeof req.body?.decision === "string" ? req.body.decision : "";
|
||||
if (!["snooze", "continue", "dismissed_false_positive"].includes(decision)) {
|
||||
res.status(400).json({ error: "Unsupported watchdog decision" });
|
||||
return;
|
||||
}
|
||||
const evaluationIssueId = typeof req.body?.evaluationIssueId === "string" ? req.body.evaluationIssueId : null;
|
||||
const reason = typeof req.body?.reason === "string" ? req.body.reason.slice(0, 4000) : null;
|
||||
const snoozedUntil = decision === "snooze"
|
||||
? new Date(String(req.body?.snoozedUntil ?? ""))
|
||||
: null;
|
||||
if (decision === "snooze" && (!snoozedUntil || Number.isNaN(snoozedUntil.getTime()) || snoozedUntil <= new Date())) {
|
||||
res.status(400).json({ error: "snoozedUntil must be a future ISO datetime" });
|
||||
return;
|
||||
}
|
||||
|
||||
const row = await recovery.recordWatchdogDecision({
|
||||
runId: existing.id,
|
||||
actor: req.actor,
|
||||
decision: decision as "snooze" | "continue" | "dismissed_false_positive",
|
||||
evaluationIssueId,
|
||||
reason,
|
||||
snoozedUntil,
|
||||
createdByRunId: req.actor.runId ?? null,
|
||||
});
|
||||
|
||||
res.json(row);
|
||||
});
|
||||
|
||||
router.get("/heartbeat-runs/:runId/events", async (req, res) => {
|
||||
const runId = req.params.runId as string;
|
||||
const run = await heartbeat.getRun(runId);
|
||||
|
|
@ -2730,11 +2791,17 @@ export function agentRoutes(
|
|||
agentId: heartbeatRuns.agentId,
|
||||
agentName: agentsTable.name,
|
||||
adapterType: agentsTable.adapterType,
|
||||
logBytes: heartbeatRuns.logBytes,
|
||||
livenessState: heartbeatRuns.livenessState,
|
||||
livenessReason: heartbeatRuns.livenessReason,
|
||||
continuationAttempt: heartbeatRuns.continuationAttempt,
|
||||
lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt,
|
||||
nextAction: heartbeatRuns.nextAction,
|
||||
lastOutputAt: heartbeatRuns.lastOutputAt,
|
||||
lastOutputSeq: heartbeatRuns.lastOutputSeq,
|
||||
lastOutputStream: heartbeatRuns.lastOutputStream,
|
||||
lastOutputBytes: heartbeatRuns.lastOutputBytes,
|
||||
processStartedAt: heartbeatRuns.processStartedAt,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.innerJoin(agentsTable, eq(heartbeatRuns.agentId, agentsTable.id))
|
||||
|
|
@ -2747,7 +2814,10 @@ export function agentRoutes(
|
|||
)
|
||||
.orderBy(desc(heartbeatRuns.createdAt));
|
||||
|
||||
res.json(liveRuns);
|
||||
res.json(await Promise.all(liveRuns.map(async (run) => ({
|
||||
...run,
|
||||
outputSilence: await heartbeat.buildRunOutputSilence({ ...run, companyId: issue.companyId }),
|
||||
}))));
|
||||
});
|
||||
|
||||
router.get("/issues/:issueId/active-run", async (req, res) => {
|
||||
|
|
@ -2795,6 +2865,7 @@ export function agentRoutes(
|
|||
agentId: agent.id,
|
||||
agentName: agent.name,
|
||||
adapterType: agent.adapterType,
|
||||
outputSilence: await heartbeat.buildRunOutputSilence({ ...run, companyId: issue.companyId }),
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,26 @@ import { validate } from "../middleware/validate.js";
|
|||
import { heartbeatService, issueService, issueTreeControlService, logActivity } from "../services/index.js";
|
||||
import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js";
|
||||
|
||||
const TREE_RUN_CANCELLATION_RESPONSE_WAIT_MS = 1_000;
|
||||
|
||||
function errorToMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
async function waitForRunCancellationTasks(tasks: Promise<void>[]) {
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
try {
|
||||
await Promise.race([
|
||||
Promise.all(tasks),
|
||||
new Promise((resolve) => {
|
||||
timeout = setTimeout(resolve, TREE_RUN_CANCELLATION_RESPONSE_WAIT_MS);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
export function issueTreeControlRoutes(db: Db) {
|
||||
const router = Router();
|
||||
const issuesSvc = issueService(db);
|
||||
|
|
@ -91,25 +111,48 @@ export function issueTreeControlRoutes(db: Db) {
|
|||
},
|
||||
});
|
||||
|
||||
const runCancellationTasks: Promise<void>[] = [];
|
||||
if (result.hold.mode === "pause" || result.hold.mode === "cancel") {
|
||||
const interruptedRunIds = [...new Set(result.preview.activeRuns.map((run) => run.id))];
|
||||
for (const runId of interruptedRunIds) {
|
||||
await heartbeat.cancelRun(runId);
|
||||
await logActivity(db, {
|
||||
companyId: root.companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
action: "issue.tree_hold_run_interrupted",
|
||||
entityType: "heartbeat_run",
|
||||
entityId: runId,
|
||||
details: {
|
||||
holdId: result.hold.id,
|
||||
rootIssueId: root.id,
|
||||
reason: result.hold.mode === "pause" ? "active_subtree_pause_hold" : "subtree_cancel_operation",
|
||||
},
|
||||
});
|
||||
for (const heartbeatRunId of interruptedRunIds) {
|
||||
const cancellationTask = (async () => {
|
||||
try {
|
||||
await heartbeat.cancelRun(heartbeatRunId);
|
||||
await logActivity(db, {
|
||||
companyId: root.companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
action: "issue.tree_hold_run_interrupted",
|
||||
entityType: "heartbeat_run",
|
||||
entityId: heartbeatRunId,
|
||||
details: {
|
||||
holdId: result.hold.id,
|
||||
rootIssueId: root.id,
|
||||
reason: result.hold.mode === "pause" ? "active_subtree_pause_hold" : "subtree_cancel_operation",
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
await Promise.resolve(logActivity(db, {
|
||||
companyId: root.companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
action: "issue.tree_hold_run_interrupt_failed",
|
||||
entityType: "heartbeat_run",
|
||||
entityId: heartbeatRunId,
|
||||
details: {
|
||||
holdId: result.hold.id,
|
||||
rootIssueId: root.id,
|
||||
reason: result.hold.mode === "pause" ? "active_subtree_pause_hold" : "subtree_cancel_operation",
|
||||
error: errorToMessage(error),
|
||||
},
|
||||
})).catch(() => null);
|
||||
}
|
||||
})();
|
||||
runCancellationTasks.push(cancellationTask);
|
||||
}
|
||||
|
||||
const cancelledWakeups = await treeControlSvc.cancelUnclaimedWakeupsForTree(
|
||||
|
|
@ -158,6 +201,10 @@ export function issueTreeControlRoutes(db: Db) {
|
|||
});
|
||||
}
|
||||
|
||||
if (runCancellationTasks.length > 0) {
|
||||
await waitForRunCancellationTasks(runCancellationTasks);
|
||||
}
|
||||
|
||||
if (result.hold.mode === "restore") {
|
||||
let statusUpdate;
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import { trackAgentTaskCompleted } from "@paperclipai/shared/telemetry";
|
|||
import { getTelemetryClient } from "../telemetry.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { validate } from "../middleware/validate.js";
|
||||
import * as serviceIndex from "../services/index.js";
|
||||
import {
|
||||
accessService,
|
||||
agentService,
|
||||
|
|
@ -184,18 +185,24 @@ function isClosedIssueStatus(status: string | null | undefined): status is "done
|
|||
return status === "done" || status === "cancelled";
|
||||
}
|
||||
|
||||
function shouldImplicitlyMoveCommentedIssueToTodoForAgent(input: {
|
||||
function shouldImplicitlyMoveCommentedIssueToTodo(input: {
|
||||
issueStatus: string | null | undefined;
|
||||
assigneeAgentId: string | null | undefined;
|
||||
actorType: "agent" | "user";
|
||||
actorId: string;
|
||||
}) {
|
||||
// Only human comments should implicitly reopen finished work.
|
||||
// Agent-authored comments remain communicative unless reopen was explicit.
|
||||
if (input.actorType !== "user") return false;
|
||||
if (!isClosedIssueStatus(input.issueStatus) && input.issueStatus !== "blocked") return false;
|
||||
if (typeof input.assigneeAgentId !== "string" || input.assigneeAgentId.length === 0) return false;
|
||||
if (input.actorType === "agent" && input.actorId === input.assigneeAgentId) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function isExplicitResumeCapableStatus(status: string | null | undefined) {
|
||||
return status === "done" || status === "blocked" || status === "todo" || status === "in_progress";
|
||||
}
|
||||
|
||||
function queueResolvedInteractionContinuationWakeup(input: {
|
||||
heartbeat: ReturnType<typeof heartbeatService>;
|
||||
issue: { id: string; assigneeAgentId: string | null; status: string };
|
||||
|
|
@ -409,6 +416,15 @@ export function issueRoutes(
|
|||
const routinesSvc = routineService(db, {
|
||||
pluginWorkerManager: opts.pluginWorkerManager,
|
||||
});
|
||||
const issueTreeControlFactory = Object.prototype.hasOwnProperty.call(
|
||||
serviceIndex,
|
||||
"issueTreeControlService",
|
||||
)
|
||||
? serviceIndex.issueTreeControlService
|
||||
: undefined;
|
||||
const treeControlSvc = issueTreeControlFactory?.(db) ?? {
|
||||
getActivePauseHoldGate: async () => null,
|
||||
};
|
||||
const feedbackExportService = opts?.feedbackExportService;
|
||||
const environmentsSvc = environmentService(db);
|
||||
const upload = multer({
|
||||
|
|
@ -627,6 +643,90 @@ export function issueRoutes(
|
|||
return true;
|
||||
}
|
||||
|
||||
async function assertExplicitResumeIntentAllowed(
|
||||
req: Request,
|
||||
res: Response,
|
||||
issue: { id: string; companyId: string; status: string; assigneeAgentId: string | null },
|
||||
) {
|
||||
if (issue.status === "cancelled") {
|
||||
res.status(409).json({
|
||||
error: "Cancelled issues must be restored through the dedicated restore flow",
|
||||
details: {
|
||||
issueId: issue.id,
|
||||
status: issue.status,
|
||||
securityPrinciples: ["Complete Mediation", "Fail Securely"],
|
||||
},
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!isExplicitResumeCapableStatus(issue.status)) {
|
||||
res.status(409).json({
|
||||
error: "Issue is not resumable through comment follow-up intent",
|
||||
details: { issueId: issue.id, status: issue.status },
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
const activePauseHold = await treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id);
|
||||
if (activePauseHold) {
|
||||
res.status(409).json({
|
||||
error: "Issue follow-up blocked by active subtree pause hold",
|
||||
details: {
|
||||
issueId: issue.id,
|
||||
holdId: activePauseHold.holdId,
|
||||
rootIssueId: activePauseHold.rootIssueId,
|
||||
mode: activePauseHold.mode,
|
||||
securityPrinciples: ["Complete Mediation", "Fail Securely", "Secure Defaults"],
|
||||
},
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
if (issue.status === "blocked") {
|
||||
const readiness = await svc.getDependencyReadiness(issue.id);
|
||||
if (readiness.unresolvedBlockerCount > 0) {
|
||||
res.status(409).json({
|
||||
error: "Issue follow-up blocked by unresolved blockers",
|
||||
details: {
|
||||
issueId: issue.id,
|
||||
unresolvedBlockerIssueIds: readiness.unresolvedBlockerIssueIds,
|
||||
},
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (req.actor.type !== "agent") return true;
|
||||
|
||||
const actorAgentId = req.actor.agentId;
|
||||
if (!actorAgentId) {
|
||||
res.status(403).json({ error: "Agent authentication required" });
|
||||
return false;
|
||||
}
|
||||
if (!issue.assigneeAgentId) {
|
||||
res.status(409).json({
|
||||
error: "Issue follow-up requires an assigned agent",
|
||||
details: { issueId: issue.id, actorAgentId },
|
||||
});
|
||||
return false;
|
||||
}
|
||||
if (issue.assigneeAgentId === actorAgentId) return true;
|
||||
if (await hasActiveCheckoutManagementOverride(actorAgentId, issue.companyId, issue.assigneeAgentId)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
res.status(403).json({
|
||||
error: "Agent cannot request follow-up for another agent's issue",
|
||||
details: {
|
||||
issueId: issue.id,
|
||||
assigneeAgentId: issue.assigneeAgentId,
|
||||
actorAgentId,
|
||||
},
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
async function resolveActiveIssueRun(issue: {
|
||||
id: string;
|
||||
assigneeAgentId: string | null;
|
||||
|
|
@ -932,6 +1032,7 @@ export function issueRoutes(
|
|||
commentCursor,
|
||||
wakeComment,
|
||||
relations,
|
||||
blockerAttention,
|
||||
attachments,
|
||||
continuationSummary,
|
||||
currentExecutionWorkspace,
|
||||
|
|
@ -942,6 +1043,7 @@ export function issueRoutes(
|
|||
svc.getCommentCursor(issue.id),
|
||||
wakeCommentId ? svc.getComment(wakeCommentId) : null,
|
||||
svc.getRelationSummaries(issue.id),
|
||||
svc.listBlockerAttention(issue.companyId, [issue]).then((map) => map.get(issue.id) ?? null),
|
||||
svc.listAttachments(issue.id),
|
||||
documentsSvc.getIssueDocumentByKey(issue.id, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY),
|
||||
currentExecutionWorkspacePromise,
|
||||
|
|
@ -954,6 +1056,7 @@ export function issueRoutes(
|
|||
title: issue.title,
|
||||
description: issue.description,
|
||||
status: issue.status,
|
||||
...(blockerAttention ? { blockerAttention } : {}),
|
||||
priority: issue.priority,
|
||||
projectId: issue.projectId,
|
||||
goalId: goal?.id ?? issue.goalId,
|
||||
|
|
@ -1023,12 +1126,13 @@ export function issueRoutes(
|
|||
return;
|
||||
}
|
||||
assertCompanyAccess(req, issue.companyId);
|
||||
const [{ project, goal }, ancestors, mentionedProjectIds, documentPayload, relations, referenceSummary] = await Promise.all([
|
||||
const [{ project, goal }, ancestors, mentionedProjectIds, documentPayload, relations, blockerAttention, referenceSummary] = await Promise.all([
|
||||
resolveIssueProjectAndGoal(issue),
|
||||
svc.getAncestors(issue.id),
|
||||
svc.findMentionedProjectIds(issue.id, { includeCommentBodies: false }),
|
||||
documentsSvc.getIssueDocumentPayload(issue),
|
||||
svc.getRelationSummaries(issue.id),
|
||||
svc.listBlockerAttention(issue.companyId, [issue]).then((map) => map.get(issue.id) ?? null),
|
||||
issueReferencesSvc.listIssueReferenceSummary(issue.id),
|
||||
]);
|
||||
const mentionedProjects = mentionedProjectIds.length > 0
|
||||
|
|
@ -1042,6 +1146,7 @@ export function issueRoutes(
|
|||
...issue,
|
||||
goalId: goal?.id ?? issue.goalId,
|
||||
ancestors,
|
||||
...(blockerAttention ? { blockerAttention } : {}),
|
||||
blockedBy: relations.blockedBy,
|
||||
blocks: relations.blocks,
|
||||
relatedWork: referenceSummary,
|
||||
|
|
@ -1800,17 +1905,27 @@ export function issueRoutes(
|
|||
comment: commentBody,
|
||||
reviewRequest,
|
||||
reopen: reopenRequested,
|
||||
resume: resumeRequested,
|
||||
interrupt: interruptRequested,
|
||||
hiddenAt: hiddenAtRaw,
|
||||
...updateFields
|
||||
} = req.body;
|
||||
if (resumeRequested === true && !commentBody) {
|
||||
res.status(400).json({ error: "Follow-up intent requires a comment" });
|
||||
return;
|
||||
}
|
||||
if (resumeRequested === true && !(await assertExplicitResumeIntentAllowed(req, res, existing))) return;
|
||||
if (resumeRequested !== true && reopenRequested === true && req.actor.type === "agent") {
|
||||
if (!(await assertExplicitResumeIntentAllowed(req, res, existing))) return;
|
||||
}
|
||||
await assertIssueEnvironmentSelection(existing.companyId, updateFields.executionWorkspaceSettings?.environmentId);
|
||||
const requestedAssigneeAgentId =
|
||||
normalizedAssigneeAgentId === undefined ? existing.assigneeAgentId : normalizedAssigneeAgentId;
|
||||
const explicitMoveToTodoRequested = reopenRequested || resumeRequested === true;
|
||||
const effectiveMoveToTodoRequested =
|
||||
reopenRequested ||
|
||||
explicitMoveToTodoRequested ||
|
||||
(!!commentBody &&
|
||||
shouldImplicitlyMoveCommentedIssueToTodoForAgent({
|
||||
shouldImplicitlyMoveCommentedIssueToTodo({
|
||||
issueStatus: existing.status,
|
||||
assigneeAgentId: requestedAssigneeAgentId,
|
||||
actorType: actor.actorType,
|
||||
|
|
@ -1823,6 +1938,10 @@ export function issueRoutes(
|
|||
isBlocked && effectiveMoveToTodoRequested
|
||||
? (await svc.getDependencyReadiness(existing.id)).unresolvedBlockerCount > 0
|
||||
: false;
|
||||
if (resumeRequested === true && isBlocked && hasUnresolvedFirstClassBlockers) {
|
||||
res.status(409).json({ error: "Issue follow-up blocked by unresolved blockers" });
|
||||
return;
|
||||
}
|
||||
let interruptedRunId: string | null = null;
|
||||
const closedExecutionWorkspace = await getClosedIssueExecutionWorkspace(existing);
|
||||
const isAgentWorkUpdate =
|
||||
|
|
@ -2078,6 +2197,7 @@ export function issueRoutes(
|
|||
...updateFields,
|
||||
identifier: issue.identifier,
|
||||
...(commentBody ? { source: "comment" } : {}),
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(reopened ? { reopened: true, reopenedFrom: reopenFromStatus } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
_previous: hasFieldChanges ? previous : undefined,
|
||||
|
|
@ -2220,6 +2340,7 @@ export function issueRoutes(
|
|||
bodySnippet: comment.body.slice(0, 120),
|
||||
identifier: issue.identifier,
|
||||
issueTitle: issue.title,
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(reopened ? { reopened: true, reopenedFrom: reopenFromStatus, source: "comment" } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
...(hasFieldChanges ? { updated: true } : {}),
|
||||
|
|
@ -2266,6 +2387,10 @@ export function issueRoutes(
|
|||
existing.status === "blocked" &&
|
||||
issue.status === "todo" &&
|
||||
(req.body.status !== undefined || reopened);
|
||||
const statusChangedFromClosedToTodo =
|
||||
isClosedIssueStatus(existing.status) &&
|
||||
issue.status === "todo" &&
|
||||
req.body.status !== undefined;
|
||||
const previousExecutionState = parseIssueExecutionState(existing.executionState);
|
||||
const nextExecutionState = parseIssueExecutionState(issue.executionState);
|
||||
const executionStageWakeup = buildExecutionStageWakeup({
|
||||
|
|
@ -2300,6 +2425,7 @@ export function issueRoutes(
|
|||
issueId: issue.id,
|
||||
...(comment ? { commentId: comment.id } : {}),
|
||||
mutation: "update",
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
requestedByActorType: actor.actorType,
|
||||
|
|
@ -2314,12 +2440,17 @@ export function issueRoutes(
|
|||
}
|
||||
: {}),
|
||||
source: "issue.update",
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (!assigneeChanged && (statusChangedFromBacklog || statusChangedFromBlockedToTodo) && issue.assigneeAgentId) {
|
||||
if (
|
||||
!assigneeChanged &&
|
||||
(statusChangedFromBacklog || statusChangedFromBlockedToTodo || statusChangedFromClosedToTodo) &&
|
||||
issue.assigneeAgentId
|
||||
) {
|
||||
addWakeup(issue.assigneeAgentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
|
|
@ -2327,6 +2458,7 @@ export function issueRoutes(
|
|||
payload: {
|
||||
issueId: issue.id,
|
||||
mutation: "update",
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
requestedByActorType: actor.actorType,
|
||||
|
|
@ -2334,6 +2466,7 @@ export function issueRoutes(
|
|||
contextSnapshot: {
|
||||
issueId: issue.id,
|
||||
source: "issue.status_change",
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
});
|
||||
|
|
@ -2355,6 +2488,7 @@ export function issueRoutes(
|
|||
commentId: comment.id,
|
||||
mutation: "comment",
|
||||
...(reopened ? { reopenedFrom: reopenFromStatus } : {}),
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
requestedByActorType: actor.actorType,
|
||||
|
|
@ -2367,6 +2501,7 @@ export function issueRoutes(
|
|||
source: reopened ? "issue.comment.reopen" : "issue.comment",
|
||||
wakeReason: reopened ? "issue_reopened_via_comment" : "issue_commented",
|
||||
...(reopened ? { reopenedFrom: reopenFromStatus } : {}),
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
});
|
||||
|
|
@ -3143,12 +3278,18 @@ export function issueRoutes(
|
|||
|
||||
const actor = getActorInfo(req);
|
||||
const reopenRequested = req.body.reopen === true;
|
||||
const resumeRequested = req.body.resume === true;
|
||||
const interruptRequested = req.body.interrupt === true;
|
||||
if (resumeRequested === true && !(await assertExplicitResumeIntentAllowed(req, res, issue))) return;
|
||||
if (resumeRequested !== true && reopenRequested === true && req.actor.type === "agent") {
|
||||
if (!(await assertExplicitResumeIntentAllowed(req, res, issue))) return;
|
||||
}
|
||||
const isClosed = isClosedIssueStatus(issue.status);
|
||||
const isBlocked = issue.status === "blocked";
|
||||
const explicitMoveToTodoRequested = reopenRequested || resumeRequested === true;
|
||||
const effectiveMoveToTodoRequested =
|
||||
reopenRequested ||
|
||||
shouldImplicitlyMoveCommentedIssueToTodoForAgent({
|
||||
explicitMoveToTodoRequested ||
|
||||
shouldImplicitlyMoveCommentedIssueToTodo({
|
||||
issueStatus: issue.status,
|
||||
assigneeAgentId: issue.assigneeAgentId,
|
||||
actorType: actor.actorType,
|
||||
|
|
@ -3158,6 +3299,10 @@ export function issueRoutes(
|
|||
isBlocked && effectiveMoveToTodoRequested
|
||||
? (await svc.getDependencyReadiness(issue.id)).unresolvedBlockerCount > 0
|
||||
: false;
|
||||
if (resumeRequested === true && isBlocked && hasUnresolvedFirstClassBlockers) {
|
||||
res.status(409).json({ error: "Issue follow-up blocked by unresolved blockers" });
|
||||
return;
|
||||
}
|
||||
let reopened = false;
|
||||
let reopenFromStatus: string | null = null;
|
||||
let interruptedRunId: string | null = null;
|
||||
|
|
@ -3188,6 +3333,7 @@ export function issueRoutes(
|
|||
reopened: true,
|
||||
reopenedFrom: reopenFromStatus,
|
||||
source: "comment",
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
identifier: currentIssue.identifier,
|
||||
},
|
||||
});
|
||||
|
|
@ -3250,6 +3396,7 @@ export function issueRoutes(
|
|||
bodySnippet: comment.body.slice(0, 120),
|
||||
identifier: currentIssue.identifier,
|
||||
issueTitle: currentIssue.title,
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(reopened ? { reopened: true, reopenedFrom: reopenFromStatus, source: "comment" } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
...summarizeIssueReferenceActivityDetails({
|
||||
|
|
@ -3293,6 +3440,7 @@ export function issueRoutes(
|
|||
commentId: comment.id,
|
||||
reopenedFrom: reopenFromStatus,
|
||||
mutation: "comment",
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
requestedByActorType: actor.actorType,
|
||||
|
|
@ -3305,6 +3453,7 @@ export function issueRoutes(
|
|||
source: "issue.comment.reopen",
|
||||
wakeReason: "issue_reopened_via_comment",
|
||||
reopenedFrom: reopenFromStatus,
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
});
|
||||
|
|
@ -3317,6 +3466,7 @@ export function issueRoutes(
|
|||
issueId: currentIssue.id,
|
||||
commentId: comment.id,
|
||||
mutation: "comment",
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
requestedByActorType: actor.actorType,
|
||||
|
|
@ -3328,6 +3478,7 @@ export function issueRoutes(
|
|||
wakeCommentId: comment.id,
|
||||
source: "issue.comment",
|
||||
wakeReason: "issue_commented",
|
||||
...(resumeRequested === true ? { resumeIntent: true, followUpRequested: true } : {}),
|
||||
...(interruptedRunId ? { interruptedRunId } : {}),
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { resolvePaperclipHomeDir } from "../home-paths.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
|
|
@ -43,25 +43,30 @@ interface AdapterSettings {
|
|||
// Paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const PAPERCLIP_DIR = path.join(os.homedir(), ".paperclip");
|
||||
const ADAPTER_PLUGINS_DIR = path.join(PAPERCLIP_DIR, "adapter-plugins");
|
||||
const ADAPTER_PLUGINS_STORE_PATH = path.join(PAPERCLIP_DIR, "adapter-plugins.json");
|
||||
const ADAPTER_SETTINGS_PATH = path.join(PAPERCLIP_DIR, "adapter-settings.json");
|
||||
function adapterPluginPaths() {
|
||||
const paperclipDir = resolvePaperclipHomeDir();
|
||||
return {
|
||||
adapterPluginsDir: path.join(paperclipDir, "adapter-plugins"),
|
||||
adapterPluginsStorePath: path.join(paperclipDir, "adapter-plugins.json"),
|
||||
adapterSettingsPath: path.join(paperclipDir, "adapter-settings.json"),
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// In-memory caches (invalidated on write)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let storeCache: AdapterPluginRecord[] | null = null;
|
||||
let settingsCache: AdapterSettings | null = null;
|
||||
let storeCache: { path: string; records: AdapterPluginRecord[] } | null = null;
|
||||
let settingsCache: { path: string; settings: AdapterSettings } | null = null;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Store functions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function ensureDirs(): void {
|
||||
fs.mkdirSync(ADAPTER_PLUGINS_DIR, { recursive: true });
|
||||
const pkgJsonPath = path.join(ADAPTER_PLUGINS_DIR, "package.json");
|
||||
function ensureDirs(): string {
|
||||
const { adapterPluginsDir } = adapterPluginPaths();
|
||||
fs.mkdirSync(adapterPluginsDir, { recursive: true });
|
||||
const pkgJsonPath = path.join(adapterPluginsDir, "package.json");
|
||||
if (!fs.existsSync(pkgJsonPath)) {
|
||||
fs.writeFileSync(pkgJsonPath, JSON.stringify({
|
||||
name: "paperclip-adapter-plugins",
|
||||
|
|
@ -70,44 +75,55 @@ function ensureDirs(): void {
|
|||
description: "Managed directory for Paperclip external adapter plugins. Do not edit manually.",
|
||||
}, null, 2) + "\n");
|
||||
}
|
||||
return adapterPluginsDir;
|
||||
}
|
||||
|
||||
function readStore(): AdapterPluginRecord[] {
|
||||
if (storeCache) return storeCache;
|
||||
const { adapterPluginsStorePath } = adapterPluginPaths();
|
||||
if (storeCache?.path === adapterPluginsStorePath) return storeCache.records;
|
||||
try {
|
||||
const raw = fs.readFileSync(ADAPTER_PLUGINS_STORE_PATH, "utf-8");
|
||||
const raw = fs.readFileSync(adapterPluginsStorePath, "utf-8");
|
||||
const parsed = JSON.parse(raw);
|
||||
storeCache = Array.isArray(parsed) ? (parsed as AdapterPluginRecord[]) : [];
|
||||
storeCache = {
|
||||
path: adapterPluginsStorePath,
|
||||
records: Array.isArray(parsed) ? (parsed as AdapterPluginRecord[]) : [],
|
||||
};
|
||||
} catch {
|
||||
storeCache = [];
|
||||
storeCache = { path: adapterPluginsStorePath, records: [] };
|
||||
}
|
||||
return storeCache;
|
||||
return storeCache.records;
|
||||
}
|
||||
|
||||
function writeStore(records: AdapterPluginRecord[]): void {
|
||||
ensureDirs();
|
||||
fs.writeFileSync(ADAPTER_PLUGINS_STORE_PATH, JSON.stringify(records, null, 2), "utf-8");
|
||||
storeCache = records;
|
||||
const { adapterPluginsStorePath } = adapterPluginPaths();
|
||||
fs.writeFileSync(adapterPluginsStorePath, JSON.stringify(records, null, 2), "utf-8");
|
||||
storeCache = { path: adapterPluginsStorePath, records };
|
||||
}
|
||||
|
||||
function readSettings(): AdapterSettings {
|
||||
if (settingsCache) return settingsCache;
|
||||
const { adapterSettingsPath } = adapterPluginPaths();
|
||||
if (settingsCache?.path === adapterSettingsPath) return settingsCache.settings;
|
||||
try {
|
||||
const raw = fs.readFileSync(ADAPTER_SETTINGS_PATH, "utf-8");
|
||||
const raw = fs.readFileSync(adapterSettingsPath, "utf-8");
|
||||
const parsed = JSON.parse(raw);
|
||||
settingsCache = parsed && Array.isArray(parsed.disabledTypes)
|
||||
? (parsed as AdapterSettings)
|
||||
: { disabledTypes: [] };
|
||||
settingsCache = {
|
||||
path: adapterSettingsPath,
|
||||
settings: parsed && Array.isArray(parsed.disabledTypes)
|
||||
? (parsed as AdapterSettings)
|
||||
: { disabledTypes: [] },
|
||||
};
|
||||
} catch {
|
||||
settingsCache = { disabledTypes: [] };
|
||||
settingsCache = { path: adapterSettingsPath, settings: { disabledTypes: [] } };
|
||||
}
|
||||
return settingsCache;
|
||||
return settingsCache.settings;
|
||||
}
|
||||
|
||||
function writeSettings(settings: AdapterSettings): void {
|
||||
ensureDirs();
|
||||
fs.writeFileSync(ADAPTER_SETTINGS_PATH, JSON.stringify(settings, null, 2), "utf-8");
|
||||
settingsCache = settings;
|
||||
const { adapterSettingsPath } = adapterPluginPaths();
|
||||
fs.writeFileSync(adapterSettingsPath, JSON.stringify(settings, null, 2), "utf-8");
|
||||
settingsCache = { path: adapterSettingsPath, settings };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -143,8 +159,7 @@ export function getAdapterPluginByType(type: string): AdapterPluginRecord | unde
|
|||
}
|
||||
|
||||
export function getAdapterPluginsDir(): string {
|
||||
ensureDirs();
|
||||
return ADAPTER_PLUGINS_DIR;
|
||||
return ensureDirs();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
48
server/src/services/agent-start-lock.ts
Normal file
48
server/src/services/agent-start-lock.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import { logger } from "../middleware/logger.js";
|
||||
|
||||
const AGENT_START_LOCK_STALE_MS = 30_000;
|
||||
const startLocksByAgent = new Map<string, { promise: Promise<void>; startedAtMs: number }>();
|
||||
|
||||
async function waitForAgentStartLock(agentId: string, lock: { promise: Promise<void>; startedAtMs: number }) {
|
||||
const elapsedMs = Date.now() - lock.startedAtMs;
|
||||
const remainingMs = AGENT_START_LOCK_STALE_MS - elapsedMs;
|
||||
if (remainingMs <= 0) {
|
||||
logger.warn({ agentId, staleMs: elapsedMs }, "agent start lock stale; continuing queued-run start");
|
||||
return;
|
||||
}
|
||||
|
||||
let timedOut = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
await Promise.race([
|
||||
lock.promise,
|
||||
new Promise<void>((resolve) => {
|
||||
timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
resolve();
|
||||
}, remainingMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
|
||||
if (timedOut) {
|
||||
logger.warn({ agentId, staleMs: AGENT_START_LOCK_STALE_MS }, "agent start lock timed out; continuing queued-run start");
|
||||
}
|
||||
}
|
||||
|
||||
export async function withAgentStartLock<T>(agentId: string, fn: () => Promise<T>) {
|
||||
const previous = startLocksByAgent.get(agentId);
|
||||
const waitForPrevious = previous ? waitForAgentStartLock(agentId, previous) : Promise.resolve();
|
||||
const run = waitForPrevious.then(fn);
|
||||
const marker = run.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
startLocksByAgent.set(agentId, { promise: marker, startedAtMs: Date.now() });
|
||||
try {
|
||||
return await run;
|
||||
} finally {
|
||||
if (startLocksByAgent.get(agentId)?.promise === marker) {
|
||||
startLocksByAgent.delete(agentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -32,7 +32,7 @@ export { routineService } from "./routines.js";
|
|||
export { costService } from "./costs.js";
|
||||
export { financeService } from "./finance.js";
|
||||
export { heartbeatService } from "./heartbeat.js";
|
||||
export { classifyIssueGraphLiveness, type IssueLivenessFinding } from "./issue-liveness.js";
|
||||
export { classifyIssueGraphLiveness, type IssueLivenessFinding } from "./recovery/index.js";
|
||||
export { dashboardService } from "./dashboard.js";
|
||||
export { sidebarBadgeService } from "./sidebar-badges.js";
|
||||
export { sidebarPreferenceService } from "./sidebar-preferences.js";
|
||||
|
|
|
|||
|
|
@ -41,12 +41,14 @@ function normalizeExperimentalSettings(raw: unknown): InstanceExperimentalSettin
|
|||
enableEnvironments: parsed.data.enableEnvironments ?? false,
|
||||
enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false,
|
||||
autoRestartDevServerWhenIdle: parsed.data.autoRestartDevServerWhenIdle ?? false,
|
||||
enableIssueGraphLivenessAutoRecovery: parsed.data.enableIssueGraphLivenessAutoRecovery ?? false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,324 +1,16 @@
|
|||
export type IssueLivenessSeverity = "warning" | "critical";
|
||||
|
||||
export type IssueLivenessState =
|
||||
| "blocked_by_unassigned_issue"
|
||||
| "blocked_by_uninvokable_assignee"
|
||||
| "blocked_by_cancelled_issue"
|
||||
| "invalid_review_participant";
|
||||
|
||||
export interface IssueLivenessIssueInput {
|
||||
id: string;
|
||||
companyId: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
status: string;
|
||||
projectId?: string | null;
|
||||
goalId?: string | null;
|
||||
parentId?: string | null;
|
||||
assigneeAgentId?: string | null;
|
||||
assigneeUserId?: string | null;
|
||||
createdByAgentId?: string | null;
|
||||
createdByUserId?: string | null;
|
||||
executionState?: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export interface IssueLivenessRelationInput {
|
||||
companyId: string;
|
||||
blockerIssueId: string;
|
||||
blockedIssueId: string;
|
||||
}
|
||||
|
||||
export interface IssueLivenessAgentInput {
|
||||
id: string;
|
||||
companyId: string;
|
||||
name: string;
|
||||
role: string;
|
||||
title?: string | null;
|
||||
status: string;
|
||||
reportsTo?: string | null;
|
||||
}
|
||||
|
||||
export interface IssueLivenessExecutionPathInput {
|
||||
companyId: string;
|
||||
issueId: string | null;
|
||||
agentId?: string | null;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface IssueLivenessDependencyPathEntry {
|
||||
issueId: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface IssueLivenessFinding {
|
||||
issueId: string;
|
||||
companyId: string;
|
||||
identifier: string | null;
|
||||
state: IssueLivenessState;
|
||||
severity: IssueLivenessSeverity;
|
||||
reason: string;
|
||||
dependencyPath: IssueLivenessDependencyPathEntry[];
|
||||
recommendedOwnerAgentId: string | null;
|
||||
recommendedOwnerCandidateAgentIds: string[];
|
||||
recommendedAction: string;
|
||||
incidentKey: string;
|
||||
}
|
||||
|
||||
export interface IssueGraphLivenessInput {
|
||||
issues: IssueLivenessIssueInput[];
|
||||
relations: IssueLivenessRelationInput[];
|
||||
agents: IssueLivenessAgentInput[];
|
||||
activeRuns?: IssueLivenessExecutionPathInput[];
|
||||
queuedWakeRequests?: IssueLivenessExecutionPathInput[];
|
||||
}
|
||||
|
||||
const INVOKABLE_AGENT_STATUSES = new Set(["active", "idle", "running", "error"]);
|
||||
const BLOCKING_AGENT_STATUSES = new Set(["paused", "terminated", "pending_approval"]);
|
||||
|
||||
function issueLabel(issue: IssueLivenessIssueInput) {
|
||||
return issue.identifier ?? issue.id;
|
||||
}
|
||||
|
||||
function pathEntry(issue: IssueLivenessIssueInput): IssueLivenessDependencyPathEntry {
|
||||
return {
|
||||
issueId: issue.id,
|
||||
identifier: issue.identifier,
|
||||
title: issue.title,
|
||||
status: issue.status,
|
||||
};
|
||||
}
|
||||
|
||||
function isInvokableAgent(agent: IssueLivenessAgentInput | null | undefined) {
|
||||
return Boolean(agent && INVOKABLE_AGENT_STATUSES.has(agent.status));
|
||||
}
|
||||
|
||||
function hasActiveExecutionPath(
|
||||
companyId: string,
|
||||
issueId: string,
|
||||
activeRuns: IssueLivenessExecutionPathInput[],
|
||||
queuedWakeRequests: IssueLivenessExecutionPathInput[],
|
||||
) {
|
||||
return [...activeRuns, ...queuedWakeRequests].some(
|
||||
(entry) => entry.companyId === companyId && entry.issueId === issueId,
|
||||
);
|
||||
}
|
||||
|
||||
function readPrincipalAgentId(principal: unknown): string | null {
|
||||
if (!principal || typeof principal !== "object") return null;
|
||||
const value = principal as Record<string, unknown>;
|
||||
return value.type === "agent" && typeof value.agentId === "string" && value.agentId.length > 0
|
||||
? value.agentId
|
||||
: null;
|
||||
}
|
||||
|
||||
function principalIsResolvableUser(principal: unknown): boolean {
|
||||
if (!principal || typeof principal !== "object") return false;
|
||||
const value = principal as Record<string, unknown>;
|
||||
return value.type === "user" && typeof value.userId === "string" && value.userId.length > 0;
|
||||
}
|
||||
|
||||
function agentChainCandidates(
|
||||
startAgentId: string | null | undefined,
|
||||
agentsById: Map<string, IssueLivenessAgentInput>,
|
||||
companyId: string,
|
||||
) {
|
||||
const candidates: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = startAgentId ? agentsById.get(startAgentId) : null;
|
||||
|
||||
while (current?.reportsTo) {
|
||||
if (seen.has(current.reportsTo)) break;
|
||||
seen.add(current.reportsTo);
|
||||
const manager = agentsById.get(current.reportsTo);
|
||||
if (!manager || manager.companyId !== companyId) break;
|
||||
if (isInvokableAgent(manager)) candidates.push(manager.id);
|
||||
current = manager;
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function fallbackExecutiveCandidates(agents: IssueLivenessAgentInput[], companyId: string) {
|
||||
const active = agents.filter((agent) => agent.companyId === companyId && isInvokableAgent(agent));
|
||||
const executive = active.filter((agent) => {
|
||||
const haystack = `${agent.role} ${agent.title ?? ""} ${agent.name}`.toLowerCase();
|
||||
return /\b(cto|chief technology|ceo|chief executive)\b/.test(haystack);
|
||||
});
|
||||
const roots = active.filter((agent) => !agent.reportsTo);
|
||||
return [...executive, ...roots, ...active].map((agent) => agent.id);
|
||||
}
|
||||
|
||||
function ownerCandidatesForIssue(
|
||||
issue: IssueLivenessIssueInput,
|
||||
agents: IssueLivenessAgentInput[],
|
||||
agentsById: Map<string, IssueLivenessAgentInput>,
|
||||
) {
|
||||
const candidates = [
|
||||
...agentChainCandidates(issue.assigneeAgentId, agentsById, issue.companyId),
|
||||
...agentChainCandidates(issue.createdByAgentId, agentsById, issue.companyId),
|
||||
...fallbackExecutiveCandidates(agents, issue.companyId),
|
||||
];
|
||||
return [...new Set(candidates)];
|
||||
}
|
||||
|
||||
function incidentKey(input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
state: IssueLivenessState;
|
||||
blockerIssueId?: string | null;
|
||||
participantAgentId?: string | null;
|
||||
}) {
|
||||
return [
|
||||
"harness_liveness",
|
||||
input.companyId,
|
||||
input.issueId,
|
||||
input.state,
|
||||
input.blockerIssueId ?? input.participantAgentId ?? "none",
|
||||
].join(":");
|
||||
}
|
||||
|
||||
function finding(input: {
|
||||
issue: IssueLivenessIssueInput;
|
||||
state: IssueLivenessState;
|
||||
severity?: IssueLivenessSeverity;
|
||||
reason: string;
|
||||
dependencyPath: IssueLivenessIssueInput[];
|
||||
recommendedOwnerCandidateAgentIds: string[];
|
||||
recommendedAction: string;
|
||||
blockerIssueId?: string | null;
|
||||
participantAgentId?: string | null;
|
||||
}): IssueLivenessFinding {
|
||||
return {
|
||||
issueId: input.issue.id,
|
||||
companyId: input.issue.companyId,
|
||||
identifier: input.issue.identifier,
|
||||
state: input.state,
|
||||
severity: input.severity ?? "critical",
|
||||
reason: input.reason,
|
||||
dependencyPath: input.dependencyPath.map(pathEntry),
|
||||
recommendedOwnerAgentId: input.recommendedOwnerCandidateAgentIds[0] ?? null,
|
||||
recommendedOwnerCandidateAgentIds: input.recommendedOwnerCandidateAgentIds,
|
||||
recommendedAction: input.recommendedAction,
|
||||
incidentKey: incidentKey({
|
||||
companyId: input.issue.companyId,
|
||||
issueId: input.issue.id,
|
||||
state: input.state,
|
||||
blockerIssueId: input.blockerIssueId,
|
||||
participantAgentId: input.participantAgentId,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function classifyIssueGraphLiveness(input: IssueGraphLivenessInput): IssueLivenessFinding[] {
|
||||
const issuesById = new Map(input.issues.map((issue) => [issue.id, issue]));
|
||||
const agentsById = new Map(input.agents.map((agent) => [agent.id, agent]));
|
||||
const blockersByBlockedIssueId = new Map<string, IssueLivenessRelationInput[]>();
|
||||
const findings: IssueLivenessFinding[] = [];
|
||||
const activeRuns = input.activeRuns ?? [];
|
||||
const queuedWakeRequests = input.queuedWakeRequests ?? [];
|
||||
|
||||
for (const relation of input.relations) {
|
||||
const list = blockersByBlockedIssueId.get(relation.blockedIssueId) ?? [];
|
||||
list.push(relation);
|
||||
blockersByBlockedIssueId.set(relation.blockedIssueId, list);
|
||||
}
|
||||
|
||||
for (const issue of input.issues) {
|
||||
const ownerCandidates = ownerCandidatesForIssue(issue, input.agents, agentsById);
|
||||
|
||||
if (issue.status === "blocked") {
|
||||
const relations = blockersByBlockedIssueId.get(issue.id) ?? [];
|
||||
for (const relation of relations) {
|
||||
if (relation.companyId !== issue.companyId) continue;
|
||||
const blocker = issuesById.get(relation.blockerIssueId);
|
||||
if (!blocker || blocker.companyId !== issue.companyId || blocker.status === "done") continue;
|
||||
|
||||
if (blocker.status === "cancelled") {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "blocked_by_cancelled_issue",
|
||||
reason: `${issueLabel(issue)} is still blocked by cancelled issue ${issueLabel(blocker)}.`,
|
||||
dependencyPath: [issue, blocker],
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Inspect ${issueLabel(blocker)} and either remove it from ${issueLabel(issue)}'s blockers or replace it with an actionable unblock issue.`,
|
||||
blockerIssueId: blocker.id,
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!blocker.assigneeAgentId && !blocker.assigneeUserId) {
|
||||
if (hasActiveExecutionPath(issue.companyId, blocker.id, activeRuns, queuedWakeRequests)) continue;
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
reason: `${issueLabel(issue)} is blocked by unassigned issue ${issueLabel(blocker)} with no user owner.`,
|
||||
dependencyPath: [issue, blocker],
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Assign ${issueLabel(blocker)} to an owner who can complete it, or remove it from ${issueLabel(issue)}'s blockers if it is no longer required.`,
|
||||
blockerIssueId: blocker.id,
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!blocker.assigneeAgentId) continue;
|
||||
if (hasActiveExecutionPath(issue.companyId, blocker.id, activeRuns, queuedWakeRequests)) continue;
|
||||
|
||||
const blockerAgent = agentsById.get(blocker.assigneeAgentId);
|
||||
if (!blockerAgent || blockerAgent.companyId !== issue.companyId || BLOCKING_AGENT_STATUSES.has(blockerAgent.status)) {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "blocked_by_uninvokable_assignee",
|
||||
reason: blockerAgent
|
||||
? `${issueLabel(issue)} is blocked by ${issueLabel(blocker)}, but its assignee is ${blockerAgent.status}.`
|
||||
: `${issueLabel(issue)} is blocked by ${issueLabel(blocker)}, but its assignee no longer exists.`,
|
||||
dependencyPath: [issue, blocker],
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Review ${issueLabel(blocker)} and assign it to an active owner or replace the blocker with an actionable issue.`,
|
||||
blockerIssueId: blocker.id,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (issue.status !== "in_review" || !issue.executionState) continue;
|
||||
const participant = issue.executionState.currentParticipant;
|
||||
const participantAgentId = readPrincipalAgentId(participant);
|
||||
if (participantAgentId) {
|
||||
const participantAgent = agentsById.get(participantAgentId);
|
||||
if (!isInvokableAgent(participantAgent) || participantAgent?.companyId !== issue.companyId) {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "invalid_review_participant",
|
||||
reason: participantAgent
|
||||
? `${issueLabel(issue)} is in review, but current participant agent is ${participantAgent.status}.`
|
||||
: `${issueLabel(issue)} is in review, but current participant agent cannot be resolved.`,
|
||||
dependencyPath: [issue],
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Repair ${issueLabel(issue)}'s review participant or return the issue to an active assignee with a clear change request.`,
|
||||
participantAgentId,
|
||||
}));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!principalIsResolvableUser(participant)) {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "invalid_review_participant",
|
||||
reason: `${issueLabel(issue)} is in review, but its current participant cannot be resolved.`,
|
||||
dependencyPath: [issue],
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Repair ${issueLabel(issue)}'s review participant or return the issue to an active assignee with a clear change request.`,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
export {
|
||||
classifyIssueGraphLiveness,
|
||||
} from "./recovery/issue-graph-liveness.js";
|
||||
export type {
|
||||
IssueGraphLivenessInput,
|
||||
IssueLivenessAgentInput,
|
||||
IssueLivenessDependencyPathEntry,
|
||||
IssueLivenessExecutionPathInput,
|
||||
IssueLivenessFinding,
|
||||
IssueLivenessIssueInput,
|
||||
IssueLivenessOwnerCandidate,
|
||||
IssueLivenessOwnerCandidateReason,
|
||||
IssueLivenessRelationInput,
|
||||
IssueLivenessSeverity,
|
||||
IssueLivenessState,
|
||||
} from "./recovery/issue-graph-liveness.js";
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import type { Db } from "@paperclipai/db";
|
|||
import {
|
||||
agentWakeupRequests,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueTreeHoldMembers,
|
||||
issueTreeHolds,
|
||||
issues,
|
||||
|
|
@ -76,6 +77,151 @@ export const ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS: ReadonlySet<string> =
|
|||
"issue_reopened_via_comment",
|
||||
"issue_comment_mentioned",
|
||||
] as const);
|
||||
const ISSUE_TREE_CONTROL_INTERACTION_WAKE_SOURCES: Readonly<Record<string, ReadonlySet<string>>> = {
|
||||
issue_commented: new Set(["issue.comment"]),
|
||||
issue_reopened_via_comment: new Set(["issue.comment.reopen"]),
|
||||
issue_comment_mentioned: new Set(["comment.mention"]),
|
||||
};
|
||||
|
||||
type VerifiedInteractionActor = {
|
||||
requestedByActorType?: string | null;
|
||||
requestedByActorId?: string | null;
|
||||
};
|
||||
|
||||
function readNonEmptyStringFromRecord(record: unknown, key: string) {
|
||||
if (!record || typeof record !== "object") return null;
|
||||
const value = (record as Record<string, unknown>)[key];
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function readInteractionWakeCommentId(record: unknown) {
|
||||
if (!record || typeof record !== "object") return null;
|
||||
const value = (record as Record<string, unknown>).wakeCommentIds;
|
||||
if (Array.isArray(value)) {
|
||||
const latest = value
|
||||
.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
|
||||
.at(-1);
|
||||
if (latest) return latest.trim();
|
||||
}
|
||||
return readNonEmptyStringFromRecord(record, "wakeCommentId") ?? readNonEmptyStringFromRecord(record, "commentId");
|
||||
}
|
||||
|
||||
function hasVerifiedInteractionSource(wakeReason: string, contextSnapshot: Record<string, unknown>) {
|
||||
const source = readNonEmptyStringFromRecord(contextSnapshot, "source");
|
||||
if (!source) return false;
|
||||
return ISSUE_TREE_CONTROL_INTERACTION_WAKE_SOURCES[wakeReason]?.has(source) ?? false;
|
||||
}
|
||||
|
||||
function actorMatchesComment(
|
||||
actor: VerifiedInteractionActor,
|
||||
comment: { authorAgentId: string | null; authorUserId: string | null },
|
||||
) {
|
||||
if (!actor.requestedByActorType) return false;
|
||||
if (actor.requestedByActorType === "system") return true;
|
||||
if (!actor.requestedByActorId) return false;
|
||||
if (actor.requestedByActorType === "agent") return comment.authorAgentId === actor.requestedByActorId;
|
||||
if (actor.requestedByActorType === "user") return comment.authorUserId === actor.requestedByActorId;
|
||||
return false;
|
||||
}
|
||||
|
||||
async function hasVerifiedInteractionWakeRequest(
|
||||
dbOrTx: Pick<Db, "select">,
|
||||
input: {
|
||||
companyId: string;
|
||||
agentId?: string | null;
|
||||
runId?: string | null;
|
||||
wakeupRequestId?: string | null;
|
||||
issueId: string;
|
||||
commentId: string;
|
||||
comment: { authorAgentId: string | null; authorUserId: string | null };
|
||||
},
|
||||
) {
|
||||
if (!input.runId && !input.wakeupRequestId) return false;
|
||||
const predicates = [
|
||||
eq(agentWakeupRequests.companyId, input.companyId),
|
||||
sql`${agentWakeupRequests.payload} ->> 'issueId' = ${input.issueId}`,
|
||||
sql`${agentWakeupRequests.payload} ->> 'commentId' = ${input.commentId}`,
|
||||
];
|
||||
if (input.agentId) predicates.push(eq(agentWakeupRequests.agentId, input.agentId));
|
||||
if (input.runId && input.wakeupRequestId) {
|
||||
const requestScope = or(
|
||||
eq(agentWakeupRequests.runId, input.runId),
|
||||
eq(agentWakeupRequests.id, input.wakeupRequestId),
|
||||
);
|
||||
if (requestScope) predicates.push(requestScope);
|
||||
} else if (input.runId) {
|
||||
predicates.push(eq(agentWakeupRequests.runId, input.runId));
|
||||
} else if (input.wakeupRequestId) {
|
||||
predicates.push(eq(agentWakeupRequests.id, input.wakeupRequestId));
|
||||
}
|
||||
|
||||
const requests = await dbOrTx
|
||||
.select({
|
||||
requestedByActorType: agentWakeupRequests.requestedByActorType,
|
||||
requestedByActorId: agentWakeupRequests.requestedByActorId,
|
||||
})
|
||||
.from(agentWakeupRequests)
|
||||
.where(and(...predicates));
|
||||
|
||||
return requests.some((request) => actorMatchesComment(request, input.comment));
|
||||
}
|
||||
|
||||
export async function isVerifiedIssueTreeControlInteractionWake(
|
||||
dbOrTx: Pick<Db, "select">,
|
||||
input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
agentId?: string | null;
|
||||
contextSnapshot: Record<string, unknown> | null | undefined;
|
||||
requestedByActorType?: "user" | "agent" | "system" | string | null;
|
||||
requestedByActorId?: string | null;
|
||||
runId?: string | null;
|
||||
wakeupRequestId?: string | null;
|
||||
},
|
||||
) {
|
||||
const contextSnapshot = input.contextSnapshot ?? null;
|
||||
const wakeReason =
|
||||
readNonEmptyStringFromRecord(contextSnapshot, "wakeReason") ??
|
||||
readNonEmptyStringFromRecord(contextSnapshot, "reason");
|
||||
if (!wakeReason || !ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS.has(wakeReason)) return false;
|
||||
if (!contextSnapshot || !hasVerifiedInteractionSource(wakeReason, contextSnapshot)) return false;
|
||||
|
||||
const commentId = readInteractionWakeCommentId(contextSnapshot);
|
||||
if (!commentId) return false;
|
||||
|
||||
const comment = await dbOrTx
|
||||
.select({
|
||||
id: issueComments.id,
|
||||
authorAgentId: issueComments.authorAgentId,
|
||||
authorUserId: issueComments.authorUserId,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.companyId, input.companyId),
|
||||
eq(issueComments.issueId, input.issueId),
|
||||
eq(issueComments.id, commentId),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!comment) return false;
|
||||
|
||||
const directActor = {
|
||||
requestedByActorType: input.requestedByActorType,
|
||||
requestedByActorId: input.requestedByActorId,
|
||||
};
|
||||
if (actorMatchesComment(directActor, comment)) return true;
|
||||
|
||||
return hasVerifiedInteractionWakeRequest(dbOrTx, {
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
runId: input.runId,
|
||||
wakeupRequestId: input.wakeupRequestId,
|
||||
issueId: input.issueId,
|
||||
commentId,
|
||||
comment,
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeReleasePolicy(
|
||||
releasePolicy: IssueTreeHoldReleasePolicy | null | undefined,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { and, asc, desc, eq, inArray, isNull, ne, or, sql } from "drizzle-orm";
|
|||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
activityLog,
|
||||
agentWakeupRequests,
|
||||
agents,
|
||||
assets,
|
||||
companies,
|
||||
|
|
@ -23,7 +24,7 @@ import {
|
|||
projectWorkspaces,
|
||||
projects,
|
||||
} from "@paperclipai/db";
|
||||
import type { IssueRelationIssueSummary } from "@paperclipai/shared";
|
||||
import type { IssueBlockerAttention, IssueRelationIssueSummary } from "@paperclipai/shared";
|
||||
import { extractAgentMentionIds, extractProjectMentionIds, isUuidLike } from "@paperclipai/shared";
|
||||
import { conflict, notFound, unprocessable } from "../errors.js";
|
||||
import {
|
||||
|
|
@ -38,7 +39,7 @@ import { redactCurrentUserText } from "../log-redaction.js";
|
|||
import { resolveIssueGoalId, resolveNextIssueGoalId } from "./issue-goal-fallback.js";
|
||||
import { getDefaultCompanyGoal } from "./goals.js";
|
||||
import {
|
||||
ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS,
|
||||
isVerifiedIssueTreeControlInteractionWake,
|
||||
issueTreeControlService,
|
||||
type ActiveIssueTreePauseHoldGate,
|
||||
} from "./issue-tree-control.js";
|
||||
|
|
@ -82,18 +83,6 @@ function readStringFromRecord(record: unknown, key: string) {
|
|||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function readLatestWakeCommentId(record: unknown) {
|
||||
if (!record || typeof record !== "object") return null;
|
||||
const value = (record as Record<string, unknown>).wakeCommentIds;
|
||||
if (Array.isArray(value)) {
|
||||
const latest = value
|
||||
.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
|
||||
.at(-1);
|
||||
if (latest) return latest.trim();
|
||||
}
|
||||
return readStringFromRecord(record, "wakeCommentId") ?? readStringFromRecord(record, "commentId");
|
||||
}
|
||||
|
||||
export interface IssueFilters {
|
||||
status?: string;
|
||||
assigneeAgentId?: string;
|
||||
|
|
@ -668,6 +657,46 @@ async function withIssueLabels(dbOrTx: any, rows: IssueRow[]): Promise<IssueWith
|
|||
}
|
||||
|
||||
const ACTIVE_RUN_STATUSES = ["queued", "running"];
|
||||
const BLOCKER_ATTENTION_ACTIVE_RUN_STATUSES = ["queued", "running"];
|
||||
const BLOCKER_ATTENTION_ACTIVE_WAKE_STATUSES = ["queued", "deferred_issue_execution"];
|
||||
const BLOCKER_ATTENTION_MAX_DEPTH = 8;
|
||||
const BLOCKER_ATTENTION_MAX_NODES = 2000;
|
||||
const BLOCKER_ATTENTION_INVOKABLE_AGENT_STATUSES = new Set(["active", "idle", "running", "error"]);
|
||||
|
||||
type IssueBlockerAttentionNode = {
|
||||
id: string;
|
||||
companyId: string;
|
||||
parentId: string | null;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
status: string;
|
||||
executionRunId?: string | null;
|
||||
assigneeAgentId: string | null;
|
||||
assigneeUserId: string | null;
|
||||
};
|
||||
type IssueBlockerAttentionInputNode =
|
||||
Pick<
|
||||
IssueBlockerAttentionNode,
|
||||
"id" | "companyId" | "parentId" | "identifier" | "title" | "status" | "assigneeAgentId" | "assigneeUserId"
|
||||
>
|
||||
& { executionRunId?: string | null };
|
||||
|
||||
type IssueBlockerAttentionEdge = {
|
||||
issueId: string;
|
||||
blockerIssueId: string;
|
||||
};
|
||||
type IssueBlockerAttentionQueryRow = IssueBlockerAttentionNode & {
|
||||
issueId: string | null;
|
||||
blockerIssueId: string;
|
||||
};
|
||||
type IssueBlockerAttentionActivePathRow = {
|
||||
issueId: string | null;
|
||||
};
|
||||
type IssueBlockerAttentionAgentRow = {
|
||||
id: string;
|
||||
companyId: string;
|
||||
status: string;
|
||||
};
|
||||
|
||||
async function activeRunMapForIssues(
|
||||
dbOrTx: any,
|
||||
|
|
@ -706,6 +735,380 @@ async function activeRunMapForIssues(
|
|||
return map;
|
||||
}
|
||||
|
||||
function createIssueBlockerAttention(input: Partial<IssueBlockerAttention> = {}): IssueBlockerAttention {
|
||||
return {
|
||||
state: input.state ?? "none",
|
||||
reason: input.reason ?? null,
|
||||
unresolvedBlockerCount: input.unresolvedBlockerCount ?? 0,
|
||||
coveredBlockerCount: input.coveredBlockerCount ?? 0,
|
||||
attentionBlockerCount: input.attentionBlockerCount ?? 0,
|
||||
sampleBlockerIdentifier: input.sampleBlockerIdentifier ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function blockerSampleIdentifier(node: IssueBlockerAttentionNode | null | undefined) {
|
||||
return node?.identifier ?? node?.id ?? null;
|
||||
}
|
||||
|
||||
function appendBlockerAttentionEdges(
|
||||
edgesByIssueId: Map<string, IssueBlockerAttentionEdge[]>,
|
||||
rows: IssueBlockerAttentionEdge[],
|
||||
) {
|
||||
for (const row of rows) {
|
||||
const existing = edgesByIssueId.get(row.issueId) ?? [];
|
||||
if (!existing.some((edge) => edge.blockerIssueId === row.blockerIssueId)) {
|
||||
existing.push(row);
|
||||
edgesByIssueId.set(row.issueId, existing);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type IssueRelationSummaryRow = {
|
||||
relatedId: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
status: string;
|
||||
priority: string;
|
||||
assigneeAgentId: string | null;
|
||||
assigneeUserId: string | null;
|
||||
};
|
||||
|
||||
function summarizeIssueRelationRow(row: IssueRelationSummaryRow): IssueRelationIssueSummary {
|
||||
return {
|
||||
id: row.relatedId,
|
||||
identifier: row.identifier,
|
||||
title: row.title,
|
||||
status: row.status as IssueRelationIssueSummary["status"],
|
||||
priority: row.priority as IssueRelationIssueSummary["priority"],
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
};
|
||||
}
|
||||
|
||||
async function terminalExplicitBlockersByRoot(
|
||||
companyId: string,
|
||||
roots: IssueRelationIssueSummary[],
|
||||
dbOrTx: DbReader,
|
||||
): Promise<Map<string, IssueRelationIssueSummary[]>> {
|
||||
const rootIds = [...new Set(roots.map((root) => root.id))];
|
||||
const terminalByRoot = new Map<string, IssueRelationIssueSummary[]>();
|
||||
if (rootIds.length === 0) return terminalByRoot;
|
||||
|
||||
const nodesById = new Map<string, IssueRelationIssueSummary>();
|
||||
const edgesByIssueId = new Map<string, string[]>();
|
||||
for (const root of roots) nodesById.set(root.id, root);
|
||||
|
||||
let frontier = rootIds;
|
||||
for (let depth = 0; frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) {
|
||||
const nextFrontier = new Set<string>();
|
||||
for (const chunk of chunkList([...new Set(frontier)], ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
|
||||
const rows = await dbOrTx
|
||||
.select({
|
||||
currentIssueId: issueRelations.relatedIssueId,
|
||||
relatedId: issues.id,
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
priority: issues.priority,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
})
|
||||
.from(issueRelations)
|
||||
.innerJoin(issues, eq(issueRelations.issueId, issues.id))
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
inArray(issueRelations.relatedIssueId, chunk),
|
||||
eq(issues.companyId, companyId),
|
||||
ne(issues.status, "done"),
|
||||
),
|
||||
);
|
||||
|
||||
for (const row of rows) {
|
||||
const existingEdges = edgesByIssueId.get(row.currentIssueId) ?? [];
|
||||
if (!existingEdges.includes(row.relatedId)) {
|
||||
existingEdges.push(row.relatedId);
|
||||
edgesByIssueId.set(row.currentIssueId, existingEdges);
|
||||
}
|
||||
if (!nodesById.has(row.relatedId)) {
|
||||
nodesById.set(row.relatedId, summarizeIssueRelationRow(row));
|
||||
nextFrontier.add(row.relatedId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) break;
|
||||
frontier = [...nextFrontier];
|
||||
}
|
||||
|
||||
const collectTerminal = (issueId: string, seen: Set<string>): IssueRelationIssueSummary[] => {
|
||||
if (seen.has(issueId)) return [];
|
||||
const node = nodesById.get(issueId);
|
||||
if (!node || node.status === "done") return [];
|
||||
const nextSeen = new Set(seen);
|
||||
nextSeen.add(issueId);
|
||||
const downstreamIds = edgesByIssueId.get(issueId) ?? [];
|
||||
if (downstreamIds.length === 0) return [node];
|
||||
return downstreamIds.flatMap((downstreamId) => collectTerminal(downstreamId, nextSeen));
|
||||
};
|
||||
|
||||
for (const rootId of rootIds) {
|
||||
const deduped = new Map<string, IssueRelationIssueSummary>();
|
||||
for (const blocker of collectTerminal(rootId, new Set())) {
|
||||
if (blocker.id !== rootId) deduped.set(blocker.id, blocker);
|
||||
}
|
||||
if (deduped.size > 0) {
|
||||
terminalByRoot.set(rootId, [...deduped.values()].sort((a, b) => a.title.localeCompare(b.title)));
|
||||
}
|
||||
}
|
||||
|
||||
return terminalByRoot;
|
||||
}
|
||||
|
||||
async function listIssueBlockerAttentionMap(
|
||||
dbOrTx: any,
|
||||
companyId: string,
|
||||
issueRows: IssueBlockerAttentionInputNode[],
|
||||
): Promise<Map<string, IssueBlockerAttention>> {
|
||||
const roots = issueRows.filter((row) => row.companyId === companyId && row.status === "blocked");
|
||||
const attentionMap = new Map<string, IssueBlockerAttention>();
|
||||
for (const row of issueRows) {
|
||||
if (row.status !== "blocked") {
|
||||
attentionMap.set(row.id, createIssueBlockerAttention());
|
||||
}
|
||||
}
|
||||
if (roots.length === 0) return attentionMap;
|
||||
|
||||
const nodesById = new Map<string, IssueBlockerAttentionNode>();
|
||||
const edgesByIssueId = new Map<string, IssueBlockerAttentionEdge[]>();
|
||||
for (const root of roots) nodesById.set(root.id, { ...root });
|
||||
|
||||
let frontier = roots.map((root) => root.id);
|
||||
let truncated = false;
|
||||
for (let depth = 0; frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) {
|
||||
const nextFrontier = new Set<string>();
|
||||
|
||||
for (const chunk of chunkList([...new Set(frontier)], ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
|
||||
const explicitBlockerRowsPromise: Promise<IssueBlockerAttentionQueryRow[]> = dbOrTx
|
||||
.select({
|
||||
issueId: issueRelations.relatedIssueId,
|
||||
blockerIssueId: issues.id,
|
||||
id: issues.id,
|
||||
companyId: issues.companyId,
|
||||
parentId: issues.parentId,
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
executionRunId: issues.executionRunId,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
})
|
||||
.from(issueRelations)
|
||||
.innerJoin(issues, eq(issueRelations.issueId, issues.id))
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
inArray(issueRelations.relatedIssueId, chunk),
|
||||
eq(issues.companyId, companyId),
|
||||
ne(issues.status, "done"),
|
||||
),
|
||||
);
|
||||
const childRowsPromise: Promise<IssueBlockerAttentionQueryRow[]> = dbOrTx
|
||||
.select({
|
||||
issueId: issues.parentId,
|
||||
blockerIssueId: issues.id,
|
||||
id: issues.id,
|
||||
companyId: issues.companyId,
|
||||
parentId: issues.parentId,
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
executionRunId: issues.executionRunId,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
})
|
||||
.from(issues)
|
||||
.where(
|
||||
and(
|
||||
eq(issues.companyId, companyId),
|
||||
inArray(issues.parentId, chunk),
|
||||
ne(issues.status, "done"),
|
||||
),
|
||||
);
|
||||
const [explicitBlockerRows, childRows] = await Promise.all([
|
||||
explicitBlockerRowsPromise,
|
||||
childRowsPromise,
|
||||
]);
|
||||
|
||||
appendBlockerAttentionEdges(edgesByIssueId, [
|
||||
...explicitBlockerRows
|
||||
.filter((row): row is IssueBlockerAttentionQueryRow & { issueId: string } => row.issueId !== null)
|
||||
.map((row) => ({ issueId: row.issueId, blockerIssueId: row.blockerIssueId })),
|
||||
...childRows
|
||||
.filter((row): row is IssueBlockerAttentionQueryRow & { issueId: string } => row.issueId !== null)
|
||||
.map((row) => ({ issueId: row.issueId, blockerIssueId: row.blockerIssueId })),
|
||||
]);
|
||||
|
||||
for (const row of [...explicitBlockerRows, ...childRows]) {
|
||||
if (!row.issueId || nodesById.has(row.blockerIssueId)) continue;
|
||||
nodesById.set(row.blockerIssueId, {
|
||||
id: row.blockerIssueId,
|
||||
companyId: row.companyId,
|
||||
parentId: row.parentId,
|
||||
identifier: row.identifier,
|
||||
title: row.title,
|
||||
status: row.status,
|
||||
executionRunId: row.executionRunId,
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
});
|
||||
nextFrontier.add(row.blockerIssueId);
|
||||
}
|
||||
}
|
||||
|
||||
if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
frontier = [...nextFrontier];
|
||||
}
|
||||
if (frontier.length > 0) truncated = true;
|
||||
|
||||
const nodeIds = [...nodesById.keys()];
|
||||
const activeIssueIds = new Set<string>();
|
||||
const agentIds = new Set<string>();
|
||||
const issueIdByExecutionRunId = new Map<string, string>();
|
||||
for (const node of nodesById.values()) {
|
||||
if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId);
|
||||
if (node.executionRunId) issueIdByExecutionRunId.set(node.executionRunId, node.id);
|
||||
}
|
||||
|
||||
for (const chunk of chunkList([...issueIdByExecutionRunId.keys()], ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
|
||||
const runRows: Array<{ id: string }> = await dbOrTx
|
||||
.select({
|
||||
id: heartbeatRuns.id,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.status, BLOCKER_ATTENTION_ACTIVE_RUN_STATUSES),
|
||||
inArray(heartbeatRuns.id, chunk),
|
||||
),
|
||||
);
|
||||
|
||||
for (const row of runRows) {
|
||||
const issueId = issueIdByExecutionRunId.get(row.id);
|
||||
if (issueId) activeIssueIds.add(issueId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const chunk of chunkList(nodeIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
|
||||
const wakeRowsPromise: Promise<IssueBlockerAttentionActivePathRow[]> = dbOrTx
|
||||
.select({
|
||||
issueId: sql<string | null>`${agentWakeupRequests.payload} ->> 'issueId'`,
|
||||
})
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
inArray(agentWakeupRequests.status, BLOCKER_ATTENTION_ACTIVE_WAKE_STATUSES),
|
||||
sql`${agentWakeupRequests.runId} is null`,
|
||||
inArray(sql<string>`${agentWakeupRequests.payload} ->> 'issueId'`, chunk),
|
||||
),
|
||||
);
|
||||
const wakeRows = await wakeRowsPromise;
|
||||
for (const row of wakeRows) {
|
||||
if (row.issueId) activeIssueIds.add(row.issueId);
|
||||
}
|
||||
}
|
||||
|
||||
const agentRows: IssueBlockerAttentionAgentRow[] = agentIds.size > 0
|
||||
? await dbOrTx
|
||||
.select({
|
||||
id: agents.id,
|
||||
companyId: agents.companyId,
|
||||
status: agents.status,
|
||||
})
|
||||
.from(agents)
|
||||
.where(and(eq(agents.companyId, companyId), inArray(agents.id, [...agentIds])))
|
||||
: [];
|
||||
const agentsById = new Map(agentRows.map((agent) => [agent.id, agent]));
|
||||
|
||||
type PathClassification = { covered: boolean; sampleBlockerIdentifier: string | null };
|
||||
const classifyPath = (
|
||||
nodeId: string,
|
||||
seen: Set<string>,
|
||||
): PathClassification => {
|
||||
if (truncated || seen.has(nodeId)) return { covered: false, sampleBlockerIdentifier: blockerSampleIdentifier(nodesById.get(nodeId)) };
|
||||
const node = nodesById.get(nodeId);
|
||||
if (!node || node.companyId !== companyId) return { covered: false, sampleBlockerIdentifier: nodeId };
|
||||
if (node.status === "done") return { covered: true, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
if (activeIssueIds.has(node.id)) return { covered: true, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
if (node.status === "cancelled") return { covered: false, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
|
||||
const downstream = (edgesByIssueId.get(node.id) ?? []).filter((edge) => nodesById.get(edge.blockerIssueId)?.status !== "done");
|
||||
if (downstream.length > 0) {
|
||||
const nextSeen = new Set(seen);
|
||||
nextSeen.add(nodeId);
|
||||
const classified = downstream.map((edge) => classifyPath(edge.blockerIssueId, nextSeen));
|
||||
const attention = classified.find((result) => !result.covered);
|
||||
if (attention) return attention;
|
||||
return {
|
||||
covered: true,
|
||||
sampleBlockerIdentifier: classified[0]?.sampleBlockerIdentifier ?? blockerSampleIdentifier(node),
|
||||
};
|
||||
}
|
||||
|
||||
if (node.assigneeAgentId) {
|
||||
const assignee = agentsById.get(node.assigneeAgentId);
|
||||
if (!assignee || assignee.companyId !== companyId || !BLOCKER_ATTENTION_INVOKABLE_AGENT_STATUSES.has(assignee.status)) {
|
||||
return { covered: false, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
}
|
||||
}
|
||||
|
||||
return { covered: false, sampleBlockerIdentifier: blockerSampleIdentifier(node) };
|
||||
};
|
||||
|
||||
for (const root of roots) {
|
||||
const topLevelEdges = (edgesByIssueId.get(root.id) ?? []).filter((edge) => nodesById.get(edge.blockerIssueId)?.status !== "done");
|
||||
if (topLevelEdges.length === 0) {
|
||||
attentionMap.set(root.id, createIssueBlockerAttention({
|
||||
state: "needs_attention",
|
||||
reason: "attention_required",
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
const classified = topLevelEdges.map((edge) => ({
|
||||
edge,
|
||||
result: classifyPath(edge.blockerIssueId, new Set([root.id])),
|
||||
}));
|
||||
const coveredBlockerCount = classified.filter((entry) => entry.result.covered).length;
|
||||
const attentionBlockerCount = classified.length - coveredBlockerCount;
|
||||
const attentionEntry = classified.find((entry) => !entry.result.covered);
|
||||
const sampleEntry = attentionEntry ?? classified[0] ?? null;
|
||||
const sampleNode = sampleEntry ? nodesById.get(sampleEntry.edge.blockerIssueId) : null;
|
||||
|
||||
attentionMap.set(root.id, createIssueBlockerAttention({
|
||||
state: attentionBlockerCount === 0 ? "covered" : "needs_attention",
|
||||
reason: attentionBlockerCount === 0
|
||||
? topLevelEdges.every((edge) => nodesById.get(edge.blockerIssueId)?.parentId === root.id)
|
||||
? "active_child"
|
||||
: "active_dependency"
|
||||
: "attention_required",
|
||||
unresolvedBlockerCount: topLevelEdges.length,
|
||||
coveredBlockerCount,
|
||||
attentionBlockerCount,
|
||||
sampleBlockerIdentifier: sampleEntry?.result.sampleBlockerIdentifier ?? blockerSampleIdentifier(sampleNode),
|
||||
}));
|
||||
}
|
||||
|
||||
return attentionMap;
|
||||
}
|
||||
|
||||
const issueListSelect = {
|
||||
id: issues.id,
|
||||
companyId: issues.companyId,
|
||||
|
|
@ -956,18 +1359,25 @@ export function issueService(db: Db) {
|
|||
) {
|
||||
if (!checkoutRunId) return false;
|
||||
const run = await db
|
||||
.select({ contextSnapshot: heartbeatRuns.contextSnapshot })
|
||||
.select({
|
||||
id: heartbeatRuns.id,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.id, checkoutRunId), eq(heartbeatRuns.companyId, companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
const wakeReason =
|
||||
readStringFromRecord(run?.contextSnapshot, "wakeReason") ??
|
||||
readStringFromRecord(run?.contextSnapshot, "reason");
|
||||
return Boolean(
|
||||
wakeReason &&
|
||||
ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS.has(wakeReason) &&
|
||||
readLatestWakeCommentId(run?.contextSnapshot),
|
||||
);
|
||||
const issueId = readStringFromRecord(run?.contextSnapshot, "issueId");
|
||||
if (!run || !issueId) return false;
|
||||
return isVerifiedIssueTreeControlInteractionWake(db, {
|
||||
companyId,
|
||||
issueId,
|
||||
agentId: run.agentId,
|
||||
runId: run.id,
|
||||
wakeupRequestId: run.wakeupRequestId,
|
||||
contextSnapshot: run.contextSnapshot as Record<string, unknown> | null | undefined,
|
||||
});
|
||||
}
|
||||
|
||||
async function assertAssignableUser(companyId: string, userId: string) {
|
||||
|
|
@ -1118,30 +1528,26 @@ export function issueService(db: Db) {
|
|||
]);
|
||||
|
||||
for (const row of blockedByRows) {
|
||||
empty.get(row.currentIssueId)?.blockedBy.push({
|
||||
id: row.relatedId,
|
||||
identifier: row.identifier,
|
||||
title: row.title,
|
||||
status: row.status as IssueRelationIssueSummary["status"],
|
||||
priority: row.priority as IssueRelationIssueSummary["priority"],
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
});
|
||||
empty.get(row.currentIssueId)?.blockedBy.push(summarizeIssueRelationRow(row));
|
||||
}
|
||||
for (const row of blockingRows) {
|
||||
empty.get(row.currentIssueId)?.blocks.push({
|
||||
id: row.relatedId,
|
||||
identifier: row.identifier,
|
||||
title: row.title,
|
||||
status: row.status as IssueRelationIssueSummary["status"],
|
||||
priority: row.priority as IssueRelationIssueSummary["priority"],
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
});
|
||||
empty.get(row.currentIssueId)?.blocks.push(summarizeIssueRelationRow(row));
|
||||
}
|
||||
|
||||
const terminalByRoot = await terminalExplicitBlockersByRoot(
|
||||
companyId,
|
||||
[...empty.values()].flatMap((relations) => relations.blockedBy),
|
||||
dbOrTx,
|
||||
);
|
||||
|
||||
for (const relations of empty.values()) {
|
||||
relations.blockedBy.sort((a, b) => a.title.localeCompare(b.title));
|
||||
for (const blocker of relations.blockedBy) {
|
||||
const terminalBlockers = terminalByRoot.get(blocker.id);
|
||||
if (terminalBlockers && terminalBlockers.length > 0) {
|
||||
blocker.terminalBlockers = terminalBlockers;
|
||||
}
|
||||
}
|
||||
relations.blocks.sort((a, b) => a.title.localeCompare(b.title));
|
||||
}
|
||||
|
||||
|
|
@ -1519,6 +1925,7 @@ export function issueService(db: Db) {
|
|||
]);
|
||||
const statsByIssueId = new Map(statsRows.map((row) => [row.issueId, row]));
|
||||
const lastActivityByIssueId = new Map(lastActivityRows.map((row) => [row.issueId, row]));
|
||||
const blockerAttentionByIssueId = await listIssueBlockerAttentionMap(db, companyId, withRuns);
|
||||
|
||||
if (!contextUserId) {
|
||||
return withRuns.map((row) => {
|
||||
|
|
@ -1531,6 +1938,7 @@ export function issueService(db: Db) {
|
|||
return {
|
||||
...row,
|
||||
lastActivityAt,
|
||||
...(blockerAttentionByIssueId.has(row.id) ? { blockerAttention: blockerAttentionByIssueId.get(row.id) } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
|
@ -1547,6 +1955,7 @@ export function issueService(db: Db) {
|
|||
return {
|
||||
...row,
|
||||
lastActivityAt,
|
||||
...(blockerAttentionByIssueId.has(row.id) ? { blockerAttention: blockerAttentionByIssueId.get(row.id) } : {}),
|
||||
...deriveIssueUserContext(row, contextUserId, {
|
||||
myLastCommentAt: statsByIssueId.get(row.id)?.myLastCommentAt ?? null,
|
||||
myLastReadAt: readByIssueId.get(row.id) ?? null,
|
||||
|
|
@ -1690,6 +2099,14 @@ export function issueService(db: Db) {
|
|||
return listIssueDependencyReadinessMap(dbOrTx, companyId, issueIds);
|
||||
},
|
||||
|
||||
listBlockerAttention: async (
|
||||
companyId: string,
|
||||
issueRows: IssueBlockerAttentionInputNode[],
|
||||
dbOrTx: any = db,
|
||||
) => {
|
||||
return listIssueBlockerAttentionMap(dbOrTx, companyId, issueRows);
|
||||
},
|
||||
|
||||
listWakeableBlockedDependents: async (blockerIssueId: string) => {
|
||||
const blockerIssue = await db
|
||||
.select({ id: issues.id, companyId: issues.companyId })
|
||||
|
|
|
|||
43
server/src/services/recovery/index.ts
Normal file
43
server/src/services/recovery/index.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
export {
|
||||
RECOVERY_KEY_PREFIXES,
|
||||
RECOVERY_ORIGIN_KINDS,
|
||||
RECOVERY_REASON_KINDS,
|
||||
buildIssueGraphLivenessIncidentKey,
|
||||
buildIssueGraphLivenessLeafKey,
|
||||
parseIssueGraphLivenessIncidentKey,
|
||||
} from "./origins.js";
|
||||
export type {
|
||||
RecoveryKeyPrefix,
|
||||
RecoveryOriginKind,
|
||||
RecoveryReasonKind,
|
||||
} from "./origins.js";
|
||||
export {
|
||||
classifyIssueGraphLiveness,
|
||||
} from "./issue-graph-liveness.js";
|
||||
export type {
|
||||
IssueGraphLivenessInput,
|
||||
IssueLivenessAgentInput,
|
||||
IssueLivenessDependencyPathEntry,
|
||||
IssueLivenessExecutionPathInput,
|
||||
IssueLivenessFinding,
|
||||
IssueLivenessIssueInput,
|
||||
IssueLivenessOwnerCandidate,
|
||||
IssueLivenessOwnerCandidateReason,
|
||||
IssueLivenessRelationInput,
|
||||
IssueLivenessSeverity,
|
||||
IssueLivenessState,
|
||||
} from "./issue-graph-liveness.js";
|
||||
export {
|
||||
recoveryService,
|
||||
} from "./service.js";
|
||||
export {
|
||||
DEFAULT_MAX_LIVENESS_CONTINUATION_ATTEMPTS,
|
||||
RUN_LIVENESS_CONTINUATION_REASON,
|
||||
buildRunLivenessContinuationIdempotencyKey,
|
||||
decideRunLivenessContinuation,
|
||||
findExistingRunLivenessContinuationWake,
|
||||
readContinuationAttempt,
|
||||
} from "./run-liveness-continuations.js";
|
||||
export type {
|
||||
RunContinuationDecision,
|
||||
} from "./run-liveness-continuations.js";
|
||||
414
server/src/services/recovery/issue-graph-liveness.ts
Normal file
414
server/src/services/recovery/issue-graph-liveness.ts
Normal file
|
|
@ -0,0 +1,414 @@
|
|||
import { buildIssueGraphLivenessIncidentKey } from "./origins.js";
|
||||
|
||||
export type IssueLivenessSeverity = "warning" | "critical";
|
||||
|
||||
export type IssueLivenessState =
|
||||
| "blocked_by_unassigned_issue"
|
||||
| "blocked_by_uninvokable_assignee"
|
||||
| "blocked_by_cancelled_issue"
|
||||
| "invalid_review_participant";
|
||||
|
||||
export interface IssueLivenessIssueInput {
|
||||
id: string;
|
||||
companyId: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
status: string;
|
||||
projectId?: string | null;
|
||||
goalId?: string | null;
|
||||
parentId?: string | null;
|
||||
assigneeAgentId?: string | null;
|
||||
assigneeUserId?: string | null;
|
||||
createdByAgentId?: string | null;
|
||||
createdByUserId?: string | null;
|
||||
executionState?: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export interface IssueLivenessRelationInput {
|
||||
companyId: string;
|
||||
blockerIssueId: string;
|
||||
blockedIssueId: string;
|
||||
}
|
||||
|
||||
export interface IssueLivenessAgentInput {
|
||||
id: string;
|
||||
companyId: string;
|
||||
name: string;
|
||||
role: string;
|
||||
title?: string | null;
|
||||
status: string;
|
||||
reportsTo?: string | null;
|
||||
}
|
||||
|
||||
export interface IssueLivenessExecutionPathInput {
|
||||
companyId: string;
|
||||
issueId: string | null;
|
||||
agentId?: string | null;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface IssueLivenessDependencyPathEntry {
|
||||
issueId: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export type IssueLivenessOwnerCandidateReason =
|
||||
| "stalled_blocker_assignee"
|
||||
| "assignee_reporting_chain"
|
||||
| "creator_reporting_chain"
|
||||
| "root_agent"
|
||||
| "ordered_invokable_fallback";
|
||||
|
||||
export interface IssueLivenessOwnerCandidate {
|
||||
agentId: string;
|
||||
reason: IssueLivenessOwnerCandidateReason;
|
||||
sourceIssueId: string;
|
||||
}
|
||||
|
||||
export interface IssueLivenessFinding {
|
||||
issueId: string;
|
||||
companyId: string;
|
||||
identifier: string | null;
|
||||
state: IssueLivenessState;
|
||||
severity: IssueLivenessSeverity;
|
||||
reason: string;
|
||||
dependencyPath: IssueLivenessDependencyPathEntry[];
|
||||
recoveryIssueId: string;
|
||||
recommendedOwnerAgentId: string | null;
|
||||
recommendedOwnerCandidateAgentIds: string[];
|
||||
recommendedOwnerCandidates: IssueLivenessOwnerCandidate[];
|
||||
recommendedAction: string;
|
||||
incidentKey: string;
|
||||
}
|
||||
|
||||
export interface IssueGraphLivenessInput {
|
||||
issues: IssueLivenessIssueInput[];
|
||||
relations: IssueLivenessRelationInput[];
|
||||
agents: IssueLivenessAgentInput[];
|
||||
activeRuns?: IssueLivenessExecutionPathInput[];
|
||||
queuedWakeRequests?: IssueLivenessExecutionPathInput[];
|
||||
}
|
||||
|
||||
const INVOKABLE_AGENT_STATUSES = new Set(["active", "idle", "running", "error"]);
|
||||
const BLOCKING_AGENT_STATUSES = new Set(["paused", "terminated", "pending_approval"]);
|
||||
|
||||
function issueLabel(issue: IssueLivenessIssueInput) {
|
||||
return issue.identifier ?? issue.id;
|
||||
}
|
||||
|
||||
function pathEntry(issue: IssueLivenessIssueInput): IssueLivenessDependencyPathEntry {
|
||||
return {
|
||||
issueId: issue.id,
|
||||
identifier: issue.identifier,
|
||||
title: issue.title,
|
||||
status: issue.status,
|
||||
};
|
||||
}
|
||||
|
||||
function isInvokableAgent(agent: IssueLivenessAgentInput | null | undefined) {
|
||||
return Boolean(agent && INVOKABLE_AGENT_STATUSES.has(agent.status));
|
||||
}
|
||||
|
||||
function hasActiveExecutionPath(
|
||||
companyId: string,
|
||||
issueId: string,
|
||||
activeRuns: IssueLivenessExecutionPathInput[],
|
||||
queuedWakeRequests: IssueLivenessExecutionPathInput[],
|
||||
) {
|
||||
return [...activeRuns, ...queuedWakeRequests].some(
|
||||
(entry) => entry.companyId === companyId && entry.issueId === issueId,
|
||||
);
|
||||
}
|
||||
|
||||
function readPrincipalAgentId(principal: unknown): string | null {
|
||||
if (!principal || typeof principal !== "object") return null;
|
||||
const value = principal as Record<string, unknown>;
|
||||
return value.type === "agent" && typeof value.agentId === "string" && value.agentId.length > 0
|
||||
? value.agentId
|
||||
: null;
|
||||
}
|
||||
|
||||
function principalIsResolvableUser(principal: unknown): boolean {
|
||||
if (!principal || typeof principal !== "object") return false;
|
||||
const value = principal as Record<string, unknown>;
|
||||
return value.type === "user" && typeof value.userId === "string" && value.userId.length > 0;
|
||||
}
|
||||
|
||||
function addOwnerCandidate(
|
||||
candidates: IssueLivenessOwnerCandidate[],
|
||||
seen: Set<string>,
|
||||
agentsById: Map<string, IssueLivenessAgentInput>,
|
||||
companyId: string,
|
||||
agentId: string | null | undefined,
|
||||
reason: IssueLivenessOwnerCandidateReason,
|
||||
sourceIssueId: string,
|
||||
) {
|
||||
if (!agentId || seen.has(agentId)) return;
|
||||
const agent = agentsById.get(agentId);
|
||||
if (!agent || agent.companyId !== companyId || !isInvokableAgent(agent)) return;
|
||||
seen.add(agentId);
|
||||
candidates.push({ agentId, reason, sourceIssueId });
|
||||
}
|
||||
|
||||
function addAgentChainCandidates(
|
||||
candidates: IssueLivenessOwnerCandidate[],
|
||||
seen: Set<string>,
|
||||
startAgentId: string | null | undefined,
|
||||
agentsById: Map<string, IssueLivenessAgentInput>,
|
||||
companyId: string,
|
||||
reason: IssueLivenessOwnerCandidateReason,
|
||||
sourceIssueId: string,
|
||||
) {
|
||||
const chainSeen = new Set<string>();
|
||||
let current = startAgentId ? agentsById.get(startAgentId) : null;
|
||||
|
||||
while (current?.reportsTo) {
|
||||
if (chainSeen.has(current.reportsTo)) break;
|
||||
chainSeen.add(current.reportsTo);
|
||||
const manager = agentsById.get(current.reportsTo);
|
||||
if (!manager || manager.companyId !== companyId) break;
|
||||
addOwnerCandidate(candidates, seen, agentsById, companyId, manager.id, reason, sourceIssueId);
|
||||
current = manager;
|
||||
}
|
||||
}
|
||||
|
||||
function orderedInvokableAgents(agents: IssueLivenessAgentInput[], companyId: string) {
|
||||
return agents
|
||||
.filter((agent) => agent.companyId === companyId && isInvokableAgent(agent))
|
||||
.sort((left, right) => left.id.localeCompare(right.id));
|
||||
}
|
||||
|
||||
function ownerCandidatesForRecoveryIssue(
|
||||
issue: IssueLivenessIssueInput,
|
||||
agents: IssueLivenessAgentInput[],
|
||||
agentsById: Map<string, IssueLivenessAgentInput>,
|
||||
options: {
|
||||
includeStalledAssignee?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
const candidates: IssueLivenessOwnerCandidate[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
if (options.includeStalledAssignee && issue.status !== "cancelled" && issue.status !== "done") {
|
||||
addOwnerCandidate(
|
||||
candidates,
|
||||
seen,
|
||||
agentsById,
|
||||
issue.companyId,
|
||||
issue.assigneeAgentId,
|
||||
"stalled_blocker_assignee",
|
||||
issue.id,
|
||||
);
|
||||
}
|
||||
|
||||
addAgentChainCandidates(
|
||||
candidates,
|
||||
seen,
|
||||
issue.assigneeAgentId,
|
||||
agentsById,
|
||||
issue.companyId,
|
||||
"assignee_reporting_chain",
|
||||
issue.id,
|
||||
);
|
||||
addAgentChainCandidates(
|
||||
candidates,
|
||||
seen,
|
||||
issue.createdByAgentId,
|
||||
agentsById,
|
||||
issue.companyId,
|
||||
"creator_reporting_chain",
|
||||
issue.id,
|
||||
);
|
||||
|
||||
const invokableAgents = orderedInvokableAgents(agents, issue.companyId);
|
||||
for (const agent of invokableAgents) {
|
||||
if (!agent.reportsTo) {
|
||||
addOwnerCandidate(candidates, seen, agentsById, issue.companyId, agent.id, "root_agent", issue.id);
|
||||
}
|
||||
}
|
||||
for (const agent of invokableAgents) {
|
||||
addOwnerCandidate(
|
||||
candidates,
|
||||
seen,
|
||||
agentsById,
|
||||
issue.companyId,
|
||||
agent.id,
|
||||
"ordered_invokable_fallback",
|
||||
issue.id,
|
||||
);
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function incidentKey(input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
state: IssueLivenessState;
|
||||
blockerIssueId?: string | null;
|
||||
participantAgentId?: string | null;
|
||||
}) {
|
||||
return buildIssueGraphLivenessIncidentKey(input);
|
||||
}
|
||||
|
||||
function finding(input: {
|
||||
issue: IssueLivenessIssueInput;
|
||||
state: IssueLivenessState;
|
||||
severity?: IssueLivenessSeverity;
|
||||
reason: string;
|
||||
dependencyPath: IssueLivenessIssueInput[];
|
||||
recoveryIssue: IssueLivenessIssueInput;
|
||||
recommendedOwnerCandidateAgentIds: string[];
|
||||
recommendedOwnerCandidates: IssueLivenessOwnerCandidate[];
|
||||
recommendedAction: string;
|
||||
blockerIssueId?: string | null;
|
||||
participantAgentId?: string | null;
|
||||
}): IssueLivenessFinding {
|
||||
return {
|
||||
issueId: input.issue.id,
|
||||
companyId: input.issue.companyId,
|
||||
identifier: input.issue.identifier,
|
||||
state: input.state,
|
||||
severity: input.severity ?? "critical",
|
||||
reason: input.reason,
|
||||
dependencyPath: input.dependencyPath.map(pathEntry),
|
||||
recoveryIssueId: input.recoveryIssue.id,
|
||||
recommendedOwnerAgentId: input.recommendedOwnerCandidateAgentIds[0] ?? null,
|
||||
recommendedOwnerCandidateAgentIds: input.recommendedOwnerCandidateAgentIds,
|
||||
recommendedOwnerCandidates: input.recommendedOwnerCandidates,
|
||||
recommendedAction: input.recommendedAction,
|
||||
incidentKey: incidentKey({
|
||||
companyId: input.issue.companyId,
|
||||
issueId: input.issue.id,
|
||||
state: input.state,
|
||||
blockerIssueId: input.blockerIssueId,
|
||||
participantAgentId: input.participantAgentId,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function classifyIssueGraphLiveness(input: IssueGraphLivenessInput): IssueLivenessFinding[] {
|
||||
const issuesById = new Map(input.issues.map((issue) => [issue.id, issue]));
|
||||
const agentsById = new Map(input.agents.map((agent) => [agent.id, agent]));
|
||||
const blockersByBlockedIssueId = new Map<string, IssueLivenessRelationInput[]>();
|
||||
const findings: IssueLivenessFinding[] = [];
|
||||
const activeRuns = input.activeRuns ?? [];
|
||||
const queuedWakeRequests = input.queuedWakeRequests ?? [];
|
||||
|
||||
for (const relation of input.relations) {
|
||||
const list = blockersByBlockedIssueId.get(relation.blockedIssueId) ?? [];
|
||||
list.push(relation);
|
||||
blockersByBlockedIssueId.set(relation.blockedIssueId, list);
|
||||
}
|
||||
|
||||
for (const issue of input.issues) {
|
||||
if (issue.status === "blocked") {
|
||||
const relations = blockersByBlockedIssueId.get(issue.id) ?? [];
|
||||
for (const relation of relations) {
|
||||
if (relation.companyId !== issue.companyId) continue;
|
||||
const blocker = issuesById.get(relation.blockerIssueId);
|
||||
if (!blocker || blocker.companyId !== issue.companyId || blocker.status === "done") continue;
|
||||
const ownerCandidates = ownerCandidatesForRecoveryIssue(blocker, input.agents, agentsById, {
|
||||
includeStalledAssignee: true,
|
||||
});
|
||||
|
||||
if (blocker.status === "cancelled") {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "blocked_by_cancelled_issue",
|
||||
reason: `${issueLabel(issue)} is still blocked by cancelled issue ${issueLabel(blocker)}.`,
|
||||
dependencyPath: [issue, blocker],
|
||||
recoveryIssue: blocker,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Inspect ${issueLabel(blocker)} and either remove it from ${issueLabel(issue)}'s blockers or replace it with an actionable unblock issue.`,
|
||||
blockerIssueId: blocker.id,
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!blocker.assigneeAgentId && !blocker.assigneeUserId) {
|
||||
if (hasActiveExecutionPath(issue.companyId, blocker.id, activeRuns, queuedWakeRequests)) continue;
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "blocked_by_unassigned_issue",
|
||||
reason: `${issueLabel(issue)} is blocked by unassigned issue ${issueLabel(blocker)} with no user owner.`,
|
||||
dependencyPath: [issue, blocker],
|
||||
recoveryIssue: blocker,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Assign ${issueLabel(blocker)} to an owner who can complete it, or remove it from ${issueLabel(issue)}'s blockers if it is no longer required.`,
|
||||
blockerIssueId: blocker.id,
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!blocker.assigneeAgentId) continue;
|
||||
if (hasActiveExecutionPath(issue.companyId, blocker.id, activeRuns, queuedWakeRequests)) continue;
|
||||
|
||||
const blockerAgent = agentsById.get(blocker.assigneeAgentId);
|
||||
if (!blockerAgent || blockerAgent.companyId !== issue.companyId || BLOCKING_AGENT_STATUSES.has(blockerAgent.status)) {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "blocked_by_uninvokable_assignee",
|
||||
reason: blockerAgent
|
||||
? `${issueLabel(issue)} is blocked by ${issueLabel(blocker)}, but its assignee is ${blockerAgent.status}.`
|
||||
: `${issueLabel(issue)} is blocked by ${issueLabel(blocker)}, but its assignee no longer exists.`,
|
||||
dependencyPath: [issue, blocker],
|
||||
recoveryIssue: blocker,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Review ${issueLabel(blocker)} and assign it to an active owner or replace the blocker with an actionable issue.`,
|
||||
blockerIssueId: blocker.id,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (issue.status !== "in_review" || !issue.executionState) continue;
|
||||
const ownerCandidates = ownerCandidatesForRecoveryIssue(issue, input.agents, agentsById);
|
||||
const participant = issue.executionState.currentParticipant;
|
||||
const participantAgentId = readPrincipalAgentId(participant);
|
||||
if (participantAgentId) {
|
||||
const participantAgent = agentsById.get(participantAgentId);
|
||||
if (!isInvokableAgent(participantAgent) || participantAgent?.companyId !== issue.companyId) {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "invalid_review_participant",
|
||||
reason: participantAgent
|
||||
? `${issueLabel(issue)} is in review, but current participant agent is ${participantAgent.status}.`
|
||||
: `${issueLabel(issue)} is in review, but current participant agent cannot be resolved.`,
|
||||
dependencyPath: [issue],
|
||||
recoveryIssue: issue,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Repair ${issueLabel(issue)}'s review participant or return the issue to an active assignee with a clear change request.`,
|
||||
participantAgentId,
|
||||
}));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!principalIsResolvableUser(participant)) {
|
||||
findings.push(finding({
|
||||
issue,
|
||||
state: "invalid_review_participant",
|
||||
reason: `${issueLabel(issue)} is in review, but its current participant cannot be resolved.`,
|
||||
dependencyPath: [issue],
|
||||
recoveryIssue: issue,
|
||||
recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId),
|
||||
recommendedOwnerCandidates: ownerCandidates,
|
||||
recommendedAction:
|
||||
`Repair ${issueLabel(issue)}'s review participant or return the issue to an active assignee with a clear change request.`,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
56
server/src/services/recovery/origins.ts
Normal file
56
server/src/services/recovery/origins.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
export const RECOVERY_ORIGIN_KINDS = {
|
||||
issueGraphLivenessEscalation: "harness_liveness_escalation",
|
||||
strandedIssueRecovery: "stranded_issue_recovery",
|
||||
staleActiveRunEvaluation: "stale_active_run_evaluation",
|
||||
} as const;
|
||||
|
||||
export const RECOVERY_REASON_KINDS = {
|
||||
runLivenessContinuation: "run_liveness_continuation",
|
||||
} as const;
|
||||
|
||||
export const RECOVERY_KEY_PREFIXES = {
|
||||
issueGraphLivenessIncident: "harness_liveness",
|
||||
issueGraphLivenessLeaf: "harness_liveness_leaf",
|
||||
} as const;
|
||||
|
||||
export type RecoveryOriginKind = typeof RECOVERY_ORIGIN_KINDS[keyof typeof RECOVERY_ORIGIN_KINDS];
|
||||
export type RecoveryReasonKind = typeof RECOVERY_REASON_KINDS[keyof typeof RECOVERY_REASON_KINDS];
|
||||
export type RecoveryKeyPrefix = typeof RECOVERY_KEY_PREFIXES[keyof typeof RECOVERY_KEY_PREFIXES];
|
||||
|
||||
export function buildIssueGraphLivenessIncidentKey(input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
state: string;
|
||||
blockerIssueId?: string | null;
|
||||
participantAgentId?: string | null;
|
||||
}) {
|
||||
return [
|
||||
RECOVERY_KEY_PREFIXES.issueGraphLivenessIncident,
|
||||
input.companyId,
|
||||
input.issueId,
|
||||
input.state,
|
||||
input.blockerIssueId ?? input.participantAgentId ?? "none",
|
||||
].join(":");
|
||||
}
|
||||
|
||||
export function parseIssueGraphLivenessIncidentKey(incidentKey: string | null | undefined) {
|
||||
if (!incidentKey) return null;
|
||||
const parts = incidentKey.split(":");
|
||||
if (parts.length !== 5 || parts[0] !== RECOVERY_KEY_PREFIXES.issueGraphLivenessIncident) return null;
|
||||
const [, companyId, issueId, state, leafIssueId] = parts;
|
||||
if (!companyId || !issueId || !state || !leafIssueId) return null;
|
||||
return { companyId, issueId, state, leafIssueId };
|
||||
}
|
||||
|
||||
export function buildIssueGraphLivenessLeafKey(input: {
|
||||
companyId: string;
|
||||
state: string;
|
||||
leafIssueId: string;
|
||||
}) {
|
||||
return [
|
||||
RECOVERY_KEY_PREFIXES.issueGraphLivenessLeaf,
|
||||
input.companyId,
|
||||
input.state,
|
||||
input.leafIssueId,
|
||||
].join(":");
|
||||
}
|
||||
14
server/src/services/recovery/pause-hold-guard.ts
Normal file
14
server/src/services/recovery/pause-hold-guard.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import type { Db } from "@paperclipai/db";
|
||||
import { issueTreeControlService } from "../issue-tree-control.js";
|
||||
|
||||
type IssueTreeControlService = ReturnType<typeof issueTreeControlService>;
|
||||
|
||||
export async function isAutomaticRecoverySuppressedByPauseHold(
|
||||
db: Db,
|
||||
companyId: string,
|
||||
issueId: string,
|
||||
treeControlSvc: IssueTreeControlService = issueTreeControlService(db),
|
||||
) {
|
||||
const activePauseHold = await treeControlSvc.getActivePauseHoldGate(companyId, issueId);
|
||||
return Boolean(activePauseHold);
|
||||
}
|
||||
189
server/src/services/recovery/run-liveness-continuations.ts
Normal file
189
server/src/services/recovery/run-liveness-continuations.ts
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agentWakeupRequests, agents, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import type { RunLivenessState } from "@paperclipai/shared";
|
||||
import { RECOVERY_REASON_KINDS } from "./origins.js";
|
||||
|
||||
export const RUN_LIVENESS_CONTINUATION_REASON = RECOVERY_REASON_KINDS.runLivenessContinuation;
|
||||
export const DEFAULT_MAX_LIVENESS_CONTINUATION_ATTEMPTS = 2;
|
||||
|
||||
const ACTIONABLE_LIVENESS_STATES = new Set<RunLivenessState>(["plan_only", "empty_response"]);
|
||||
const CONTINUATION_ACTIVE_ISSUE_STATUSES = new Set(["todo", "in_progress"]);
|
||||
// A prior adapter error should not permanently suppress bounded liveness
|
||||
// continuations; the max-attempt/idempotency guards prevent unbounded retries.
|
||||
const CONTINUATION_AGENT_STATUSES = new Set(["active", "idle", "running", "error"]);
|
||||
const IDEMPOTENT_WAKE_STATUSES = ["queued", "deferred_issue_execution", "completed"];
|
||||
|
||||
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
|
||||
type IssueRow = Pick<
|
||||
typeof issues.$inferSelect,
|
||||
"id" | "companyId" | "identifier" | "title" | "status" | "assigneeAgentId" | "executionState" | "projectId"
|
||||
>;
|
||||
type AgentRow = Pick<typeof agents.$inferSelect, "id" | "companyId" | "status">;
|
||||
|
||||
export type RunContinuationDecision =
|
||||
| {
|
||||
kind: "enqueue";
|
||||
nextAttempt: number;
|
||||
idempotencyKey: string;
|
||||
payload: Record<string, unknown>;
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
}
|
||||
| {
|
||||
kind: "exhausted";
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
comment: string;
|
||||
}
|
||||
| {
|
||||
kind: "skip";
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export function readContinuationAttempt(value: unknown): number {
|
||||
const numeric = typeof value === "number" ? value : Number.parseInt(String(value ?? ""), 10);
|
||||
return Number.isFinite(numeric) && numeric > 0 ? Math.floor(numeric) : 0;
|
||||
}
|
||||
|
||||
export function buildRunLivenessContinuationIdempotencyKey(input: {
|
||||
issueId: string;
|
||||
sourceRunId: string;
|
||||
livenessState: RunLivenessState;
|
||||
nextAttempt: number;
|
||||
}) {
|
||||
return [
|
||||
RUN_LIVENESS_CONTINUATION_REASON,
|
||||
input.issueId,
|
||||
input.sourceRunId,
|
||||
input.livenessState,
|
||||
String(input.nextAttempt),
|
||||
].join(":");
|
||||
}
|
||||
|
||||
export async function findExistingRunLivenessContinuationWake(
|
||||
db: Db,
|
||||
input: {
|
||||
companyId: string;
|
||||
idempotencyKey: string;
|
||||
},
|
||||
) {
|
||||
return db
|
||||
.select({ id: agentWakeupRequests.id, status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, input.companyId),
|
||||
eq(agentWakeupRequests.idempotencyKey, input.idempotencyKey),
|
||||
inArray(agentWakeupRequests.status, IDEMPOTENT_WAKE_STATUSES),
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
}
|
||||
|
||||
export function decideRunLivenessContinuation(input: {
|
||||
run: HeartbeatRunRow;
|
||||
issue: IssueRow | null;
|
||||
agent: AgentRow | null;
|
||||
livenessState: RunLivenessState | null;
|
||||
livenessReason: string | null;
|
||||
nextAction: string | null;
|
||||
budgetBlocked: boolean;
|
||||
idempotentWakeExists: boolean;
|
||||
maxAttempts?: number;
|
||||
}): RunContinuationDecision {
|
||||
const {
|
||||
run,
|
||||
issue,
|
||||
agent,
|
||||
livenessState,
|
||||
livenessReason,
|
||||
nextAction,
|
||||
budgetBlocked,
|
||||
idempotentWakeExists,
|
||||
} = input;
|
||||
const maxAttempts = input.maxAttempts ?? DEFAULT_MAX_LIVENESS_CONTINUATION_ATTEMPTS;
|
||||
|
||||
if (!livenessState || !ACTIONABLE_LIVENESS_STATES.has(livenessState)) {
|
||||
return { kind: "skip", reason: "liveness state is not actionable for continuation" };
|
||||
}
|
||||
if (!issue) return { kind: "skip", reason: "issue not found" };
|
||||
if (!agent) return { kind: "skip", reason: "agent not found" };
|
||||
if (issue.companyId !== run.companyId || agent.companyId !== run.companyId) {
|
||||
return { kind: "skip", reason: "company scope mismatch" };
|
||||
}
|
||||
if (issue.assigneeAgentId !== run.agentId) {
|
||||
return { kind: "skip", reason: "issue is no longer assigned to the source run agent" };
|
||||
}
|
||||
if (!CONTINUATION_ACTIVE_ISSUE_STATUSES.has(issue.status)) {
|
||||
return { kind: "skip", reason: `issue status ${issue.status} is not continuable` };
|
||||
}
|
||||
if (issue.executionState) {
|
||||
return { kind: "skip", reason: "issue is blocked by execution policy state" };
|
||||
}
|
||||
if (!CONTINUATION_AGENT_STATUSES.has(agent.status)) {
|
||||
return { kind: "skip", reason: `agent status ${agent.status} is not invokable` };
|
||||
}
|
||||
if (budgetBlocked) {
|
||||
return { kind: "skip", reason: "budget hard stop blocks continuation" };
|
||||
}
|
||||
|
||||
const currentAttempt = readContinuationAttempt(run.continuationAttempt);
|
||||
if (currentAttempt >= maxAttempts) {
|
||||
return {
|
||||
kind: "exhausted",
|
||||
attempt: currentAttempt,
|
||||
maxAttempts,
|
||||
comment: [
|
||||
"Bounded liveness continuation exhausted",
|
||||
"",
|
||||
`- Last liveness state: \`${livenessState}\``,
|
||||
`- Attempts used: ${currentAttempt}/${maxAttempts}`,
|
||||
`- Reason: ${livenessReason ?? "Run ended without concrete progress"}`,
|
||||
"- Next action: a human or manager should inspect the run and either clarify the task, mark it blocked, or assign a concrete follow-up.",
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
const nextAttempt = currentAttempt + 1;
|
||||
const idempotencyKey = buildRunLivenessContinuationIdempotencyKey({
|
||||
issueId: issue.id,
|
||||
sourceRunId: run.id,
|
||||
livenessState,
|
||||
nextAttempt,
|
||||
});
|
||||
if (idempotentWakeExists) {
|
||||
return { kind: "skip", reason: "continuation wake already exists for this source run and attempt" };
|
||||
}
|
||||
|
||||
const payload = {
|
||||
issueId: issue.id,
|
||||
sourceRunId: run.id,
|
||||
livenessState,
|
||||
livenessReason,
|
||||
continuationAttempt: nextAttempt,
|
||||
maxContinuationAttempts: maxAttempts,
|
||||
instruction:
|
||||
nextAction ??
|
||||
"The previous run ended without concrete progress. Take the first concrete action now or mark the issue blocked with a specific unblock request.",
|
||||
};
|
||||
|
||||
return {
|
||||
kind: "enqueue",
|
||||
nextAttempt,
|
||||
idempotencyKey,
|
||||
payload,
|
||||
contextSnapshot: {
|
||||
issueId: issue.id,
|
||||
taskId: issue.id,
|
||||
taskKey: issue.id,
|
||||
wakeReason: RUN_LIVENESS_CONTINUATION_REASON,
|
||||
livenessContinuationAttempt: nextAttempt,
|
||||
livenessContinuationMaxAttempts: maxAttempts,
|
||||
livenessContinuationSourceRunId: run.id,
|
||||
livenessContinuationState: livenessState,
|
||||
livenessContinuationReason: livenessReason,
|
||||
livenessContinuationInstruction: payload.instruction,
|
||||
},
|
||||
};
|
||||
}
|
||||
2143
server/src/services/recovery/service.ts
Normal file
2143
server/src/services/recovery/service.ts
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,188 +1,11 @@
|
|||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agentWakeupRequests, agents, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import type { RunLivenessState } from "@paperclipai/shared";
|
||||
|
||||
export const RUN_LIVENESS_CONTINUATION_REASON = "run_liveness_continuation";
|
||||
export const DEFAULT_MAX_LIVENESS_CONTINUATION_ATTEMPTS = 2;
|
||||
|
||||
const ACTIONABLE_LIVENESS_STATES = new Set<RunLivenessState>(["plan_only", "empty_response"]);
|
||||
const CONTINUATION_ACTIVE_ISSUE_STATUSES = new Set(["todo", "in_progress"]);
|
||||
// A prior adapter error should not permanently suppress bounded liveness
|
||||
// continuations; the max-attempt/idempotency guards prevent unbounded retries.
|
||||
const CONTINUATION_AGENT_STATUSES = new Set(["active", "idle", "running", "error"]);
|
||||
const IDEMPOTENT_WAKE_STATUSES = ["queued", "deferred_issue_execution", "completed"];
|
||||
|
||||
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
|
||||
type IssueRow = Pick<
|
||||
typeof issues.$inferSelect,
|
||||
"id" | "companyId" | "identifier" | "title" | "status" | "assigneeAgentId" | "executionState" | "projectId"
|
||||
>;
|
||||
type AgentRow = Pick<typeof agents.$inferSelect, "id" | "companyId" | "status">;
|
||||
|
||||
export type RunContinuationDecision =
|
||||
| {
|
||||
kind: "enqueue";
|
||||
nextAttempt: number;
|
||||
idempotencyKey: string;
|
||||
payload: Record<string, unknown>;
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
}
|
||||
| {
|
||||
kind: "exhausted";
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
comment: string;
|
||||
}
|
||||
| {
|
||||
kind: "skip";
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export function readContinuationAttempt(value: unknown): number {
|
||||
const numeric = typeof value === "number" ? value : Number.parseInt(String(value ?? ""), 10);
|
||||
return Number.isFinite(numeric) && numeric > 0 ? Math.floor(numeric) : 0;
|
||||
}
|
||||
|
||||
export function buildRunLivenessContinuationIdempotencyKey(input: {
|
||||
issueId: string;
|
||||
sourceRunId: string;
|
||||
livenessState: RunLivenessState;
|
||||
nextAttempt: number;
|
||||
}) {
|
||||
return [
|
||||
"run_liveness_continuation",
|
||||
input.issueId,
|
||||
input.sourceRunId,
|
||||
input.livenessState,
|
||||
String(input.nextAttempt),
|
||||
].join(":");
|
||||
}
|
||||
|
||||
export async function findExistingRunLivenessContinuationWake(
|
||||
db: Db,
|
||||
input: {
|
||||
companyId: string;
|
||||
idempotencyKey: string;
|
||||
},
|
||||
) {
|
||||
return db
|
||||
.select({ id: agentWakeupRequests.id, status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, input.companyId),
|
||||
eq(agentWakeupRequests.idempotencyKey, input.idempotencyKey),
|
||||
inArray(agentWakeupRequests.status, IDEMPOTENT_WAKE_STATUSES),
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
}
|
||||
|
||||
export function decideRunLivenessContinuation(input: {
|
||||
run: HeartbeatRunRow;
|
||||
issue: IssueRow | null;
|
||||
agent: AgentRow | null;
|
||||
livenessState: RunLivenessState | null;
|
||||
livenessReason: string | null;
|
||||
nextAction: string | null;
|
||||
budgetBlocked: boolean;
|
||||
idempotentWakeExists: boolean;
|
||||
maxAttempts?: number;
|
||||
}): RunContinuationDecision {
|
||||
const {
|
||||
run,
|
||||
issue,
|
||||
agent,
|
||||
livenessState,
|
||||
livenessReason,
|
||||
nextAction,
|
||||
budgetBlocked,
|
||||
idempotentWakeExists,
|
||||
} = input;
|
||||
const maxAttempts = input.maxAttempts ?? DEFAULT_MAX_LIVENESS_CONTINUATION_ATTEMPTS;
|
||||
|
||||
if (!livenessState || !ACTIONABLE_LIVENESS_STATES.has(livenessState)) {
|
||||
return { kind: "skip", reason: "liveness state is not actionable for continuation" };
|
||||
}
|
||||
if (!issue) return { kind: "skip", reason: "issue not found" };
|
||||
if (!agent) return { kind: "skip", reason: "agent not found" };
|
||||
if (issue.companyId !== run.companyId || agent.companyId !== run.companyId) {
|
||||
return { kind: "skip", reason: "company scope mismatch" };
|
||||
}
|
||||
if (issue.assigneeAgentId !== run.agentId) {
|
||||
return { kind: "skip", reason: "issue is no longer assigned to the source run agent" };
|
||||
}
|
||||
if (!CONTINUATION_ACTIVE_ISSUE_STATUSES.has(issue.status)) {
|
||||
return { kind: "skip", reason: `issue status ${issue.status} is not continuable` };
|
||||
}
|
||||
if (issue.executionState) {
|
||||
return { kind: "skip", reason: "issue is blocked by execution policy state" };
|
||||
}
|
||||
if (!CONTINUATION_AGENT_STATUSES.has(agent.status)) {
|
||||
return { kind: "skip", reason: `agent status ${agent.status} is not invokable` };
|
||||
}
|
||||
if (budgetBlocked) {
|
||||
return { kind: "skip", reason: "budget hard stop blocks continuation" };
|
||||
}
|
||||
|
||||
const currentAttempt = readContinuationAttempt(run.continuationAttempt);
|
||||
if (currentAttempt >= maxAttempts) {
|
||||
return {
|
||||
kind: "exhausted",
|
||||
attempt: currentAttempt,
|
||||
maxAttempts,
|
||||
comment: [
|
||||
"Bounded liveness continuation exhausted",
|
||||
"",
|
||||
`- Last liveness state: \`${livenessState}\``,
|
||||
`- Attempts used: ${currentAttempt}/${maxAttempts}`,
|
||||
`- Reason: ${livenessReason ?? "Run ended without concrete progress"}`,
|
||||
"- Next action: a human or manager should inspect the run and either clarify the task, mark it blocked, or assign a concrete follow-up.",
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
const nextAttempt = currentAttempt + 1;
|
||||
const idempotencyKey = buildRunLivenessContinuationIdempotencyKey({
|
||||
issueId: issue.id,
|
||||
sourceRunId: run.id,
|
||||
livenessState,
|
||||
nextAttempt,
|
||||
});
|
||||
if (idempotentWakeExists) {
|
||||
return { kind: "skip", reason: "continuation wake already exists for this source run and attempt" };
|
||||
}
|
||||
|
||||
const payload = {
|
||||
issueId: issue.id,
|
||||
sourceRunId: run.id,
|
||||
livenessState,
|
||||
livenessReason,
|
||||
continuationAttempt: nextAttempt,
|
||||
maxContinuationAttempts: maxAttempts,
|
||||
instruction:
|
||||
nextAction ??
|
||||
"The previous run ended without concrete progress. Take the first concrete action now or mark the issue blocked with a specific unblock request.",
|
||||
};
|
||||
|
||||
return {
|
||||
kind: "enqueue",
|
||||
nextAttempt,
|
||||
idempotencyKey,
|
||||
payload,
|
||||
contextSnapshot: {
|
||||
issueId: issue.id,
|
||||
taskId: issue.id,
|
||||
taskKey: issue.id,
|
||||
wakeReason: RUN_LIVENESS_CONTINUATION_REASON,
|
||||
livenessContinuationAttempt: nextAttempt,
|
||||
livenessContinuationMaxAttempts: maxAttempts,
|
||||
livenessContinuationSourceRunId: run.id,
|
||||
livenessContinuationState: livenessState,
|
||||
livenessContinuationReason: livenessReason,
|
||||
livenessContinuationInstruction: payload.instruction,
|
||||
},
|
||||
};
|
||||
}
|
||||
export {
|
||||
DEFAULT_MAX_LIVENESS_CONTINUATION_ATTEMPTS,
|
||||
RUN_LIVENESS_CONTINUATION_REASON,
|
||||
buildRunLivenessContinuationIdempotencyKey,
|
||||
decideRunLivenessContinuation,
|
||||
findExistingRunLivenessContinuationWake,
|
||||
readContinuationAttempt,
|
||||
} from "./recovery/run-liveness-continuations.js";
|
||||
export type {
|
||||
RunContinuationDecision,
|
||||
} from "./recovery/run-liveness-continuations.js";
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
import type { HeartbeatRunStatus, IssueStatus, RunLivenessState } from "@paperclipai/shared";
|
||||
|
||||
export type RunLivenessActionability =
|
||||
| "runnable"
|
||||
| "manager_review"
|
||||
| "blocked_external"
|
||||
| "approval_required"
|
||||
| "unknown";
|
||||
|
||||
export interface RunLivenessIssueInput {
|
||||
status: IssueStatus | string;
|
||||
title: string;
|
||||
|
|
@ -21,6 +28,8 @@ export interface RunLivenessClassificationInput {
|
|||
runStatus: HeartbeatRunStatus | string;
|
||||
issue: RunLivenessIssueInput | null;
|
||||
resultJson?: Record<string, unknown> | null;
|
||||
issueCommentBodies?: string[] | null;
|
||||
continuationSummaryBody?: string | null;
|
||||
stdoutExcerpt?: string | null;
|
||||
stderrExcerpt?: string | null;
|
||||
error?: string | null;
|
||||
|
|
@ -35,6 +44,7 @@ export interface RunLivenessClassification {
|
|||
continuationAttempt: number;
|
||||
lastUsefulActionAt: Date | null;
|
||||
nextAction: string | null;
|
||||
actionability: RunLivenessActionability;
|
||||
}
|
||||
|
||||
const DEFAULT_EVIDENCE: RunLivenessEvidenceInput = {
|
||||
|
|
@ -54,6 +64,14 @@ const NEXT_STEPS_RE = /^\s*(?:next steps?|plan)\s*:/im;
|
|||
const BLOCKER_RE =
|
||||
/\b(?:blocked|can't proceed|cannot proceed|unable to proceed|waiting on|need(?:s|ed)? .{0,80}\b(?:approval|access|credential|credentials|secret|api key|token|input|clarification)|requires? .{0,80}\b(?:approval|access|credential|credentials|secret|api key|token|input|clarification))\b/i;
|
||||
const NEGATED_BLOCKER_RE = /\b(?:not blocked|no blocker|no blockers|unblocked)\b/i;
|
||||
const APPROVAL_REQUIRED_RE =
|
||||
/\b(?:approval required|requires? .{0,80}\bapproval|need(?:s|ed)? .{0,80}\bapproval|waiting on .{0,80}\bapproval|pending approval|board approval|human approval|user approval|operator approval)\b/i;
|
||||
const EXTERNAL_BLOCKER_RE =
|
||||
/\b(?:can't proceed|cannot proceed|unable to proceed|waiting on|blocked by|blocked on|need(?:s|ed)?|requires?) .{0,120}\b(?:access|credential|credentials|secret|secrets|api key|token|password|login|account|permission|permissions|input|clarification)\b/i;
|
||||
const MANAGER_REVIEW_RE =
|
||||
/\b(?:manager review|human review|manual review|security review|escalate|production deploy|deploy(?:ing)? to production|deploy(?:ing)? to prod|prod deploy|production access|rotate .{0,40}\b(?:secret|key|token)|delete .{0,40}\bproduction|security-sensitive|credentialed operation|budget-sensitive|cost approval|spend approval)\b/i;
|
||||
const RUNNABLE_RE =
|
||||
/\b(?:(?:run|rerun|execute)\s+(?:pnpm|npm|yarn|bun|vitest|jest|pytest|cargo|go test|curl|tests?|typecheck|build|lint|package|verification)|(?:inspect|check|review|look|investigate|analy[sz]e|open|read|start|begin|continue|implement|fix|test|update|create|add|write|verify|validate|report)\b)/i;
|
||||
const PLAN_TASK_TITLE_RE = /\b(?:plan|planning|analysis|investigation|research|report|proposal|design doc|write-?up)\b/i;
|
||||
const PLAN_TASK_DESCRIPTION_RE =
|
||||
/\b(?:create|write|produce|draft|update|revise|prepare)\s+(?:a\s+|the\s+)?(?:plan|analysis|investigation|research report|report|proposal|design doc|write-?up)\b/i;
|
||||
|
|
@ -76,12 +94,22 @@ function readText(value: unknown): string | null {
|
|||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
function resultText(resultJson: Record<string, unknown> | null | undefined) {
|
||||
function resultFinalText(resultJson: Record<string, unknown> | null | undefined) {
|
||||
if (!resultJson) return "";
|
||||
return [
|
||||
readText(resultJson.nextAction),
|
||||
readText(resultJson.summary),
|
||||
readText(resultJson.result),
|
||||
readText(resultJson.message),
|
||||
readText(resultJson.error),
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function resultRawText(resultJson: Record<string, unknown> | null | undefined) {
|
||||
if (!resultJson) return "";
|
||||
return [
|
||||
readText(resultJson.stdout),
|
||||
readText(resultJson.stderr),
|
||||
]
|
||||
|
|
@ -89,16 +117,34 @@ function resultText(resultJson: Record<string, unknown> | null | undefined) {
|
|||
.join("\n");
|
||||
}
|
||||
|
||||
function combinedOutput(input: RunLivenessClassificationInput) {
|
||||
function highSignalSources(input: RunLivenessClassificationInput) {
|
||||
return [
|
||||
resultText(input.resultJson),
|
||||
...(input.issueCommentBodies ?? []).map(readText),
|
||||
readText(resultFinalText(input.resultJson)),
|
||||
readText(input.continuationSummaryBody),
|
||||
].filter((value): value is string => Boolean(value));
|
||||
}
|
||||
|
||||
function rawSources(input: RunLivenessClassificationInput) {
|
||||
return [
|
||||
readText(resultRawText(input.resultJson)),
|
||||
readText(input.stdoutExcerpt),
|
||||
readText(input.stderrExcerpt),
|
||||
readText(input.error),
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join("\n")
|
||||
.trim();
|
||||
.map(stripNoisyTranscriptLines)
|
||||
.filter((value) => value.length > 0);
|
||||
}
|
||||
|
||||
function combinedOutput(input: RunLivenessClassificationInput) {
|
||||
return [...highSignalSources(input), ...rawSources(input)].join("\n").trim();
|
||||
}
|
||||
|
||||
function actionabilityText(input: RunLivenessClassificationInput) {
|
||||
const highSignal = highSignalSources(input).join("\n").trim();
|
||||
if (highSignal) return highSignal;
|
||||
return rawSources(input).join("\n").trim();
|
||||
}
|
||||
|
||||
export function hasUsefulOutput(input: RunLivenessClassificationInput) {
|
||||
|
|
@ -107,15 +153,14 @@ export function hasUsefulOutput(input: RunLivenessClassificationInput) {
|
|||
|
||||
export function declaredBlocker(input: RunLivenessClassificationInput) {
|
||||
if (input.issue?.status === "blocked") return true;
|
||||
const text = combinedOutput(input);
|
||||
if (!text || NEGATED_BLOCKER_RE.test(text)) return false;
|
||||
return BLOCKER_RE.test(text);
|
||||
const actionability = classifyRunActionability(input);
|
||||
return actionability === "blocked_external" || actionability === "approval_required";
|
||||
}
|
||||
|
||||
export function looksLikePlanningOnly(input: RunLivenessClassificationInput) {
|
||||
const text = combinedOutput(input);
|
||||
const text = actionabilityText(input);
|
||||
if (!text) return false;
|
||||
return PLANNING_ONLY_RE.test(text) || NEXT_STEPS_RE.test(text);
|
||||
return PLANNING_ONLY_RE.test(text) || NEXT_STEPS_RE.test(text) || /^\s*next(?: steps?| action)?\s*:/im.test(text);
|
||||
}
|
||||
|
||||
export function isPlanningOrDocumentTask(issue: RunLivenessIssueInput | null | undefined) {
|
||||
|
|
@ -163,20 +208,92 @@ function evidenceReason(evidence: RunLivenessEvidenceInput) {
|
|||
return parts.join(", ");
|
||||
}
|
||||
|
||||
function extractNextAction(input: RunLivenessClassificationInput) {
|
||||
const text = combinedOutput(input);
|
||||
if (!text) return null;
|
||||
const line = text
|
||||
function stripMarkdownListPrefix(line: string) {
|
||||
return line.replace(/^\s*(?:[-*]|\d+\.)\s+/, "").trim();
|
||||
}
|
||||
|
||||
function isNoisyTranscriptLine(line: string) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) return true;
|
||||
return (
|
||||
/^(?:command|status|exit_code|tool|tool_call|tool_result|stdout|stderr|event|payload|session|cwd|ref_id)\s*:/i.test(trimmed) ||
|
||||
/^(?:\{|\[).{0,80}(?:tool|event|stdout|stderr|cmd|command|payload)/i.test(trimmed) ||
|
||||
/^\$?\s*(?:rg|sed|cat|ls|git|pnpm|npm|yarn|curl|node|python)\b/i.test(trimmed)
|
||||
);
|
||||
}
|
||||
|
||||
function stripNoisyTranscriptLines(text: string) {
|
||||
return text
|
||||
.split(/\r?\n/)
|
||||
.map((entry) => entry.trim())
|
||||
.find((entry) => PLANNING_ONLY_RE.test(entry) || /^next(?: steps?| action)?\s*:/i.test(entry));
|
||||
if (!line) return null;
|
||||
return line.length <= 500 ? line : `${line.slice(0, 497)}...`;
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => !isNoisyTranscriptLine(line))
|
||||
.join("\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function nextNonNoiseLine(lines: string[], startIndex: number) {
|
||||
for (let i = startIndex + 1; i < lines.length; i += 1) {
|
||||
const line = stripMarkdownListPrefix(lines[i] ?? "");
|
||||
if (!line || isNoisyTranscriptLine(line)) continue;
|
||||
return line;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractNextActionFromText(text: string) {
|
||||
const lines = text.split(/\r?\n/).map((entry) => entry.trim());
|
||||
for (let i = 0; i < lines.length; i += 1) {
|
||||
const rawLine = lines[i] ?? "";
|
||||
if (!rawLine || isNoisyTranscriptLine(rawLine)) continue;
|
||||
const line = stripMarkdownListPrefix(rawLine);
|
||||
const labeled = line.match(/^next(?: steps?| action)?\s*:\s*(.*)$/i);
|
||||
if (labeled) {
|
||||
const sameLine = stripMarkdownListPrefix(labeled[1] ?? "");
|
||||
return sameLine || nextNonNoiseLine(lines, i);
|
||||
}
|
||||
if (PLANNING_ONLY_RE.test(line)) return line;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractNextAction(input: RunLivenessClassificationInput) {
|
||||
const structuredNextAction = readText(input.resultJson?.nextAction);
|
||||
const candidates = [
|
||||
...(input.issueCommentBodies ?? []),
|
||||
structuredNextAction ? `Next action: ${structuredNextAction}` : null,
|
||||
resultFinalText(input.resultJson),
|
||||
input.continuationSummaryBody,
|
||||
...rawSources(input),
|
||||
].filter((value): value is string => Boolean(readText(value)));
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const line = extractNextActionFromText(candidate);
|
||||
if (!line) continue;
|
||||
return line.length <= 500 ? line : `${line.slice(0, 497)}...`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function classifyRunActionability(input: RunLivenessClassificationInput): RunLivenessActionability {
|
||||
const text = actionabilityText(input);
|
||||
if (!text) return "unknown";
|
||||
if (NEGATED_BLOCKER_RE.test(text)) {
|
||||
return RUNNABLE_RE.test(text) ? "runnable" : "unknown";
|
||||
}
|
||||
if (APPROVAL_REQUIRED_RE.test(text)) return "approval_required";
|
||||
if (EXTERNAL_BLOCKER_RE.test(text) || BLOCKER_RE.test(text) && /\b(?:credential|secret|api key|token|access|input|clarification)\b/i.test(text)) {
|
||||
return "blocked_external";
|
||||
}
|
||||
if (MANAGER_REVIEW_RE.test(text)) return "manager_review";
|
||||
if (RUNNABLE_RE.test(text)) return "runnable";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
export function classifyRunLiveness(input: RunLivenessClassificationInput): RunLivenessClassification {
|
||||
const evidence = normalizeEvidence(input.evidence);
|
||||
const continuationAttempt = normalizeContinuationAttempt(input.continuationAttempt);
|
||||
const actionability = classifyRunActionability(input);
|
||||
const nextAction = extractNextAction(input);
|
||||
const issueStatus = input.issue?.status ?? null;
|
||||
const usefulOutput = hasUsefulOutput(input);
|
||||
const concreteEvidence = hasConcreteActionEvidence(evidence);
|
||||
|
|
@ -189,6 +306,7 @@ export function classifyRunLiveness(input: RunLivenessClassificationInput): RunL
|
|||
continuationAttempt,
|
||||
lastUsefulActionAt: state === "advanced" || state === "completed" || state === "blocked" ? lastUsefulActionAt : null,
|
||||
nextAction,
|
||||
actionability,
|
||||
});
|
||||
|
||||
if (input.runStatus !== "succeeded") {
|
||||
|
|
@ -200,7 +318,7 @@ export function classifyRunLiveness(input: RunLivenessClassificationInput): RunL
|
|||
}
|
||||
|
||||
if (declaredBlocker(input)) {
|
||||
return output("blocked", issueStatus === "blocked" ? "Issue status is blocked" : "Run output declared a concrete blocker", extractNextAction(input));
|
||||
return output("blocked", issueStatus === "blocked" ? "Issue status is blocked" : "Run output declared a concrete blocker", nextAction);
|
||||
}
|
||||
|
||||
if (!usefulOutput && !concreteEvidence) {
|
||||
|
|
@ -215,12 +333,15 @@ export function classifyRunLiveness(input: RunLivenessClassificationInput): RunL
|
|||
return output("advanced", "Planning/document task produced useful output and is exempt from plan-only classification");
|
||||
}
|
||||
|
||||
if (looksLikePlanningOnly(input)) {
|
||||
return output("plan_only", "Run described future work without concrete action evidence", extractNextAction(input));
|
||||
if (looksLikePlanningOnly(input) || nextAction) {
|
||||
if (actionability === "runnable") {
|
||||
return output("plan_only", "Run described runnable future work without concrete action evidence", nextAction);
|
||||
}
|
||||
return output("needs_followup", "Run described future work that is not safe to auto-continue", nextAction);
|
||||
}
|
||||
|
||||
if (usefulOutput) {
|
||||
return output("needs_followup", "Run produced useful output but no concrete action evidence", extractNextAction(input));
|
||||
return output("needs_followup", "Run produced useful output but no concrete action evidence", nextAction);
|
||||
}
|
||||
|
||||
return output("empty_response", "Run succeeded without useful output");
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ export interface RunLogStore {
|
|||
append(
|
||||
handle: RunLogHandle,
|
||||
event: { stream: "stdout" | "stderr" | "system"; chunk: string; ts: string },
|
||||
): Promise<void>;
|
||||
): Promise<number>;
|
||||
finalize(handle: RunLogHandle): Promise<RunLogFinalizeSummary>;
|
||||
read(handle: RunLogHandle, opts?: RunLogReadOptions): Promise<RunLogReadResult>;
|
||||
}
|
||||
|
|
@ -107,14 +107,16 @@ function createLocalFileRunLogStore(basePath: string): RunLogStore {
|
|||
},
|
||||
|
||||
async append(handle, event) {
|
||||
if (handle.store !== "local_file") return;
|
||||
if (handle.store !== "local_file") return 0;
|
||||
const absPath = resolveWithin(basePath, handle.logRef);
|
||||
const line = JSON.stringify({
|
||||
ts: event.ts,
|
||||
stream: event.stream,
|
||||
chunk: event.chunk,
|
||||
});
|
||||
await fs.appendFile(absPath, `${line}\n`, "utf8");
|
||||
const persisted = `${line}\n`;
|
||||
await fs.appendFile(absPath, persisted, "utf8");
|
||||
return Buffer.byteLength(persisted, "utf8");
|
||||
},
|
||||
|
||||
async finalize(handle) {
|
||||
|
|
@ -153,4 +155,3 @@ export function getRunLogStore() {
|
|||
cachedStore = createLocalFileRunLogStore(basePath);
|
||||
return cachedStore;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue