[codex] Add runtime lifecycle recovery and live issue visibility (#4419)

This commit is contained in:
Dotta 2026-04-24 15:50:32 -05:00 committed by GitHub
parent 9a8d219949
commit 5a0c1979cf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
121 changed files with 9625 additions and 2044 deletions

View file

@ -347,6 +347,198 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
expect(blockedWakeRequestCount).toBeGreaterThanOrEqual(2);
});
it("cancels stale queued runs when issue blockers are still unresolved", async () => {
const companyId = randomUUID();
const agentId = randomUUID();
const blockerId = randomUUID();
const blockedIssueId = randomUUID();
const readyIssueId = randomUUID();
const blockedWakeupRequestId = randomUUID();
const readyWakeupRequestId = randomUUID();
const blockedRunId = randomUUID();
const readyRunId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
id: agentId,
companyId,
name: "QAChecker",
role: "qa",
status: "active",
adapterType: "codex_local",
adapterConfig: {},
runtimeConfig: {
heartbeat: {
wakeOnDemand: true,
maxConcurrentRuns: 2,
},
},
permissions: {},
});
await db.insert(issues).values([
{
id: blockerId,
companyId,
title: "Security review",
status: "blocked",
priority: "high",
},
{
id: blockedIssueId,
companyId,
title: "QA validation",
status: "blocked",
priority: "medium",
assigneeAgentId: agentId,
},
{
id: readyIssueId,
companyId,
title: "Ready QA task",
status: "todo",
priority: "low",
assigneeAgentId: agentId,
},
]);
await db.insert(issueRelations).values({
companyId,
issueId: blockerId,
relatedIssueId: blockedIssueId,
type: "blocks",
});
await db.insert(agentWakeupRequests).values([
{
id: blockedWakeupRequestId,
companyId,
agentId,
source: "automation",
triggerDetail: "system",
reason: "transient_failure_retry",
payload: { issueId: blockedIssueId },
status: "queued",
},
{
id: readyWakeupRequestId,
companyId,
agentId,
source: "assignment",
triggerDetail: "system",
reason: "issue_assigned",
payload: { issueId: readyIssueId },
status: "queued",
},
]);
await db.insert(heartbeatRuns).values([
{
id: blockedRunId,
companyId,
agentId,
invocationSource: "automation",
triggerDetail: "system",
status: "queued",
wakeupRequestId: blockedWakeupRequestId,
contextSnapshot: {
issueId: blockedIssueId,
wakeReason: "transient_failure_retry",
},
},
{
id: readyRunId,
companyId,
agentId,
invocationSource: "assignment",
triggerDetail: "system",
status: "queued",
wakeupRequestId: readyWakeupRequestId,
contextSnapshot: {
issueId: readyIssueId,
wakeReason: "issue_assigned",
},
},
]);
await db
.update(agentWakeupRequests)
.set({ runId: blockedRunId })
.where(eq(agentWakeupRequests.id, blockedWakeupRequestId));
await db
.update(agentWakeupRequests)
.set({ runId: readyRunId })
.where(eq(agentWakeupRequests.id, readyWakeupRequestId));
await db
.update(issues)
.set({
executionRunId: blockedRunId,
executionAgentNameKey: "qa-checker",
executionLockedAt: new Date(),
})
.where(eq(issues.id, blockedIssueId));
await heartbeat.resumeQueuedRuns();
await waitForCondition(async () => {
const run = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, readyRunId))
.then((rows) => rows[0] ?? null);
return run?.status === "succeeded";
});
const [blockedRun, blockedWakeup, blockedIssue, readyRun] = await Promise.all([
db
.select({
status: heartbeatRuns.status,
errorCode: heartbeatRuns.errorCode,
finishedAt: heartbeatRuns.finishedAt,
resultJson: heartbeatRuns.resultJson,
})
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, blockedRunId))
.then((rows) => rows[0] ?? null),
db
.select({
status: agentWakeupRequests.status,
error: agentWakeupRequests.error,
})
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.id, blockedWakeupRequestId))
.then((rows) => rows[0] ?? null),
db
.select({
executionRunId: issues.executionRunId,
executionAgentNameKey: issues.executionAgentNameKey,
executionLockedAt: issues.executionLockedAt,
})
.from(issues)
.where(eq(issues.id, blockedIssueId))
.then((rows) => rows[0] ?? null),
db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, readyRunId))
.then((rows) => rows[0] ?? null),
]);
expect(blockedRun?.status).toBe("cancelled");
expect(blockedRun?.errorCode).toBe("issue_dependencies_blocked");
expect(blockedRun?.finishedAt).toBeTruthy();
expect(blockedRun?.resultJson).toMatchObject({ stopReason: "issue_dependencies_blocked" });
expect(blockedWakeup?.status).toBe("skipped");
expect(blockedWakeup?.error).toContain("dependencies are still blocked");
expect(blockedIssue).toMatchObject({
executionRunId: null,
executionAgentNameKey: null,
executionLockedAt: null,
});
expect(readyRun?.status).toBe("succeeded");
expect(mockAdapterExecute).toHaveBeenCalledTimes(1);
});
it("suppresses normal wakeups while allowing comment interaction wakes under a pause hold", async () => {
const companyId = randomUUID();
const agentId = randomUUID();
@ -425,12 +617,39 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
.then((rows) => rows[0] ?? null);
expect(skippedWake).toMatchObject({ status: "skipped", reason: "issue_tree_hold_active" });
const childCommentId = randomUUID();
await db.insert(issueComments).values({
id: childCommentId,
companyId,
issueId: childIssueId,
authorUserId: "board-user",
body: "Please respond while this hold is active.",
});
const forgedChildCommentWake = await heartbeat.wakeup(agentId, {
source: "on_demand",
triggerDetail: "manual",
reason: "issue_commented",
payload: { issueId: childIssueId, commentId: childCommentId },
requestedByActorType: "agent",
requestedByActorId: agentId,
});
expect(forgedChildCommentWake).toBeNull();
const childCommentWake = await heartbeat.wakeup(agentId, {
source: "automation",
triggerDetail: "system",
reason: "issue_commented",
payload: { issueId: childIssueId, commentId: randomUUID() },
contextSnapshot: { issueId: childIssueId, wakeReason: "issue_commented" },
payload: { issueId: childIssueId, commentId: childCommentId },
requestedByActorType: "user",
requestedByActorId: "board-user",
contextSnapshot: {
issueId: childIssueId,
commentId: childCommentId,
wakeCommentId: childCommentId,
wakeReason: "issue_commented",
source: "issue.comment",
},
});
expect(childCommentWake).not.toBeNull();
@ -494,12 +713,29 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
releasePolicy: { strategy: "manual", note: "full_pause" },
});
const rootCommentId = randomUUID();
await db.insert(issueComments).values({
id: rootCommentId,
companyId,
issueId: rootIssueId,
authorUserId: "board-user",
body: "Please respond while this hold is active.",
});
const rootCommentWake = await heartbeat.wakeup(agentId, {
source: "automation",
triggerDetail: "system",
reason: "issue_commented",
payload: { issueId: rootIssueId, commentId: randomUUID() },
contextSnapshot: { issueId: rootIssueId, wakeReason: "issue_commented" },
payload: { issueId: rootIssueId, commentId: rootCommentId },
requestedByActorType: "user",
requestedByActorId: "board-user",
contextSnapshot: {
issueId: rootIssueId,
commentId: rootCommentId,
wakeCommentId: rootCommentId,
wakeReason: "issue_commented",
source: "issue.comment",
},
});
expect(rootCommentWake).not.toBeNull();