mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-15 10:30:37 +09:00
[codex] Add run liveness continuations (#4083)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies. > - Heartbeat runs are the control-plane record of each agent execution window. > - Long-running local agents can exhaust context or stop while still holding useful next-step state. > - Operators need that stop reason, next action, and continuation path to be durable and visible. > - This pull request adds run liveness metadata, continuation summaries, and UI surfaces for issue run ledgers. > - The benefit is that interrupted or long-running work can resume with clearer context instead of losing the agent's last useful handoff. ## What Changed - Added heartbeat-run liveness fields, continuation attempt tracking, and an idempotent `0058` migration. - Added server services and tests for run liveness, continuation summaries, stop metadata, and activity backfill. - Wired local and HTTP adapters to surface continuation/liveness context through shared adapter utilities. - Added shared constants, validators, and heartbeat types for liveness continuation state. - Added issue-detail UI surfaces for continuation handoffs and the run ledger, with component tests. - Updated agent runtime docs, heartbeat protocol docs, prompt guidance, onboarding assets, and skills instructions to explain continuation behavior. - Addressed Greptile feedback by scoping document evidence by run, excluding system continuation-summary documents from liveness evidence, importing shared liveness types, surfacing hidden ledger run counts, documenting bounded retry behavior, and moving run-ledger liveness backfill off the request path. ## Verification - `pnpm exec vitest run packages/adapter-utils/src/server-utils.test.ts server/src/__tests__/run-continuations.test.ts server/src/__tests__/run-liveness.test.ts server/src/__tests__/activity-service.test.ts server/src/__tests__/documents-service.test.ts server/src/__tests__/issue-continuation-summary.test.ts server/src/services/heartbeat-stop-metadata.test.ts ui/src/components/IssueRunLedger.test.tsx ui/src/components/IssueContinuationHandoff.test.tsx ui/src/components/IssueDocumentsSection.test.tsx` - `pnpm --filter @paperclipai/db build` - `pnpm exec vitest run server/src/__tests__/activity-service.test.ts ui/src/components/IssueRunLedger.test.tsx` - `pnpm --filter @paperclipai/ui typecheck` - `pnpm --filter @paperclipai/server typecheck` - `pnpm exec vitest run server/src/__tests__/activity-service.test.ts server/src/__tests__/run-continuations.test.ts ui/src/components/IssueRunLedger.test.tsx` - `pnpm exec vitest run server/src/__tests__/heartbeat-process-recovery.test.ts -t "treats a plan document update"` - `pnpm exec vitest run server/src/__tests__/activity-service.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts -t "activity service|treats a plan document update"` - Remote PR checks on head `e53b1a1d`: `verify`, `e2e`, `policy`, and Snyk all passed. - Confirmed `public-gh/master` is an ancestor of this branch after fetching `public-gh master`. - Confirmed `pnpm-lock.yaml` is not included in the branch diff. - Confirmed migration `0058_wealthy_starbolt.sql` is ordered after `0057` and uses `IF NOT EXISTS` guards for repeat application. - Greptile inline review threads are resolved. ## Risks - Medium risk: this touches heartbeat execution, liveness recovery, activity rendering, issue routes, shared contracts, docs, and UI. - Migration risk is mitigated by additive columns/indexes and idempotent guards. - Run-ledger liveness backfill is now asynchronous, so the first ledger response can briefly show historical missing liveness until the background backfill completes. - UI screenshot coverage is not included in this packaging pass; validation is currently through focused component tests. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5.4, local tool-use coding agent with terminal, git, GitHub connector, GitHub CLI, and Paperclip API access. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge Screenshot note: no before/after screenshots were captured in this PR packaging pass; the UI changes are covered by focused component tests listed above. --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
b9a80dcf22
commit
236d11d36f
71 changed files with 18254 additions and 85 deletions
|
|
@ -1,6 +1,20 @@
|
|||
import { and, desc, eq, isNull, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { activityLog, agents, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import {
|
||||
activityLog,
|
||||
agents,
|
||||
documentRevisions,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
issues,
|
||||
issueWorkProducts,
|
||||
workspaceOperations,
|
||||
} from "@paperclipai/db";
|
||||
import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "@paperclipai/shared";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { classifyRunLiveness } from "./run-liveness.js";
|
||||
|
||||
export interface ActivityFilters {
|
||||
companyId: string;
|
||||
|
|
@ -10,6 +24,7 @@ export interface ActivityFilters {
|
|||
}
|
||||
|
||||
export function activityService(db: Db) {
|
||||
const scheduledLivenessBackfills = new Set<string>();
|
||||
const issueIdAsText = sql<string>`${issues.id}::text`;
|
||||
const summarizedUsageJson = sql<Record<string, unknown> | null>`
|
||||
case
|
||||
|
|
@ -74,11 +89,230 @@ export function activityService(db: Db) {
|
|||
${heartbeatRuns.resultJson} -> 'total_cost_usd',
|
||||
${heartbeatRuns.resultJson} -> 'cost_usd',
|
||||
${heartbeatRuns.resultJson} -> 'costUsd'
|
||||
)
|
||||
),
|
||||
'stopReason', ${heartbeatRuns.resultJson} -> 'stopReason',
|
||||
'effectiveTimeoutSec', ${heartbeatRuns.resultJson} -> 'effectiveTimeoutSec',
|
||||
'effectiveTimeoutMs', ${heartbeatRuns.resultJson} -> 'effectiveTimeoutMs',
|
||||
'timeoutConfigured', ${heartbeatRuns.resultJson} -> 'timeoutConfigured',
|
||||
'timeoutSource', ${heartbeatRuns.resultJson} -> 'timeoutSource',
|
||||
'timeoutFired', ${heartbeatRuns.resultJson} -> 'timeoutFired'
|
||||
))
|
||||
end
|
||||
`.as("resultJson");
|
||||
|
||||
function countValue(value: unknown) {
|
||||
const parsed = Number(value ?? 0);
|
||||
return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;
|
||||
}
|
||||
|
||||
function dateValue(value: unknown) {
|
||||
if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value;
|
||||
if (typeof value === "string" || typeof value === "number") {
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function latestDate(...values: unknown[]) {
|
||||
let latest: Date | null = null;
|
||||
for (const value of values) {
|
||||
const parsed = dateValue(value);
|
||||
if (!parsed) continue;
|
||||
if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;
|
||||
}
|
||||
return latest;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function readNumber(value: unknown) {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
async function backfillMissingRunLivenessForIssue(companyId: string, issueId: string) {
|
||||
const runs = await db
|
||||
.select({
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
status: heartbeatRuns.status,
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||||
resultJson: heartbeatRuns.resultJson,
|
||||
stdoutExcerpt: heartbeatRuns.stdoutExcerpt,
|
||||
stderrExcerpt: heartbeatRuns.stderrExcerpt,
|
||||
error: heartbeatRuns.error,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
continuationAttempt: heartbeatRuns.continuationAttempt,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
isNull(heartbeatRuns.livenessState),
|
||||
sql`${heartbeatRuns.status} not in ('queued', 'running')`,
|
||||
or(
|
||||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
|
||||
sql`exists (
|
||||
select 1
|
||||
from ${activityLog}
|
||||
where ${activityLog.companyId} = ${companyId}
|
||||
and ${activityLog.entityType} = 'issue'
|
||||
and ${activityLog.entityId} = ${issueId}
|
||||
and ${activityLog.runId} = ${heartbeatRuns.id}
|
||||
)`,
|
||||
),
|
||||
),
|
||||
)
|
||||
.limit(20);
|
||||
|
||||
if (runs.length === 0) return;
|
||||
|
||||
const issue = await db
|
||||
.select({
|
||||
status: issues.status,
|
||||
title: issues.title,
|
||||
description: issues.description,
|
||||
})
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.id, issueId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
for (const run of runs) {
|
||||
const context = asRecord(run.contextSnapshot);
|
||||
const continuationAttempt =
|
||||
readNumber(context?.continuationAttempt) ??
|
||||
readNumber(context?.livenessContinuationAttempt) ??
|
||||
run.continuationAttempt ??
|
||||
0;
|
||||
|
||||
const [commentStats] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
latestAt: sql<Date | null>`max(${issueComments.createdAt})`,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.companyId, companyId),
|
||||
eq(issueComments.issueId, issueId),
|
||||
eq(issueComments.createdByRunId, run.id),
|
||||
),
|
||||
);
|
||||
|
||||
const [documentStats] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,
|
||||
latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,
|
||||
})
|
||||
.from(documentRevisions)
|
||||
.innerJoin(issueDocuments, eq(documentRevisions.documentId, issueDocuments.documentId))
|
||||
.where(
|
||||
and(
|
||||
eq(documentRevisions.companyId, companyId),
|
||||
eq(documentRevisions.createdByRunId, run.id),
|
||||
eq(issueDocuments.companyId, companyId),
|
||||
eq(issueDocuments.issueId, issueId),
|
||||
sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,
|
||||
),
|
||||
);
|
||||
|
||||
const [workProductStats] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,
|
||||
})
|
||||
.from(issueWorkProducts)
|
||||
.where(
|
||||
and(
|
||||
eq(issueWorkProducts.companyId, companyId),
|
||||
eq(issueWorkProducts.issueId, issueId),
|
||||
eq(issueWorkProducts.createdByRunId, run.id),
|
||||
),
|
||||
);
|
||||
|
||||
const [workspaceOperationStats] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,
|
||||
})
|
||||
.from(workspaceOperations)
|
||||
.where(and(eq(workspaceOperations.companyId, companyId), eq(workspaceOperations.heartbeatRunId, run.id)));
|
||||
|
||||
const [activityStats] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
latestAt: sql<Date | null>`max(${activityLog.createdAt})`,
|
||||
})
|
||||
.from(activityLog)
|
||||
.where(and(eq(activityLog.companyId, companyId), eq(activityLog.runId, run.id)));
|
||||
|
||||
const [eventStats] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,
|
||||
latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,
|
||||
})
|
||||
.from(heartbeatRunEvents)
|
||||
.where(and(eq(heartbeatRunEvents.companyId, companyId), eq(heartbeatRunEvents.runId, run.id)));
|
||||
|
||||
const classification = classifyRunLiveness({
|
||||
runStatus: run.status,
|
||||
issue,
|
||||
resultJson: asRecord(run.resultJson),
|
||||
stdoutExcerpt: run.stdoutExcerpt,
|
||||
stderrExcerpt: run.stderrExcerpt,
|
||||
error: run.error,
|
||||
errorCode: run.errorCode,
|
||||
continuationAttempt,
|
||||
evidence: {
|
||||
issueCommentsCreated: countValue(commentStats?.count),
|
||||
documentRevisionsCreated: countValue(documentStats?.count),
|
||||
planDocumentRevisionsCreated: countValue(documentStats?.planCount),
|
||||
workProductsCreated: countValue(workProductStats?.count),
|
||||
workspaceOperationsCreated: countValue(workspaceOperationStats?.count),
|
||||
activityEventsCreated: countValue(activityStats?.count),
|
||||
toolOrActionEventsCreated: countValue(eventStats?.count),
|
||||
latestEvidenceAt: latestDate(
|
||||
commentStats?.latestAt,
|
||||
documentStats?.latestAt,
|
||||
workProductStats?.latestAt,
|
||||
workspaceOperationStats?.latestAt,
|
||||
activityStats?.latestAt,
|
||||
eventStats?.latestAt,
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
livenessState: classification.livenessState,
|
||||
livenessReason: classification.livenessReason,
|
||||
continuationAttempt: classification.continuationAttempt,
|
||||
lastUsefulActionAt: classification.lastUsefulActionAt,
|
||||
nextAction: classification.nextAction,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(and(eq(heartbeatRuns.id, run.id), isNull(heartbeatRuns.livenessState)));
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleRunLivenessBackfill(companyId: string, issueId: string) {
|
||||
const key = `${companyId}:${issueId}`;
|
||||
if (scheduledLivenessBackfills.has(key)) return;
|
||||
scheduledLivenessBackfills.add(key);
|
||||
void backfillMissingRunLivenessForIssue(companyId, issueId)
|
||||
.catch((err: unknown) => {
|
||||
logger.warn({ err, companyId, issueId }, "run liveness backfill failed");
|
||||
})
|
||||
.finally(() => {
|
||||
scheduledLivenessBackfills.delete(key);
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
list: (filters: ActivityFilters) => {
|
||||
const conditions = [eq(activityLog.companyId, filters.companyId)];
|
||||
|
|
@ -128,8 +362,9 @@ export function activityService(db: Db) {
|
|||
)
|
||||
.orderBy(desc(activityLog.createdAt)),
|
||||
|
||||
runsForIssue: (companyId: string, issueId: string) =>
|
||||
db
|
||||
runsForIssue: async (companyId: string, issueId: string) => {
|
||||
scheduleRunLivenessBackfill(companyId, issueId);
|
||||
return db
|
||||
.select({
|
||||
runId: heartbeatRuns.id,
|
||||
status: heartbeatRuns.status,
|
||||
|
|
@ -142,6 +377,11 @@ export function activityService(db: Db) {
|
|||
usageJson: summarizedUsageJson,
|
||||
resultJson: summarizedResultJson,
|
||||
logBytes: heartbeatRuns.logBytes,
|
||||
livenessState: heartbeatRuns.livenessState,
|
||||
livenessReason: heartbeatRuns.livenessReason,
|
||||
continuationAttempt: heartbeatRuns.continuationAttempt,
|
||||
lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt,
|
||||
nextAction: heartbeatRuns.nextAction,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.innerJoin(
|
||||
|
|
@ -167,7 +407,8 @@ export function activityService(db: Db) {
|
|||
),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(heartbeatRuns.createdAt)),
|
||||
.orderBy(desc(heartbeatRuns.createdAt));
|
||||
},
|
||||
|
||||
issuesForRun: async (runId: string) => {
|
||||
const run = await db
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { and, asc, desc, eq } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { documentRevisions, documents, issueDocuments, issues } from "@paperclipai/db";
|
||||
import { issueDocumentKeySchema } from "@paperclipai/shared";
|
||||
import { isSystemIssueDocumentKey, issueDocumentKeySchema } from "@paperclipai/shared";
|
||||
import { conflict, notFound, unprocessable } from "../errors.js";
|
||||
|
||||
function normalizeDocumentKey(key: string) {
|
||||
|
|
@ -83,8 +83,14 @@ const issueDocumentSelect = {
|
|||
};
|
||||
|
||||
export function documentService(db: Db) {
|
||||
const filterSystemDocuments = <T extends { key: string }>(rows: T[], includeSystem: boolean) =>
|
||||
includeSystem ? rows : rows.filter((row) => !isSystemIssueDocumentKey(row.key));
|
||||
|
||||
return {
|
||||
getIssueDocumentPayload: async (issue: { id: string; description: string | null }) => {
|
||||
getIssueDocumentPayload: async (
|
||||
issue: { id: string; description: string | null },
|
||||
options: { includeSystem?: boolean } = {},
|
||||
) => {
|
||||
const [planDocument, documentSummaries] = await Promise.all([
|
||||
db
|
||||
.select(issueDocumentSelect)
|
||||
|
|
@ -104,7 +110,8 @@ export function documentService(db: Db) {
|
|||
|
||||
return {
|
||||
planDocument: planDocument ? mapIssueDocumentRow(planDocument, true) : null,
|
||||
documentSummaries: documentSummaries.map((row) => mapIssueDocumentRow(row, false)),
|
||||
documentSummaries: filterSystemDocuments(documentSummaries, options.includeSystem ?? false)
|
||||
.map((row) => mapIssueDocumentRow(row, false)),
|
||||
legacyPlanDocument: legacyPlanBody
|
||||
? {
|
||||
key: "plan" as const,
|
||||
|
|
@ -115,14 +122,14 @@ export function documentService(db: Db) {
|
|||
};
|
||||
},
|
||||
|
||||
listIssueDocuments: async (issueId: string) => {
|
||||
listIssueDocuments: async (issueId: string, options: { includeSystem?: boolean } = {}) => {
|
||||
const rows = await db
|
||||
.select(issueDocumentSelect)
|
||||
.from(issueDocuments)
|
||||
.innerJoin(documents, eq(issueDocuments.documentId, documents.id))
|
||||
.where(eq(issueDocuments.issueId, issueId))
|
||||
.orderBy(asc(issueDocuments.key), desc(documents.updatedAt));
|
||||
return rows.map((row) => mapIssueDocumentRow(row, true));
|
||||
return filterSystemDocuments(rows, options.includeSystem ?? false).map((row) => mapIssueDocumentRow(row, true));
|
||||
},
|
||||
|
||||
getIssueDocumentByKey: async (issueId: string, rawKey: string) => {
|
||||
|
|
|
|||
|
|
@ -69,6 +69,26 @@ export function summarizeHeartbeatRunResultJson(
|
|||
}
|
||||
}
|
||||
|
||||
for (const key of ["stopReason", "timeoutSource"] as const) {
|
||||
const value = readCommentText(resultJson[key]);
|
||||
if (value !== null) {
|
||||
summary[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of ["effectiveTimeoutSec", "effectiveTimeoutMs"] as const) {
|
||||
const value = readNumericField(resultJson, key);
|
||||
if (value !== undefined && value !== null) {
|
||||
summary[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of ["timeoutConfigured", "timeoutFired"] as const) {
|
||||
if (typeof resultJson[key] === "boolean") {
|
||||
summary[key] = resultJson[key];
|
||||
}
|
||||
}
|
||||
|
||||
return Object.keys(summary).length > 0 ? summary : null;
|
||||
}
|
||||
|
||||
|
|
|
|||
86
server/src/services/heartbeat-stop-metadata.test.ts
Normal file
86
server/src/services/heartbeat-stop-metadata.test.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildHeartbeatRunStopMetadata,
|
||||
mergeHeartbeatRunStopMetadata,
|
||||
resolveHeartbeatRunTimeoutPolicy,
|
||||
} from "./heartbeat-stop-metadata.js";
|
||||
|
||||
describe("heartbeat stop metadata", () => {
|
||||
it("keeps local coding adapters at no timeout by default", () => {
|
||||
for (const adapterType of [
|
||||
"codex_local",
|
||||
"claude_local",
|
||||
"cursor",
|
||||
"gemini_local",
|
||||
"opencode_local",
|
||||
"pi_local",
|
||||
"process",
|
||||
]) {
|
||||
expect(resolveHeartbeatRunTimeoutPolicy(adapterType, {})).toEqual({
|
||||
effectiveTimeoutSec: 0,
|
||||
timeoutConfigured: false,
|
||||
timeoutSource: "default",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("records configured timeout policy and timeout stop reason", () => {
|
||||
const metadata = buildHeartbeatRunStopMetadata({
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: { timeoutSec: 45 },
|
||||
outcome: "timed_out",
|
||||
errorCode: "timeout",
|
||||
errorMessage: "Timed out after 45s",
|
||||
});
|
||||
|
||||
expect(metadata).toEqual({
|
||||
effectiveTimeoutSec: 45,
|
||||
timeoutConfigured: true,
|
||||
timeoutSource: "config",
|
||||
stopReason: "timeout",
|
||||
timeoutFired: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("distinguishes budget cancellation from manual cancellation", () => {
|
||||
expect(
|
||||
buildHeartbeatRunStopMetadata({
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
outcome: "cancelled",
|
||||
errorCode: "cancelled",
|
||||
errorMessage: "Cancelled due to budget pause",
|
||||
}).stopReason,
|
||||
).toBe("budget_paused");
|
||||
|
||||
expect(
|
||||
buildHeartbeatRunStopMetadata({
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
outcome: "cancelled",
|
||||
errorCode: "cancelled",
|
||||
errorMessage: "Cancelled by control plane",
|
||||
}).stopReason,
|
||||
).toBe("cancelled");
|
||||
});
|
||||
|
||||
it("preserves existing result fields when merging stop metadata", () => {
|
||||
const result = mergeHeartbeatRunStopMetadata(
|
||||
{ summary: "done" },
|
||||
buildHeartbeatRunStopMetadata({
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {},
|
||||
outcome: "succeeded",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
summary: "done",
|
||||
stopReason: "completed",
|
||||
effectiveTimeoutSec: 120,
|
||||
timeoutConfigured: true,
|
||||
timeoutSource: "default",
|
||||
timeoutFired: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
119
server/src/services/heartbeat-stop-metadata.ts
Normal file
119
server/src/services/heartbeat-stop-metadata.ts
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
export type HeartbeatRunOutcome = "succeeded" | "failed" | "cancelled" | "timed_out";
|
||||
|
||||
export type HeartbeatRunStopReason =
|
||||
| "completed"
|
||||
| "timeout"
|
||||
| "cancelled"
|
||||
| "budget_paused"
|
||||
| "paused"
|
||||
| "process_lost"
|
||||
| "adapter_failed";
|
||||
|
||||
export interface HeartbeatRunTimeoutPolicy {
|
||||
effectiveTimeoutSec: number | null;
|
||||
effectiveTimeoutMs?: number | null;
|
||||
timeoutConfigured: boolean;
|
||||
timeoutSource: "config" | "default" | "unknown";
|
||||
}
|
||||
|
||||
export interface HeartbeatRunStopMetadata extends HeartbeatRunTimeoutPolicy {
|
||||
stopReason: HeartbeatRunStopReason;
|
||||
timeoutFired: boolean;
|
||||
}
|
||||
|
||||
function readFiniteNumber(value: unknown): number | null {
|
||||
if (typeof value === "number") {
|
||||
return Number.isFinite(value) ? value : null;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
const parsed = Number(value.trim());
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function hasOwn(record: Record<string, unknown>, key: string) {
|
||||
return Object.prototype.hasOwnProperty.call(record, key);
|
||||
}
|
||||
|
||||
function defaultTimeoutSecForAdapter(adapterType: string) {
|
||||
return adapterType === "openclaw_gateway" ? 120 : 0;
|
||||
}
|
||||
|
||||
export function resolveHeartbeatRunTimeoutPolicy(
|
||||
adapterType: string,
|
||||
adapterConfig: Record<string, unknown> | null | undefined,
|
||||
): HeartbeatRunTimeoutPolicy {
|
||||
const config = adapterConfig ?? {};
|
||||
|
||||
if (adapterType === "http") {
|
||||
const hasTimeoutMs = hasOwn(config, "timeoutMs");
|
||||
const rawTimeoutMs = hasTimeoutMs ? readFiniteNumber(config.timeoutMs) : 0;
|
||||
const timeoutMs = Math.max(0, Math.floor(rawTimeoutMs ?? 0));
|
||||
return {
|
||||
effectiveTimeoutSec: timeoutMs / 1000,
|
||||
effectiveTimeoutMs: timeoutMs,
|
||||
timeoutConfigured: timeoutMs > 0,
|
||||
timeoutSource: hasTimeoutMs ? "config" : "default",
|
||||
};
|
||||
}
|
||||
|
||||
const hasTimeoutSec = hasOwn(config, "timeoutSec");
|
||||
const defaultTimeoutSec = defaultTimeoutSecForAdapter(adapterType);
|
||||
const rawTimeoutSec = hasTimeoutSec ? readFiniteNumber(config.timeoutSec) : defaultTimeoutSec;
|
||||
const timeoutSec = Math.max(0, Math.floor(rawTimeoutSec ?? defaultTimeoutSec));
|
||||
|
||||
return {
|
||||
effectiveTimeoutSec: timeoutSec,
|
||||
timeoutConfigured: timeoutSec > 0,
|
||||
timeoutSource: hasTimeoutSec ? "config" : "default",
|
||||
};
|
||||
}
|
||||
|
||||
export function inferHeartbeatRunStopReason(input: {
|
||||
outcome: HeartbeatRunOutcome;
|
||||
errorCode?: string | null;
|
||||
errorMessage?: string | null;
|
||||
}): HeartbeatRunStopReason {
|
||||
if (input.outcome === "succeeded") return "completed";
|
||||
if (input.outcome === "timed_out") return "timeout";
|
||||
if (input.outcome === "failed" && input.errorCode === "process_lost") return "process_lost";
|
||||
if (input.outcome === "cancelled") {
|
||||
const message = (input.errorMessage ?? "").toLowerCase();
|
||||
if (message.includes("budget")) return "budget_paused";
|
||||
if (message.includes("pause") || message.includes("paused")) return "paused";
|
||||
return "cancelled";
|
||||
}
|
||||
return "adapter_failed";
|
||||
}
|
||||
|
||||
export function buildHeartbeatRunStopMetadata(input: {
|
||||
adapterType: string;
|
||||
adapterConfig: Record<string, unknown> | null | undefined;
|
||||
outcome: HeartbeatRunOutcome;
|
||||
errorCode?: string | null;
|
||||
errorMessage?: string | null;
|
||||
}): HeartbeatRunStopMetadata {
|
||||
const timeoutPolicy = resolveHeartbeatRunTimeoutPolicy(input.adapterType, input.adapterConfig);
|
||||
const stopReason = inferHeartbeatRunStopReason(input);
|
||||
return {
|
||||
...timeoutPolicy,
|
||||
stopReason,
|
||||
timeoutFired: stopReason === "timeout",
|
||||
};
|
||||
}
|
||||
|
||||
export function mergeHeartbeatRunStopMetadata(
|
||||
resultJson: Record<string, unknown> | null | undefined,
|
||||
metadata: HeartbeatRunStopMetadata,
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
...(resultJson ?? {}),
|
||||
stopReason: metadata.stopReason,
|
||||
effectiveTimeoutSec: metadata.effectiveTimeoutSec,
|
||||
timeoutConfigured: metadata.timeoutConfigured,
|
||||
timeoutSource: metadata.timeoutSource,
|
||||
timeoutFired: metadata.timeoutFired,
|
||||
...(metadata.effectiveTimeoutMs != null ? { effectiveTimeoutMs: metadata.effectiveTimeoutMs } : {}),
|
||||
};
|
||||
}
|
||||
|
|
@ -4,19 +4,25 @@ import { execFile as execFileCallback } from "node:child_process";
|
|||
import { promisify } from "node:util";
|
||||
import { and, asc, desc, eq, getTableColumns, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { BillingType, ExecutionWorkspace, ExecutionWorkspaceConfig } from "@paperclipai/shared";
|
||||
import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "@paperclipai/shared";
|
||||
import type { BillingType, ExecutionWorkspace, ExecutionWorkspaceConfig, RunLivenessState } from "@paperclipai/shared";
|
||||
import {
|
||||
agents,
|
||||
agentRuntimeState,
|
||||
agentTaskSessions,
|
||||
agentWakeupRequests,
|
||||
activityLog,
|
||||
companySkills as companySkillsTable,
|
||||
documentRevisions,
|
||||
issueDocuments,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issues,
|
||||
issueWorkProducts,
|
||||
projects,
|
||||
projectWorkspaces,
|
||||
workspaceOperations,
|
||||
} from "@paperclipai/db";
|
||||
import { conflict, HttpError, notFound } from "../errors.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
|
|
@ -40,6 +46,14 @@ import {
|
|||
HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES,
|
||||
mergeHeartbeatRunResultJson,
|
||||
} from "./heartbeat-run-summary.js";
|
||||
import {
|
||||
buildHeartbeatRunStopMetadata,
|
||||
mergeHeartbeatRunStopMetadata,
|
||||
} from "./heartbeat-stop-metadata.js";
|
||||
import {
|
||||
classifyRunLiveness,
|
||||
type RunLivenessClassificationInput,
|
||||
} from "./run-liveness.js";
|
||||
import { logActivity, type LogActivityInput } from "./activity-log.js";
|
||||
import {
|
||||
buildWorkspaceReadyComment,
|
||||
|
|
@ -53,6 +67,10 @@ import {
|
|||
sanitizeRuntimeServiceBaseEnv,
|
||||
} from "./workspace-runtime.js";
|
||||
import { issueService } from "./issues.js";
|
||||
import {
|
||||
getIssueContinuationSummaryDocument,
|
||||
refreshIssueContinuationSummary,
|
||||
} from "./issue-continuation-summary.js";
|
||||
import { executionWorkspaceService, mergeExecutionWorkspaceConfig } from "./execution-workspaces.js";
|
||||
import { workspaceOperationService } from "./workspace-operations.js";
|
||||
import { isProcessGroupAlive, terminateLocalService } from "./local-service-supervisor.js";
|
||||
|
|
@ -65,6 +83,13 @@ import {
|
|||
resolveExecutionWorkspaceMode,
|
||||
} from "./execution-workspace-policy.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import {
|
||||
RUN_LIVENESS_CONTINUATION_REASON,
|
||||
buildRunLivenessContinuationIdempotencyKey,
|
||||
decideRunLivenessContinuation,
|
||||
findExistingRunLivenessContinuationWake,
|
||||
readContinuationAttempt,
|
||||
} from "./run-continuations.js";
|
||||
import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js";
|
||||
import {
|
||||
hasSessionCompactionThresholds,
|
||||
|
|
@ -397,6 +422,11 @@ const heartbeatRunListColumns = {
|
|||
processStartedAt: heartbeatRuns.processStartedAt,
|
||||
retryOfRunId: heartbeatRuns.retryOfRunId,
|
||||
processLossRetryCount: heartbeatRuns.processLossRetryCount,
|
||||
livenessState: heartbeatRuns.livenessState,
|
||||
livenessReason: heartbeatRuns.livenessReason,
|
||||
continuationAttempt: heartbeatRuns.continuationAttempt,
|
||||
lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt,
|
||||
nextAction: heartbeatRuns.nextAction,
|
||||
createdAt: heartbeatRuns.createdAt,
|
||||
updatedAt: heartbeatRuns.updatedAt,
|
||||
} as const;
|
||||
|
|
@ -490,6 +520,11 @@ const heartbeatRunIssueSummaryColumns = {
|
|||
finishedAt: heartbeatRuns.finishedAt,
|
||||
createdAt: heartbeatRuns.createdAt,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
livenessState: heartbeatRuns.livenessState,
|
||||
livenessReason: heartbeatRuns.livenessReason,
|
||||
continuationAttempt: heartbeatRuns.continuationAttempt,
|
||||
lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt,
|
||||
nextAction: heartbeatRuns.nextAction,
|
||||
issueId: sql<string | null>`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("issueId"),
|
||||
} as const;
|
||||
|
||||
|
|
@ -1204,6 +1239,14 @@ async function buildPaperclipWakePayload(input: {
|
|||
db: Db;
|
||||
companyId: string;
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
continuationSummary?:
|
||||
| {
|
||||
key: string;
|
||||
title: string | null;
|
||||
body: string;
|
||||
updatedAt: Date;
|
||||
}
|
||||
| null;
|
||||
issueSummary?:
|
||||
| {
|
||||
id: string;
|
||||
|
|
@ -1217,6 +1260,7 @@ async function buildPaperclipWakePayload(input: {
|
|||
const executionStage = parseObject(input.contextSnapshot.executionStage);
|
||||
const commentIds = extractWakeCommentIds(input.contextSnapshot);
|
||||
const issueId = readNonEmptyString(input.contextSnapshot.issueId);
|
||||
const continuationSummary = input.continuationSummary ?? null;
|
||||
const issueSummary =
|
||||
input.issueSummary ??
|
||||
(issueId
|
||||
|
|
@ -1309,8 +1353,37 @@ async function buildPaperclipWakePayload(input: {
|
|||
priority: issueSummary.priority,
|
||||
}
|
||||
: null,
|
||||
childIssueSummaries: Array.isArray(input.contextSnapshot.childIssueSummaries)
|
||||
? input.contextSnapshot.childIssueSummaries
|
||||
: [],
|
||||
childIssueSummaryTruncated: input.contextSnapshot.childIssueSummaryTruncated === true,
|
||||
livenessContinuation: readNonEmptyString(input.contextSnapshot.livenessContinuationState) ||
|
||||
readNonEmptyString(input.contextSnapshot.livenessContinuationInstruction) ||
|
||||
readNonEmptyString(input.contextSnapshot.livenessContinuationSourceRunId) ||
|
||||
typeof input.contextSnapshot.livenessContinuationAttempt === "number"
|
||||
? {
|
||||
attempt: input.contextSnapshot.livenessContinuationAttempt,
|
||||
maxAttempts: input.contextSnapshot.livenessContinuationMaxAttempts,
|
||||
sourceRunId: readNonEmptyString(input.contextSnapshot.livenessContinuationSourceRunId),
|
||||
state: readNonEmptyString(input.contextSnapshot.livenessContinuationState),
|
||||
reason: readNonEmptyString(input.contextSnapshot.livenessContinuationReason),
|
||||
instruction: readNonEmptyString(input.contextSnapshot.livenessContinuationInstruction),
|
||||
}
|
||||
: null,
|
||||
checkedOutByHarness: input.contextSnapshot[PAPERCLIP_HARNESS_CHECKOUT_KEY] === true,
|
||||
executionStage: Object.keys(executionStage).length > 0 ? executionStage : null,
|
||||
continuationSummary: continuationSummary
|
||||
? {
|
||||
key: continuationSummary.key,
|
||||
title: continuationSummary.title,
|
||||
body:
|
||||
continuationSummary.body.length > 4_000
|
||||
? continuationSummary.body.slice(0, 4_000)
|
||||
: continuationSummary.body,
|
||||
bodyTruncated: continuationSummary.body.length > 4_000,
|
||||
updatedAt: continuationSummary.updatedAt.toISOString(),
|
||||
}
|
||||
: null,
|
||||
commentIds,
|
||||
latestCommentId: commentIds[commentIds.length - 1] ?? null,
|
||||
comments,
|
||||
|
|
@ -1643,6 +1716,7 @@ export function heartbeatService(db: Db) {
|
|||
agent: typeof agents.$inferSelect;
|
||||
sessionId: string | null;
|
||||
issueId: string | null;
|
||||
continuationSummaryBody?: string | null;
|
||||
}): Promise<SessionCompactionDecision> {
|
||||
const { agent, sessionId, issueId } = input;
|
||||
if (!sessionId) {
|
||||
|
|
@ -1746,6 +1820,9 @@ export function heartbeatService(db: Db) {
|
|||
issueId ? `- Issue: ${issueId}` : "",
|
||||
`- Rotation reason: ${reason}`,
|
||||
latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "",
|
||||
input.continuationSummaryBody
|
||||
? `- Issue continuation summary: ${input.continuationSummaryBody.slice(0, 1_500)}`
|
||||
: "",
|
||||
"Continue from the current task state. Rebuild only the minimum context you need.",
|
||||
]
|
||||
.filter(Boolean)
|
||||
|
|
@ -2170,6 +2247,136 @@ export function heartbeatService(db: Db) {
|
|||
.where(eq(agentWakeupRequests.id, wakeupRequestId));
|
||||
}
|
||||
|
||||
async function addContinuationExhaustedCommentOnce(input: {
|
||||
run: typeof heartbeatRuns.$inferSelect;
|
||||
issueId: string;
|
||||
comment: string;
|
||||
}) {
|
||||
const existing = await db
|
||||
.select({ id: issueComments.id })
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.companyId, input.run.companyId),
|
||||
eq(issueComments.issueId, input.issueId),
|
||||
eq(issueComments.createdByRunId, input.run.id),
|
||||
sql`${issueComments.body} like 'Bounded liveness continuation exhausted%'`,
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (existing) return;
|
||||
await issuesSvc.addComment(input.issueId, input.comment, {
|
||||
agentId: input.run.agentId,
|
||||
runId: input.run.id,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleRunLivenessContinuation(run: typeof heartbeatRuns.$inferSelect) {
|
||||
const livenessState = run.livenessState as RunLivenessState | null;
|
||||
if (livenessState !== "plan_only" && livenessState !== "empty_response") return;
|
||||
|
||||
const context = parseObject(run.contextSnapshot);
|
||||
const issueId = readNonEmptyString(context.issueId);
|
||||
if (!issueId) return;
|
||||
|
||||
const [issue, agent] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
id: issues.id,
|
||||
companyId: issues.companyId,
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
executionState: issues.executionState,
|
||||
projectId: issues.projectId,
|
||||
})
|
||||
.from(issues)
|
||||
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db
|
||||
.select({
|
||||
id: agents.id,
|
||||
companyId: agents.companyId,
|
||||
status: agents.status,
|
||||
})
|
||||
.from(agents)
|
||||
.where(eq(agents.id, run.agentId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
|
||||
const budgetBlock =
|
||||
issue && agent
|
||||
? await budgets.getInvocationBlock(issue.companyId, agent.id, {
|
||||
issueId: issue.id,
|
||||
projectId: issue.projectId,
|
||||
})
|
||||
: null;
|
||||
|
||||
const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1;
|
||||
const idempotencyKey = issue
|
||||
? buildRunLivenessContinuationIdempotencyKey({
|
||||
issueId: issue.id,
|
||||
sourceRunId: run.id,
|
||||
livenessState,
|
||||
nextAttempt,
|
||||
})
|
||||
: null;
|
||||
const existingWake = idempotencyKey
|
||||
? await findExistingRunLivenessContinuationWake(db, {
|
||||
companyId: run.companyId,
|
||||
idempotencyKey,
|
||||
})
|
||||
: null;
|
||||
|
||||
const decision = decideRunLivenessContinuation({
|
||||
run,
|
||||
issue,
|
||||
agent,
|
||||
livenessState,
|
||||
livenessReason: run.livenessReason,
|
||||
nextAction: run.nextAction,
|
||||
budgetBlocked: Boolean(budgetBlock),
|
||||
idempotentWakeExists: Boolean(existingWake),
|
||||
});
|
||||
|
||||
if (decision.kind === "exhausted") {
|
||||
await setRunStatus(run.id, run.status, {
|
||||
livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation attempts exhausted`,
|
||||
});
|
||||
await addContinuationExhaustedCommentOnce({
|
||||
run,
|
||||
issueId,
|
||||
comment: decision.comment,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (decision.kind !== "enqueue") return;
|
||||
|
||||
const continuationRun = await enqueueWakeup(run.agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: RUN_LIVENESS_CONTINUATION_REASON,
|
||||
payload: decision.payload,
|
||||
contextSnapshot: decision.contextSnapshot,
|
||||
idempotencyKey: decision.idempotencyKey,
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: "heartbeat",
|
||||
});
|
||||
|
||||
if (continuationRun) {
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
continuationAttempt: decision.nextAttempt,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(heartbeatRuns.id, continuationRun.id));
|
||||
}
|
||||
}
|
||||
|
||||
async function appendRunEvent(
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
seq: number,
|
||||
|
|
@ -2298,6 +2505,47 @@ export function heartbeatService(db: Db) {
|
|||
.then((rows) => rows[0] ?? null);
|
||||
}
|
||||
|
||||
async function refreshContinuationSummaryForRun(
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
agent: typeof agents.$inferSelect,
|
||||
) {
|
||||
const contextSnapshot = parseObject(run.contextSnapshot);
|
||||
const issueId = readNonEmptyString(contextSnapshot.issueId);
|
||||
if (!issueId) return null;
|
||||
try {
|
||||
return await refreshIssueContinuationSummary({
|
||||
db,
|
||||
issueId,
|
||||
run: {
|
||||
id: run.id,
|
||||
status: run.status,
|
||||
error: run.error,
|
||||
errorCode: run.errorCode,
|
||||
resultJson: run.resultJson as Record<string, unknown> | null,
|
||||
stdoutExcerpt: run.stdoutExcerpt,
|
||||
stderrExcerpt: run.stderrExcerpt,
|
||||
finishedAt: run.finishedAt,
|
||||
},
|
||||
agent: {
|
||||
id: agent.id,
|
||||
name: agent.name,
|
||||
adapterType: agent.adapterType,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{
|
||||
err,
|
||||
runId: run.id,
|
||||
issueId,
|
||||
agentId: agent.id,
|
||||
},
|
||||
"failed to refresh issue continuation summary",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function enqueueMissingIssueCommentRetry(
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
agent: typeof agents.$inferSelect,
|
||||
|
|
@ -2737,6 +2985,194 @@ export function heartbeatService(db: Db) {
|
|||
}
|
||||
}
|
||||
|
||||
function mergeRunStopMetadataForAgent(
|
||||
agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,
|
||||
outcome: "succeeded" | "failed" | "cancelled" | "timed_out",
|
||||
options?: {
|
||||
resultJson?: Record<string, unknown> | null;
|
||||
errorCode?: string | null;
|
||||
errorMessage?: string | null;
|
||||
},
|
||||
) {
|
||||
const stopMetadata = buildHeartbeatRunStopMetadata({
|
||||
adapterType: agent.adapterType,
|
||||
adapterConfig: parseObject(agent.adapterConfig),
|
||||
outcome,
|
||||
errorCode: options?.errorCode ?? null,
|
||||
errorMessage: options?.errorMessage ?? null,
|
||||
});
|
||||
return mergeHeartbeatRunStopMetadata(options?.resultJson ?? null, stopMetadata);
|
||||
}
|
||||
|
||||
function countValue(value: unknown) {
|
||||
const parsed = Number(value ?? 0);
|
||||
return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;
|
||||
}
|
||||
|
||||
function dateValue(value: unknown) {
|
||||
if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value;
|
||||
if (typeof value === "string" || typeof value === "number") {
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function latestDate(...values: unknown[]) {
|
||||
let latest: Date | null = null;
|
||||
for (const value of values) {
|
||||
const parsed = dateValue(value);
|
||||
if (!parsed) continue;
|
||||
if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;
|
||||
}
|
||||
return latest;
|
||||
}
|
||||
|
||||
async function buildRunLivenessInput(
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
resultJson: Record<string, unknown> | null | undefined,
|
||||
): Promise<RunLivenessClassificationInput> {
|
||||
const context = parseObject(run.contextSnapshot);
|
||||
const contextIssueId = readNonEmptyString(context.issueId);
|
||||
const continuationAttempt = asNumber(context.continuationAttempt, run.continuationAttempt ?? 0);
|
||||
|
||||
const issue = contextIssueId
|
||||
? await db
|
||||
.select({
|
||||
status: issues.status,
|
||||
title: issues.title,
|
||||
description: issues.description,
|
||||
})
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, run.companyId), eq(issues.id, contextIssueId)))
|
||||
.then((rows) => rows[0] ?? null)
|
||||
: null;
|
||||
|
||||
const [commentStats] = contextIssueId
|
||||
? await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
latestAt: sql<Date | null>`max(${issueComments.createdAt})`,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.companyId, run.companyId),
|
||||
eq(issueComments.issueId, contextIssueId),
|
||||
eq(issueComments.createdByRunId, run.id),
|
||||
),
|
||||
)
|
||||
: [{ count: 0, latestAt: null }];
|
||||
|
||||
const [documentStats] = contextIssueId
|
||||
? await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,
|
||||
latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,
|
||||
})
|
||||
.from(documentRevisions)
|
||||
.innerJoin(issueDocuments, eq(documentRevisions.documentId, issueDocuments.documentId))
|
||||
.where(
|
||||
and(
|
||||
eq(documentRevisions.companyId, run.companyId),
|
||||
eq(documentRevisions.createdByRunId, run.id),
|
||||
eq(issueDocuments.companyId, run.companyId),
|
||||
eq(issueDocuments.issueId, contextIssueId),
|
||||
sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,
|
||||
),
|
||||
)
|
||||
: [{ count: 0, planCount: 0, latestAt: null }];
|
||||
|
||||
const [workProductStats] = contextIssueId
|
||||
? await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,
|
||||
})
|
||||
.from(issueWorkProducts)
|
||||
.where(
|
||||
and(
|
||||
eq(issueWorkProducts.companyId, run.companyId),
|
||||
eq(issueWorkProducts.issueId, contextIssueId),
|
||||
eq(issueWorkProducts.createdByRunId, run.id),
|
||||
),
|
||||
)
|
||||
: [{ count: 0, latestAt: null }];
|
||||
|
||||
const [workspaceOperationStats] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,
|
||||
})
|
||||
.from(workspaceOperations)
|
||||
.where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.heartbeatRunId, run.id)));
|
||||
|
||||
const [activityStats] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*)::int`,
|
||||
latestAt: sql<Date | null>`max(${activityLog.createdAt})`,
|
||||
})
|
||||
.from(activityLog)
|
||||
.where(and(eq(activityLog.companyId, run.companyId), eq(activityLog.runId, run.id)));
|
||||
|
||||
const [eventStats] = await db
|
||||
.select({
|
||||
count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,
|
||||
latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,
|
||||
})
|
||||
.from(heartbeatRunEvents)
|
||||
.where(and(eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id)));
|
||||
|
||||
return {
|
||||
runStatus: run.status,
|
||||
issue,
|
||||
resultJson: resultJson ?? run.resultJson ?? null,
|
||||
stdoutExcerpt: run.stdoutExcerpt ?? null,
|
||||
stderrExcerpt: run.stderrExcerpt ?? null,
|
||||
error: run.error ?? null,
|
||||
errorCode: run.errorCode ?? null,
|
||||
continuationAttempt,
|
||||
evidence: {
|
||||
issueCommentsCreated: countValue(commentStats?.count),
|
||||
documentRevisionsCreated: countValue(documentStats?.count),
|
||||
planDocumentRevisionsCreated: countValue(documentStats?.planCount),
|
||||
workProductsCreated: countValue(workProductStats?.count),
|
||||
workspaceOperationsCreated: countValue(workspaceOperationStats?.count),
|
||||
activityEventsCreated: countValue(activityStats?.count),
|
||||
toolOrActionEventsCreated: countValue(eventStats?.count),
|
||||
latestEvidenceAt: latestDate(
|
||||
commentStats?.latestAt,
|
||||
documentStats?.latestAt,
|
||||
workProductStats?.latestAt,
|
||||
workspaceOperationStats?.latestAt,
|
||||
activityStats?.latestAt,
|
||||
eventStats?.latestAt,
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function classifyAndPersistRunLiveness(
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
resultJson?: Record<string, unknown> | null,
|
||||
) {
|
||||
const classification = classifyRunLiveness(await buildRunLivenessInput(run, resultJson));
|
||||
return db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
livenessState: classification.livenessState,
|
||||
livenessReason: classification.livenessReason,
|
||||
continuationAttempt: classification.continuationAttempt,
|
||||
lastUsefulActionAt: classification.lastUsefulActionAt,
|
||||
nextAction: classification.nextAction,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(heartbeatRuns.id, run.id))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
}
|
||||
|
||||
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) {
|
||||
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
|
||||
const now = new Date();
|
||||
|
|
@ -2746,6 +3182,7 @@ export function heartbeatService(db: Db) {
|
|||
.select({
|
||||
run: heartbeatRuns,
|
||||
adapterType: agents.adapterType,
|
||||
adapterConfig: agents.adapterConfig,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
|
||||
|
|
@ -2753,7 +3190,7 @@ export function heartbeatService(db: Db) {
|
|||
|
||||
const reaped: string[] = [];
|
||||
|
||||
for (const { run, adapterType } of activeRuns) {
|
||||
for (const { run, adapterType, adapterConfig } of activeRuns) {
|
||||
if (runningProcesses.has(run.id) || activeRunExecutions.has(run.id)) continue;
|
||||
|
||||
// Apply staleness threshold to avoid false positives
|
||||
|
|
@ -2803,6 +3240,15 @@ export function heartbeatService(db: Db) {
|
|||
error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
|
||||
errorCode: "process_lost",
|
||||
finishedAt: now,
|
||||
resultJson: mergeRunStopMetadataForAgent(
|
||||
{ adapterType, adapterConfig },
|
||||
"failed",
|
||||
{
|
||||
resultJson: parseObject(run.resultJson),
|
||||
errorCode: "process_lost",
|
||||
errorMessage: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
|
||||
},
|
||||
),
|
||||
});
|
||||
await setWakeupStatus(run.wakeupRequestId, "failed", {
|
||||
finishedAt: now,
|
||||
|
|
@ -2810,6 +3256,7 @@ export function heartbeatService(db: Db) {
|
|||
});
|
||||
if (!finalizedRun) finalizedRun = await getRun(run.id);
|
||||
if (!finalizedRun) continue;
|
||||
finalizedRun = await classifyAndPersistRunLiveness(finalizedRun, parseObject(finalizedRun.resultJson)) ?? finalizedRun;
|
||||
|
||||
let retriedRun: typeof heartbeatRuns.$inferSelect | null = null;
|
||||
if (shouldRetry) {
|
||||
|
|
@ -3340,10 +3787,24 @@ export function heartbeatService(db: Db) {
|
|||
executionWorkspacePreference: issueContext.executionWorkspacePreference,
|
||||
}
|
||||
: null;
|
||||
const continuationSummary = issueRef
|
||||
? await getIssueContinuationSummaryDocument(db, issueRef.id)
|
||||
: null;
|
||||
if (continuationSummary) {
|
||||
context.paperclipContinuationSummary = {
|
||||
key: continuationSummary.key,
|
||||
title: continuationSummary.title,
|
||||
body: continuationSummary.body,
|
||||
updatedAt: continuationSummary.updatedAt.toISOString(),
|
||||
};
|
||||
} else {
|
||||
delete context.paperclipContinuationSummary;
|
||||
}
|
||||
const paperclipWakePayload = await buildPaperclipWakePayload({
|
||||
db,
|
||||
companyId: agent.companyId,
|
||||
contextSnapshot: context,
|
||||
continuationSummary,
|
||||
issueSummary: issueRef
|
||||
? {
|
||||
id: issueRef.id,
|
||||
|
|
@ -3656,6 +4117,7 @@ export function heartbeatService(db: Db) {
|
|||
agent,
|
||||
sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter,
|
||||
issueId,
|
||||
continuationSummaryBody: continuationSummary?.body ?? null,
|
||||
});
|
||||
if (sessionCompaction.rotate) {
|
||||
context.paperclipSessionHandoffMarkdown = sessionCompaction.handoffMarkdown;
|
||||
|
|
@ -3962,6 +4424,23 @@ export function heartbeatService(db: Db) {
|
|||
} else {
|
||||
outcome = "failed";
|
||||
}
|
||||
const runErrorMessage =
|
||||
outcome === "cancelled"
|
||||
? (latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled")
|
||||
: outcome === "succeeded"
|
||||
? null
|
||||
: redactCurrentUserText(
|
||||
adapterResult.errorMessage ?? (outcome === "timed_out" ? "Timed out" : "Adapter failed"),
|
||||
currentUserRedactionOptions,
|
||||
);
|
||||
const runErrorCode =
|
||||
outcome === "timed_out"
|
||||
? "timeout"
|
||||
: outcome === "cancelled"
|
||||
? (latestRun?.errorCode ?? "cancelled")
|
||||
: outcome === "failed"
|
||||
? (adapterResult.errorCode ?? "adapter_failed")
|
||||
: null;
|
||||
|
||||
let logSummary: { bytes: number; sha256?: string; compressed: boolean } | null = null;
|
||||
if (handle) {
|
||||
|
|
@ -4004,27 +4483,18 @@ export function heartbeatService(db: Db) {
|
|||
: null;
|
||||
|
||||
const persistedResultJson = mergeHeartbeatRunResultJson(
|
||||
adapterResult.resultJson ?? null,
|
||||
mergeRunStopMetadataForAgent(agent, outcome, {
|
||||
resultJson: adapterResult.resultJson ?? null,
|
||||
errorCode: runErrorCode,
|
||||
errorMessage: runErrorMessage,
|
||||
}),
|
||||
adapterResult.summary ?? null,
|
||||
);
|
||||
|
||||
await setRunStatus(run.id, status, {
|
||||
let persistedRun = await setRunStatus(run.id, status, {
|
||||
finishedAt: new Date(),
|
||||
error:
|
||||
outcome === "succeeded"
|
||||
? null
|
||||
: redactCurrentUserText(
|
||||
adapterResult.errorMessage ?? (outcome === "timed_out" ? "Timed out" : "Adapter failed"),
|
||||
currentUserRedactionOptions,
|
||||
),
|
||||
errorCode:
|
||||
outcome === "timed_out"
|
||||
? "timeout"
|
||||
: outcome === "cancelled"
|
||||
? "cancelled"
|
||||
: outcome === "failed"
|
||||
? (adapterResult.errorCode ?? "adapter_failed")
|
||||
: null,
|
||||
error: runErrorMessage,
|
||||
errorCode: runErrorCode,
|
||||
exitCode: adapterResult.exitCode,
|
||||
signal: adapterResult.signal,
|
||||
usageJson,
|
||||
|
|
@ -4036,13 +4506,16 @@ export function heartbeatService(db: Db) {
|
|||
logSha256: logSummary?.sha256,
|
||||
logCompressed: logSummary?.compressed ?? false,
|
||||
});
|
||||
if (persistedRun) {
|
||||
persistedRun = await classifyAndPersistRunLiveness(persistedRun, persistedResultJson) ?? persistedRun;
|
||||
}
|
||||
|
||||
await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, {
|
||||
finishedAt: new Date(),
|
||||
error: adapterResult.errorMessage ?? null,
|
||||
error: runErrorMessage,
|
||||
});
|
||||
|
||||
const finalizedRun = await getRun(run.id);
|
||||
const finalizedRun = persistedRun ?? (await getRun(run.id));
|
||||
if (finalizedRun) {
|
||||
await appendRunEvent(finalizedRun, seq++, {
|
||||
eventType: "lifecycle",
|
||||
|
|
@ -4054,13 +4527,15 @@ export function heartbeatService(db: Db) {
|
|||
exitCode: adapterResult.exitCode,
|
||||
},
|
||||
});
|
||||
const livenessRun = finalizedRun;
|
||||
await refreshContinuationSummaryForRun(livenessRun, agent);
|
||||
if (issueId && outcome === "succeeded") {
|
||||
try {
|
||||
const existingRunComment = await findRunIssueComment(finalizedRun.id, finalizedRun.companyId, issueId);
|
||||
const existingRunComment = await findRunIssueComment(livenessRun.id, livenessRun.companyId, issueId);
|
||||
if (!existingRunComment) {
|
||||
const issueComment = buildHeartbeatRunIssueComment(persistedResultJson);
|
||||
if (issueComment) {
|
||||
await issuesSvc.addComment(issueId, issueComment, { agentId: agent.id, runId: finalizedRun.id });
|
||||
await issuesSvc.addComment(issueId, issueComment, { agentId: agent.id, runId: livenessRun.id });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
|
|
@ -4070,8 +4545,9 @@ export function heartbeatService(db: Db) {
|
|||
);
|
||||
}
|
||||
}
|
||||
await finalizeIssueCommentPolicy(finalizedRun, agent);
|
||||
await releaseIssueExecutionAndPromote(finalizedRun);
|
||||
await finalizeIssueCommentPolicy(livenessRun, agent);
|
||||
await releaseIssueExecutionAndPromote(livenessRun);
|
||||
await handleRunLivenessContinuation(livenessRun);
|
||||
}
|
||||
|
||||
if (finalizedRun) {
|
||||
|
|
@ -4119,6 +4595,10 @@ export function heartbeatService(db: Db) {
|
|||
error: message,
|
||||
errorCode: "adapter_failed",
|
||||
finishedAt: new Date(),
|
||||
resultJson: mergeRunStopMetadataForAgent(agent, "failed", {
|
||||
errorCode: "adapter_failed",
|
||||
errorMessage: message,
|
||||
}),
|
||||
stdoutExcerpt,
|
||||
stderrExcerpt,
|
||||
logBytes: logSummary?.bytes,
|
||||
|
|
@ -4137,10 +4617,12 @@ export function heartbeatService(db: Db) {
|
|||
level: "error",
|
||||
message,
|
||||
});
|
||||
await finalizeIssueCommentPolicy(failedRun, agent);
|
||||
await releaseIssueExecutionAndPromote(failedRun);
|
||||
const livenessRun = await classifyAndPersistRunLiveness(failedRun) ?? failedRun;
|
||||
await refreshContinuationSummaryForRun(livenessRun, agent);
|
||||
await finalizeIssueCommentPolicy(livenessRun, agent);
|
||||
await releaseIssueExecutionAndPromote(livenessRun);
|
||||
|
||||
await updateRuntimeState(agent, failedRun, {
|
||||
await updateRuntimeState(agent, livenessRun, {
|
||||
exitCode: null,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
|
|
@ -4170,10 +4652,17 @@ export function heartbeatService(db: Db) {
|
|||
// The inner catch did not fire, so we must record the failure here.
|
||||
const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure";
|
||||
logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
|
||||
const setupFailureAgent = await getAgent(run.agentId).catch(() => null);
|
||||
await setRunStatus(runId, "failed", {
|
||||
error: message,
|
||||
errorCode: "adapter_failed",
|
||||
finishedAt: new Date(),
|
||||
...(setupFailureAgent ? {
|
||||
resultJson: mergeRunStopMetadataForAgent(setupFailureAgent, "failed", {
|
||||
errorCode: "adapter_failed",
|
||||
errorMessage: message,
|
||||
}),
|
||||
} : {}),
|
||||
}).catch(() => undefined);
|
||||
await setWakeupStatus(run.wakeupRequestId, "failed", {
|
||||
finishedAt: new Date(),
|
||||
|
|
@ -4189,11 +4678,13 @@ export function heartbeatService(db: Db) {
|
|||
level: "error",
|
||||
message,
|
||||
}).catch(() => undefined);
|
||||
const failedAgent = await getAgent(run.agentId).catch(() => null);
|
||||
const livenessRun = await classifyAndPersistRunLiveness(failedRun).catch(() => failedRun);
|
||||
const failedAgent = setupFailureAgent ?? await getAgent(run.agentId).catch(() => null);
|
||||
if (failedAgent) {
|
||||
await finalizeIssueCommentPolicy(failedRun, failedAgent).catch(() => undefined);
|
||||
await refreshContinuationSummaryForRun(livenessRun, failedAgent).catch(() => undefined);
|
||||
await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined);
|
||||
}
|
||||
await releaseIssueExecutionAndPromote(failedRun).catch(() => undefined);
|
||||
await releaseIssueExecutionAndPromote(livenessRun).catch(() => undefined);
|
||||
}
|
||||
// Ensure the agent is not left stuck in "running" if the inner catch handler's
|
||||
// DB calls threw (e.g. a transient DB error in finalizeAgentStatus).
|
||||
|
|
@ -4363,6 +4854,9 @@ export function heartbeatService(db: Db) {
|
|||
const sessionBefore =
|
||||
readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ??
|
||||
await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey);
|
||||
const promotedContinuationAttempt = readContinuationAttempt(
|
||||
promotedContextSnapshot.livenessContinuationAttempt,
|
||||
);
|
||||
const now = new Date();
|
||||
const newRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
|
|
@ -4375,6 +4869,7 @@ export function heartbeatService(db: Db) {
|
|||
wakeupRequestId: deferred.id,
|
||||
contextSnapshot: promotedContextSnapshot,
|
||||
sessionIdBefore: sessionBefore,
|
||||
continuationAttempt: promotedContinuationAttempt,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
|
@ -4473,6 +4968,7 @@ export function heartbeatService(db: Db) {
|
|||
const sessionBefore =
|
||||
explicitResumeSession?.sessionDisplayId ??
|
||||
await resolveSessionBeforeForWakeup(agent, effectiveTaskKey);
|
||||
const continuationAttempt = readContinuationAttempt(enrichedContextSnapshot.livenessContinuationAttempt);
|
||||
|
||||
const writeSkippedRequest = async (skipReason: string) => {
|
||||
await db.insert(agentWakeupRequests).values({
|
||||
|
|
@ -4771,6 +5267,7 @@ export function heartbeatService(db: Db) {
|
|||
wakeupRequestId: wakeupRequest.id,
|
||||
contextSnapshot: enrichedContextSnapshot,
|
||||
sessionIdBefore: sessionBefore,
|
||||
continuationAttempt,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
|
@ -4890,6 +5387,7 @@ export function heartbeatService(db: Db) {
|
|||
wakeupRequestId: wakeupRequest.id,
|
||||
contextSnapshot: enrichedContextSnapshot,
|
||||
sessionIdBefore: sessionBefore,
|
||||
continuationAttempt,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
|
@ -5022,6 +5520,7 @@ export function heartbeatService(db: Db) {
|
|||
const run = await getRun(runId);
|
||||
if (!run) throw notFound("Heartbeat run not found");
|
||||
if (run.status !== "running" && run.status !== "queued") return run;
|
||||
const agent = await getAgent(run.agentId);
|
||||
|
||||
const running = runningProcesses.get(run.id);
|
||||
if (running) {
|
||||
|
|
@ -5041,6 +5540,13 @@ export function heartbeatService(db: Db) {
|
|||
finishedAt: new Date(),
|
||||
error: reason,
|
||||
errorCode: "cancelled",
|
||||
...(agent ? {
|
||||
resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", {
|
||||
resultJson: parseObject(run.resultJson),
|
||||
errorCode: "cancelled",
|
||||
errorMessage: reason,
|
||||
}),
|
||||
} : {}),
|
||||
});
|
||||
|
||||
await setWakeupStatus(run.wakeupRequestId, "cancelled", {
|
||||
|
|
@ -5065,6 +5571,7 @@ export function heartbeatService(db: Db) {
|
|||
}
|
||||
|
||||
async function cancelActiveForAgentInternal(agentId: string, reason = "Cancelled due to agent pause") {
|
||||
const agent = await getAgent(agentId);
|
||||
const runs = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
|
|
@ -5075,6 +5582,13 @@ export function heartbeatService(db: Db) {
|
|||
finishedAt: new Date(),
|
||||
error: reason,
|
||||
errorCode: "cancelled",
|
||||
...(agent ? {
|
||||
resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", {
|
||||
resultJson: parseObject(run.resultJson),
|
||||
errorCode: "cancelled",
|
||||
errorMessage: reason,
|
||||
}),
|
||||
} : {}),
|
||||
});
|
||||
|
||||
await setWakeupStatus(run.wakeupRequestId, "cancelled", {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,12 @@ export { agentService, deduplicateAgentName } from "./agents.js";
|
|||
export { agentInstructionsService, syncInstructionsBundleConfigFromFilePath } from "./agent-instructions.js";
|
||||
export { assetService } from "./assets.js";
|
||||
export { documentService, extractLegacyPlanBody } from "./documents.js";
|
||||
export {
|
||||
ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY,
|
||||
buildContinuationSummaryMarkdown,
|
||||
getIssueContinuationSummaryDocument,
|
||||
refreshIssueContinuationSummary,
|
||||
} from "./issue-continuation-summary.js";
|
||||
export { projectService } from "./projects.js";
|
||||
export { issueService, type IssueFilters } from "./issues.js";
|
||||
export { issueApprovalService } from "./issue-approvals.js";
|
||||
|
|
|
|||
269
server/src/services/issue-continuation-summary.ts
Normal file
269
server/src/services/issue-continuation-summary.ts
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
import { and, eq } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { documents, issueDocuments, issues } from "@paperclipai/db";
|
||||
import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "@paperclipai/shared";
|
||||
import { documentService } from "./documents.js";
|
||||
|
||||
export { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY };
|
||||
export const ISSUE_CONTINUATION_SUMMARY_TITLE = "Continuation Summary";
|
||||
export const ISSUE_CONTINUATION_SUMMARY_MAX_BODY_CHARS = 8_000;
|
||||
const SUMMARY_SECTION_MAX_CHARS = 1_200;
|
||||
const PATH_CANDIDATE_RE = /(?:^|[\s`"'(])((?:server|ui|packages|doc|scripts|\.github)\/[A-Za-z0-9._/-]+)/g;
|
||||
|
||||
type IssueSummaryInput = {
|
||||
id: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
description: string | null;
|
||||
status: string;
|
||||
priority: string;
|
||||
};
|
||||
|
||||
type RunSummaryInput = {
|
||||
id: string;
|
||||
status: string;
|
||||
error: string | null;
|
||||
errorCode?: string | null;
|
||||
resultJson?: Record<string, unknown> | null;
|
||||
stdoutExcerpt?: string | null;
|
||||
stderrExcerpt?: string | null;
|
||||
finishedAt?: Date | null;
|
||||
};
|
||||
|
||||
type AgentSummaryInput = {
|
||||
id: string;
|
||||
name: string;
|
||||
adapterType: string | null;
|
||||
};
|
||||
|
||||
export type IssueContinuationSummaryDocument = {
|
||||
key: typeof ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY;
|
||||
title: string | null;
|
||||
body: string;
|
||||
latestRevisionId: string | null;
|
||||
latestRevisionNumber: number;
|
||||
updatedAt: Date;
|
||||
};
|
||||
|
||||
function truncateText(value: string, maxChars: number) {
|
||||
const trimmed = value.trim();
|
||||
if (trimmed.length <= maxChars) return trimmed;
|
||||
return `${trimmed.slice(0, Math.max(0, maxChars - 20)).trimEnd()}\n[truncated]`;
|
||||
}
|
||||
|
||||
function asNonEmptyString(value: unknown) {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function readResultSummary(resultJson: Record<string, unknown> | null | undefined) {
|
||||
if (!resultJson || typeof resultJson !== "object" || Array.isArray(resultJson)) return null;
|
||||
return (
|
||||
asNonEmptyString(resultJson.summary) ??
|
||||
asNonEmptyString(resultJson.result) ??
|
||||
asNonEmptyString(resultJson.message) ??
|
||||
asNonEmptyString(resultJson.error) ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
function extractMarkdownSection(markdown: string | null | undefined, heading: string) {
|
||||
if (!markdown) return null;
|
||||
const escaped = heading.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const re = new RegExp(`^##\\s+${escaped}\\s*$([\\s\\S]*?)(?=^##\\s+|(?![\\s\\S]))`, "im");
|
||||
const match = re.exec(markdown);
|
||||
const section = match?.[1]?.trim();
|
||||
return section ? truncateText(section, SUMMARY_SECTION_MAX_CHARS) : null;
|
||||
}
|
||||
|
||||
function extractPathCandidates(...texts: Array<string | null | undefined>) {
|
||||
const seen = new Set<string>();
|
||||
for (const text of texts) {
|
||||
if (!text) continue;
|
||||
for (const match of text.matchAll(PATH_CANDIDATE_RE)) {
|
||||
const path = match[1]?.replace(/[),.;:]+$/, "");
|
||||
if (path) seen.add(path);
|
||||
if (seen.size >= 12) break;
|
||||
}
|
||||
if (seen.size >= 12) break;
|
||||
}
|
||||
return [...seen];
|
||||
}
|
||||
|
||||
function inferMode(issue: IssueSummaryInput, run: RunSummaryInput) {
|
||||
if (issue.status === "done" || issue.status === "in_review") return "review";
|
||||
if (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") return "implementation";
|
||||
if (issue.status === "backlog" || issue.status === "todo") return "plan";
|
||||
return "implementation";
|
||||
}
|
||||
|
||||
function inferNextAction(issue: IssueSummaryInput, run: RunSummaryInput, previousNextAction: string | null) {
|
||||
if (issue.status === "done") return "Review the completed issue output and close any remaining follow-up comments.";
|
||||
if (issue.status === "in_review") return "Wait for reviewer feedback or approval before continuing executor work.";
|
||||
if (run.status === "failed" || run.status === "timed_out") {
|
||||
return "Inspect the failed run, fix the cause, and resume from the most recent concrete action above.";
|
||||
}
|
||||
if (run.status === "cancelled") return "Confirm the cancellation reason before starting another run.";
|
||||
return previousNextAction ?? "Resume implementation from the acceptance criteria, latest comments, and this summary.";
|
||||
}
|
||||
|
||||
function bulletList(items: string[], empty: string) {
|
||||
if (items.length === 0) return `- ${empty}`;
|
||||
return items.map((item) => `- ${item}`).join("\n");
|
||||
}
|
||||
|
||||
function extractPreviousNextAction(previousBody: string | null | undefined) {
|
||||
const section = extractMarkdownSection(previousBody, "Next Action");
|
||||
if (!section) return null;
|
||||
return section
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.replace(/^[-*]\s+/, "").trim())
|
||||
.find(Boolean) ?? null;
|
||||
}
|
||||
|
||||
export function buildContinuationSummaryMarkdown(input: {
|
||||
issue: IssueSummaryInput;
|
||||
run: RunSummaryInput;
|
||||
agent: AgentSummaryInput;
|
||||
previousSummaryBody?: string | null;
|
||||
}) {
|
||||
const { issue, run, agent } = input;
|
||||
const resultSummary = readResultSummary(run.resultJson);
|
||||
const recentActions = [
|
||||
`Run \`${run.id}\` finished with status \`${run.status}\`${run.finishedAt ? ` at ${run.finishedAt.toISOString()}` : ""}.`,
|
||||
resultSummary ? truncateText(resultSummary, SUMMARY_SECTION_MAX_CHARS) : "No adapter-provided result summary was captured for this run.",
|
||||
];
|
||||
if (run.error) {
|
||||
recentActions.push(`Latest run error${run.errorCode ? ` (${run.errorCode})` : ""}: ${truncateText(run.error, 500)}`);
|
||||
}
|
||||
|
||||
const paths = extractPathCandidates(resultSummary, run.stdoutExcerpt, run.stderrExcerpt, input.previousSummaryBody);
|
||||
const objective = extractMarkdownSection(issue.description, "Objective") ?? issue.description?.trim() ?? "No objective captured.";
|
||||
const acceptanceCriteria = extractMarkdownSection(issue.description, "Acceptance Criteria") ?? "No explicit acceptance criteria captured.";
|
||||
const mode = inferMode(issue, run);
|
||||
const nextAction = inferNextAction(issue, run, extractPreviousNextAction(input.previousSummaryBody));
|
||||
|
||||
const body = [
|
||||
"# Continuation Summary",
|
||||
"",
|
||||
`- Issue: ${issue.identifier ?? issue.id} — ${issue.title}`,
|
||||
`- Status: ${issue.status}`,
|
||||
`- Priority: ${issue.priority}`,
|
||||
`- Current mode: ${mode}`,
|
||||
`- Last updated by run: ${run.id}`,
|
||||
`- Agent: ${agent.name} (${agent.adapterType ?? "unknown"})`,
|
||||
"",
|
||||
"## Objective",
|
||||
"",
|
||||
truncateText(objective, SUMMARY_SECTION_MAX_CHARS),
|
||||
"",
|
||||
"## Acceptance Criteria",
|
||||
"",
|
||||
acceptanceCriteria,
|
||||
"",
|
||||
"## Recent Concrete Actions",
|
||||
"",
|
||||
bulletList(recentActions, "No recent actions captured."),
|
||||
"",
|
||||
"## Files / Routes Touched",
|
||||
"",
|
||||
bulletList(paths.map((path) => `\`${path}\``), "No file or route paths were detected in the captured run summary."),
|
||||
"",
|
||||
"## Commands Run",
|
||||
"",
|
||||
bulletList(
|
||||
[
|
||||
`Heartbeat run \`${run.id}\` invoked adapter \`${agent.adapterType ?? "unknown"}\`.`,
|
||||
"Detailed shell/tool commands remain in the run log and transcript.",
|
||||
],
|
||||
"No command metadata captured.",
|
||||
),
|
||||
"",
|
||||
"## Blockers / Decisions",
|
||||
"",
|
||||
bulletList(
|
||||
run.error
|
||||
? [`Latest run ended with \`${run.status}\`; inspect the error before continuing.`]
|
||||
: ["No new blocker was recorded by the latest run."],
|
||||
"No blockers or decisions captured.",
|
||||
),
|
||||
"",
|
||||
"## Next Action",
|
||||
"",
|
||||
`- ${nextAction}`,
|
||||
].join("\n");
|
||||
|
||||
return truncateText(body, ISSUE_CONTINUATION_SUMMARY_MAX_BODY_CHARS);
|
||||
}
|
||||
|
||||
export async function getIssueContinuationSummaryDocument(
|
||||
db: Db,
|
||||
issueId: string,
|
||||
): Promise<IssueContinuationSummaryDocument | null> {
|
||||
const row = await db
|
||||
.select({
|
||||
key: issueDocuments.key,
|
||||
title: documents.title,
|
||||
body: documents.latestBody,
|
||||
latestRevisionId: documents.latestRevisionId,
|
||||
latestRevisionNumber: documents.latestRevisionNumber,
|
||||
updatedAt: documents.updatedAt,
|
||||
})
|
||||
.from(issueDocuments)
|
||||
.innerJoin(documents, eq(issueDocuments.documentId, documents.id))
|
||||
.where(and(eq(issueDocuments.issueId, issueId), eq(issueDocuments.key, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
if (!row) return null;
|
||||
return {
|
||||
key: ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY,
|
||||
title: row.title,
|
||||
body: row.body,
|
||||
latestRevisionId: row.latestRevisionId,
|
||||
latestRevisionNumber: row.latestRevisionNumber,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export async function refreshIssueContinuationSummary(input: {
|
||||
db: Db;
|
||||
issueId: string;
|
||||
run: RunSummaryInput;
|
||||
agent: AgentSummaryInput;
|
||||
}) {
|
||||
const { db, issueId, run, agent } = input;
|
||||
const [issue, existing] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
id: issues.id,
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
description: issues.description,
|
||||
status: issues.status,
|
||||
priority: issues.priority,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
getIssueContinuationSummaryDocument(db, issueId),
|
||||
]);
|
||||
|
||||
if (!issue) return null;
|
||||
const body = buildContinuationSummaryMarkdown({
|
||||
issue,
|
||||
run,
|
||||
agent,
|
||||
previousSummaryBody: existing?.body ?? null,
|
||||
});
|
||||
const result = await documentService(db).upsertIssueDocument({
|
||||
issueId,
|
||||
key: ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY,
|
||||
title: ISSUE_CONTINUATION_SUMMARY_TITLE,
|
||||
format: "markdown",
|
||||
body,
|
||||
baseRevisionId: existing?.latestRevisionId ?? null,
|
||||
changeSummary: `Refresh continuation summary after run ${run.id}`,
|
||||
createdByAgentId: agent.id,
|
||||
createdByRunId: run.id,
|
||||
});
|
||||
return result.document;
|
||||
}
|
||||
|
|
@ -38,6 +38,9 @@ import { getDefaultCompanyGoal } from "./goals.js";
|
|||
|
||||
const ALL_ISSUE_STATUSES = ["backlog", "todo", "in_progress", "in_review", "blocked", "done", "cancelled"];
|
||||
const MAX_ISSUE_COMMENT_PAGE_LIMIT = 500;
|
||||
export const MAX_CHILD_ISSUES_CREATED_BY_HELPER = 25;
|
||||
const MAX_CHILD_COMPLETION_SUMMARIES = 20;
|
||||
const CHILD_COMPLETION_SUMMARY_BODY_MAX_CHARS = 500;
|
||||
|
||||
function assertTransition(from: string, to: string) {
|
||||
if (from === to) return;
|
||||
|
|
@ -121,10 +124,27 @@ type IssueCreateInput = Omit<typeof issues.$inferInsert, "companyId"> & {
|
|||
blockedByIssueIds?: string[];
|
||||
inheritExecutionWorkspaceFromIssueId?: string | null;
|
||||
};
|
||||
type IssueChildCreateInput = IssueCreateInput & {
|
||||
acceptanceCriteria?: string[];
|
||||
blockParentUntilDone?: boolean;
|
||||
actorAgentId?: string | null;
|
||||
actorUserId?: string | null;
|
||||
};
|
||||
type IssueRelationSummaryMap = {
|
||||
blockedBy: IssueRelationIssueSummary[];
|
||||
blocks: IssueRelationIssueSummary[];
|
||||
};
|
||||
export type ChildIssueCompletionSummary = {
|
||||
id: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
status: string;
|
||||
priority: string;
|
||||
assigneeAgentId: string | null;
|
||||
assigneeUserId: string | null;
|
||||
updatedAt: Date;
|
||||
summary: string | null;
|
||||
};
|
||||
|
||||
function sameRunLock(checkoutRunId: string | null, actorRunId: string | null) {
|
||||
if (actorRunId) return checkoutRunId === actorRunId;
|
||||
|
|
@ -138,6 +158,20 @@ function escapeLikePattern(value: string): string {
|
|||
return value.replace(/[\\%_]/g, "\\$&");
|
||||
}
|
||||
|
||||
function truncateInlineSummary(value: string | null | undefined, maxChars = CHILD_COMPLETION_SUMMARY_BODY_MAX_CHARS) {
|
||||
const normalized = value?.trim();
|
||||
if (!normalized) return null;
|
||||
return normalized.length > maxChars ? `${normalized.slice(0, Math.max(0, maxChars - 15)).trimEnd()} [truncated]` : normalized;
|
||||
}
|
||||
|
||||
function appendAcceptanceCriteriaToDescription(description: string | null | undefined, acceptanceCriteria: string[] | undefined) {
|
||||
const criteria = (acceptanceCriteria ?? []).map((item) => item.trim()).filter(Boolean);
|
||||
if (criteria.length === 0) return description ?? null;
|
||||
const base = description?.trim() ?? "";
|
||||
const criteriaMarkdown = ["## Acceptance Criteria", "", ...criteria.map((item) => `- ${item}`)].join("\n");
|
||||
return base ? `${base}\n\n${criteriaMarkdown}` : criteriaMarkdown;
|
||||
}
|
||||
|
||||
async function getProjectDefaultGoalId(
|
||||
db: ProjectGoalReader,
|
||||
companyId: string,
|
||||
|
|
@ -1406,18 +1440,110 @@ export function issueService(db: Db) {
|
|||
}
|
||||
|
||||
const children = await db
|
||||
.select({ id: issues.id, status: issues.status })
|
||||
.select({
|
||||
id: issues.id,
|
||||
identifier: issues.identifier,
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
priority: issues.priority,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
updatedAt: issues.updatedAt,
|
||||
})
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, parent.companyId), eq(issues.parentId, parentIssueId)));
|
||||
.where(and(eq(issues.companyId, parent.companyId), eq(issues.parentId, parentIssueId)))
|
||||
.orderBy(asc(issues.issueNumber), asc(issues.createdAt));
|
||||
if (children.length === 0) return null;
|
||||
if (!children.every((child) => child.status === "done" || child.status === "cancelled")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const childIdsForSummaries = children.slice(0, MAX_CHILD_COMPLETION_SUMMARIES).map((child) => child.id);
|
||||
const commentRows = childIdsForSummaries.length > 0
|
||||
? await db
|
||||
.select({
|
||||
issueId: issueComments.issueId,
|
||||
body: issueComments.body,
|
||||
createdAt: issueComments.createdAt,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(and(eq(issueComments.companyId, parent.companyId), inArray(issueComments.issueId, childIdsForSummaries)))
|
||||
.orderBy(desc(issueComments.createdAt), desc(issueComments.id))
|
||||
: [];
|
||||
const latestCommentByIssueId = new Map<string, string>();
|
||||
for (const comment of commentRows) {
|
||||
if (!latestCommentByIssueId.has(comment.issueId)) {
|
||||
latestCommentByIssueId.set(comment.issueId, comment.body);
|
||||
}
|
||||
}
|
||||
const childIssueSummaries: ChildIssueCompletionSummary[] = children
|
||||
.slice(0, MAX_CHILD_COMPLETION_SUMMARIES)
|
||||
.map((child) => ({
|
||||
...child,
|
||||
summary: truncateInlineSummary(latestCommentByIssueId.get(child.id)),
|
||||
}));
|
||||
|
||||
return {
|
||||
id: parent.id,
|
||||
assigneeAgentId: parent.assigneeAgentId,
|
||||
childIssueIds: children.map((child) => child.id),
|
||||
childIssueSummaries,
|
||||
childIssueSummaryTruncated: children.length > childIssueSummaries.length,
|
||||
};
|
||||
},
|
||||
|
||||
createChild: async (
|
||||
parentIssueId: string,
|
||||
data: IssueChildCreateInput,
|
||||
) => {
|
||||
const parent = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(eq(issues.id, parentIssueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!parent) throw notFound("Parent issue not found");
|
||||
|
||||
const [{ childCount }] = await db
|
||||
.select({ childCount: sql<number>`count(*)::int` })
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, parent.companyId), eq(issues.parentId, parent.id)));
|
||||
if (childCount >= MAX_CHILD_ISSUES_CREATED_BY_HELPER) {
|
||||
throw unprocessable(`Parent issue already has the maximum ${MAX_CHILD_ISSUES_CREATED_BY_HELPER} child issues for this helper`);
|
||||
}
|
||||
|
||||
const {
|
||||
acceptanceCriteria,
|
||||
blockParentUntilDone,
|
||||
actorAgentId,
|
||||
actorUserId,
|
||||
...issueData
|
||||
} = data;
|
||||
const child = await issueService(db).create(parent.companyId, {
|
||||
...issueData,
|
||||
parentId: parent.id,
|
||||
projectId: issueData.projectId ?? parent.projectId,
|
||||
goalId: issueData.goalId ?? parent.goalId,
|
||||
requestDepth: Math.max(parent.requestDepth + 1, issueData.requestDepth ?? 0),
|
||||
description: appendAcceptanceCriteriaToDescription(issueData.description, acceptanceCriteria),
|
||||
inheritExecutionWorkspaceFromIssueId: parent.id,
|
||||
});
|
||||
|
||||
if (blockParentUntilDone) {
|
||||
const existingBlockers = await db
|
||||
.select({ blockerIssueId: issueRelations.issueId })
|
||||
.from(issueRelations)
|
||||
.where(and(eq(issueRelations.companyId, parent.companyId), eq(issueRelations.relatedIssueId, parent.id), eq(issueRelations.type, "blocks")));
|
||||
await syncBlockedByIssueIds(
|
||||
parent.id,
|
||||
parent.companyId,
|
||||
[...new Set([...existingBlockers.map((row) => row.blockerIssueId), child.id])],
|
||||
{ agentId: actorAgentId ?? null, userId: actorUserId ?? null },
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
issue: child,
|
||||
parentBlockerAdded: Boolean(blockParentUntilDone),
|
||||
};
|
||||
},
|
||||
|
||||
|
|
|
|||
188
server/src/services/run-continuations.ts
Normal file
188
server/src/services/run-continuations.ts
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agentWakeupRequests, agents, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import type { RunLivenessState } from "@paperclipai/shared";
|
||||
|
||||
export const RUN_LIVENESS_CONTINUATION_REASON = "run_liveness_continuation";
|
||||
export const DEFAULT_MAX_LIVENESS_CONTINUATION_ATTEMPTS = 2;
|
||||
|
||||
const ACTIONABLE_LIVENESS_STATES = new Set<RunLivenessState>(["plan_only", "empty_response"]);
|
||||
const CONTINUATION_ACTIVE_ISSUE_STATUSES = new Set(["todo", "in_progress"]);
|
||||
// A prior adapter error should not permanently suppress bounded liveness
|
||||
// continuations; the max-attempt/idempotency guards prevent unbounded retries.
|
||||
const CONTINUATION_AGENT_STATUSES = new Set(["active", "idle", "running", "error"]);
|
||||
const IDEMPOTENT_WAKE_STATUSES = ["queued", "deferred_issue_execution", "completed"];
|
||||
|
||||
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
|
||||
type IssueRow = Pick<
|
||||
typeof issues.$inferSelect,
|
||||
"id" | "companyId" | "identifier" | "title" | "status" | "assigneeAgentId" | "executionState" | "projectId"
|
||||
>;
|
||||
type AgentRow = Pick<typeof agents.$inferSelect, "id" | "companyId" | "status">;
|
||||
|
||||
export type RunContinuationDecision =
|
||||
| {
|
||||
kind: "enqueue";
|
||||
nextAttempt: number;
|
||||
idempotencyKey: string;
|
||||
payload: Record<string, unknown>;
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
}
|
||||
| {
|
||||
kind: "exhausted";
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
comment: string;
|
||||
}
|
||||
| {
|
||||
kind: "skip";
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export function readContinuationAttempt(value: unknown): number {
|
||||
const numeric = typeof value === "number" ? value : Number.parseInt(String(value ?? ""), 10);
|
||||
return Number.isFinite(numeric) && numeric > 0 ? Math.floor(numeric) : 0;
|
||||
}
|
||||
|
||||
export function buildRunLivenessContinuationIdempotencyKey(input: {
|
||||
issueId: string;
|
||||
sourceRunId: string;
|
||||
livenessState: RunLivenessState;
|
||||
nextAttempt: number;
|
||||
}) {
|
||||
return [
|
||||
"run_liveness_continuation",
|
||||
input.issueId,
|
||||
input.sourceRunId,
|
||||
input.livenessState,
|
||||
String(input.nextAttempt),
|
||||
].join(":");
|
||||
}
|
||||
|
||||
export async function findExistingRunLivenessContinuationWake(
|
||||
db: Db,
|
||||
input: {
|
||||
companyId: string;
|
||||
idempotencyKey: string;
|
||||
},
|
||||
) {
|
||||
return db
|
||||
.select({ id: agentWakeupRequests.id, status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, input.companyId),
|
||||
eq(agentWakeupRequests.idempotencyKey, input.idempotencyKey),
|
||||
inArray(agentWakeupRequests.status, IDEMPOTENT_WAKE_STATUSES),
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
}
|
||||
|
||||
export function decideRunLivenessContinuation(input: {
|
||||
run: HeartbeatRunRow;
|
||||
issue: IssueRow | null;
|
||||
agent: AgentRow | null;
|
||||
livenessState: RunLivenessState | null;
|
||||
livenessReason: string | null;
|
||||
nextAction: string | null;
|
||||
budgetBlocked: boolean;
|
||||
idempotentWakeExists: boolean;
|
||||
maxAttempts?: number;
|
||||
}): RunContinuationDecision {
|
||||
const {
|
||||
run,
|
||||
issue,
|
||||
agent,
|
||||
livenessState,
|
||||
livenessReason,
|
||||
nextAction,
|
||||
budgetBlocked,
|
||||
idempotentWakeExists,
|
||||
} = input;
|
||||
const maxAttempts = input.maxAttempts ?? DEFAULT_MAX_LIVENESS_CONTINUATION_ATTEMPTS;
|
||||
|
||||
if (!livenessState || !ACTIONABLE_LIVENESS_STATES.has(livenessState)) {
|
||||
return { kind: "skip", reason: "liveness state is not actionable for continuation" };
|
||||
}
|
||||
if (!issue) return { kind: "skip", reason: "issue not found" };
|
||||
if (!agent) return { kind: "skip", reason: "agent not found" };
|
||||
if (issue.companyId !== run.companyId || agent.companyId !== run.companyId) {
|
||||
return { kind: "skip", reason: "company scope mismatch" };
|
||||
}
|
||||
if (issue.assigneeAgentId !== run.agentId) {
|
||||
return { kind: "skip", reason: "issue is no longer assigned to the source run agent" };
|
||||
}
|
||||
if (!CONTINUATION_ACTIVE_ISSUE_STATUSES.has(issue.status)) {
|
||||
return { kind: "skip", reason: `issue status ${issue.status} is not continuable` };
|
||||
}
|
||||
if (issue.executionState) {
|
||||
return { kind: "skip", reason: "issue is blocked by execution policy state" };
|
||||
}
|
||||
if (!CONTINUATION_AGENT_STATUSES.has(agent.status)) {
|
||||
return { kind: "skip", reason: `agent status ${agent.status} is not invokable` };
|
||||
}
|
||||
if (budgetBlocked) {
|
||||
return { kind: "skip", reason: "budget hard stop blocks continuation" };
|
||||
}
|
||||
|
||||
const currentAttempt = readContinuationAttempt(run.continuationAttempt);
|
||||
if (currentAttempt >= maxAttempts) {
|
||||
return {
|
||||
kind: "exhausted",
|
||||
attempt: currentAttempt,
|
||||
maxAttempts,
|
||||
comment: [
|
||||
"Bounded liveness continuation exhausted",
|
||||
"",
|
||||
`- Last liveness state: \`${livenessState}\``,
|
||||
`- Attempts used: ${currentAttempt}/${maxAttempts}`,
|
||||
`- Reason: ${livenessReason ?? "Run ended without concrete progress"}`,
|
||||
"- Next action: a human or manager should inspect the run and either clarify the task, mark it blocked, or assign a concrete follow-up.",
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
const nextAttempt = currentAttempt + 1;
|
||||
const idempotencyKey = buildRunLivenessContinuationIdempotencyKey({
|
||||
issueId: issue.id,
|
||||
sourceRunId: run.id,
|
||||
livenessState,
|
||||
nextAttempt,
|
||||
});
|
||||
if (idempotentWakeExists) {
|
||||
return { kind: "skip", reason: "continuation wake already exists for this source run and attempt" };
|
||||
}
|
||||
|
||||
const payload = {
|
||||
issueId: issue.id,
|
||||
sourceRunId: run.id,
|
||||
livenessState,
|
||||
livenessReason,
|
||||
continuationAttempt: nextAttempt,
|
||||
maxContinuationAttempts: maxAttempts,
|
||||
instruction:
|
||||
nextAction ??
|
||||
"The previous run ended without concrete progress. Take the first concrete action now or mark the issue blocked with a specific unblock request.",
|
||||
};
|
||||
|
||||
return {
|
||||
kind: "enqueue",
|
||||
nextAttempt,
|
||||
idempotencyKey,
|
||||
payload,
|
||||
contextSnapshot: {
|
||||
issueId: issue.id,
|
||||
taskId: issue.id,
|
||||
taskKey: issue.id,
|
||||
wakeReason: RUN_LIVENESS_CONTINUATION_REASON,
|
||||
livenessContinuationAttempt: nextAttempt,
|
||||
livenessContinuationMaxAttempts: maxAttempts,
|
||||
livenessContinuationSourceRunId: run.id,
|
||||
livenessContinuationState: livenessState,
|
||||
livenessContinuationReason: livenessReason,
|
||||
livenessContinuationInstruction: payload.instruction,
|
||||
},
|
||||
};
|
||||
}
|
||||
227
server/src/services/run-liveness.ts
Normal file
227
server/src/services/run-liveness.ts
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
import type { HeartbeatRunStatus, IssueStatus, RunLivenessState } from "@paperclipai/shared";
|
||||
|
||||
export interface RunLivenessIssueInput {
|
||||
status: IssueStatus | string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
}
|
||||
|
||||
export interface RunLivenessEvidenceInput {
|
||||
issueCommentsCreated: number;
|
||||
documentRevisionsCreated: number;
|
||||
planDocumentRevisionsCreated: number;
|
||||
workProductsCreated: number;
|
||||
workspaceOperationsCreated: number;
|
||||
activityEventsCreated: number;
|
||||
toolOrActionEventsCreated: number;
|
||||
latestEvidenceAt: Date | null;
|
||||
}
|
||||
|
||||
export interface RunLivenessClassificationInput {
|
||||
runStatus: HeartbeatRunStatus | string;
|
||||
issue: RunLivenessIssueInput | null;
|
||||
resultJson?: Record<string, unknown> | null;
|
||||
stdoutExcerpt?: string | null;
|
||||
stderrExcerpt?: string | null;
|
||||
error?: string | null;
|
||||
errorCode?: string | null;
|
||||
continuationAttempt?: number | null;
|
||||
evidence?: Partial<RunLivenessEvidenceInput> | null;
|
||||
}
|
||||
|
||||
export interface RunLivenessClassification {
|
||||
livenessState: RunLivenessState;
|
||||
livenessReason: string;
|
||||
continuationAttempt: number;
|
||||
lastUsefulActionAt: Date | null;
|
||||
nextAction: string | null;
|
||||
}
|
||||
|
||||
const DEFAULT_EVIDENCE: RunLivenessEvidenceInput = {
|
||||
issueCommentsCreated: 0,
|
||||
documentRevisionsCreated: 0,
|
||||
planDocumentRevisionsCreated: 0,
|
||||
workProductsCreated: 0,
|
||||
workspaceOperationsCreated: 0,
|
||||
activityEventsCreated: 0,
|
||||
toolOrActionEventsCreated: 0,
|
||||
latestEvidenceAt: null,
|
||||
};
|
||||
|
||||
const PLANNING_ONLY_RE =
|
||||
/\b(?:i(?:'ll| will| am going to|'m going to)|let me|i need to|next(?:,| i will| i'll)?|my next step is|the next step is)\s+(?:first\s+)?(?:inspect|check|review|look|investigate|analy[sz]e|open|read|start|begin|work on|implement|fix|test|update|create|add)\b/i;
|
||||
const NEXT_STEPS_RE = /^\s*(?:next steps?|plan)\s*:/im;
|
||||
const BLOCKER_RE =
|
||||
/\b(?:blocked|can't proceed|cannot proceed|unable to proceed|waiting on|need(?:s|ed)? .{0,80}\b(?:approval|access|credential|credentials|secret|api key|token|input|clarification)|requires? .{0,80}\b(?:approval|access|credential|credentials|secret|api key|token|input|clarification))\b/i;
|
||||
const NEGATED_BLOCKER_RE = /\b(?:not blocked|no blocker|no blockers|unblocked)\b/i;
|
||||
const PLAN_TASK_TITLE_RE = /\b(?:plan|planning|analysis|investigation|research|report|proposal|design doc|write-?up)\b/i;
|
||||
const PLAN_TASK_DESCRIPTION_RE =
|
||||
/\b(?:create|write|produce|draft|update|revise|prepare)\s+(?:a\s+|the\s+)?(?:plan|analysis|investigation|research report|report|proposal|design doc|write-?up)\b/i;
|
||||
|
||||
function compactReason(reason: string) {
|
||||
return reason.length <= 500 ? reason : `${reason.slice(0, 497)}...`;
|
||||
}
|
||||
|
||||
function normalizeCount(value: unknown) {
|
||||
return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0;
|
||||
}
|
||||
|
||||
function normalizeContinuationAttempt(value: unknown) {
|
||||
return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0;
|
||||
}
|
||||
|
||||
function readText(value: unknown): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
function resultText(resultJson: Record<string, unknown> | null | undefined) {
|
||||
if (!resultJson) return "";
|
||||
return [
|
||||
readText(resultJson.summary),
|
||||
readText(resultJson.result),
|
||||
readText(resultJson.message),
|
||||
readText(resultJson.stdout),
|
||||
readText(resultJson.stderr),
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function combinedOutput(input: RunLivenessClassificationInput) {
|
||||
return [
|
||||
resultText(input.resultJson),
|
||||
readText(input.stdoutExcerpt),
|
||||
readText(input.stderrExcerpt),
|
||||
readText(input.error),
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join("\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function hasUsefulOutput(input: RunLivenessClassificationInput) {
|
||||
return combinedOutput(input).length > 0;
|
||||
}
|
||||
|
||||
export function declaredBlocker(input: RunLivenessClassificationInput) {
|
||||
if (input.issue?.status === "blocked") return true;
|
||||
const text = combinedOutput(input);
|
||||
if (!text || NEGATED_BLOCKER_RE.test(text)) return false;
|
||||
return BLOCKER_RE.test(text);
|
||||
}
|
||||
|
||||
export function looksLikePlanningOnly(input: RunLivenessClassificationInput) {
|
||||
const text = combinedOutput(input);
|
||||
if (!text) return false;
|
||||
return PLANNING_ONLY_RE.test(text) || NEXT_STEPS_RE.test(text);
|
||||
}
|
||||
|
||||
export function isPlanningOrDocumentTask(issue: RunLivenessIssueInput | null | undefined) {
|
||||
if (!issue) return false;
|
||||
if (PLAN_TASK_TITLE_RE.test(issue.title)) return true;
|
||||
return PLAN_TASK_DESCRIPTION_RE.test(issue.description ?? "");
|
||||
}
|
||||
|
||||
function normalizeEvidence(evidence: Partial<RunLivenessEvidenceInput> | null | undefined): RunLivenessEvidenceInput {
|
||||
return {
|
||||
issueCommentsCreated: normalizeCount(evidence?.issueCommentsCreated),
|
||||
documentRevisionsCreated: normalizeCount(evidence?.documentRevisionsCreated),
|
||||
planDocumentRevisionsCreated: normalizeCount(evidence?.planDocumentRevisionsCreated),
|
||||
workProductsCreated: normalizeCount(evidence?.workProductsCreated),
|
||||
workspaceOperationsCreated: normalizeCount(evidence?.workspaceOperationsCreated),
|
||||
activityEventsCreated: normalizeCount(evidence?.activityEventsCreated),
|
||||
toolOrActionEventsCreated: normalizeCount(evidence?.toolOrActionEventsCreated),
|
||||
latestEvidenceAt: evidence?.latestEvidenceAt instanceof Date ? evidence.latestEvidenceAt : null,
|
||||
};
|
||||
}
|
||||
|
||||
export function hasConcreteActionEvidence(evidence: Partial<RunLivenessEvidenceInput> | null | undefined) {
|
||||
const normalized = normalizeEvidence(evidence);
|
||||
// Workspace creation is setup evidence, not task progress by itself. It can
|
||||
// appear in reasons alongside durable activity, but it must not prevent a
|
||||
// planning-only or empty run from receiving a bounded continuation.
|
||||
return (
|
||||
normalized.issueCommentsCreated +
|
||||
normalized.documentRevisionsCreated +
|
||||
normalized.workProductsCreated +
|
||||
normalized.activityEventsCreated +
|
||||
normalized.toolOrActionEventsCreated >
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
function evidenceReason(evidence: RunLivenessEvidenceInput) {
|
||||
const parts: string[] = [];
|
||||
if (evidence.issueCommentsCreated > 0) parts.push(`${evidence.issueCommentsCreated} issue comment(s)`);
|
||||
if (evidence.documentRevisionsCreated > 0) parts.push(`${evidence.documentRevisionsCreated} document revision(s)`);
|
||||
if (evidence.workProductsCreated > 0) parts.push(`${evidence.workProductsCreated} work product(s)`);
|
||||
if (evidence.workspaceOperationsCreated > 0) parts.push(`${evidence.workspaceOperationsCreated} workspace operation(s)`);
|
||||
if (evidence.activityEventsCreated > 0) parts.push(`${evidence.activityEventsCreated} activity event(s)`);
|
||||
if (evidence.toolOrActionEventsCreated > 0) parts.push(`${evidence.toolOrActionEventsCreated} tool/action event(s)`);
|
||||
return parts.join(", ");
|
||||
}
|
||||
|
||||
function extractNextAction(input: RunLivenessClassificationInput) {
|
||||
const text = combinedOutput(input);
|
||||
if (!text) return null;
|
||||
const line = text
|
||||
.split(/\r?\n/)
|
||||
.map((entry) => entry.trim())
|
||||
.find((entry) => PLANNING_ONLY_RE.test(entry) || /^next(?: steps?| action)?\s*:/i.test(entry));
|
||||
if (!line) return null;
|
||||
return line.length <= 500 ? line : `${line.slice(0, 497)}...`;
|
||||
}
|
||||
|
||||
export function classifyRunLiveness(input: RunLivenessClassificationInput): RunLivenessClassification {
|
||||
const evidence = normalizeEvidence(input.evidence);
|
||||
const continuationAttempt = normalizeContinuationAttempt(input.continuationAttempt);
|
||||
const issueStatus = input.issue?.status ?? null;
|
||||
const usefulOutput = hasUsefulOutput(input);
|
||||
const concreteEvidence = hasConcreteActionEvidence(evidence);
|
||||
const planExempt = isPlanningOrDocumentTask(input.issue) || evidence.planDocumentRevisionsCreated > 0;
|
||||
const lastUsefulActionAt = concreteEvidence ? evidence.latestEvidenceAt : null;
|
||||
|
||||
const output = (state: RunLivenessState, reason: string, nextAction: string | null = null): RunLivenessClassification => ({
|
||||
livenessState: state,
|
||||
livenessReason: compactReason(reason),
|
||||
continuationAttempt,
|
||||
lastUsefulActionAt: state === "advanced" || state === "completed" || state === "blocked" ? lastUsefulActionAt : null,
|
||||
nextAction,
|
||||
});
|
||||
|
||||
if (input.runStatus !== "succeeded") {
|
||||
return output("failed", input.errorCode ? `Run ended with ${input.runStatus} (${input.errorCode})` : `Run ended with ${input.runStatus}`);
|
||||
}
|
||||
|
||||
if (issueStatus === "done" || issueStatus === "cancelled") {
|
||||
return output("completed", `Issue is ${issueStatus}`);
|
||||
}
|
||||
|
||||
if (declaredBlocker(input)) {
|
||||
return output("blocked", issueStatus === "blocked" ? "Issue status is blocked" : "Run output declared a concrete blocker", extractNextAction(input));
|
||||
}
|
||||
|
||||
if (!usefulOutput && !concreteEvidence) {
|
||||
return output("empty_response", "Run succeeded without useful output or concrete action evidence");
|
||||
}
|
||||
|
||||
if (concreteEvidence) {
|
||||
return output("advanced", `Run produced concrete action evidence: ${evidenceReason(evidence)}`);
|
||||
}
|
||||
|
||||
if (planExempt && usefulOutput) {
|
||||
return output("advanced", "Planning/document task produced useful output and is exempt from plan-only classification");
|
||||
}
|
||||
|
||||
if (looksLikePlanningOnly(input)) {
|
||||
return output("plan_only", "Run described future work without concrete action evidence", extractNextAction(input));
|
||||
}
|
||||
|
||||
if (usefulOutput) {
|
||||
return output("needs_followup", "Run produced useful output but no concrete action evidence", extractNextAction(input));
|
||||
}
|
||||
|
||||
return output("empty_response", "Run succeeded without useful output");
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue