import type { HeartbeatRunStatus, IssueStatus, RunLivenessState } from "@paperclipai/shared"; export type RunLivenessActionability = | "runnable" | "manager_review" | "blocked_external" | "approval_required" | "unknown"; export interface RunLivenessIssueInput { status: IssueStatus | string; title: string; description: string | null; } export interface RunLivenessEvidenceInput { issueCommentsCreated: number; documentRevisionsCreated: number; planDocumentRevisionsCreated: number; workProductsCreated: number; workspaceOperationsCreated: number; activityEventsCreated: number; toolOrActionEventsCreated: number; latestEvidenceAt: Date | null; } export interface RunLivenessClassificationInput { runStatus: HeartbeatRunStatus | string; issue: RunLivenessIssueInput | null; resultJson?: Record | null; issueCommentBodies?: string[] | null; continuationSummaryBody?: string | null; stdoutExcerpt?: string | null; stderrExcerpt?: string | null; error?: string | null; errorCode?: string | null; continuationAttempt?: number | null; evidence?: Partial | null; } export interface RunLivenessClassification { livenessState: RunLivenessState; livenessReason: string; continuationAttempt: number; lastUsefulActionAt: Date | null; nextAction: string | null; actionability: RunLivenessActionability; } const DEFAULT_EVIDENCE: RunLivenessEvidenceInput = { issueCommentsCreated: 0, documentRevisionsCreated: 0, planDocumentRevisionsCreated: 0, workProductsCreated: 0, workspaceOperationsCreated: 0, activityEventsCreated: 0, toolOrActionEventsCreated: 0, latestEvidenceAt: null, }; const PLANNING_ONLY_RE = /\b(?:i(?:'ll| will| am going to|'m going to)|let me|i need to|next(?:,| i will| i'll)?|my next step is|the next step is)\s+(?:first\s+)?(?:inspect|check|review|look|investigate|analy[sz]e|open|read|start|begin|work on|implement|fix|test|update|create|add)\b/i; const NEXT_STEPS_RE = /^\s*(?:next steps?|plan)\s*:/im; const BLOCKER_RE = /\b(?:blocked|can't proceed|cannot proceed|unable to proceed|waiting on|need(?:s|ed)? .{0,80}\b(?:approval|access|credential|credentials|secret|api key|token|input|clarification)|requires? .{0,80}\b(?:approval|access|credential|credentials|secret|api key|token|input|clarification))\b/i; const NEGATED_BLOCKER_RE = /\b(?:not blocked|no blocker|no blockers|unblocked)\b/i; const APPROVAL_REQUIRED_RE = /\b(?:approval required|requires? .{0,80}\bapproval|need(?:s|ed)? .{0,80}\bapproval|waiting on .{0,80}\bapproval|pending approval|board approval|human approval|user approval|operator approval)\b/i; const EXTERNAL_BLOCKER_RE = /\b(?:can't proceed|cannot proceed|unable to proceed|waiting on|blocked by|blocked on|need(?:s|ed)?|requires?) .{0,120}\b(?:access|credential|credentials|secret|secrets|api key|token|password|login|account|permission|permissions|input|clarification)\b/i; const MANAGER_REVIEW_RE = /\b(?:manager review|human review|manual review|security review|escalate|production deploy|deploy(?:ing)? to production|deploy(?:ing)? to prod|prod deploy|production access|rotate .{0,40}\b(?:secret|key|token)|delete .{0,40}\bproduction|security-sensitive|credentialed operation|budget-sensitive|cost approval|spend approval)\b/i; const RUNNABLE_RE = /\b(?:(?:run|rerun|execute)\s+(?:pnpm|npm|yarn|bun|vitest|jest|pytest|cargo|go test|curl|tests?|typecheck|build|lint|package|verification)|(?:inspect|check|review|look|investigate|analy[sz]e|open|read|start|begin|continue|implement|fix|test|update|create|add|write|verify|validate|report)\b)/i; const PLAN_TASK_TITLE_RE = /\b(?:plan|planning|analysis|investigation|research|report|proposal|design doc|write-?up)\b/i; const PLAN_TASK_DESCRIPTION_RE = /\b(?:create|write|produce|draft|update|revise|prepare)\s+(?:a\s+|the\s+)?(?:plan|analysis|investigation|research report|report|proposal|design doc|write-?up)\b/i; function compactReason(reason: string) { return reason.length <= 500 ? reason : `${reason.slice(0, 497)}...`; } function normalizeCount(value: unknown) { return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0; } function normalizeContinuationAttempt(value: unknown) { return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0; } function readText(value: unknown): string | null { if (typeof value !== "string") return null; const trimmed = value.trim(); return trimmed.length > 0 ? trimmed : null; } function resultFinalText(resultJson: Record | null | undefined) { if (!resultJson) return ""; return [ readText(resultJson.nextAction), readText(resultJson.summary), readText(resultJson.result), readText(resultJson.message), readText(resultJson.error), ] .filter((value): value is string => Boolean(value)) .join("\n"); } function resultRawText(resultJson: Record | null | undefined) { if (!resultJson) return ""; return [ readText(resultJson.stdout), readText(resultJson.stderr), ] .filter((value): value is string => Boolean(value)) .join("\n"); } function highSignalSources(input: RunLivenessClassificationInput) { return [ ...(input.issueCommentBodies ?? []).map(readText), readText(resultFinalText(input.resultJson)), readText(input.continuationSummaryBody), ].filter((value): value is string => Boolean(value)); } function rawSources(input: RunLivenessClassificationInput) { return [ readText(resultRawText(input.resultJson)), readText(input.stdoutExcerpt), readText(input.stderrExcerpt), readText(input.error), ] .filter((value): value is string => Boolean(value)) .map(stripNoisyTranscriptLines) .filter((value) => value.length > 0); } function combinedOutput(input: RunLivenessClassificationInput) { return [...highSignalSources(input), ...rawSources(input)].join("\n").trim(); } function actionabilityText(input: RunLivenessClassificationInput) { const highSignal = highSignalSources(input).join("\n").trim(); if (highSignal) return highSignal; return rawSources(input).join("\n").trim(); } export function hasUsefulOutput(input: RunLivenessClassificationInput) { return combinedOutput(input).length > 0; } export function declaredBlocker(input: RunLivenessClassificationInput) { if (input.issue?.status === "blocked") return true; const actionability = classifyRunActionability(input); return actionability === "blocked_external" || actionability === "approval_required"; } export function looksLikePlanningOnly(input: RunLivenessClassificationInput) { const text = actionabilityText(input); if (!text) return false; return PLANNING_ONLY_RE.test(text) || NEXT_STEPS_RE.test(text) || /^\s*next(?: steps?| action)?\s*:/im.test(text); } export function isPlanningOrDocumentTask(issue: RunLivenessIssueInput | null | undefined) { if (!issue) return false; if (PLAN_TASK_TITLE_RE.test(issue.title)) return true; return PLAN_TASK_DESCRIPTION_RE.test(issue.description ?? ""); } function normalizeEvidence(evidence: Partial | null | undefined): RunLivenessEvidenceInput { return { issueCommentsCreated: normalizeCount(evidence?.issueCommentsCreated), documentRevisionsCreated: normalizeCount(evidence?.documentRevisionsCreated), planDocumentRevisionsCreated: normalizeCount(evidence?.planDocumentRevisionsCreated), workProductsCreated: normalizeCount(evidence?.workProductsCreated), workspaceOperationsCreated: normalizeCount(evidence?.workspaceOperationsCreated), activityEventsCreated: normalizeCount(evidence?.activityEventsCreated), toolOrActionEventsCreated: normalizeCount(evidence?.toolOrActionEventsCreated), latestEvidenceAt: evidence?.latestEvidenceAt instanceof Date ? evidence.latestEvidenceAt : null, }; } export function hasConcreteActionEvidence(evidence: Partial | null | undefined) { const normalized = normalizeEvidence(evidence); // Workspace creation is setup evidence, not task progress by itself. It can // appear in reasons alongside durable activity, but it must not prevent a // planning-only or empty run from receiving a bounded continuation. return ( normalized.issueCommentsCreated + normalized.documentRevisionsCreated + normalized.workProductsCreated + normalized.activityEventsCreated + normalized.toolOrActionEventsCreated > 0 ); } function evidenceReason(evidence: RunLivenessEvidenceInput) { const parts: string[] = []; if (evidence.issueCommentsCreated > 0) parts.push(`${evidence.issueCommentsCreated} issue comment(s)`); if (evidence.documentRevisionsCreated > 0) parts.push(`${evidence.documentRevisionsCreated} document revision(s)`); if (evidence.workProductsCreated > 0) parts.push(`${evidence.workProductsCreated} work product(s)`); if (evidence.workspaceOperationsCreated > 0) parts.push(`${evidence.workspaceOperationsCreated} workspace operation(s)`); if (evidence.activityEventsCreated > 0) parts.push(`${evidence.activityEventsCreated} activity event(s)`); if (evidence.toolOrActionEventsCreated > 0) parts.push(`${evidence.toolOrActionEventsCreated} tool/action event(s)`); return parts.join(", "); } function stripMarkdownListPrefix(line: string) { return line.replace(/^\s*(?:[-*]|\d+\.)\s+/, "").trim(); } function isNoisyTranscriptLine(line: string) { const trimmed = line.trim(); if (!trimmed) return true; return ( /^(?:command|status|exit_code|tool|tool_call|tool_result|stdout|stderr|event|payload|session|cwd|ref_id)\s*:/i.test(trimmed) || /^(?:\{|\[).{0,80}(?:tool|event|stdout|stderr|cmd|command|payload)/i.test(trimmed) || /^\$?\s*(?:rg|sed|cat|ls|git|pnpm|npm|yarn|curl|node|python)\b/i.test(trimmed) ); } function stripNoisyTranscriptLines(text: string) { return text .split(/\r?\n/) .map((line) => line.trim()) .filter((line) => !isNoisyTranscriptLine(line)) .join("\n") .trim(); } function nextNonNoiseLine(lines: string[], startIndex: number) { for (let i = startIndex + 1; i < lines.length; i += 1) { const line = stripMarkdownListPrefix(lines[i] ?? ""); if (!line || isNoisyTranscriptLine(line)) continue; return line; } return null; } function extractNextActionFromText(text: string) { const lines = text.split(/\r?\n/).map((entry) => entry.trim()); for (let i = 0; i < lines.length; i += 1) { const rawLine = lines[i] ?? ""; if (!rawLine || isNoisyTranscriptLine(rawLine)) continue; const line = stripMarkdownListPrefix(rawLine); const labeled = line.match(/^next(?: steps?| action)?\s*:\s*(.*)$/i); if (labeled) { const sameLine = stripMarkdownListPrefix(labeled[1] ?? ""); return sameLine || nextNonNoiseLine(lines, i); } if (PLANNING_ONLY_RE.test(line)) return line; } return null; } function extractNextAction(input: RunLivenessClassificationInput) { const structuredNextAction = readText(input.resultJson?.nextAction); const candidates = [ ...(input.issueCommentBodies ?? []), structuredNextAction ? `Next action: ${structuredNextAction}` : null, resultFinalText(input.resultJson), input.continuationSummaryBody, ...rawSources(input), ].filter((value): value is string => Boolean(readText(value))); for (const candidate of candidates) { const line = extractNextActionFromText(candidate); if (!line) continue; return line.length <= 500 ? line : `${line.slice(0, 497)}...`; } return null; } export function classifyRunActionability(input: RunLivenessClassificationInput): RunLivenessActionability { const text = actionabilityText(input); if (!text) return "unknown"; if (NEGATED_BLOCKER_RE.test(text)) { return RUNNABLE_RE.test(text) ? "runnable" : "unknown"; } if (APPROVAL_REQUIRED_RE.test(text)) return "approval_required"; if (EXTERNAL_BLOCKER_RE.test(text) || BLOCKER_RE.test(text) && /\b(?:credential|secret|api key|token|access|input|clarification)\b/i.test(text)) { return "blocked_external"; } if (MANAGER_REVIEW_RE.test(text)) return "manager_review"; if (RUNNABLE_RE.test(text)) return "runnable"; return "unknown"; } export function classifyRunLiveness(input: RunLivenessClassificationInput): RunLivenessClassification { const evidence = normalizeEvidence(input.evidence); const continuationAttempt = normalizeContinuationAttempt(input.continuationAttempt); const actionability = classifyRunActionability(input); const nextAction = extractNextAction(input); const issueStatus = input.issue?.status ?? null; const usefulOutput = hasUsefulOutput(input); const concreteEvidence = hasConcreteActionEvidence(evidence); const planExempt = isPlanningOrDocumentTask(input.issue) || evidence.planDocumentRevisionsCreated > 0; const lastUsefulActionAt = concreteEvidence ? evidence.latestEvidenceAt : null; const output = (state: RunLivenessState, reason: string, nextAction: string | null = null): RunLivenessClassification => ({ livenessState: state, livenessReason: compactReason(reason), continuationAttempt, lastUsefulActionAt: state === "advanced" || state === "completed" || state === "blocked" ? lastUsefulActionAt : null, nextAction, actionability, }); if (input.runStatus !== "succeeded") { return output("failed", input.errorCode ? `Run ended with ${input.runStatus} (${input.errorCode})` : `Run ended with ${input.runStatus}`); } if (issueStatus === "done" || issueStatus === "cancelled") { return output("completed", `Issue is ${issueStatus}`); } if (declaredBlocker(input)) { return output("blocked", issueStatus === "blocked" ? "Issue status is blocked" : "Run output declared a concrete blocker", nextAction); } if (!usefulOutput && !concreteEvidence) { return output("empty_response", "Run succeeded without useful output or concrete action evidence"); } if (concreteEvidence) { return output("advanced", `Run produced concrete action evidence: ${evidenceReason(evidence)}`); } if (planExempt && usefulOutput) { return output("advanced", "Planning/document task produced useful output and is exempt from plan-only classification"); } if (looksLikePlanningOnly(input) || nextAction) { if (actionability === "runnable") { return output("plan_only", "Run described runnable future work without concrete action evidence", nextAction); } return output("needs_followup", "Run described future work that is not safe to auto-continue", nextAction); } if (usefulOutput) { return output("needs_followup", "Run produced useful output but no concrete action evidence", nextAction); } return output("empty_response", "Run succeeded without useful output"); }