import fs from "node:fs/promises"; import path from "node:path"; import { execFile as execFileCallback } from "node:child_process"; import { promisify } from "node:util"; import { randomUUID } from "node:crypto"; import { and, asc, desc, eq, getTableColumns, gt, inArray, isNull, lte, notInArray, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, isEnvironmentDriverSupportedForAdapter, type BillingType, type EnvironmentLeaseStatus, type ExecutionWorkspace, type ExecutionWorkspaceConfig, type RunLivenessState, } from "@paperclipai/shared"; import { ensureSshWorkspaceReady, findReachablePaperclipApiUrlOverSsh, type SshRemoteExecutionSpec, } from "@paperclipai/adapter-utils/ssh"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { agents, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, companySkills as companySkillsTable, documentRevisions, issueDocuments, heartbeatRunEvents, heartbeatRuns, issueComments, issueRelations, issues, issueWorkProducts, projects, projectWorkspaces, workspaceOperations, } from "@paperclipai/db"; import { conflict, HttpError, notFound } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { publishLiveEvent } from "./live-events.js"; import { getRunLogStore, type RunLogHandle } from "./run-log-store.js"; import { getServerAdapter, runningProcesses } from "../adapters/index.js"; import type { AdapterExecutionResult, AdapterInvocationMeta, AdapterSessionCodec, UsageSummary } from "../adapters/index.js"; import { createLocalAgentJwt } from "../agent-auth-jwt.js"; import { parseObject, asBoolean, asNumber, appendWithByteCap, MAX_EXCERPT_BYTES } from "../adapters/utils.js"; import { costService } from "./costs.js"; import { trackAgentFirstHeartbeat } from "@paperclipai/shared/telemetry"; import { getTelemetryClient } from "../telemetry.js"; import { companySkillService } from "./company-skills.js"; import { budgetService, type BudgetEnforcementScope } from "./budgets.js"; import { secretService } from "./secrets.js"; import { resolveDefaultAgentWorkspaceDir, resolveManagedProjectWorkspaceDir } from "../home-paths.js"; import { buildHeartbeatRunIssueComment, HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS, HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS, HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES, mergeHeartbeatRunResultJson, } from "./heartbeat-run-summary.js"; import { buildHeartbeatRunStopMetadata, mergeHeartbeatRunStopMetadata, } from "./heartbeat-stop-metadata.js"; import { classifyRunLiveness, type RunLivenessClassificationInput, } from "./run-liveness.js"; import { classifyIssueGraphLiveness, type IssueLivenessFinding, } from "./issue-liveness.js"; import { logActivity, publishPluginDomainEvent, type LogActivityInput } from "./activity-log.js"; import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, type ExecutionWorkspaceInput, type RealizedExecutionWorkspace, sanitizeRuntimeServiceBaseEnv, } from "./workspace-runtime.js"; import { issueService } from "./issues.js"; import { ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, issueTreeControlService, } from "./issue-tree-control.js"; import { getIssueContinuationSummaryDocument, refreshIssueContinuationSummary, } from "./issue-continuation-summary.js"; import { executionWorkspaceService, mergeExecutionWorkspaceConfig } from "./execution-workspaces.js"; import { environmentService } from "./environments.js"; import { workspaceOperationService } from "./workspace-operations.js"; import { isProcessGroupAlive, terminateLocalService } from "./local-service-supervisor.js"; import { buildExecutionWorkspaceAdapterConfig, gateProjectExecutionWorkspacePolicy, issueExecutionWorkspaceModeForPersistedWorkspace, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, resolveExecutionWorkspaceEnvironmentId, resolveExecutionWorkspaceMode, } from "./execution-workspace-policy.js"; import { resolveEnvironmentDriverConfigForRuntime } from "./environment-config.js"; import { instanceSettingsService } from "./instance-settings.js"; import { RUN_LIVENESS_CONTINUATION_REASON, buildRunLivenessContinuationIdempotencyKey, decideRunLivenessContinuation, findExistingRunLivenessContinuationWake, readContinuationAttempt, } from "./run-continuations.js"; import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js"; import { hasSessionCompactionThresholds, resolveSessionCompactionPolicy, type SessionCompactionPolicy, } from "@paperclipai/adapter-utils"; import { readPaperclipSkillSyncPreference, writePaperclipSkillSyncPreference, } from "@paperclipai/adapter-utils/server-utils"; import { extractSkillMentionIds } from "@paperclipai/shared"; const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024; const MAX_PERSISTED_LOG_CHUNK_CHARS = 64 * 1024; const MAX_RUN_EVENT_PAYLOAD_STRING_CHARS = 16 * 1024; const MAX_RUN_EVENT_PAYLOAD_ARRAY_ITEMS = 50; const MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS = 100; const MAX_RUN_EVENT_PAYLOAD_DEPTH = 6; const HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT = AGENT_DEFAULT_MAX_CONCURRENT_RUNS; const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = 10; const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [ "environment.lease_acquired", "environment.lease_released", ]; const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; const WAKE_COMMENT_IDS_KEY = "wakeCommentIds"; const PAPERCLIP_WAKE_PAYLOAD_KEY = "paperclipWake"; const PAPERCLIP_HARNESS_CHECKOUT_KEY = "paperclipHarnessCheckedOut"; const DETACHED_PROCESS_ERROR_CODE = "process_detached"; const startLocksByAgent = new Map>(); const REPO_ONLY_CWD_SENTINEL = "/__paperclip_repo_only__"; const MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS = 10 * 60 * 1000; const MAX_INLINE_WAKE_COMMENTS = 8; const MAX_INLINE_WAKE_COMMENT_BODY_CHARS = 4_000; const MAX_INLINE_WAKE_COMMENT_BODY_TOTAL_CHARS = 12_000; const execFile = promisify(execFileCallback); const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; const CANCELLABLE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"] as const; const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const; export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS = [ 2 * 60 * 1000, 10 * 60 * 1000, 30 * 60 * 1000, 2 * 60 * 60 * 1000, ] as const; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0.25; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; type CodexTransientFallbackMode = | "same_session" | "safer_invocation" | "fresh_session" | "fresh_session_safer_invocation"; function resolveCodexTransientFallbackMode(attempt: number): CodexTransientFallbackMode { if (attempt <= 1) return "same_session"; if (attempt === 2) return "safer_invocation"; if (attempt === 3) return "fresh_session"; return "fresh_session_safer_invocation"; } const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set(["approval_approved"]); const SESSIONED_LOCAL_ADAPTERS = new Set([ "claude_local", "codex_local", "cursor", "gemini_local", "hermes_local", "opencode_local", "pi_local", ]); const INLINE_BASE64_IMAGE_DATA_RE = /("type":"image","source":\{"type":"base64","data":")([A-Za-z0-9+/=]{1024,})(")/g; type RuntimeConfigSecretResolver = Pick< ReturnType, "resolveAdapterConfigForRuntime" | "resolveEnvBindings" >; export async function resolveExecutionRunAdapterConfig(input: { companyId: string; executionRunConfig: Record; projectEnv: unknown; secretsSvc: RuntimeConfigSecretResolver; }) { const { config: resolvedConfig, secretKeys } = await input.secretsSvc.resolveAdapterConfigForRuntime( input.companyId, input.executionRunConfig, ); const projectEnvResolution = input.projectEnv ? await input.secretsSvc.resolveEnvBindings(input.companyId, input.projectEnv) : { env: {}, secretKeys: new Set() }; if (Object.keys(projectEnvResolution.env).length > 0) { resolvedConfig.env = { ...parseObject(resolvedConfig.env), ...projectEnvResolution.env, }; for (const key of projectEnvResolution.secretKeys) { secretKeys.add(key); } } return { resolvedConfig, secretKeys }; } export function extractMentionedSkillIdsFromSources( sources: Array, ): string[] { const mentionedIds = new Set(); for (const source of sources) { if (typeof source !== "string" || source.length === 0) continue; for (const skillId of extractSkillMentionIds(source)) { mentionedIds.add(skillId); } } return [...mentionedIds]; } export function applyRunScopedMentionedSkillKeys( config: Record, skillKeys: string[], ): Record { const normalizedSkillKeys = Array.from( new Set( skillKeys .map((value) => value.trim()) .filter(Boolean), ), ); if (normalizedSkillKeys.length === 0) return config; const existingPreference = readPaperclipSkillSyncPreference(config); return writePaperclipSkillSyncPreference(config, [ ...existingPreference.desiredSkills, ...normalizedSkillKeys, ]); } export function computeBoundedTransientHeartbeatRetrySchedule( attempt: number, now = new Date(), random: () => number = Math.random, ) { if (!Number.isInteger(attempt) || attempt <= 0) return null; const baseDelayMs = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS[attempt - 1]; if (typeof baseDelayMs !== "number") return null; const sample = Math.min(1, Math.max(0, random())); const jitterMultiplier = 1 + (((sample * 2) - 1) * BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO); const delayMs = Math.max(1_000, Math.round(baseDelayMs * jitterMultiplier)); return { attempt, baseDelayMs, delayMs, dueAt: new Date(now.getTime() + delayMs), maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, }; } async function resolveRunScopedMentionedSkillKeys(input: { db: Db; companyId: string; issueId: string | null; }): Promise { if (!input.issueId) return []; const issue = await input.db .select({ title: issues.title, description: issues.description, }) .from(issues) .where(and(eq(issues.id, input.issueId), eq(issues.companyId, input.companyId))) .then((rows) => rows[0] ?? null); if (!issue) return []; const comments = await input.db .select({ body: issueComments.body }) .from(issueComments) .where( and( eq(issueComments.issueId, input.issueId), eq(issueComments.companyId, input.companyId), ), ); const mentionedSkillIds = extractMentionedSkillIdsFromSources([ issue.title, issue.description ?? "", ...comments.map((comment) => comment.body), ]); if (mentionedSkillIds.length === 0) return []; const skillRows = await input.db .select({ id: companySkillsTable.id, key: companySkillsTable.key, }) .from(companySkillsTable) .where( and( eq(companySkillsTable.companyId, input.companyId), inArray(companySkillsTable.id, mentionedSkillIds), ), ); const skillKeyById = new Map(skillRows.map((row) => [row.id, row.key])); return mentionedSkillIds .map((skillId) => skillKeyById.get(skillId) ?? null) .filter((skillKey): skillKey is string => Boolean(skillKey)); } function leaseReleaseStatusForRunStatus( status: string | null | undefined, ): Extract { return status === "failed" || status === "timed_out" ? "failed" : "released"; } function runtimeApiUrlCandidates() { const candidates = [ process.env.PAPERCLIP_RUNTIME_API_URL, process.env.PAPERCLIP_API_URL, process.env.PUBLIC_BASE_URL, ].filter((value): value is string => typeof value === "string" && value.trim().length > 0); const encoded = process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON; if (!encoded) return candidates; try { const parsed = JSON.parse(encoded); if (Array.isArray(parsed)) { candidates.push( ...parsed.filter((value): value is string => typeof value === "string" && value.trim().length > 0), ); } } catch { logger.warn("Ignoring invalid PAPERCLIP_RUNTIME_API_CANDIDATES_JSON"); } return candidates; } export function applyPersistedExecutionWorkspaceConfig(input: { config: Record; workspaceConfig: ExecutionWorkspaceConfig | null; mode: ReturnType; }) { const nextConfig = { ...input.config }; if (input.mode !== "agent_default") { if (input.workspaceConfig?.workspaceRuntime === null) { delete nextConfig.workspaceRuntime; } else if (input.workspaceConfig?.workspaceRuntime) { nextConfig.workspaceRuntime = { ...input.workspaceConfig.workspaceRuntime }; } if (input.workspaceConfig?.desiredState === null) { delete nextConfig.desiredState; } else if (input.workspaceConfig?.desiredState) { nextConfig.desiredState = input.workspaceConfig.desiredState; } if (input.workspaceConfig?.serviceStates === null) { delete nextConfig.serviceStates; } else if (input.workspaceConfig?.serviceStates) { nextConfig.serviceStates = { ...input.workspaceConfig.serviceStates }; } } if (input.workspaceConfig && input.mode === "isolated_workspace") { const nextStrategy = parseObject(nextConfig.workspaceStrategy); if (input.workspaceConfig.provisionCommand === null) delete nextStrategy.provisionCommand; else nextStrategy.provisionCommand = input.workspaceConfig.provisionCommand; if (input.workspaceConfig.teardownCommand === null) delete nextStrategy.teardownCommand; else nextStrategy.teardownCommand = input.workspaceConfig.teardownCommand; nextConfig.workspaceStrategy = nextStrategy; } return nextConfig; } export function stripWorkspaceRuntimeFromExecutionRunConfig(config: Record) { const nextConfig = { ...config }; delete nextConfig.workspaceRuntime; return nextConfig; } export function buildRealizedExecutionWorkspaceFromPersisted(input: { base: ExecutionWorkspaceInput; workspace: ExecutionWorkspace; }): RealizedExecutionWorkspace | null { const cwd = readNonEmptyString(input.workspace.cwd) ?? readNonEmptyString(input.workspace.providerRef); if (!cwd) { return null; } const strategy = input.workspace.strategyType === "git_worktree" ? "git_worktree" : "project_primary"; return { baseCwd: input.base.baseCwd, source: input.workspace.mode === "shared_workspace" ? "project_primary" : "task_session", projectId: input.workspace.projectId ?? input.base.projectId, workspaceId: input.workspace.projectWorkspaceId ?? input.base.workspaceId, repoUrl: input.workspace.repoUrl ?? input.base.repoUrl, repoRef: input.workspace.baseRef ?? input.base.repoRef, strategy, cwd, branchName: input.workspace.branchName ?? null, worktreePath: strategy === "git_worktree" ? (readNonEmptyString(input.workspace.providerRef) ?? cwd) : null, warnings: [], created: false, }; } function buildExecutionWorkspaceConfigSnapshot( config: Record, environmentId?: string | null, ): Partial | null { const strategy = parseObject(config.workspaceStrategy); const snapshot: Partial = {}; // Persist the resolved environment onto the workspace so reused sessions stay on the // environment they were created against until the workspace itself is recreated/reset. const hasExplicitEnvironmentSelection = environmentId !== undefined; if (hasExplicitEnvironmentSelection) { snapshot.environmentId = environmentId ?? null; } if ("workspaceStrategy" in config) { snapshot.provisionCommand = typeof strategy.provisionCommand === "string" ? strategy.provisionCommand : null; snapshot.teardownCommand = typeof strategy.teardownCommand === "string" ? strategy.teardownCommand : null; } if ("workspaceRuntime" in config) { const workspaceRuntime = parseObject(config.workspaceRuntime); snapshot.workspaceRuntime = Object.keys(workspaceRuntime).length > 0 ? workspaceRuntime : null; } if ("desiredState" in config) { snapshot.desiredState = config.desiredState === "running" || config.desiredState === "stopped" || config.desiredState === "manual" ? config.desiredState : null; } if ("serviceStates" in config) { const serviceStates = parseObject(config.serviceStates); snapshot.serviceStates = Object.keys(serviceStates).length > 0 ? Object.fromEntries( Object.entries(serviceStates).filter(([, state]) => state === "running" || state === "stopped" || state === "manual" ), ) as ExecutionWorkspaceConfig["serviceStates"] : null; } const hasSnapshot = Object.values(snapshot).some((value) => { if (value === null) return false; if (typeof value === "object") return Object.keys(value).length > 0; return true; }); return hasSnapshot || hasExplicitEnvironmentSelection ? snapshot : null; } function deriveRepoNameFromRepoUrl(repoUrl: string | null): string | null { const trimmed = repoUrl?.trim() ?? ""; if (!trimmed) return null; try { const parsed = new URL(trimmed); const cleanedPath = parsed.pathname.replace(/\/+$/, ""); const repoName = cleanedPath.split("/").filter(Boolean).pop()?.replace(/\.git$/i, "") ?? ""; return repoName || null; } catch { return null; } } async function ensureManagedProjectWorkspace(input: { companyId: string; projectId: string; repoUrl: string | null; }): Promise<{ cwd: string; warning: string | null }> { const cwd = resolveManagedProjectWorkspaceDir({ companyId: input.companyId, projectId: input.projectId, repoName: deriveRepoNameFromRepoUrl(input.repoUrl), }); await fs.mkdir(path.dirname(cwd), { recursive: true }); const stats = await fs.stat(cwd).catch(() => null); if (!input.repoUrl) { if (!stats) { await fs.mkdir(cwd, { recursive: true }); } return { cwd, warning: null }; } const gitDirExists = await fs .stat(path.resolve(cwd, ".git")) .then((entry) => entry.isDirectory()) .catch(() => false); if (gitDirExists) { return { cwd, warning: null }; } if (stats) { const entries = await fs.readdir(cwd).catch(() => []); if (entries.length > 0) { return { cwd, warning: `Managed workspace path "${cwd}" already exists but is not a git checkout. Using it as-is.`, }; } await fs.rm(cwd, { recursive: true, force: true }); } try { await execFile("git", ["clone", input.repoUrl, cwd], { env: sanitizeRuntimeServiceBaseEnv(process.env), timeout: MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS, }); return { cwd, warning: null }; } catch (error) { const reason = error instanceof Error ? error.message : String(error); throw new Error(`Failed to prepare managed checkout for "${input.repoUrl}" at "${cwd}": ${reason}`); } } const heartbeatRunProcessGroupIdColumn = heartbeatRuns.processGroupId ?? sql`NULL`.as("processGroupId"); const heartbeatRunListColumns = { id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, invocationSource: heartbeatRuns.invocationSource, triggerDetail: heartbeatRuns.triggerDetail, status: heartbeatRuns.status, startedAt: heartbeatRuns.startedAt, finishedAt: heartbeatRuns.finishedAt, error: heartbeatRuns.error, wakeupRequestId: heartbeatRuns.wakeupRequestId, exitCode: heartbeatRuns.exitCode, signal: heartbeatRuns.signal, usageJson: heartbeatRuns.usageJson, sessionIdBefore: heartbeatRuns.sessionIdBefore, sessionIdAfter: heartbeatRuns.sessionIdAfter, logStore: heartbeatRuns.logStore, logRef: heartbeatRuns.logRef, logBytes: heartbeatRuns.logBytes, logSha256: heartbeatRuns.logSha256, logCompressed: heartbeatRuns.logCompressed, stdoutExcerpt: sql`NULL`.as("stdoutExcerpt"), stderrExcerpt: sql`NULL`.as("stderrExcerpt"), errorCode: heartbeatRuns.errorCode, externalRunId: heartbeatRuns.externalRunId, processPid: heartbeatRuns.processPid, processGroupId: heartbeatRunProcessGroupIdColumn, processStartedAt: heartbeatRuns.processStartedAt, retryOfRunId: heartbeatRuns.retryOfRunId, processLossRetryCount: heartbeatRuns.processLossRetryCount, scheduledRetryAt: heartbeatRuns.scheduledRetryAt, scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, livenessState: heartbeatRuns.livenessState, livenessReason: heartbeatRuns.livenessReason, continuationAttempt: heartbeatRuns.continuationAttempt, lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt, nextAction: heartbeatRuns.nextAction, createdAt: heartbeatRuns.createdAt, updatedAt: heartbeatRuns.updatedAt, } as const; const heartbeatRunListContextColumns = { contextIssueId: sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("contextIssueId"), contextTaskId: sql`${heartbeatRuns.contextSnapshot} ->> 'taskId'`.as("contextTaskId"), contextTaskKey: sql`${heartbeatRuns.contextSnapshot} ->> 'taskKey'`.as("contextTaskKey"), contextCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"), contextWakeCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as("contextWakeCommentId"), contextWakeReason: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeReason'`.as("contextWakeReason"), contextWakeSource: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeSource'`.as("contextWakeSource"), contextWakeTriggerDetail: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeTriggerDetail'`.as("contextWakeTriggerDetail"), } as const; const heartbeatRunListResultColumns = { resultSummary: sql`left(${heartbeatRuns.resultJson} ->> 'summary', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as("resultSummary"), resultResult: sql`left(${heartbeatRuns.resultJson} ->> 'result', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as("resultResult"), resultMessage: sql`left(${heartbeatRuns.resultJson} ->> 'message', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as("resultMessage"), resultError: sql`left(${heartbeatRuns.resultJson} ->> 'error', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as("resultError"), resultTotalCostUsd: sql`${heartbeatRuns.resultJson} ->> 'total_cost_usd'`.as("resultTotalCostUsd"), resultCostUsd: sql`${heartbeatRuns.resultJson} ->> 'cost_usd'`.as("resultCostUsd"), resultCostUsdCamel: sql`${heartbeatRuns.resultJson} ->> 'costUsd'`.as("resultCostUsdCamel"), } as const; const heartbeatRunSafeResultJsonColumn = sql | null>` case when ${heartbeatRuns.resultJson} is null then null when pg_column_size(${heartbeatRuns.resultJson}) <= ${HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES} then ${heartbeatRuns.resultJson} else jsonb_strip_nulls( jsonb_build_object( 'summary', left(${heartbeatRuns.resultJson} ->> 'summary', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), 'result', left(${heartbeatRuns.resultJson} ->> 'result', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), 'message', left(${heartbeatRuns.resultJson} ->> 'message', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), 'error', left(${heartbeatRuns.resultJson} ->> 'error', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), 'stdout', left(${heartbeatRuns.resultJson} ->> 'stdout', ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS}), 'stderr', left(${heartbeatRuns.resultJson} ->> 'stderr', ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS}), 'stdoutTruncated', case when length(${heartbeatRuns.resultJson} ->> 'stdout') > ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS} then to_jsonb(true) else null end, 'stderrTruncated', case when length(${heartbeatRuns.resultJson} ->> 'stderr') > ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS} then to_jsonb(true) else null end, 'costUsd', coalesce( ${heartbeatRuns.resultJson} -> 'costUsd', ${heartbeatRuns.resultJson} -> 'cost_usd', ${heartbeatRuns.resultJson} -> 'total_cost_usd' ), 'cost_usd', coalesce( ${heartbeatRuns.resultJson} -> 'cost_usd', ${heartbeatRuns.resultJson} -> 'costUsd', ${heartbeatRuns.resultJson} -> 'total_cost_usd' ), 'total_cost_usd', coalesce( ${heartbeatRuns.resultJson} -> 'total_cost_usd', ${heartbeatRuns.resultJson} -> 'cost_usd', ${heartbeatRuns.resultJson} -> 'costUsd' ), 'truncated', true, 'truncationReason', 'oversized_result_json', 'originalSizeBytes', pg_column_size(${heartbeatRuns.resultJson}) ) ) end `.as("resultJson"); const heartbeatRunSafeColumns = { ...getTableColumns(heartbeatRuns), processGroupId: heartbeatRunProcessGroupIdColumn, resultJson: heartbeatRunSafeResultJsonColumn, } as const; const heartbeatRunSqlAsciiSafeColumns = { ...getTableColumns(heartbeatRuns), processGroupId: heartbeatRunProcessGroupIdColumn, error: sql`NULL`.as("error"), resultJson: sql | null>`NULL`.as("resultJson"), stdoutExcerpt: sql`NULL`.as("stdoutExcerpt"), stderrExcerpt: sql`NULL`.as("stderrExcerpt"), } as const; const heartbeatRunLogAccessColumns = { id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, logStore: heartbeatRuns.logStore, logRef: heartbeatRuns.logRef, } as const; const heartbeatRunIssueSummaryColumns = { id: heartbeatRuns.id, status: heartbeatRuns.status, invocationSource: heartbeatRuns.invocationSource, triggerDetail: heartbeatRuns.triggerDetail, startedAt: heartbeatRuns.startedAt, finishedAt: heartbeatRuns.finishedAt, createdAt: heartbeatRuns.createdAt, agentId: heartbeatRuns.agentId, livenessState: heartbeatRuns.livenessState, livenessReason: heartbeatRuns.livenessReason, continuationAttempt: heartbeatRuns.continuationAttempt, lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt, nextAction: heartbeatRuns.nextAction, issueId: sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("issueId"), } as const; function appendExcerpt(prev: string, chunk: string) { return appendWithByteCap(prev, chunk, MAX_EXCERPT_BYTES); } function truncateRunEventString(value: string) { if (value.length <= MAX_RUN_EVENT_PAYLOAD_STRING_CHARS) return value; const omittedChars = value.length - MAX_RUN_EVENT_PAYLOAD_STRING_CHARS; return `${value.slice(0, MAX_RUN_EVENT_PAYLOAD_STRING_CHARS)}\n[truncated ${omittedChars} chars]`; } function boundRunEventValue(value: unknown, depth: number, seen: WeakSet): unknown { if (typeof value === "string") { return truncateRunEventString(value); } if ( value === null || typeof value === "number" || typeof value === "boolean" ) { return value; } if (value instanceof Date) { return value.toISOString(); } if (Array.isArray(value)) { if (depth >= MAX_RUN_EVENT_PAYLOAD_DEPTH) { return { _truncated: true, type: "array", originalLength: value.length, }; } const bounded = value .slice(0, MAX_RUN_EVENT_PAYLOAD_ARRAY_ITEMS) .map((entry) => boundRunEventValue(entry, depth + 1, seen)); if (value.length > MAX_RUN_EVENT_PAYLOAD_ARRAY_ITEMS) { bounded.push({ _truncated: true, omittedItems: value.length - MAX_RUN_EVENT_PAYLOAD_ARRAY_ITEMS, }); } return bounded; } if (typeof value !== "object" || value === undefined) { return null; } if (seen.has(value)) { return "[Circular]"; } seen.add(value); const entries = Object.entries(value as Record); if (depth >= MAX_RUN_EVENT_PAYLOAD_DEPTH) { const bounded = { _truncated: true, type: "object", keys: entries.map(([key]) => key).slice(0, 20), }; seen.delete(value); return bounded; } const out: Record = {}; for (const [key, entryValue] of entries.slice(0, MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS)) { out[key] = boundRunEventValue(entryValue, depth + 1, seen); } if (entries.length > MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS) { out._truncated = true; out._omittedKeys = entries.length - MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS; } seen.delete(value); return out; } export function boundHeartbeatRunEventPayloadForStorage(payload: Record): Record { const bounded = boundRunEventValue(payload, 0, new WeakSet()); return parseObject(bounded) ?? { _truncated: true }; } function redactInlineBase64ImageData(chunk: string) { return chunk.replace(INLINE_BASE64_IMAGE_DATA_RE, (_match, prefix: string, data: string, suffix: string) => `${prefix}[omitted base64 image data: ${data.length} chars]${suffix}`, ); } export function compactRunLogChunk(chunk: string, maxChars = MAX_PERSISTED_LOG_CHUNK_CHARS) { const normalized = redactInlineBase64ImageData(chunk); if (normalized.length <= maxChars) return normalized; const headChars = Math.max(0, Math.floor(maxChars * 0.6)); const tailChars = Math.max(0, Math.floor(maxChars * 0.25)); const omittedChars = Math.max(0, normalized.length - headChars - tailChars); const marker = `\n[paperclip truncated run log chunk: omitted ${omittedChars} chars]\n`; return `${normalized.slice(0, headChars)}${marker}${normalized.slice(normalized.length - tailChars)}`; } function normalizeMaxConcurrentRuns(value: unknown) { const parsed = Math.floor(asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT)); if (!Number.isFinite(parsed)) return HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT; return Math.max(HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT, Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed)); } async function withAgentStartLock(agentId: string, fn: () => Promise) { const previous = startLocksByAgent.get(agentId) ?? Promise.resolve(); const run = previous.then(fn); const marker = run.then( () => undefined, () => undefined, ); startLocksByAgent.set(agentId, marker); try { return await run; } finally { if (startLocksByAgent.get(agentId) === marker) { startLocksByAgent.delete(agentId); } } } interface WakeupOptions { source?: "timer" | "assignment" | "on_demand" | "automation"; triggerDetail?: "manual" | "ping" | "callback" | "system"; reason?: string | null; payload?: Record | null; idempotencyKey?: string | null; requestedByActorType?: "user" | "agent" | "system"; requestedByActorId?: string | null; contextSnapshot?: Record; } type UsageTotals = { inputTokens: number; cachedInputTokens: number; outputTokens: number; }; type SessionCompactionDecision = { rotate: boolean; reason: string | null; handoffMarkdown: string | null; previousRunId: string | null; }; interface ParsedIssueAssigneeAdapterOverrides { adapterConfig: Record | null; useProjectWorkspace: boolean | null; } export type ResolvedWorkspaceForRun = { cwd: string; source: "project_primary" | "task_session" | "agent_home"; projectId: string | null; workspaceId: string | null; repoUrl: string | null; repoRef: string | null; workspaceHints: Array<{ workspaceId: string; cwd: string | null; repoUrl: string | null; repoRef: string | null; }>; warnings: string[]; }; type ProjectWorkspaceCandidate = { id: string; }; export function prioritizeProjectWorkspaceCandidatesForRun( rows: T[], preferredWorkspaceId: string | null | undefined, ): T[] { if (!preferredWorkspaceId) return rows; const preferredIndex = rows.findIndex((row) => row.id === preferredWorkspaceId); if (preferredIndex <= 0) return rows; return [rows[preferredIndex]!, ...rows.slice(0, preferredIndex), ...rows.slice(preferredIndex + 1)]; } function readNonEmptyString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } export function summarizeHeartbeatRunContextSnapshot( contextSnapshot: Record | null | undefined, ): Record | null { const summary: Record = {}; const allowedKeys = [ "issueId", "taskId", "taskKey", "commentId", "wakeCommentId", "wakeReason", "wakeSource", "wakeTriggerDetail", ] as const; for (const key of allowedKeys) { const value = readNonEmptyString(contextSnapshot?.[key]); if (value) summary[key] = value; } return Object.keys(summary).length > 0 ? summary : null; } export function summarizeHeartbeatRunListResultJson(input: { summary?: string | null; result?: string | null; message?: string | null; error?: string | null; totalCostUsd?: string | null; costUsd?: string | null; costUsdCamel?: string | null; }): Record | null { const summary: Record = {}; for (const [key, value] of [ ["summary", input.summary], ["result", input.result], ["message", input.message], ["error", input.error], ] as const) { const normalized = readNonEmptyString(value); if (normalized) summary[key] = normalized; } for (const [key, value] of [ ["total_cost_usd", input.totalCostUsd], ["cost_usd", input.costUsd], ["costUsd", input.costUsdCamel], ] as const) { const normalized = readNonEmptyString(value); if (!normalized) continue; const parsed = Number(normalized); if (Number.isFinite(parsed)) summary[key] = parsed; } return Object.keys(summary).length > 0 ? summary : null; } function summarizeRunFailureForIssueComment( run: Pick | null | undefined, ) { if (!run) return null; const errorCode = readNonEmptyString(run.errorCode)?.trim() ?? null; const rawError = readNonEmptyString(run.error)?.trim() ?? null; const apiMessageMatch = rawError?.match(/"message"\s*:\s*"([^"]+)"/); const firstLine = rawError ?.split(/\r?\n/) .map((line) => line.trim()) .find(Boolean) ?? null; const summarySource = apiMessageMatch?.[1] ?? firstLine; const summary = summarySource && summarySource.length > 240 ? `${summarySource.slice(0, 237)}...` : summarySource; if (errorCode && summary) return ` Latest retry failure: \`${errorCode}\` - ${summary}.`; if (errorCode) return ` Latest retry failure: \`${errorCode}\`.`; if (summary) return ` Latest retry failure: ${summary}.`; return null; } function didAutomaticRecoveryFail( latestRun: Pick | null, expectedRetryReason: "assignment_recovery" | "issue_continuation_needed", ) { if (!latestRun) return false; const latestContext = parseObject(latestRun.contextSnapshot); const latestRetryReason = readNonEmptyString(latestContext.retryReason); return ( latestRetryReason === expectedRetryReason && UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.includes( latestRun.status as (typeof UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES)[number], ) ); } function normalizeLedgerBillingType(value: unknown): BillingType { const raw = readNonEmptyString(value); switch (raw) { case "api": case "metered_api": return "metered_api"; case "subscription": case "subscription_included": return "subscription_included"; case "subscription_overage": return "subscription_overage"; case "credits": return "credits"; case "fixed": return "fixed"; default: return "unknown"; } } function resolveLedgerBiller(result: AdapterExecutionResult): string { return readNonEmptyString(result.biller) ?? readNonEmptyString(result.provider) ?? "unknown"; } function normalizeBilledCostCents(costUsd: number | null | undefined, billingType: BillingType): number { if (billingType === "subscription_included") return 0; if (typeof costUsd !== "number" || !Number.isFinite(costUsd)) return 0; return Math.max(0, Math.round(costUsd * 100)); } async function resolveLedgerScopeForRun( db: Db, companyId: string, run: typeof heartbeatRuns.$inferSelect, ) { const context = parseObject(run.contextSnapshot); const contextIssueId = readNonEmptyString(context.issueId); const contextProjectId = readNonEmptyString(context.projectId); if (!contextIssueId) { return { issueId: null, projectId: contextProjectId, }; } const issue = await db .select({ id: issues.id, projectId: issues.projectId, }) .from(issues) .where(and(eq(issues.id, contextIssueId), eq(issues.companyId, companyId))) .then((rows) => rows[0] ?? null); return { issueId: issue?.id ?? null, projectId: issue?.projectId ?? contextProjectId, }; } type ResumeSessionRow = { sessionParamsJson: Record | null; sessionDisplayId: string | null; lastRunId: string | null; }; export function buildExplicitResumeSessionOverride(input: { resumeFromRunId: string; resumeRunSessionIdBefore: string | null; resumeRunSessionIdAfter: string | null; taskSession: ResumeSessionRow | null; sessionCodec: AdapterSessionCodec; }) { const desiredDisplayId = truncateDisplayId( input.resumeRunSessionIdAfter ?? input.resumeRunSessionIdBefore, ); const taskSessionParams = normalizeSessionParams( input.sessionCodec.deserialize(input.taskSession?.sessionParamsJson ?? null), ); const taskSessionDisplayId = truncateDisplayId( input.taskSession?.sessionDisplayId ?? (input.sessionCodec.getDisplayId ? input.sessionCodec.getDisplayId(taskSessionParams) : null) ?? readNonEmptyString(taskSessionParams?.sessionId), ); const canReuseTaskSessionParams = input.taskSession != null && ( input.taskSession.lastRunId === input.resumeFromRunId || (!!desiredDisplayId && taskSessionDisplayId === desiredDisplayId) ); const sessionParams = canReuseTaskSessionParams ? taskSessionParams : desiredDisplayId ? { sessionId: desiredDisplayId } : null; const sessionDisplayId = desiredDisplayId ?? (canReuseTaskSessionParams ? taskSessionDisplayId : null); if (!sessionDisplayId && !sessionParams) return null; return { sessionDisplayId, sessionParams, }; } function normalizeUsageTotals(usage: UsageSummary | null | undefined): UsageTotals | null { if (!usage) return null; return { inputTokens: Math.max(0, Math.floor(asNumber(usage.inputTokens, 0))), cachedInputTokens: Math.max(0, Math.floor(asNumber(usage.cachedInputTokens, 0))), outputTokens: Math.max(0, Math.floor(asNumber(usage.outputTokens, 0))), }; } function readRawUsageTotals(usageJson: unknown): UsageTotals | null { const parsed = parseObject(usageJson); if (Object.keys(parsed).length === 0) return null; const inputTokens = Math.max( 0, Math.floor(asNumber(parsed.rawInputTokens, asNumber(parsed.inputTokens, 0))), ); const cachedInputTokens = Math.max( 0, Math.floor(asNumber(parsed.rawCachedInputTokens, asNumber(parsed.cachedInputTokens, 0))), ); const outputTokens = Math.max( 0, Math.floor(asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0))), ); if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) { return null; } return { inputTokens, cachedInputTokens, outputTokens, }; } function deriveNormalizedUsageDelta(current: UsageTotals | null, previous: UsageTotals | null): UsageTotals | null { if (!current) return null; if (!previous) return { ...current }; const inputTokens = current.inputTokens >= previous.inputTokens ? current.inputTokens - previous.inputTokens : current.inputTokens; const cachedInputTokens = current.cachedInputTokens >= previous.cachedInputTokens ? current.cachedInputTokens - previous.cachedInputTokens : current.cachedInputTokens; const outputTokens = current.outputTokens >= previous.outputTokens ? current.outputTokens - previous.outputTokens : current.outputTokens; return { inputTokens: Math.max(0, inputTokens), cachedInputTokens: Math.max(0, cachedInputTokens), outputTokens: Math.max(0, outputTokens), }; } function formatCount(value: number | null | undefined) { if (typeof value !== "number" || !Number.isFinite(value)) return "0"; return value.toLocaleString("en-US"); } export function parseSessionCompactionPolicy(agent: typeof agents.$inferSelect): SessionCompactionPolicy { return resolveSessionCompactionPolicy(agent.adapterType, agent.runtimeConfig).policy; } export function resolveRuntimeSessionParamsForWorkspace(input: { agentId: string; previousSessionParams: Record | null; resolvedWorkspace: ResolvedWorkspaceForRun; }) { const { agentId, previousSessionParams, resolvedWorkspace } = input; const previousSessionId = readNonEmptyString(previousSessionParams?.sessionId); const previousCwd = readNonEmptyString(previousSessionParams?.cwd); if (!previousSessionId || !previousCwd) { return { sessionParams: previousSessionParams, warning: null as string | null, }; } if (resolvedWorkspace.source !== "project_primary") { return { sessionParams: previousSessionParams, warning: null as string | null, }; } const projectCwd = readNonEmptyString(resolvedWorkspace.cwd); if (!projectCwd) { return { sessionParams: previousSessionParams, warning: null as string | null, }; } const fallbackAgentHomeCwd = resolveDefaultAgentWorkspaceDir(agentId); if (path.resolve(previousCwd) !== path.resolve(fallbackAgentHomeCwd)) { return { sessionParams: previousSessionParams, warning: null as string | null, }; } if (path.resolve(projectCwd) === path.resolve(previousCwd)) { return { sessionParams: previousSessionParams, warning: null as string | null, }; } const previousWorkspaceId = readNonEmptyString(previousSessionParams?.workspaceId); if ( previousWorkspaceId && resolvedWorkspace.workspaceId && previousWorkspaceId !== resolvedWorkspace.workspaceId ) { return { sessionParams: previousSessionParams, warning: null as string | null, }; } const migratedSessionParams: Record = { ...(previousSessionParams ?? {}), cwd: projectCwd, }; if (resolvedWorkspace.workspaceId) migratedSessionParams.workspaceId = resolvedWorkspace.workspaceId; if (resolvedWorkspace.repoUrl) migratedSessionParams.repoUrl = resolvedWorkspace.repoUrl; if (resolvedWorkspace.repoRef) migratedSessionParams.repoRef = resolvedWorkspace.repoRef; return { sessionParams: migratedSessionParams, warning: `Project workspace "${projectCwd}" is now available. ` + `Attempting to resume session "${previousSessionId}" that was previously saved in fallback workspace "${previousCwd}".`, }; } function parseIssueAssigneeAdapterOverrides( raw: unknown, ): ParsedIssueAssigneeAdapterOverrides | null { const parsed = parseObject(raw); const parsedAdapterConfig = parseObject(parsed.adapterConfig); const adapterConfig = Object.keys(parsedAdapterConfig).length > 0 ? parsedAdapterConfig : null; const useProjectWorkspace = typeof parsed.useProjectWorkspace === "boolean" ? parsed.useProjectWorkspace : null; if (!adapterConfig && useProjectWorkspace === null) return null; return { adapterConfig, useProjectWorkspace, }; } /** * Synthetic task key for timer/heartbeat wakes that have no issue context. * This allows timer wakes to participate in the `agentTaskSessions` system * and benefit from robust session resume, instead of relying solely on the * simpler `agentRuntimeState.sessionId` fallback. */ const HEARTBEAT_TASK_KEY = "__heartbeat__"; function deriveTaskKey( contextSnapshot: Record | null | undefined, payload: Record | null | undefined, ) { return ( readNonEmptyString(contextSnapshot?.taskKey) ?? readNonEmptyString(contextSnapshot?.taskId) ?? readNonEmptyString(contextSnapshot?.issueId) ?? readNonEmptyString(payload?.taskKey) ?? readNonEmptyString(payload?.taskId) ?? readNonEmptyString(payload?.issueId) ?? null ); } /** * Extended task key derivation that falls back to a stable synthetic key * for timer/heartbeat wakes. This ensures timer wakes can resume their * previous session via `agentTaskSessions` instead of starting fresh. * * The synthetic key is only used when: * - No explicit task/issue key exists in the context * - The wake source is "timer" (scheduled heartbeat) */ export function deriveTaskKeyWithHeartbeatFallback( contextSnapshot: Record | null | undefined, payload: Record | null | undefined, ) { const explicit = deriveTaskKey(contextSnapshot, payload); if (explicit) return explicit; const wakeSource = readNonEmptyString(contextSnapshot?.wakeSource); if (wakeSource === "timer") return HEARTBEAT_TASK_KEY; return null; } export function shouldResetTaskSessionForWake( contextSnapshot: Record | null | undefined, ) { if (contextSnapshot?.forceFreshSession === true) return true; const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); if ( wakeReason === "issue_assigned" || wakeReason === "execution_review_requested" || wakeReason === "execution_approval_requested" || wakeReason === "execution_changes_requested" ) { return true; } return false; } function shouldRequireIssueCommentForWake( contextSnapshot: Record | null | undefined, ) { const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); return ( wakeReason === "issue_assigned" || wakeReason === "execution_review_requested" || wakeReason === "execution_approval_requested" || wakeReason === "execution_changes_requested" ); } function allowsIssueInteractionWake( contextSnapshot: Record | null | undefined, ) { const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); if (!wakeReason || !ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS.has(wakeReason)) return false; return Boolean(deriveCommentId(contextSnapshot, null)); } async function listUnresolvedBlockerSummaries( dbOrTx: Pick, companyId: string, issueId: string, unresolvedBlockerIssueIds: string[], ) { const ids = [...new Set(unresolvedBlockerIssueIds.filter(Boolean))]; if (ids.length === 0) return []; return dbOrTx .select({ id: issues.id, identifier: issues.identifier, title: issues.title, status: issues.status, priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, }) .from(issueRelations) .innerJoin(issues, eq(issueRelations.issueId, issues.id)) .where( and( eq(issueRelations.companyId, companyId), eq(issueRelations.type, "blocks"), eq(issueRelations.relatedIssueId, issueId), inArray(issues.id, ids), ), ) .orderBy(asc(issues.title)); } export function formatRuntimeWorkspaceWarningLog(warning: string) { return { stream: "stdout" as const, chunk: `[paperclip] ${warning}\n`, }; } function describeSessionResetReason( contextSnapshot: Record | null | undefined, ) { if (contextSnapshot?.forceFreshSession === true) return "forceFreshSession was requested"; const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); if (wakeReason === "issue_assigned") return "wake reason is issue_assigned"; if (wakeReason === "execution_review_requested") return "wake reason is execution_review_requested"; if (wakeReason === "execution_approval_requested") return "wake reason is execution_approval_requested"; if (wakeReason === "execution_changes_requested") return "wake reason is execution_changes_requested"; return null; } function shouldAutoCheckoutIssueForWake(input: { contextSnapshot: Record | null | undefined; issueStatus: string | null; issueAssigneeAgentId: string | null; isDependencyReady: boolean; agentId: string; }) { if (input.issueAssigneeAgentId !== input.agentId) return false; if (!input.isDependencyReady) return false; const issueStatus = readNonEmptyString(input.issueStatus); if ( issueStatus !== "todo" && issueStatus !== "backlog" && issueStatus !== "blocked" && issueStatus !== "in_progress" ) { return false; } const wakeReason = readNonEmptyString(input.contextSnapshot?.wakeReason); if (!wakeReason) return false; if (wakeReason === "issue_comment_mentioned") return false; if (wakeReason.startsWith("execution_")) return false; return true; } function shouldQueueFollowupForRunningIssueWake(input: { contextSnapshot: Record | null | undefined; wakeCommentId: string | null; }) { if (input.wakeCommentId) return true; const wakeReason = readNonEmptyString(input.contextSnapshot?.wakeReason); return Boolean(wakeReason && RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP.has(wakeReason)); } function isCheckoutConflictError(error: unknown): boolean { return error instanceof HttpError && error.status === 409 && error.message === "Issue checkout conflict"; } function deriveCommentId( contextSnapshot: Record | null | undefined, payload: Record | null | undefined, ) { const batchedCommentId = extractWakeCommentIds(contextSnapshot).at(-1); return ( batchedCommentId ?? readNonEmptyString(contextSnapshot?.wakeCommentId) ?? readNonEmptyString(contextSnapshot?.commentId) ?? readNonEmptyString(payload?.commentId) ?? null ); } export function extractWakeCommentIds( contextSnapshot: Record | null | undefined, ): string[] { const raw = contextSnapshot?.[WAKE_COMMENT_IDS_KEY]; if (!Array.isArray(raw)) return []; const out: string[] = []; for (const entry of raw) { const value = readNonEmptyString(entry); if (!value || out.includes(value)) continue; out.push(value); } return out; } function mergeWakeCommentIds(...values: Array): string[] { const merged: string[] = []; const append = (value: unknown) => { const normalized = readNonEmptyString(value); if (!normalized || merged.includes(normalized)) return; merged.push(normalized); }; for (const value of values) { if (Array.isArray(value)) { for (const entry of value) append(entry); continue; } if (typeof value === "object" && value !== null) { const candidate = value as Record; const batched = extractWakeCommentIds(candidate); if (batched.length > 0) { for (const entry of batched) append(entry); continue; } append(candidate.wakeCommentId); append(candidate.commentId); continue; } append(value); } return merged; } function enrichWakeContextSnapshot(input: { contextSnapshot: Record; reason: string | null; source: WakeupOptions["source"]; triggerDetail: WakeupOptions["triggerDetail"] | null; payload: Record | null; }) { const { contextSnapshot, reason, source, triggerDetail, payload } = input; const issueIdFromPayload = readNonEmptyString(payload?.["issueId"]); const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]); const taskKey = deriveTaskKey(contextSnapshot, payload); const wakeCommentId = deriveCommentId(contextSnapshot, payload); const wakeCommentIds = mergeWakeCommentIds(contextSnapshot, commentIdFromPayload); if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) { contextSnapshot.wakeReason = reason; } if (!readNonEmptyString(contextSnapshot["issueId"]) && issueIdFromPayload) { contextSnapshot.issueId = issueIdFromPayload; } if (!readNonEmptyString(contextSnapshot["taskId"]) && issueIdFromPayload) { contextSnapshot.taskId = issueIdFromPayload; } if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) { contextSnapshot.taskKey = taskKey; } if (!readNonEmptyString(contextSnapshot["commentId"]) && commentIdFromPayload) { contextSnapshot.commentId = commentIdFromPayload; } if (wakeCommentIds.length > 0) { const latestCommentId = wakeCommentIds[wakeCommentIds.length - 1]; contextSnapshot[WAKE_COMMENT_IDS_KEY] = wakeCommentIds; contextSnapshot.commentId = latestCommentId; contextSnapshot.wakeCommentId = latestCommentId; // Once comment ids are normalized into the snapshot, rebuild the structured // wake payload from those ids later instead of carrying forward stale data. delete contextSnapshot[PAPERCLIP_WAKE_PAYLOAD_KEY]; } else if (!readNonEmptyString(contextSnapshot["wakeCommentId"]) && wakeCommentId) { contextSnapshot.wakeCommentId = wakeCommentId; } if (!readNonEmptyString(contextSnapshot["wakeSource"]) && source) { contextSnapshot.wakeSource = source; } if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) { contextSnapshot.wakeTriggerDetail = triggerDetail; } return { contextSnapshot, issueIdFromPayload, commentIdFromPayload, taskKey, wakeCommentId, }; } export function mergeCoalescedContextSnapshot( existingRaw: unknown, incoming: Record, ) { const existing = parseObject(existingRaw); const merged: Record = { ...existing, ...incoming, }; const mergedCommentIds = mergeWakeCommentIds(existing, incoming); if (mergedCommentIds.length > 0) { const latestCommentId = mergedCommentIds[mergedCommentIds.length - 1]; merged[WAKE_COMMENT_IDS_KEY] = mergedCommentIds; merged.commentId = latestCommentId; merged.wakeCommentId = latestCommentId; // The merged context should carry canonical comment ids; the next wake will // regenerate any structured payload from those ids. delete merged[PAPERCLIP_WAKE_PAYLOAD_KEY]; } return merged; } async function buildPaperclipWakePayload(input: { db: Db; companyId: string; contextSnapshot: Record; continuationSummary?: | { key: string; title: string | null; body: string; updatedAt: Date; } | null; issueSummary?: | { id: string; identifier: string | null; title: string; status: string; priority: string; } | null; }) { const executionStage = parseObject(input.contextSnapshot.executionStage); const commentIds = extractWakeCommentIds(input.contextSnapshot); const issueId = readNonEmptyString(input.contextSnapshot.issueId); const continuationSummary = input.continuationSummary ?? null; const issueSummary = input.issueSummary ?? (issueId ? await input.db .select({ id: issues.id, identifier: issues.identifier, title: issues.title, status: issues.status, priority: issues.priority, }) .from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, input.companyId))) .then((rows) => rows[0] ?? null) : null); if (commentIds.length === 0 && Object.keys(executionStage).length === 0 && !issueSummary) return null; const commentRows = commentIds.length === 0 ? [] : await input.db .select({ id: issueComments.id, issueId: issueComments.issueId, body: issueComments.body, authorAgentId: issueComments.authorAgentId, authorUserId: issueComments.authorUserId, createdAt: issueComments.createdAt, }) .from(issueComments) .where( and( eq(issueComments.companyId, input.companyId), inArray(issueComments.id, commentIds), ), ); const commentsById = new Map(commentRows.map((comment) => [comment.id, comment])); const comments: Array> = []; let remainingBodyChars = MAX_INLINE_WAKE_COMMENT_BODY_TOTAL_CHARS; let truncated = false; let missingCommentCount = 0; for (const commentId of commentIds) { const row = commentsById.get(commentId); if (!row) { truncated = true; missingCommentCount += 1; continue; } if (comments.length >= MAX_INLINE_WAKE_COMMENTS) { truncated = true; break; } const fullBody = row.body; const allowedBodyChars = Math.min(MAX_INLINE_WAKE_COMMENT_BODY_CHARS, remainingBodyChars); if (allowedBodyChars <= 0) { truncated = true; break; } const body = fullBody.length > allowedBodyChars ? fullBody.slice(0, allowedBodyChars) : fullBody; const bodyTruncated = body.length < fullBody.length; if (bodyTruncated) truncated = true; remainingBodyChars -= body.length; comments.push({ id: row.id, issueId: row.issueId, body, bodyTruncated, createdAt: row.createdAt.toISOString(), author: row.authorAgentId ? { type: "agent", id: row.authorAgentId } : row.authorUserId ? { type: "user", id: row.authorUserId } : { type: "system", id: null }, }); } return { reason: readNonEmptyString(input.contextSnapshot.wakeReason), issue: issueSummary ? { id: issueSummary.id, identifier: issueSummary.identifier, title: issueSummary.title, status: issueSummary.status, priority: issueSummary.priority, } : null, childIssueSummaries: Array.isArray(input.contextSnapshot.childIssueSummaries) ? input.contextSnapshot.childIssueSummaries : [], childIssueSummaryTruncated: input.contextSnapshot.childIssueSummaryTruncated === true, livenessContinuation: readNonEmptyString(input.contextSnapshot.livenessContinuationState) || readNonEmptyString(input.contextSnapshot.livenessContinuationInstruction) || readNonEmptyString(input.contextSnapshot.livenessContinuationSourceRunId) || typeof input.contextSnapshot.livenessContinuationAttempt === "number" ? { attempt: input.contextSnapshot.livenessContinuationAttempt, maxAttempts: input.contextSnapshot.livenessContinuationMaxAttempts, sourceRunId: readNonEmptyString(input.contextSnapshot.livenessContinuationSourceRunId), state: readNonEmptyString(input.contextSnapshot.livenessContinuationState), reason: readNonEmptyString(input.contextSnapshot.livenessContinuationReason), instruction: readNonEmptyString(input.contextSnapshot.livenessContinuationInstruction), } : null, checkedOutByHarness: input.contextSnapshot[PAPERCLIP_HARNESS_CHECKOUT_KEY] === true, dependencyBlockedInteraction: input.contextSnapshot.dependencyBlockedInteraction === true, treeHoldInteraction: input.contextSnapshot.treeHoldInteraction === true, activeTreeHold: parseObject(input.contextSnapshot.activeTreeHold), unresolvedBlockerIssueIds: Array.isArray(input.contextSnapshot.unresolvedBlockerIssueIds) ? input.contextSnapshot.unresolvedBlockerIssueIds.filter((value): value is string => typeof value === "string" && value.length > 0) : [], unresolvedBlockerSummaries: Array.isArray(input.contextSnapshot.unresolvedBlockerSummaries) ? input.contextSnapshot.unresolvedBlockerSummaries : [], executionStage: Object.keys(executionStage).length > 0 ? executionStage : null, continuationSummary: continuationSummary ? { key: continuationSummary.key, title: continuationSummary.title, body: continuationSummary.body.length > 4_000 ? continuationSummary.body.slice(0, 4_000) : continuationSummary.body, bodyTruncated: continuationSummary.body.length > 4_000, updatedAt: continuationSummary.updatedAt.toISOString(), } : null, commentIds, latestCommentId: commentIds[commentIds.length - 1] ?? null, comments, commentWindow: { requestedCount: commentIds.length, includedCount: comments.length, missingCount: missingCommentCount, }, truncated, fallbackFetchNeeded: truncated || missingCommentCount > 0, }; } function runTaskKey(run: typeof heartbeatRuns.$inferSelect) { return deriveTaskKey(run.contextSnapshot as Record | null, null); } function isSameTaskScope(left: string | null, right: string | null) { return (left ?? null) === (right ?? null); } function isTrackedLocalChildProcessAdapter(adapterType: string) { return SESSIONED_LOCAL_ADAPTERS.has(adapterType); } function isHeartbeatRunTerminalStatus( status: string | null | undefined, ): status is (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number] { return HEARTBEAT_RUN_TERMINAL_STATUSES.includes( status as (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number], ); } // A positive liveness check means some process currently owns the PID. // On Linux, PIDs can be recycled, so this is a best-effort signal rather // than proof that the original child is still alive. function isProcessAlive(pid: number | null | undefined) { if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false; try { process.kill(pid, 0); return true; } catch (error) { const code = (error as NodeJS.ErrnoException | undefined)?.code; if (code === "EPERM") return true; if (code === "ESRCH") return false; return false; } } async function terminateHeartbeatRunProcess(input: { pid: number | null | undefined; processGroupId: number | null | undefined; graceMs?: number; }) { const pid = input.pid ?? null; const processGroupId = input.processGroupId ?? null; if (typeof pid !== "number" && typeof processGroupId !== "number") return; await terminateLocalService( { pid: typeof pid === "number" && Number.isInteger(pid) && pid > 0 ? pid : (processGroupId ?? 0), processGroupId: typeof processGroupId === "number" && Number.isInteger(processGroupId) && processGroupId > 0 ? processGroupId : null, }, input.graceMs ? { forceAfterMs: input.graceMs } : undefined, ); } function buildProcessLossMessage(run: { processPid: number | null; processGroupId: number | null; }, options?: { descendantOnly?: boolean }) { if (options?.descendantOnly && run.processGroupId) { return `Process lost -- parent pid ${run.processPid ?? "unknown"} exited, but descendant process group ${run.processGroupId} was still alive and was terminated`; } if (run.processPid) { return `Process lost -- child pid ${run.processPid} is no longer running`; } if (run.processGroupId) { return `Process lost -- process group ${run.processGroupId} is no longer running`; } return "Process lost -- server may have restarted"; } function truncateDisplayId(value: string | null | undefined, max = 128) { if (!value) return null; return value.length > max ? value.slice(0, max) : value; } function normalizeAgentNameKey(value: string | null | undefined) { if (typeof value !== "string") return null; const normalized = value.trim().toLowerCase(); return normalized.length > 0 ? normalized : null; } const defaultSessionCodec: AdapterSessionCodec = { deserialize(raw: unknown) { const asObj = parseObject(raw); if (Object.keys(asObj).length > 0) return asObj; const sessionId = readNonEmptyString((raw as Record | null)?.sessionId); if (sessionId) return { sessionId }; return null; }, serialize(params: Record | null) { if (!params || Object.keys(params).length === 0) return null; return params; }, getDisplayId(params: Record | null) { return readNonEmptyString(params?.sessionId); }, }; function getAdapterSessionCodec(adapterType: string) { const adapter = getServerAdapter(adapterType); return adapter.sessionCodec ?? defaultSessionCodec; } function normalizeSessionParams(params: Record | null | undefined) { if (!params) return null; return Object.keys(params).length > 0 ? params : null; } function resolveNextSessionState(input: { codec: AdapterSessionCodec; adapterResult: AdapterExecutionResult; previousParams: Record | null; previousDisplayId: string | null; previousLegacySessionId: string | null; }) { const { codec, adapterResult, previousParams, previousDisplayId, previousLegacySessionId } = input; if (adapterResult.clearSession) { return { params: null as Record | null, displayId: null as string | null, legacySessionId: null as string | null, }; } const explicitParams = adapterResult.sessionParams; const hasExplicitParams = adapterResult.sessionParams !== undefined; const hasExplicitSessionId = adapterResult.sessionId !== undefined; const explicitSessionId = readNonEmptyString(adapterResult.sessionId); const hasExplicitDisplay = adapterResult.sessionDisplayId !== undefined; const explicitDisplayId = readNonEmptyString(adapterResult.sessionDisplayId); const shouldUsePrevious = !hasExplicitParams && !hasExplicitSessionId && !hasExplicitDisplay; const candidateParams = hasExplicitParams ? explicitParams : hasExplicitSessionId ? (explicitSessionId ? { sessionId: explicitSessionId } : null) : previousParams; const serialized = normalizeSessionParams(codec.serialize(normalizeSessionParams(candidateParams) ?? null)); const deserialized = normalizeSessionParams(codec.deserialize(serialized)); const displayId = truncateDisplayId( explicitDisplayId ?? (codec.getDisplayId ? codec.getDisplayId(deserialized) : null) ?? readNonEmptyString(deserialized?.sessionId) ?? (shouldUsePrevious ? previousDisplayId : null) ?? explicitSessionId ?? (shouldUsePrevious ? previousLegacySessionId : null), ); const legacySessionId = explicitSessionId ?? readNonEmptyString(deserialized?.sessionId) ?? displayId ?? (shouldUsePrevious ? previousLegacySessionId : null); return { params: serialized, displayId, legacySessionId, }; } export function heartbeatService(db: Db) { const instanceSettings = instanceSettingsService(db); const getCurrentUserRedactionOptions = async () => ({ enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs, }); const runLogStore = getRunLogStore(); const secretsSvc = secretService(db); const companySkills = companySkillService(db); const issuesSvc = issueService(db); const treeControlSvc = issueTreeControlService(db); const executionWorkspacesSvc = executionWorkspaceService(db); const environmentsSvc = environmentService(db); const workspaceOperationsSvc = workspaceOperationService(db); const activeRunExecutions = new Set(); const budgetHooks = { cancelWorkForScope: cancelBudgetScopeWork, }; const budgets = budgetService(db, budgetHooks); let unsafeTextProjectionPromise: Promise | null = null; async function hasUnsafeTextProjectionDatabase() { if (!unsafeTextProjectionPromise) { unsafeTextProjectionPromise = db .execute(sql`select current_setting('server_encoding') as server_encoding`) .then((rows) => { const first = Array.isArray(rows) ? rows[0] : null; const serverEncoding = typeof first === "object" && first !== null ? (first as Record).server_encoding : null; return typeof serverEncoding === "string" && serverEncoding.toUpperCase() === "SQL_ASCII"; }) .catch((err) => { logger.warn({ err }, "failed to inspect database server encoding; using conservative heartbeat result projection"); return true; }); } return unsafeTextProjectionPromise; } async function getAgent(agentId: string) { return db .select() .from(agents) .where(eq(agents.id, agentId)) .then((rows) => rows[0] ?? null); } async function getRun(runId: string, opts?: { unsafeFullResultJson?: boolean }) { const safeForLegacyEncoding = !opts?.unsafeFullResultJson && await hasUnsafeTextProjectionDatabase(); return db .select( opts?.unsafeFullResultJson ? getTableColumns(heartbeatRuns) : safeForLegacyEncoding ? heartbeatRunSqlAsciiSafeColumns : heartbeatRunSafeColumns, ) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)) .then((rows) => rows[0] ?? null); } async function getRunLogAccess(runId: string) { return db .select(heartbeatRunLogAccessColumns) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)) .then((rows) => rows[0] ?? null); } async function getIssueExecutionContext(companyId: string, issueId: string) { return db .select({ id: issues.id, identifier: issues.identifier, title: issues.title, status: issues.status, priority: issues.priority, projectId: issues.projectId, projectWorkspaceId: issues.projectWorkspaceId, executionWorkspaceId: issues.executionWorkspaceId, executionWorkspacePreference: issues.executionWorkspacePreference, assigneeAgentId: issues.assigneeAgentId, assigneeAdapterOverrides: issues.assigneeAdapterOverrides, executionWorkspaceSettings: issues.executionWorkspaceSettings, }) .from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))) .then((rows) => rows[0] ?? null); } async function getRuntimeState(agentId: string) { return db .select() .from(agentRuntimeState) .where(eq(agentRuntimeState.agentId, agentId)) .then((rows) => rows[0] ?? null); } async function getTaskSession( companyId: string, agentId: string, adapterType: string, taskKey: string, ) { return db .select() .from(agentTaskSessions) .where( and( eq(agentTaskSessions.companyId, companyId), eq(agentTaskSessions.agentId, agentId), eq(agentTaskSessions.adapterType, adapterType), eq(agentTaskSessions.taskKey, taskKey), ), ) .then((rows) => rows[0] ?? null); } async function getLatestRunForSession( agentId: string, sessionId: string, opts?: { excludeRunId?: string | null }, ) { const conditions = [ eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.sessionIdAfter, sessionId), ]; if (opts?.excludeRunId) { conditions.push(sql`${heartbeatRuns.id} <> ${opts.excludeRunId}`); } return db .select({ id: heartbeatRuns.id, usageJson: heartbeatRuns.usageJson, }) .from(heartbeatRuns) .where(and(...conditions)) .orderBy(desc(heartbeatRuns.createdAt)) .limit(1) .then((rows) => rows[0] ?? null); } async function getOldestRunForSession(agentId: string, sessionId: string) { return db .select({ id: heartbeatRuns.id, createdAt: heartbeatRuns.createdAt, }) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.sessionIdAfter, sessionId))) .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) .limit(1) .then((rows) => rows[0] ?? null); } async function resolveNormalizedUsageForSession(input: { agentId: string; runId: string; sessionId: string | null; rawUsage: UsageTotals | null; }) { const { agentId, runId, sessionId, rawUsage } = input; if (!sessionId || !rawUsage) { return { normalizedUsage: rawUsage, previousRawUsage: null as UsageTotals | null, derivedFromSessionTotals: false, }; } const previousRun = await getLatestRunForSession(agentId, sessionId, { excludeRunId: runId }); const previousRawUsage = readRawUsageTotals(previousRun?.usageJson); return { normalizedUsage: deriveNormalizedUsageDelta(rawUsage, previousRawUsage), previousRawUsage, derivedFromSessionTotals: previousRawUsage !== null, }; } async function evaluateSessionCompaction(input: { agent: typeof agents.$inferSelect; sessionId: string | null; issueId: string | null; continuationSummaryBody?: string | null; }): Promise { const { agent, sessionId, issueId } = input; if (!sessionId) { return { rotate: false, reason: null, handoffMarkdown: null, previousRunId: null, }; } const policy = parseSessionCompactionPolicy(agent); if (!policy.enabled || !hasSessionCompactionThresholds(policy)) { return { rotate: false, reason: null, handoffMarkdown: null, previousRunId: null, }; } const fetchLimit = Math.max(policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, 4); const runs = await db .select({ id: heartbeatRuns.id, createdAt: heartbeatRuns.createdAt, usageJson: heartbeatRuns.usageJson, error: heartbeatRuns.error, ...heartbeatRunListResultColumns, }) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.agentId, agent.id), eq(heartbeatRuns.sessionIdAfter, sessionId))) .orderBy(desc(heartbeatRuns.createdAt)) .limit(fetchLimit); if (runs.length === 0) { return { rotate: false, reason: null, handoffMarkdown: null, previousRunId: null, }; } const latestRun = runs[0] ?? null; const oldestRun = policy.maxSessionAgeHours > 0 ? await getOldestRunForSession(agent.id, sessionId) : runs[runs.length - 1] ?? latestRun; const latestRawUsage = readRawUsageTotals(latestRun?.usageJson); const sessionAgeHours = latestRun && oldestRun ? Math.max( 0, (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60), ) : 0; let reason: string | null = null; if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) { reason = `session exceeded ${policy.maxSessionRuns} runs`; } else if ( policy.maxRawInputTokens > 0 && latestRawUsage && latestRawUsage.inputTokens >= policy.maxRawInputTokens ) { reason = `session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` + `(threshold ${formatCount(policy.maxRawInputTokens)})`; } else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) { reason = `session age reached ${Math.floor(sessionAgeHours)} hours`; } if (!reason || !latestRun) { return { rotate: false, reason: null, handoffMarkdown: null, previousRunId: latestRun?.id ?? null, }; } const latestSummary = summarizeHeartbeatRunListResultJson({ summary: latestRun?.resultSummary, result: latestRun?.resultResult, message: latestRun?.resultMessage, error: latestRun?.resultError, totalCostUsd: latestRun?.resultTotalCostUsd, costUsd: latestRun?.resultCostUsd, costUsdCamel: latestRun?.resultCostUsdCamel, }); const latestTextSummary = readNonEmptyString(latestSummary?.summary) ?? readNonEmptyString(latestSummary?.result) ?? readNonEmptyString(latestSummary?.message) ?? readNonEmptyString(latestRun.error); const handoffMarkdown = [ "Paperclip session handoff:", `- Previous session: ${sessionId}`, issueId ? `- Issue: ${issueId}` : "", `- Rotation reason: ${reason}`, latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "", input.continuationSummaryBody ? `- Issue continuation summary: ${input.continuationSummaryBody.slice(0, 1_500)}` : "", "Continue from the current task state. Rebuild only the minimum context you need.", ] .filter(Boolean) .join("\n"); return { rotate: true, reason, handoffMarkdown, previousRunId: latestRun.id, }; } async function resolveSessionBeforeForWakeup( agent: typeof agents.$inferSelect, taskKey: string | null, ) { if (taskKey) { const codec = getAdapterSessionCodec(agent.adapterType); const existingTaskSession = await getTaskSession( agent.companyId, agent.id, agent.adapterType, taskKey, ); const parsedParams = normalizeSessionParams( codec.deserialize(existingTaskSession?.sessionParamsJson ?? null), ); return truncateDisplayId( existingTaskSession?.sessionDisplayId ?? (codec.getDisplayId ? codec.getDisplayId(parsedParams) : null) ?? readNonEmptyString(parsedParams?.sessionId), ); } const runtimeForRun = await getRuntimeState(agent.id); return runtimeForRun?.sessionId ?? null; } async function resolveExplicitResumeSessionOverride( agent: typeof agents.$inferSelect, payload: Record | null, taskKey: string | null, ) { const resumeFromRunId = readNonEmptyString(payload?.resumeFromRunId); if (!resumeFromRunId) return null; const resumeRun = await db .select({ id: heartbeatRuns.id, contextSnapshot: heartbeatRuns.contextSnapshot, sessionIdBefore: heartbeatRuns.sessionIdBefore, sessionIdAfter: heartbeatRuns.sessionIdAfter, }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, resumeFromRunId), eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), ), ) .then((rows) => rows[0] ?? null); if (!resumeRun) return null; const resumeContext = parseObject(resumeRun.contextSnapshot); const resumeTaskKey = deriveTaskKey(resumeContext, null) ?? taskKey; const resumeTaskSession = resumeTaskKey ? await getTaskSession(agent.companyId, agent.id, agent.adapterType, resumeTaskKey) : null; const sessionCodec = getAdapterSessionCodec(agent.adapterType); const sessionOverride = buildExplicitResumeSessionOverride({ resumeFromRunId, resumeRunSessionIdBefore: resumeRun.sessionIdBefore, resumeRunSessionIdAfter: resumeRun.sessionIdAfter, taskSession: resumeTaskSession, sessionCodec, }); if (!sessionOverride) return null; return { resumeFromRunId, taskKey: resumeTaskKey, issueId: readNonEmptyString(resumeContext.issueId), taskId: readNonEmptyString(resumeContext.taskId) ?? readNonEmptyString(resumeContext.issueId), sessionDisplayId: sessionOverride.sessionDisplayId, sessionParams: sessionOverride.sessionParams, }; } async function resolveWorkspaceForRun( agent: typeof agents.$inferSelect, context: Record, previousSessionParams: Record | null, opts?: { useProjectWorkspace?: boolean | null }, ): Promise { const issueId = readNonEmptyString(context.issueId); const contextProjectId = readNonEmptyString(context.projectId); const contextProjectWorkspaceId = readNonEmptyString(context.projectWorkspaceId); const issueProjectRef = issueId ? await db .select({ projectId: issues.projectId, projectWorkspaceId: issues.projectWorkspaceId, }) .from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId))) .then((rows) => rows[0] ?? null) : null; const issueProjectId = issueProjectRef?.projectId ?? null; const preferredProjectWorkspaceId = issueProjectRef?.projectWorkspaceId ?? contextProjectWorkspaceId ?? null; const resolvedProjectId = issueProjectId ?? contextProjectId; const useProjectWorkspace = opts?.useProjectWorkspace !== false; const workspaceProjectId = useProjectWorkspace ? resolvedProjectId : null; const unorderedProjectWorkspaceRows = workspaceProjectId ? await db .select() .from(projectWorkspaces) .where( and( eq(projectWorkspaces.companyId, agent.companyId), eq(projectWorkspaces.projectId, workspaceProjectId), ), ) .orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id)) : []; const projectWorkspaceRows = prioritizeProjectWorkspaceCandidatesForRun( unorderedProjectWorkspaceRows, preferredProjectWorkspaceId, ); const workspaceHints = projectWorkspaceRows.map((workspace) => ({ workspaceId: workspace.id, cwd: readNonEmptyString(workspace.cwd), repoUrl: readNonEmptyString(workspace.repoUrl), repoRef: readNonEmptyString(workspace.repoRef), })); if (projectWorkspaceRows.length > 0) { const preferredWorkspace = preferredProjectWorkspaceId ? projectWorkspaceRows.find((workspace) => workspace.id === preferredProjectWorkspaceId) ?? null : null; const missingProjectCwds: string[] = []; let hasConfiguredProjectCwd = false; let preferredWorkspaceWarning: string | null = null; if (preferredProjectWorkspaceId && !preferredWorkspace) { preferredWorkspaceWarning = `Selected project workspace "${preferredProjectWorkspaceId}" is not available on this project.`; } for (const workspace of projectWorkspaceRows) { let projectCwd = readNonEmptyString(workspace.cwd); let managedWorkspaceWarning: string | null = null; if (!projectCwd || projectCwd === REPO_ONLY_CWD_SENTINEL) { try { const managedWorkspace = await ensureManagedProjectWorkspace({ companyId: agent.companyId, projectId: workspaceProjectId ?? resolvedProjectId ?? workspace.projectId, repoUrl: readNonEmptyString(workspace.repoUrl), }); projectCwd = managedWorkspace.cwd; managedWorkspaceWarning = managedWorkspace.warning; } catch (error) { if (preferredWorkspace?.id === workspace.id) { preferredWorkspaceWarning = error instanceof Error ? error.message : String(error); } continue; } } hasConfiguredProjectCwd = true; const projectCwdExists = await fs .stat(projectCwd) .then((stats) => stats.isDirectory()) .catch(() => false); if (projectCwdExists) { return { cwd: projectCwd, source: "project_primary" as const, projectId: resolvedProjectId, workspaceId: workspace.id, repoUrl: workspace.repoUrl, repoRef: workspace.repoRef, workspaceHints, warnings: [preferredWorkspaceWarning, managedWorkspaceWarning].filter( (value): value is string => Boolean(value), ), }; } if (preferredWorkspace?.id === workspace.id) { preferredWorkspaceWarning = `Selected project workspace path "${projectCwd}" is not available yet.`; } missingProjectCwds.push(projectCwd); } const fallbackCwd = resolveDefaultAgentWorkspaceDir(agent.id); await fs.mkdir(fallbackCwd, { recursive: true }); const warnings: string[] = []; if (preferredWorkspaceWarning) { warnings.push(preferredWorkspaceWarning); } if (missingProjectCwds.length > 0) { const firstMissing = missingProjectCwds[0]; const extraMissingCount = Math.max(0, missingProjectCwds.length - 1); warnings.push( extraMissingCount > 0 ? `Project workspace path "${firstMissing}" and ${extraMissingCount} other configured path(s) are not available yet. Using fallback workspace "${fallbackCwd}" for this run.` : `Project workspace path "${firstMissing}" is not available yet. Using fallback workspace "${fallbackCwd}" for this run.`, ); } else if (!hasConfiguredProjectCwd) { warnings.push( `Project workspace has no local cwd configured. Using fallback workspace "${fallbackCwd}" for this run.`, ); } return { cwd: fallbackCwd, source: "project_primary" as const, projectId: resolvedProjectId, workspaceId: projectWorkspaceRows[0]?.id ?? null, repoUrl: projectWorkspaceRows[0]?.repoUrl ?? null, repoRef: projectWorkspaceRows[0]?.repoRef ?? null, workspaceHints, warnings, }; } if (workspaceProjectId) { const managedWorkspace = await ensureManagedProjectWorkspace({ companyId: agent.companyId, projectId: workspaceProjectId, repoUrl: null, }); return { cwd: managedWorkspace.cwd, source: "project_primary" as const, projectId: resolvedProjectId, workspaceId: null, repoUrl: null, repoRef: null, workspaceHints, warnings: managedWorkspace.warning ? [managedWorkspace.warning] : [], }; } const sessionCwd = readNonEmptyString(previousSessionParams?.cwd); if (sessionCwd) { const sessionCwdExists = await fs .stat(sessionCwd) .then((stats) => stats.isDirectory()) .catch(() => false); if (sessionCwdExists) { return { cwd: sessionCwd, source: "task_session" as const, projectId: resolvedProjectId, workspaceId: readNonEmptyString(previousSessionParams?.workspaceId), repoUrl: readNonEmptyString(previousSessionParams?.repoUrl), repoRef: readNonEmptyString(previousSessionParams?.repoRef), workspaceHints, warnings: [], }; } } const cwd = resolveDefaultAgentWorkspaceDir(agent.id); await fs.mkdir(cwd, { recursive: true }); const warnings: string[] = []; if (sessionCwd) { warnings.push( `Saved session workspace "${sessionCwd}" is not available. Using fallback workspace "${cwd}" for this run.`, ); } else if (resolvedProjectId) { warnings.push( `No project workspace directory is currently available for this issue. Using fallback workspace "${cwd}" for this run.`, ); } else { warnings.push( `No project or prior session workspace was available. Using fallback workspace "${cwd}" for this run.`, ); } return { cwd, source: "agent_home" as const, projectId: resolvedProjectId, workspaceId: null, repoUrl: null, repoRef: null, workspaceHints, warnings, }; } async function upsertTaskSession(input: { companyId: string; agentId: string; adapterType: string; taskKey: string; sessionParamsJson: Record | null; sessionDisplayId: string | null; lastRunId: string | null; lastError: string | null; }) { const existing = await getTaskSession( input.companyId, input.agentId, input.adapterType, input.taskKey, ); if (existing) { return db .update(agentTaskSessions) .set({ sessionParamsJson: input.sessionParamsJson, sessionDisplayId: input.sessionDisplayId, lastRunId: input.lastRunId, lastError: input.lastError, updatedAt: new Date(), }) .where(eq(agentTaskSessions.id, existing.id)) .returning() .then((rows) => rows[0] ?? null); } return db .insert(agentTaskSessions) .values({ companyId: input.companyId, agentId: input.agentId, adapterType: input.adapterType, taskKey: input.taskKey, sessionParamsJson: input.sessionParamsJson, sessionDisplayId: input.sessionDisplayId, lastRunId: input.lastRunId, lastError: input.lastError, }) .returning() .then((rows) => rows[0] ?? null); } async function clearTaskSessions( companyId: string, agentId: string, opts?: { taskKey?: string | null; adapterType?: string | null }, ) { const conditions = [ eq(agentTaskSessions.companyId, companyId), eq(agentTaskSessions.agentId, agentId), ]; if (opts?.taskKey) { conditions.push(eq(agentTaskSessions.taskKey, opts.taskKey)); } if (opts?.adapterType) { conditions.push(eq(agentTaskSessions.adapterType, opts.adapterType)); } return db .delete(agentTaskSessions) .where(and(...conditions)) .returning() .then((rows) => rows.length); } async function ensureRuntimeState(agent: typeof agents.$inferSelect) { const existing = await getRuntimeState(agent.id); if (existing) return existing; return db .insert(agentRuntimeState) .values({ agentId: agent.id, companyId: agent.companyId, adapterType: agent.adapterType, stateJson: {}, }) .returning() .then((rows) => rows[0]); } async function setRunStatus( runId: string, status: string, patch?: Partial, ) { const updated = await db .update(heartbeatRuns) .set({ status, ...patch, updatedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)) .returning() .then((rows) => rows[0] ?? null); if (updated) { publishLiveEvent({ companyId: updated.companyId, type: "heartbeat.run.status", payload: { runId: updated.id, agentId: updated.agentId, status: updated.status, invocationSource: updated.invocationSource, triggerDetail: updated.triggerDetail, error: updated.error ?? null, errorCode: updated.errorCode ?? null, startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null, finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null, }, }); publishRunLifecyclePluginEvent(updated); } return updated; } function publishRunLifecyclePluginEvent(run: typeof heartbeatRuns.$inferSelect) { const eventType = run.status === "running" ? "agent.run.started" : run.status === "succeeded" ? "agent.run.finished" : run.status === "failed" || run.status === "timed_out" ? "agent.run.failed" : run.status === "cancelled" ? "agent.run.cancelled" : null; if (!eventType) return; publishPluginDomainEvent({ eventId: randomUUID(), eventType, occurredAt: new Date().toISOString(), actorId: run.agentId, actorType: "agent", entityId: run.id, entityType: "heartbeat_run", companyId: run.companyId, payload: { runId: run.id, agentId: run.agentId, status: run.status, invocationSource: run.invocationSource, triggerDetail: run.triggerDetail, error: run.error ?? null, errorCode: run.errorCode ?? null, issueId: typeof run.contextSnapshot === "object" && run.contextSnapshot !== null ? (run.contextSnapshot as Record).issueId ?? null : null, startedAt: run.startedAt ? new Date(run.startedAt).toISOString() : null, finishedAt: run.finishedAt ? new Date(run.finishedAt).toISOString() : null, }, }); } async function setWakeupStatus( wakeupRequestId: string | null | undefined, status: string, patch?: Partial, ) { if (!wakeupRequestId) return; await db .update(agentWakeupRequests) .set({ status, ...patch, updatedAt: new Date() }) .where(eq(agentWakeupRequests.id, wakeupRequestId)); } async function addContinuationExhaustedCommentOnce(input: { run: typeof heartbeatRuns.$inferSelect; issueId: string; comment: string; }) { const existing = await db .select({ id: issueComments.id }) .from(issueComments) .where( and( eq(issueComments.companyId, input.run.companyId), eq(issueComments.issueId, input.issueId), eq(issueComments.createdByRunId, input.run.id), sql`${issueComments.body} like 'Bounded liveness continuation exhausted%'`, ), ) .limit(1) .then((rows) => rows[0] ?? null); if (existing) return; await issuesSvc.addComment(input.issueId, input.comment, { agentId: input.run.agentId, runId: input.run.id, }); } async function handleRunLivenessContinuation(run: typeof heartbeatRuns.$inferSelect) { const livenessState = run.livenessState as RunLivenessState | null; if (livenessState !== "plan_only" && livenessState !== "empty_response") return; const context = parseObject(run.contextSnapshot); const issueId = readNonEmptyString(context.issueId); if (!issueId) return; const [issue, agent] = await Promise.all([ db .select({ id: issues.id, companyId: issues.companyId, identifier: issues.identifier, title: issues.title, status: issues.status, assigneeAgentId: issues.assigneeAgentId, executionState: issues.executionState, projectId: issues.projectId, }) .from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) .then((rows) => rows[0] ?? null), db .select({ id: agents.id, companyId: agents.companyId, status: agents.status, }) .from(agents) .where(eq(agents.id, run.agentId)) .then((rows) => rows[0] ?? null), ]); const budgetBlock = issue && agent ? await budgets.getInvocationBlock(issue.companyId, agent.id, { issueId: issue.id, projectId: issue.projectId, }) : null; const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1; const idempotencyKey = issue ? buildRunLivenessContinuationIdempotencyKey({ issueId: issue.id, sourceRunId: run.id, livenessState, nextAttempt, }) : null; const existingWake = idempotencyKey ? await findExistingRunLivenessContinuationWake(db, { companyId: run.companyId, idempotencyKey, }) : null; const decision = decideRunLivenessContinuation({ run, issue, agent, livenessState, livenessReason: run.livenessReason, nextAction: run.nextAction, budgetBlocked: Boolean(budgetBlock), idempotentWakeExists: Boolean(existingWake), }); if (decision.kind === "exhausted") { await setRunStatus(run.id, run.status, { livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation attempts exhausted`, }); await addContinuationExhaustedCommentOnce({ run, issueId, comment: decision.comment, }); return; } if (decision.kind !== "enqueue") return; const continuationRun = await enqueueWakeup(run.agentId, { source: "automation", triggerDetail: "system", reason: RUN_LIVENESS_CONTINUATION_REASON, payload: decision.payload, contextSnapshot: decision.contextSnapshot, idempotencyKey: decision.idempotencyKey, requestedByActorType: "system", requestedByActorId: "heartbeat", }); if (continuationRun) { await db .update(heartbeatRuns) .set({ continuationAttempt: decision.nextAttempt, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, continuationRun.id)); } } async function appendRunEvent( run: typeof heartbeatRuns.$inferSelect, seq: number, event: { eventType: string; stream?: "system" | "stdout" | "stderr"; level?: "info" | "warn" | "error"; color?: string; message?: string; payload?: Record; }, ) { const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); const sanitizedMessage = event.message ? redactCurrentUserText(event.message, currentUserRedactionOptions) : event.message; const boundedPayload = event.payload ? boundHeartbeatRunEventPayloadForStorage(event.payload) : event.payload; const sanitizedPayload = boundedPayload ? redactCurrentUserValue(boundedPayload, currentUserRedactionOptions) : boundedPayload; await db.insert(heartbeatRunEvents).values({ companyId: run.companyId, runId: run.id, agentId: run.agentId, seq, eventType: event.eventType, stream: event.stream, level: event.level, color: event.color, message: sanitizedMessage, payload: sanitizedPayload, }); publishLiveEvent({ companyId: run.companyId, type: "heartbeat.run.event", payload: { runId: run.id, agentId: run.agentId, seq, eventType: event.eventType, stream: event.stream ?? null, level: event.level ?? null, color: event.color ?? null, message: sanitizedMessage ?? null, payload: sanitizedPayload ?? null, }, }); } async function nextRunEventSeq(runId: string) { const [row] = await db .select({ maxSeq: sql`max(${heartbeatRunEvents.seq})` }) .from(heartbeatRunEvents) .where(eq(heartbeatRunEvents.runId, runId)); return Number(row?.maxSeq ?? 0) + 1; } async function persistRunProcessMetadata( runId: string, meta: { pid: number; processGroupId: number | null; startedAt: string }, ) { const startedAt = new Date(meta.startedAt); return db .update(heartbeatRuns) .set({ processPid: meta.pid, processGroupId: meta.processGroupId, processStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)) .returning() .then((rows) => rows[0] ?? null); } async function clearDetachedRunWarning(runId: string) { const updated = await db .update(heartbeatRuns) .set({ error: null, errorCode: null, updatedAt: new Date(), }) .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE))) .returning() .then((rows) => rows[0] ?? null); if (!updated) return null; await appendRunEvent(updated, await nextRunEventSeq(updated.id), { eventType: "lifecycle", stream: "system", level: "info", message: "Detached child process reported activity; cleared detached warning", }); return updated; } async function patchRunIssueCommentStatus( runId: string, patch: Partial>, ) { return db .update(heartbeatRuns) .set({ ...patch, updatedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)) .returning() .then((rows) => rows[0] ?? null); } async function findRunIssueComment(runId: string, companyId: string, issueId: string) { return db .select({ id: issueComments.id, }) .from(issueComments) .where( and( eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), eq(issueComments.createdByRunId, runId), ), ) .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) .limit(1) .then((rows) => rows[0] ?? null); } async function refreshContinuationSummaryForRun( run: typeof heartbeatRuns.$inferSelect, agent: typeof agents.$inferSelect, ) { const contextSnapshot = parseObject(run.contextSnapshot); const issueId = readNonEmptyString(contextSnapshot.issueId); if (!issueId) return null; try { return await refreshIssueContinuationSummary({ db, issueId, run: { id: run.id, status: run.status, error: run.error, errorCode: run.errorCode, resultJson: run.resultJson as Record | null, stdoutExcerpt: run.stdoutExcerpt, stderrExcerpt: run.stderrExcerpt, finishedAt: run.finishedAt, }, agent: { id: agent.id, name: agent.name, adapterType: agent.adapterType, }, }); } catch (err) { logger.warn( { err, runId: run.id, issueId, agentId: agent.id, }, "failed to refresh issue continuation summary", ); return null; } } async function enqueueMissingIssueCommentRetry( run: typeof heartbeatRuns.$inferSelect, agent: typeof agents.$inferSelect, issueId: string, ) { const contextSnapshot = parseObject(run.contextSnapshot); const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); const retryContextSnapshot = { ...contextSnapshot, retryOfRunId: run.id, wakeReason: "missing_issue_comment", retryReason: "missing_issue_comment", missingIssueCommentForRunId: run.id, }; const now = new Date(); const retryRun = await db.transaction(async (tx) => { await tx.execute( sql`select id from issues where company_id = ${run.companyId} and execution_run_id = ${run.id} for update`, ); const issue = await tx .select({ id: issues.id }) .from(issues) .where(and(eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id))) .then((rows) => rows[0] ?? null); if (!issue) return null; const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: "missing_issue_comment", payload: { issueId, retryOfRunId: run.id, retryReason: "missing_issue_comment", }, status: "queued", requestedByActorType: "system", requestedByActorId: null, updatedAt: now, }) .returning() .then((rows) => rows[0]); const queuedRun = await tx .insert(heartbeatRuns) .values({ companyId: run.companyId, agentId: run.agentId, invocationSource: "automation", triggerDetail: "system", status: "queued", wakeupRequestId: wakeupRequest.id, contextSnapshot: retryContextSnapshot, sessionIdBefore: sessionBefore, retryOfRunId: run.id, issueCommentStatus: "not_applicable", updatedAt: now, }) .returning() .then((rows) => rows[0]); await tx .update(agentWakeupRequests) .set({ runId: queuedRun.id, updatedAt: now, }) .where(eq(agentWakeupRequests.id, wakeupRequest.id)); await tx .update(issues) .set({ executionRunId: queuedRun.id, executionAgentNameKey: normalizeAgentNameKey(agent.name), executionLockedAt: now, updatedAt: now, }) .where(eq(issues.id, issue.id)); await tx .update(heartbeatRuns) .set({ issueCommentStatus: "retry_queued", issueCommentRetryQueuedAt: now, updatedAt: now, }) .where(eq(heartbeatRuns.id, run.id)); return queuedRun; }); if (!retryRun) return null; publishLiveEvent({ companyId: retryRun.companyId, type: "heartbeat.run.queued", payload: { runId: retryRun.id, agentId: retryRun.agentId, invocationSource: retryRun.invocationSource, triggerDetail: retryRun.triggerDetail, wakeupRequestId: retryRun.wakeupRequestId, }, }); return retryRun; } async function hasDeferredIssueCommentWake(companyId: string, issueId: string, agentId: string) { const deferredPayloads = await db .select({ payload: agentWakeupRequests.payload }) .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, ), ); return deferredPayloads.some(({ payload }) => { const parsedPayload = parseObject(payload); const deferredContext = parseObject(parsedPayload[DEFERRED_WAKE_CONTEXT_KEY]); return Boolean(deriveCommentId(deferredContext, parsedPayload)); }); } async function finalizeIssueCommentPolicy( run: typeof heartbeatRuns.$inferSelect, agent: typeof agents.$inferSelect, ) { const contextSnapshot = parseObject(run.contextSnapshot); const issueId = readNonEmptyString(contextSnapshot.issueId); if (!issueId) { if (run.issueCommentStatus !== "not_applicable") { await patchRunIssueCommentStatus(run.id, { issueCommentStatus: "not_applicable", issueCommentSatisfiedByCommentId: null, issueCommentRetryQueuedAt: null, }); } return { outcome: "not_applicable" as const, queuedRun: null }; } const postedComment = await findRunIssueComment(run.id, run.companyId, issueId); if (postedComment) { await patchRunIssueCommentStatus(run.id, { issueCommentStatus: "satisfied", issueCommentSatisfiedByCommentId: postedComment.id, issueCommentRetryQueuedAt: null, }); return { outcome: "satisfied" as const, queuedRun: null }; } if (readNonEmptyString(contextSnapshot.retryReason) === "missing_issue_comment") { await patchRunIssueCommentStatus(run.id, { issueCommentStatus: "retry_exhausted", issueCommentSatisfiedByCommentId: null, }); await appendRunEvent(run, await nextRunEventSeq(run.id), { eventType: "lifecycle", stream: "system", level: "warn", message: "Run ended without an issue comment after one retry; no further comment wake will be queued", }); return { outcome: "retry_exhausted" as const, queuedRun: null }; } if (!shouldRequireIssueCommentForWake(contextSnapshot)) { if (run.issueCommentStatus !== "not_applicable") { await patchRunIssueCommentStatus(run.id, { issueCommentStatus: "not_applicable", issueCommentSatisfiedByCommentId: null, issueCommentRetryQueuedAt: null, }); } return { outcome: "not_applicable" as const, queuedRun: null }; } if (await hasDeferredIssueCommentWake(run.companyId, issueId, run.agentId)) { await patchRunIssueCommentStatus(run.id, { issueCommentStatus: "not_applicable", issueCommentSatisfiedByCommentId: null, issueCommentRetryQueuedAt: null, }); await appendRunEvent(run, await nextRunEventSeq(run.id), { eventType: "lifecycle", stream: "system", level: "info", message: "Run ended without an issue comment; a deferred comment wake already exists for this issue", }); return { outcome: "not_applicable" as const, queuedRun: null }; } const queuedRun = await enqueueMissingIssueCommentRetry(run, agent, issueId); if (queuedRun) { await appendRunEvent(run, await nextRunEventSeq(run.id), { eventType: "lifecycle", stream: "system", level: "warn", message: "Run ended without an issue comment; queued one follow-up wake to require a comment", }); return { outcome: "retry_queued" as const, queuedRun }; } await patchRunIssueCommentStatus(run.id, { issueCommentStatus: "retry_exhausted", issueCommentSatisfiedByCommentId: null, }); return { outcome: "retry_exhausted" as const, queuedRun: null }; } async function enqueueProcessLossRetry( run: typeof heartbeatRuns.$inferSelect, agent: typeof agents.$inferSelect, now: Date, ) { const contextSnapshot = parseObject(run.contextSnapshot); const issueId = readNonEmptyString(contextSnapshot.issueId); const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); const retryContextSnapshot = { ...contextSnapshot, retryOfRunId: run.id, wakeReason: "process_lost_retry", retryReason: "process_lost", }; const queued = await db.transaction(async (tx) => { const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: "process_lost_retry", payload: { ...(issueId ? { issueId } : {}), retryOfRunId: run.id, }, status: "queued", requestedByActorType: "system", requestedByActorId: null, updatedAt: now, }) .returning() .then((rows) => rows[0]); const retryRun = await tx .insert(heartbeatRuns) .values({ companyId: run.companyId, agentId: run.agentId, invocationSource: "automation", triggerDetail: "system", status: "queued", wakeupRequestId: wakeupRequest.id, contextSnapshot: retryContextSnapshot, sessionIdBefore: sessionBefore, retryOfRunId: run.id, processLossRetryCount: (run.processLossRetryCount ?? 0) + 1, updatedAt: now, }) .returning() .then((rows) => rows[0]); await tx .update(agentWakeupRequests) .set({ runId: retryRun.id, updatedAt: now, }) .where(eq(agentWakeupRequests.id, wakeupRequest.id)); if (issueId) { await tx .update(issues) .set({ executionRunId: retryRun.id, executionAgentNameKey: normalizeAgentNameKey(agent.name), executionLockedAt: now, updatedAt: now, }) .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id))); } return retryRun; }); publishLiveEvent({ companyId: queued.companyId, type: "heartbeat.run.queued", payload: { runId: queued.id, agentId: queued.agentId, invocationSource: queued.invocationSource, triggerDetail: queued.triggerDetail, wakeupRequestId: queued.wakeupRequestId, }, }); await appendRunEvent(queued, 1, { eventType: "lifecycle", stream: "system", level: "warn", message: "Queued automatic retry after orphaned child process was confirmed dead", payload: { retryOfRunId: run.id, }, }); return queued; } async function scheduleBoundedRetryForRun( run: typeof heartbeatRuns.$inferSelect, agent: typeof agents.$inferSelect, opts?: { now?: Date; random?: () => number; retryReason?: string; wakeReason?: string; }, ) { const now = opts?.now ?? new Date(); const retryReason = opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON; const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; const nextAttempt = (run.scheduledRetryAttempt ?? 0) + 1; const schedule = computeBoundedTransientHeartbeatRetrySchedule(nextAttempt, now, opts?.random); const codexTransientFallbackMode = agent.adapterType === "codex_local" && retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON && run.errorCode === "codex_transient_upstream" ? resolveCodexTransientFallbackMode(nextAttempt) : null; if (!schedule) { await appendRunEvent(run, await nextRunEventSeq(run.id), { eventType: "lifecycle", stream: "system", level: "warn", message: `Bounded retry exhausted after ${run.scheduledRetryAttempt ?? 0} scheduled attempts; no further automatic retry will be queued`, payload: { retryReason, scheduledRetryAttempt: run.scheduledRetryAttempt ?? 0, maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, }, }); return { outcome: "retry_exhausted" as const, attempt: nextAttempt, maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, }; } const contextSnapshot = parseObject(run.contextSnapshot); const issueId = readNonEmptyString(contextSnapshot.issueId); const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); const retryContextSnapshot: Record = { ...contextSnapshot, retryOfRunId: run.id, wakeReason, retryReason, scheduledRetryAttempt: schedule.attempt, scheduledRetryAt: schedule.dueAt.toISOString(), ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), }; const retryRun = await db.transaction(async (tx) => { const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: wakeReason, payload: { ...(issueId ? { issueId } : {}), retryOfRunId: run.id, retryReason, scheduledRetryAttempt: schedule.attempt, scheduledRetryAt: schedule.dueAt.toISOString(), ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), }, status: "queued", requestedByActorType: "system", requestedByActorId: null, updatedAt: now, }) .returning() .then((rows) => rows[0]); const scheduledRun = await tx .insert(heartbeatRuns) .values({ companyId: run.companyId, agentId: run.agentId, invocationSource: "automation", triggerDetail: "system", status: "scheduled_retry", wakeupRequestId: wakeupRequest.id, contextSnapshot: retryContextSnapshot, sessionIdBefore: sessionBefore, retryOfRunId: run.id, scheduledRetryAt: schedule.dueAt, scheduledRetryAttempt: schedule.attempt, scheduledRetryReason: retryReason, continuationAttempt: readContinuationAttempt(retryContextSnapshot.livenessContinuationAttempt), updatedAt: now, }) .returning() .then((rows) => rows[0]); await tx .update(agentWakeupRequests) .set({ runId: scheduledRun.id, updatedAt: now, }) .where(eq(agentWakeupRequests.id, wakeupRequest.id)); if (issueId) { await tx .update(issues) .set({ executionRunId: scheduledRun.id, executionAgentNameKey: normalizeAgentNameKey(agent.name), executionLockedAt: now, updatedAt: now, }) .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id))); } return scheduledRun; }); await appendRunEvent(run, await nextRunEventSeq(run.id), { eventType: "lifecycle", stream: "system", level: "warn", message: `Scheduled bounded retry ${schedule.attempt}/${schedule.maxAttempts} for ${schedule.dueAt.toISOString()}`, payload: { retryRunId: retryRun.id, retryReason, scheduledRetryAttempt: schedule.attempt, scheduledRetryAt: schedule.dueAt.toISOString(), baseDelayMs: schedule.baseDelayMs, delayMs: schedule.delayMs, ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), }, }); return { outcome: "scheduled" as const, run: retryRun, dueAt: schedule.dueAt, attempt: schedule.attempt, maxAttempts: schedule.maxAttempts, }; } async function promoteDueScheduledRetries(now = new Date()) { const dueRuns = await db .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.status, "scheduled_retry"), lte(heartbeatRuns.scheduledRetryAt, now), ), ) .orderBy(asc(heartbeatRuns.scheduledRetryAt), asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) .limit(50); const promotedRunIds: string[] = []; for (const dueRun of dueRuns) { const promoted = await db .update(heartbeatRuns) .set({ status: "queued", updatedAt: now, }) .where( and( eq(heartbeatRuns.id, dueRun.id), eq(heartbeatRuns.status, "scheduled_retry"), lte(heartbeatRuns.scheduledRetryAt, now), ), ) .returning() .then((rows) => rows[0] ?? null); if (!promoted) continue; promotedRunIds.push(promoted.id); await appendRunEvent(promoted, await nextRunEventSeq(promoted.id), { eventType: "lifecycle", stream: "system", level: "info", message: "Scheduled retry became due and was promoted to the queued run pool", payload: { scheduledRetryAttempt: promoted.scheduledRetryAttempt, scheduledRetryAt: promoted.scheduledRetryAt ? new Date(promoted.scheduledRetryAt).toISOString() : null, scheduledRetryReason: promoted.scheduledRetryReason, }, }); publishLiveEvent({ companyId: promoted.companyId, type: "heartbeat.run.queued", payload: { runId: promoted.id, agentId: promoted.agentId, invocationSource: promoted.invocationSource, triggerDetail: promoted.triggerDetail, wakeupRequestId: promoted.wakeupRequestId, }, }); } return { promoted: promotedRunIds.length, runIds: promotedRunIds, }; } function parseHeartbeatPolicy(agent: typeof agents.$inferSelect) { const runtimeConfig = parseObject(agent.runtimeConfig); const heartbeat = parseObject(runtimeConfig.heartbeat); return { enabled: asBoolean(heartbeat.enabled, false), intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)), wakeOnDemand: asBoolean(heartbeat.wakeOnDemand ?? heartbeat.wakeOnAssignment ?? heartbeat.wakeOnOnDemand ?? heartbeat.wakeOnAutomation, true), maxConcurrentRuns: normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns), }; } function issueRunPriorityRank(priority: string | null | undefined) { switch (priority) { case "critical": return 0; case "high": return 1; case "medium": return 2; case "low": return 3; default: return 4; } } async function listQueuedRunDependencyReadiness( companyId: string, queuedRuns: Array, ) { const issueIds = [...new Set( queuedRuns .map((run) => readNonEmptyString(parseObject(run.contextSnapshot).issueId)) .filter((issueId): issueId is string => Boolean(issueId)), )]; if (issueIds.length === 0) { return new Map>>(); } return issuesSvc.listDependencyReadiness(companyId, issueIds); } async function countRunningRunsForAgent(agentId: string) { const [{ count }] = await db .select({ count: sql`count(*)` }) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running"))); return Number(count ?? 0); } async function claimQueuedRun(run: typeof heartbeatRuns.$inferSelect) { if (run.status !== "queued") return run; const agent = await getAgent(run.agentId); if (!agent) { await cancelRunInternal(run.id, "Cancelled because the agent no longer exists"); return null; } if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") { await cancelRunInternal(run.id, "Cancelled because the agent is not invokable"); return null; } const context = parseObject(run.contextSnapshot); const budgetBlock = await budgets.getInvocationBlock(run.companyId, run.agentId, { issueId: readNonEmptyString(context.issueId), projectId: readNonEmptyString(context.projectId), }); if (budgetBlock) { await cancelRunInternal(run.id, budgetBlock.reason); return null; } const issueId = readNonEmptyString(context.issueId); if (issueId) { const activePauseHold = await treeControlSvc.getActivePauseHoldGate(run.companyId, issueId); const treeHoldInteractionWake = activePauseHold && allowsIssueInteractionWake(context); if (activePauseHold && !treeHoldInteractionWake) { await cancelRunInternal(run.id, "Cancelled because issue is held by an active subtree pause hold"); await logActivity(db, { companyId: run.companyId, actorType: "system", actorId: "system", agentId: run.agentId, runId: run.id, action: "issue.tree_hold_run_interrupted", entityType: "heartbeat_run", entityId: run.id, details: { issueId, holdId: activePauseHold.holdId, rootIssueId: activePauseHold.rootIssueId, source: "heartbeat.claim_queued_run", securityPrinciples: ["Complete Mediation", "Fail Securely", "Secure Defaults"], }, }); return null; } const dependencyReadiness = await issuesSvc.listDependencyReadiness(run.companyId, [issueId]); const unresolvedBlockerCount = dependencyReadiness.get(issueId)?.unresolvedBlockerCount ?? 0; if (unresolvedBlockerCount > 0 && !allowsIssueInteractionWake(context)) { logger.debug({ runId: run.id, issueId, unresolvedBlockerCount }, "claimQueuedRun: skipping blocked run"); return null; } } const claimedAt = new Date(); const claimed = await db .update(heartbeatRuns) .set({ status: "running", startedAt: run.startedAt ?? claimedAt, updatedAt: claimedAt, }) .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued"))) .returning() .then((rows) => rows[0] ?? null); if (!claimed) return null; publishLiveEvent({ companyId: claimed.companyId, type: "heartbeat.run.status", payload: { runId: claimed.id, agentId: claimed.agentId, status: claimed.status, invocationSource: claimed.invocationSource, triggerDetail: claimed.triggerDetail, error: claimed.error ?? null, errorCode: claimed.errorCode ?? null, startedAt: claimed.startedAt ? new Date(claimed.startedAt).toISOString() : null, finishedAt: claimed.finishedAt ? new Date(claimed.finishedAt).toISOString() : null, }, }); publishRunLifecyclePluginEvent(claimed); await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt }); // Fix A (lazy locking): stamp executionRunId now that the run is actually running, // not at queue time. Guard is idempotent — safe if called more than once. const claimedIssueId = readNonEmptyString(parseObject(claimed.contextSnapshot).issueId); if (claimedIssueId) { const claimedAgent = await getAgent(claimed.agentId); await db .update(issues) .set({ executionRunId: claimed.id, executionAgentNameKey: normalizeAgentNameKey(claimedAgent?.name), executionLockedAt: claimedAt, updatedAt: claimedAt, }) .where( and( eq(issues.id, claimedIssueId), eq(issues.companyId, claimed.companyId), or(isNull(issues.executionRunId), eq(issues.executionRunId, claimed.id)), ), ); } return claimed; } async function finalizeAgentStatus( agentId: string, outcome: "succeeded" | "failed" | "cancelled" | "timed_out", ) { const existing = await getAgent(agentId); if (!existing) return; if (existing.status === "paused" || existing.status === "terminated") { return; } const isFirstHeartbeat = !existing.lastHeartbeatAt; const runningCount = await countRunningRunsForAgent(agentId); const nextStatus = runningCount > 0 ? "running" : outcome === "succeeded" || outcome === "cancelled" ? "idle" : "error"; const updated = await db .update(agents) .set({ status: nextStatus, lastHeartbeatAt: new Date(), updatedAt: new Date(), }) .where(eq(agents.id, agentId)) .returning() .then((rows) => rows[0] ?? null); if (isFirstHeartbeat && updated) { const tc = getTelemetryClient(); if (tc) trackAgentFirstHeartbeat(tc, { agentRole: updated.role, agentId: updated.id }); } if (updated) { publishLiveEvent({ companyId: updated.companyId, type: "agent.status", payload: { agentId: updated.id, status: updated.status, lastHeartbeatAt: updated.lastHeartbeatAt ? new Date(updated.lastHeartbeatAt).toISOString() : null, outcome, }, }); } } function mergeRunStopMetadataForAgent( agent: Pick, outcome: "succeeded" | "failed" | "cancelled" | "timed_out", options?: { resultJson?: Record | null; errorCode?: string | null; errorMessage?: string | null; }, ) { const stopMetadata = buildHeartbeatRunStopMetadata({ adapterType: agent.adapterType, adapterConfig: parseObject(agent.adapterConfig), outcome, errorCode: options?.errorCode ?? null, errorMessage: options?.errorMessage ?? null, }); return mergeHeartbeatRunStopMetadata(options?.resultJson ?? null, stopMetadata); } function countValue(value: unknown) { const parsed = Number(value ?? 0); return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0; } function dateValue(value: unknown) { if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value; if (typeof value === "string" || typeof value === "number") { const parsed = new Date(value); return Number.isNaN(parsed.getTime()) ? null : parsed; } return null; } function latestDate(...values: unknown[]) { let latest: Date | null = null; for (const value of values) { const parsed = dateValue(value); if (!parsed) continue; if (!latest || parsed.getTime() > latest.getTime()) latest = parsed; } return latest; } async function buildRunLivenessInput( run: typeof heartbeatRuns.$inferSelect, resultJson: Record | null | undefined, ): Promise { const context = parseObject(run.contextSnapshot); const contextIssueId = readNonEmptyString(context.issueId); const continuationAttempt = asNumber(context.continuationAttempt, run.continuationAttempt ?? 0); const issue = contextIssueId ? await db .select({ status: issues.status, title: issues.title, description: issues.description, }) .from(issues) .where(and(eq(issues.companyId, run.companyId), eq(issues.id, contextIssueId))) .then((rows) => rows[0] ?? null) : null; const [commentStats] = contextIssueId ? await db .select({ count: sql`count(*)::int`, latestAt: sql`max(${issueComments.createdAt})`, }) .from(issueComments) .where( and( eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, contextIssueId), eq(issueComments.createdByRunId, run.id), ), ) : [{ count: 0, latestAt: null }]; const [documentStats] = contextIssueId ? await db .select({ count: sql`count(*)::int`, planCount: sql`count(*) filter (where ${issueDocuments.key} = 'plan')::int`, latestAt: sql`max(${documentRevisions.createdAt})`, }) .from(documentRevisions) .innerJoin(issueDocuments, eq(documentRevisions.documentId, issueDocuments.documentId)) .where( and( eq(documentRevisions.companyId, run.companyId), eq(documentRevisions.createdByRunId, run.id), eq(issueDocuments.companyId, run.companyId), eq(issueDocuments.issueId, contextIssueId), sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`, ), ) : [{ count: 0, planCount: 0, latestAt: null }]; const [workProductStats] = contextIssueId ? await db .select({ count: sql`count(*)::int`, latestAt: sql`max(${issueWorkProducts.createdAt})`, }) .from(issueWorkProducts) .where( and( eq(issueWorkProducts.companyId, run.companyId), eq(issueWorkProducts.issueId, contextIssueId), eq(issueWorkProducts.createdByRunId, run.id), ), ) : [{ count: 0, latestAt: null }]; const [workspaceOperationStats] = await db .select({ count: sql`count(*)::int`, latestAt: sql`max(${workspaceOperations.startedAt})`, }) .from(workspaceOperations) .where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.heartbeatRunId, run.id))); const [activityStats] = await db .select({ count: sql`count(*)::int`, latestAt: sql`max(${activityLog.createdAt})`, }) .from(activityLog) .where( and( eq(activityLog.companyId, run.companyId), eq(activityLog.runId, run.id), notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS), ), ); const [eventStats] = await db .select({ count: sql`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`, latestAt: sql`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`, }) .from(heartbeatRunEvents) .where(and(eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id))); return { runStatus: run.status, issue, resultJson: resultJson ?? run.resultJson ?? null, stdoutExcerpt: run.stdoutExcerpt ?? null, stderrExcerpt: run.stderrExcerpt ?? null, error: run.error ?? null, errorCode: run.errorCode ?? null, continuationAttempt, evidence: { issueCommentsCreated: countValue(commentStats?.count), documentRevisionsCreated: countValue(documentStats?.count), planDocumentRevisionsCreated: countValue(documentStats?.planCount), workProductsCreated: countValue(workProductStats?.count), workspaceOperationsCreated: countValue(workspaceOperationStats?.count), activityEventsCreated: countValue(activityStats?.count), toolOrActionEventsCreated: countValue(eventStats?.count), latestEvidenceAt: latestDate( commentStats?.latestAt, documentStats?.latestAt, workProductStats?.latestAt, workspaceOperationStats?.latestAt, activityStats?.latestAt, eventStats?.latestAt, ), }, }; } async function classifyAndPersistRunLiveness( run: typeof heartbeatRuns.$inferSelect, resultJson?: Record | null, ) { const classification = classifyRunLiveness(await buildRunLivenessInput(run, resultJson)); return db .update(heartbeatRuns) .set({ livenessState: classification.livenessState, livenessReason: classification.livenessReason, continuationAttempt: classification.continuationAttempt, lastUsefulActionAt: classification.lastUsefulActionAt, nextAction: classification.nextAction, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.id)) .returning() .then((rows) => rows[0] ?? null); } async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) { const staleThresholdMs = opts?.staleThresholdMs ?? 0; const now = new Date(); // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them) const activeRuns = await db .select({ run: heartbeatRuns, adapterType: agents.adapterType, adapterConfig: agents.adapterConfig, }) .from(heartbeatRuns) .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) .where(eq(heartbeatRuns.status, "running")); const reaped: string[] = []; for (const { run, adapterType, adapterConfig } of activeRuns) { if (runningProcesses.has(run.id) || activeRunExecutions.has(run.id)) continue; // Apply staleness threshold to avoid false positives if (staleThresholdMs > 0) { const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0; if (now.getTime() - refTime < staleThresholdMs) continue; } const tracksLocalChild = isTrackedLocalChildProcessAdapter(adapterType); const processPidAlive = tracksLocalChild && run.processPid && isProcessAlive(run.processPid); const processGroupAlive = tracksLocalChild && run.processGroupId && isProcessGroupAlive(run.processGroupId); if (processPidAlive) { if (run.errorCode !== DETACHED_PROCESS_ERROR_CODE) { const detachedMessage = `Lost in-memory process handle, but child pid ${run.processPid} is still alive`; const detachedRun = await setRunStatus(run.id, "running", { error: detachedMessage, errorCode: DETACHED_PROCESS_ERROR_CODE, }); if (detachedRun) { await appendRunEvent(detachedRun, await nextRunEventSeq(detachedRun.id), { eventType: "lifecycle", stream: "system", level: "warn", message: detachedMessage, payload: { processPid: run.processPid, }, }); } } continue; } let descendantOnlyCleanup = false; if (processGroupAlive) { descendantOnlyCleanup = true; await terminateHeartbeatRunProcess({ pid: run.processPid, processGroupId: run.processGroupId, }); } const shouldRetry = tracksLocalChild && (!!run.processPid || !!run.processGroupId) && (run.processLossRetryCount ?? 0) < 1; const baseMessage = buildProcessLossMessage(run, descendantOnlyCleanup ? { descendantOnly: true } : undefined); let finalizedRun = await setRunStatus(run.id, "failed", { error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage, errorCode: "process_lost", finishedAt: now, resultJson: mergeRunStopMetadataForAgent( { adapterType, adapterConfig }, "failed", { resultJson: parseObject(run.resultJson), errorCode: "process_lost", errorMessage: shouldRetry ? `${baseMessage}; retrying once` : baseMessage, }, ), }); await setWakeupStatus(run.wakeupRequestId, "failed", { finishedAt: now, error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage, }); if (!finalizedRun) finalizedRun = await getRun(run.id); if (!finalizedRun) continue; finalizedRun = await classifyAndPersistRunLiveness(finalizedRun, parseObject(finalizedRun.resultJson)) ?? finalizedRun; let retriedRun: typeof heartbeatRuns.$inferSelect | null = null; if (shouldRetry) { const agent = await getAgent(run.agentId); if (agent) { retriedRun = await enqueueProcessLossRetry(finalizedRun, agent, now); } } else { await releaseIssueExecutionAndPromote(finalizedRun); } await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), { eventType: "lifecycle", stream: "system", level: "error", message: shouldRetry ? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim() : baseMessage, payload: { ...(run.processPid ? { processPid: run.processPid } : {}), ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}), ...(descendantOnlyCleanup ? { descendantOnlyCleanup: true } : {}), ...(retriedRun ? { retryRunId: retriedRun.id } : {}), }, }); await finalizeAgentStatus(run.agentId, "failed"); await startNextQueuedRunForAgent(run.agentId); runningProcesses.delete(run.id); reaped.push(run.id); } if (reaped.length > 0) { logger.warn({ reapedCount: reaped.length, runIds: reaped }, "reaped orphaned heartbeat runs"); } return { reaped: reaped.length, runIds: reaped }; } async function resumeQueuedRuns() { const queuedRuns = await db .select({ agentId: heartbeatRuns.agentId }) .from(heartbeatRuns) .where(eq(heartbeatRuns.status, "queued")); const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))]; for (const agentId of agentIds) { await startNextQueuedRunForAgent(agentId); } } async function getLatestIssueRun(companyId: string, issueId: string) { return db .select({ id: heartbeatRuns.id, status: heartbeatRuns.status, error: heartbeatRuns.error, errorCode: heartbeatRuns.errorCode, contextSnapshot: heartbeatRuns.contextSnapshot, }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, ), ) .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)) .limit(1) .then((rows) => rows[0] ?? null); } async function hasActiveExecutionPath(companyId: string, issueId: string) { const [run, deferredWake] = await Promise.all([ db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, ), ) .limit(1) .then((rows) => rows[0] ?? null), db .select({ id: agentWakeupRequests.id }) .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, ), ) .limit(1) .then((rows) => rows[0] ?? null), ]); return Boolean(run || deferredWake); } async function enqueueStrandedIssueRecovery(input: { issueId: string; agentId: string; reason: "issue_assignment_recovery" | "issue_continuation_needed"; retryReason: "assignment_recovery" | "issue_continuation_needed"; source: string; retryOfRunId?: string | null; }) { const queued = await enqueueWakeup(input.agentId, { source: "automation", triggerDetail: "system", reason: input.reason, payload: { issueId: input.issueId, ...(input.retryOfRunId ? { retryOfRunId: input.retryOfRunId } : {}), }, requestedByActorType: "system", requestedByActorId: null, contextSnapshot: { issueId: input.issueId, taskId: input.issueId, wakeReason: input.reason, retryReason: input.retryReason, source: input.source, ...(input.retryOfRunId ? { retryOfRunId: input.retryOfRunId } : {}), }, }); if (queued && input.retryOfRunId) { return db .update(heartbeatRuns) .set({ retryOfRunId: input.retryOfRunId, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, queued.id)) .returning() .then((rows) => rows[0] ?? queued); } return queued; } function formatIssueLinksForComment(relations: Array<{ identifier?: string | null }>) { const identifiers = [ ...new Set( relations .map((relation) => relation.identifier) .filter((identifier): identifier is string => Boolean(identifier)), ), ]; if (identifiers.length === 0) return "another open issue"; return identifiers .slice(0, 5) .map((identifier) => { const prefix = identifier.split("-")[0] || "PAP"; return `[${identifier}](/${prefix}/issues/${identifier})`; }) .join(", "); } async function reconcileUnassignedBlockingIssues() { const candidates = await db .select({ id: issues.id, companyId: issues.companyId, identifier: issues.identifier, status: issues.status, createdByAgentId: issues.createdByAgentId, }) .from(issueRelations) .innerJoin(issues, eq(issueRelations.issueId, issues.id)) .where( and( eq(issueRelations.type, "blocks"), inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId), sql`${issues.createdByAgentId} is not null`, sql`exists ( select 1 from issues blocked_issue where blocked_issue.id = ${issueRelations.relatedIssueId} and blocked_issue.company_id = ${issues.companyId} and blocked_issue.status not in ('done', 'cancelled') )`, ), ); let assigned = 0; let skipped = 0; const issueIds: string[] = []; const seen = new Set(); for (const candidate of candidates) { if (seen.has(candidate.id)) continue; seen.add(candidate.id); const creatorAgentId = candidate.createdByAgentId; if (!creatorAgentId) { skipped += 1; continue; } const creatorAgent = await getAgent(creatorAgentId); if ( !creatorAgent || creatorAgent.companyId !== candidate.companyId || creatorAgent.status === "paused" || creatorAgent.status === "terminated" || creatorAgent.status === "pending_approval" ) { skipped += 1; continue; } const relations = await issuesSvc.getRelationSummaries(candidate.id); const blockingLinks = formatIssueLinksForComment(relations.blocks); const updated = await issuesSvc.update(candidate.id, { assigneeAgentId: creatorAgent.id, assigneeUserId: null, }); if (!updated) { skipped += 1; continue; } await issuesSvc.addComment( candidate.id, [ "## Assigned Orphan Blocker", "", `Paperclip found this issue is blocking ${blockingLinks} but had no assignee, so no heartbeat could pick it up.`, "", "- Assigned it back to the agent that created the blocker.", "- Next action: resolve this blocker or reassign it to the right owner.", ].join("\n"), {}, ); await logActivity(db, { companyId: candidate.companyId, actorType: "system", actorId: "system", agentId: null, runId: null, action: "issue.updated", entityType: "issue", entityId: candidate.id, details: { identifier: candidate.identifier, assigneeAgentId: creatorAgent.id, source: "heartbeat.reconcile_unassigned_blocking_issue", }, }); const queued = await enqueueWakeup(creatorAgent.id, { source: "automation", triggerDetail: "system", reason: "issue_assigned", payload: { issueId: candidate.id, mutation: "unassigned_blocker_recovery", }, requestedByActorType: "system", requestedByActorId: null, contextSnapshot: { issueId: candidate.id, taskId: candidate.id, wakeReason: "issue_assigned", source: "issue.unassigned_blocker_recovery", }, }); if (queued) { assigned += 1; issueIds.push(candidate.id); } else { skipped += 1; } } return { assigned, skipped, issueIds }; } async function escalateStrandedAssignedIssue(input: { issue: typeof issues.$inferSelect; previousStatus: "todo" | "in_progress"; latestRun: Pick< typeof heartbeatRuns.$inferSelect, "id" | "status" | "error" | "errorCode" | "contextSnapshot" > | null; comment: string; }) { const updated = await issuesSvc.update(input.issue.id, { status: "blocked", }); if (!updated) return null; await issuesSvc.addComment(input.issue.id, input.comment, {}); await logActivity(db, { companyId: input.issue.companyId, actorType: "system", actorId: "system", agentId: null, runId: null, action: "issue.updated", entityType: "issue", entityId: input.issue.id, details: { identifier: input.issue.identifier, status: "blocked", previousStatus: input.previousStatus, source: "heartbeat.reconcile_stranded_assigned_issue", latestRunId: input.latestRun?.id ?? null, latestRunStatus: input.latestRun?.status ?? null, latestRunErrorCode: input.latestRun?.errorCode ?? null, }, }); return updated; } async function reconcileStrandedAssignedIssues() { const candidates = await db .select() .from(issues) .where( and( isNull(issues.assigneeUserId), inArray(issues.status, ["todo", "in_progress"]), sql`${issues.assigneeAgentId} is not null`, ), ); const result = { dispatchRequeued: 0, continuationRequeued: 0, orphanBlockersAssigned: 0, escalated: 0, skipped: 0, issueIds: [] as string[], }; for (const issue of candidates) { const agentId = issue.assigneeAgentId; if (!agentId) { result.skipped += 1; continue; } const agent = await getAgent(agentId); if (!agent || agent.companyId !== issue.companyId) { result.skipped += 1; continue; } if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") { result.skipped += 1; continue; } if (await hasActiveExecutionPath(issue.companyId, issue.id)) { result.skipped += 1; continue; } const latestRun = await getLatestIssueRun(issue.companyId, issue.id); if (issue.status === "todo") { if (!latestRun || latestRun.status === "succeeded") { result.skipped += 1; continue; } if (didAutomaticRecoveryFail(latestRun, "assignment_recovery")) { const failureSummary = summarizeRunFailureForIssueComment(latestRun); const updated = await escalateStrandedAssignedIssue({ issue, previousStatus: "todo", latestRun, comment: "Paperclip automatically retried dispatch for this assigned `todo` issue after a lost wake/run, " + `but it still has no live execution path.${failureSummary ?? ""} ` + "Moving it to `blocked` so it is visible for intervention.", }); if (updated) { result.escalated += 1; result.issueIds.push(issue.id); } else { result.skipped += 1; } continue; } const queued = await enqueueStrandedIssueRecovery({ issueId: issue.id, agentId, reason: "issue_assignment_recovery", retryReason: "assignment_recovery", source: "issue.assignment_recovery", retryOfRunId: latestRun.id, }); if (queued) { result.dispatchRequeued += 1; result.issueIds.push(issue.id); } else { result.skipped += 1; } continue; } if (!latestRun && !issue.checkoutRunId && !issue.executionRunId) { result.skipped += 1; continue; } if (didAutomaticRecoveryFail(latestRun, "issue_continuation_needed")) { const failureSummary = summarizeRunFailureForIssueComment(latestRun); const updated = await escalateStrandedAssignedIssue({ issue, previousStatus: "in_progress", latestRun, comment: "Paperclip automatically retried continuation for this assigned `in_progress` issue after its live " + `execution disappeared, but it still has no live execution path.${failureSummary ?? ""} ` + "Moving it to `blocked` so it is visible for intervention.", }); if (updated) { result.escalated += 1; result.issueIds.push(issue.id); } else { result.skipped += 1; } continue; } const queued = await enqueueStrandedIssueRecovery({ issueId: issue.id, agentId, reason: "issue_continuation_needed", retryReason: "issue_continuation_needed", source: "issue.continuation_recovery", retryOfRunId: latestRun?.id ?? issue.checkoutRunId ?? null, }); if (queued) { result.continuationRequeued += 1; result.issueIds.push(issue.id); } else { result.skipped += 1; } } const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(); result.orphanBlockersAssigned = orphanBlockerRecovery.assigned; result.skipped += orphanBlockerRecovery.skipped; result.issueIds.push(...orphanBlockerRecovery.issueIds); return result; } function issueIdFromRunContext(contextSnapshot: unknown) { const context = parseObject(contextSnapshot); return readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); } function issueIdFromWakePayload(payload: unknown) { const parsed = parseObject(payload); const nestedContext = parseObject(parsed[DEFERRED_WAKE_CONTEXT_KEY]); return readNonEmptyString(parsed.issueId) ?? readNonEmptyString(nestedContext.issueId) ?? readNonEmptyString(nestedContext.taskId); } async function collectIssueGraphLivenessFindings() { const [issueRows, relationRows, agentRows, activeRunRows, wakeRows] = await Promise.all([ db .select({ id: issues.id, companyId: issues.companyId, identifier: issues.identifier, title: issues.title, status: issues.status, projectId: issues.projectId, goalId: issues.goalId, parentId: issues.parentId, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, createdByAgentId: issues.createdByAgentId, createdByUserId: issues.createdByUserId, executionState: issues.executionState, }) .from(issues) .where(isNull(issues.hiddenAt)), db .select({ companyId: issueRelations.companyId, blockerIssueId: issueRelations.issueId, blockedIssueId: issueRelations.relatedIssueId, }) .from(issueRelations) .where(eq(issueRelations.type, "blocks")), db .select({ id: agents.id, companyId: agents.companyId, name: agents.name, role: agents.role, title: agents.title, status: agents.status, reportsTo: agents.reportsTo, }) .from(agents), db .select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, status: heartbeatRuns.status, contextSnapshot: heartbeatRuns.contextSnapshot, }) .from(heartbeatRuns) .where(inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES])), db .select({ companyId: agentWakeupRequests.companyId, agentId: agentWakeupRequests.agentId, status: agentWakeupRequests.status, payload: agentWakeupRequests.payload, }) .from(agentWakeupRequests) .where(inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"])), ]); return classifyIssueGraphLiveness({ issues: issueRows, relations: relationRows, agents: agentRows, activeRuns: activeRunRows.map((row) => ({ companyId: row.companyId, agentId: row.agentId, status: row.status, issueId: issueIdFromRunContext(row.contextSnapshot), })), queuedWakeRequests: wakeRows.map((row) => ({ companyId: row.companyId, agentId: row.agentId, status: row.status, issueId: issueIdFromWakePayload(row.payload), })), }); } async function findOpenLivenessEscalation(companyId: string, incidentKey: string) { return db .select() .from(issues) .where( and( eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation"), eq(issues.originId, incidentKey), isNull(issues.hiddenAt), notInArray(issues.status, ["done", "cancelled"]), ), ) .limit(1) .then((rows) => rows[0] ?? null); } async function existingBlockerIssueIds(companyId: string, issueId: string) { return db .select({ blockerIssueId: issueRelations.issueId }) .from(issueRelations) .where( and( eq(issueRelations.companyId, companyId), eq(issueRelations.relatedIssueId, issueId), eq(issueRelations.type, "blocks"), ), ) .then((rows) => rows.map((row) => row.blockerIssueId)); } function formatDependencyPath(finding: IssueLivenessFinding) { return finding.dependencyPath .map((entry) => entry.identifier ?? entry.issueId) .join(" -> "); } function buildLivenessEscalationDescription(finding: IssueLivenessFinding) { return [ "Paperclip detected a harness-level issue graph liveness incident.", "", `- Incident key: \`${finding.incidentKey}\``, `- Finding: \`${finding.state}\``, `- Dependency path: ${formatDependencyPath(finding)}`, `- Reason: ${finding.reason}`, `- Requested action: ${finding.recommendedAction}`, "", "Resolve the blocked chain, then mark this escalation issue done so the original issue can resume when all blockers are cleared.", ].join("\n"); } function buildLivenessOriginalIssueComment(finding: IssueLivenessFinding, escalation: typeof issues.$inferSelect) { return [ "Paperclip detected a harness-level liveness incident in this issue's dependency graph.", "", `- Escalation issue: ${escalation.identifier ?? escalation.id}`, `- Incident key: \`${finding.incidentKey}\``, `- Finding: \`${finding.state}\``, `- Dependency path: ${formatDependencyPath(finding)}`, `- Reason: ${finding.reason}`, `- Manager action requested: ${finding.recommendedAction}`, "", "This issue now keeps its existing blockers and is also blocked by the escalation issue so dependency wakeups remain explicit.", ].join("\n"); } async function resolveEscalationOwnerAgentId( finding: IssueLivenessFinding, issue: typeof issues.$inferSelect, ) { const candidates = [ finding.recommendedOwnerAgentId, ...finding.recommendedOwnerCandidateAgentIds, ].filter((candidate): candidate is string => Boolean(candidate)); for (const candidate of [...new Set(candidates)]) { const budgetBlock = await budgets.getInvocationBlock(issue.companyId, candidate, { issueId: issue.id, projectId: issue.projectId, }); if (!budgetBlock) return candidate; } return null; } async function ensureIssueBlockedByEscalation(input: { issue: typeof issues.$inferSelect; escalationIssueId: string; finding: IssueLivenessFinding; runId?: string | null; }) { const blockerIds = await existingBlockerIssueIds(input.issue.companyId, input.issue.id); const nextBlockerIds = [...new Set([...blockerIds, input.escalationIssueId])]; const update: Partial & { blockedByIssueIds: string[] } = { blockedByIssueIds: nextBlockerIds, }; if (input.issue.status !== "blocked") { update.status = "blocked"; } const updated = await issuesSvc.update(input.issue.id, update); if (!updated) return null; await logActivity(db, { companyId: input.issue.companyId, actorType: "system", actorId: "system", agentId: null, runId: input.runId ?? null, action: "issue.blockers.updated", entityType: "issue", entityId: input.issue.id, details: { source: "heartbeat.reconcile_issue_graph_liveness", incidentKey: input.finding.incidentKey, findingState: input.finding.state, blockerIssueIds: nextBlockerIds, escalationIssueId: input.escalationIssueId, status: update.status ?? input.issue.status, previousStatus: input.issue.status, }, }); return updated; } async function createIssueGraphLivenessEscalation(input: { finding: IssueLivenessFinding; runId?: string | null; }) { const issue = await db .select() .from(issues) .where(eq(issues.id, input.finding.issueId)) .then((rows) => rows[0] ?? null); if (!issue || issue.companyId !== input.finding.companyId) return { kind: "skipped" as const }; const existing = await findOpenLivenessEscalation(issue.companyId, input.finding.incidentKey); if (existing) { await ensureIssueBlockedByEscalation({ issue, escalationIssueId: existing.id, finding: input.finding, runId: input.runId ?? null, }); return { kind: "existing" as const, escalationIssueId: existing.id }; } const ownerAgentId = await resolveEscalationOwnerAgentId(input.finding, issue); if (!ownerAgentId) return { kind: "skipped" as const }; const escalation = await issuesSvc.create(issue.companyId, { title: `Unblock liveness incident for ${issue.identifier ?? issue.title}`, description: buildLivenessEscalationDescription(input.finding), status: "todo", priority: "high", parentId: issue.id, projectId: issue.projectId, goalId: issue.goalId, assigneeAgentId: ownerAgentId, originKind: "harness_liveness_escalation", originId: input.finding.incidentKey, billingCode: issue.billingCode, inheritExecutionWorkspaceFromIssueId: issue.id, }); await ensureIssueBlockedByEscalation({ issue, escalationIssueId: escalation.id, finding: input.finding, runId: input.runId ?? null, }); await issuesSvc.addComment( issue.id, buildLivenessOriginalIssueComment(input.finding, escalation), { runId: input.runId ?? null }, ); await logActivity(db, { companyId: issue.companyId, actorType: "system", actorId: "system", agentId: ownerAgentId, runId: input.runId ?? null, action: "issue.harness_liveness_escalation_created", entityType: "issue", entityId: escalation.id, details: { source: "heartbeat.reconcile_issue_graph_liveness", incidentKey: input.finding.incidentKey, findingState: input.finding.state, sourceIssueId: issue.id, sourceIdentifier: issue.identifier, escalationIssueId: escalation.id, escalationIdentifier: escalation.identifier, dependencyPath: input.finding.dependencyPath, }, }); const wake = await enqueueWakeup(ownerAgentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned", payload: { issueId: escalation.id, sourceIssueId: issue.id, incidentKey: input.finding.incidentKey, }, requestedByActorType: "system", requestedByActorId: null, contextSnapshot: { issueId: escalation.id, taskId: escalation.id, wakeReason: "issue_assigned", source: "harness_liveness_escalation", sourceIssueId: issue.id, incidentKey: input.finding.incidentKey, }, }); logger.warn({ incidentKey: input.finding.incidentKey, findingState: input.finding.state, sourceIssueId: issue.id, escalationIssueId: escalation.id, ownerAgentId, wakeupRunId: wake?.id ?? null, }, "created issue graph liveness escalation"); return { kind: "created" as const, escalationIssueId: escalation.id }; } async function reconcileIssueGraphLiveness(opts?: { runId?: string | null }) { const findings = await collectIssueGraphLivenessFindings(); const result = { findings: findings.length, escalationsCreated: 0, existingEscalations: 0, skipped: 0, issueIds: [] as string[], escalationIssueIds: [] as string[], }; for (const finding of findings) { const escalation = await createIssueGraphLivenessEscalation({ finding, runId: opts?.runId ?? null, }); if (escalation.kind === "created") { result.escalationsCreated += 1; result.issueIds.push(finding.issueId); result.escalationIssueIds.push(escalation.escalationIssueId); } else if (escalation.kind === "existing") { result.existingEscalations += 1; result.issueIds.push(finding.issueId); result.escalationIssueIds.push(escalation.escalationIssueId); } else { result.skipped += 1; } } return result; } async function updateRuntimeState( agent: typeof agents.$inferSelect, run: typeof heartbeatRuns.$inferSelect, result: AdapterExecutionResult, session: { legacySessionId: string | null }, normalizedUsage?: UsageTotals | null, ) { await ensureRuntimeState(agent); const usage = normalizedUsage ?? normalizeUsageTotals(result.usage); const inputTokens = usage?.inputTokens ?? 0; const outputTokens = usage?.outputTokens ?? 0; const cachedInputTokens = usage?.cachedInputTokens ?? 0; const billingType = normalizeLedgerBillingType(result.billingType); const additionalCostCents = normalizeBilledCostCents(result.costUsd, billingType); const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0; const provider = result.provider ?? "unknown"; const biller = resolveLedgerBiller(result); const ledgerScope = await resolveLedgerScopeForRun(db, agent.companyId, run); await db .update(agentRuntimeState) .set({ adapterType: agent.adapterType, sessionId: session.legacySessionId, lastRunId: run.id, lastRunStatus: run.status, lastError: result.errorMessage ?? null, totalInputTokens: sql`${agentRuntimeState.totalInputTokens} + ${inputTokens}`, totalOutputTokens: sql`${agentRuntimeState.totalOutputTokens} + ${outputTokens}`, totalCachedInputTokens: sql`${agentRuntimeState.totalCachedInputTokens} + ${cachedInputTokens}`, totalCostCents: sql`${agentRuntimeState.totalCostCents} + ${additionalCostCents}`, updatedAt: new Date(), }) .where(eq(agentRuntimeState.agentId, agent.id)); if (additionalCostCents > 0 || hasTokenUsage) { const costs = costService(db, budgetHooks); await costs.createEvent(agent.companyId, { heartbeatRunId: run.id, agentId: agent.id, issueId: ledgerScope.issueId, projectId: ledgerScope.projectId, provider, biller, billingType, model: result.model ?? "unknown", inputTokens, cachedInputTokens, outputTokens, costCents: additionalCostCents, occurredAt: new Date(), }); } } async function startNextQueuedRunForAgent(agentId: string) { return withAgentStartLock(agentId, async () => { const agent = await getAgent(agentId); if (!agent) return []; if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") { return []; } const policy = parseHeartbeatPolicy(agent); const runningCount = await countRunningRunsForAgent(agentId); const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount); if (availableSlots <= 0) return []; const queuedRuns = await db .select() .from(heartbeatRuns) .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"))) .orderBy(asc(heartbeatRuns.createdAt)); if (queuedRuns.length === 0) return []; const dependencyReadiness = await listQueuedRunDependencyReadiness(agent.companyId, queuedRuns); const queuedIssueIds = [...new Set( queuedRuns .map((run) => readNonEmptyString(parseObject(run.contextSnapshot).issueId)) .filter((issueId): issueId is string => Boolean(issueId)), )]; const issueRows = await db .select({ id: issues.id, status: issues.status, priority: issues.priority, }) .from(issues) .where( queuedIssueIds.length > 0 ? and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds)) : sql`false`, ); const issueById = new Map(issueRows.map((row) => [row.id, row])); const prioritizedRuns = [...queuedRuns].sort((left, right) => { const leftIssueId = readNonEmptyString(parseObject(left.contextSnapshot).issueId); const rightIssueId = readNonEmptyString(parseObject(right.contextSnapshot).issueId); const leftReadiness = leftIssueId ? dependencyReadiness.get(leftIssueId) : null; const rightReadiness = rightIssueId ? dependencyReadiness.get(rightIssueId) : null; const leftReady = leftIssueId ? (leftReadiness?.isDependencyReady ?? true) : true; const rightReady = rightIssueId ? (rightReadiness?.isDependencyReady ?? true) : true; const leftIssue = leftIssueId ? issueById.get(leftIssueId) : null; const rightIssue = rightIssueId ? issueById.get(rightIssueId) : null; const leftRank = leftIssueId ? (leftReady ? (leftIssue?.status === "in_progress" ? 0 : 1) : 3) : 2; const rightRank = rightIssueId ? (rightReady ? (rightIssue?.status === "in_progress" ? 0 : 1) : 3) : 2; if (leftRank !== rightRank) return leftRank - rightRank; const leftPriorityRank = issueRunPriorityRank(leftIssue?.priority); const rightPriorityRank = issueRunPriorityRank(rightIssue?.priority); if (leftPriorityRank !== rightPriorityRank) return leftPriorityRank - rightPriorityRank; return left.createdAt.getTime() - right.createdAt.getTime(); }); const claimedRuns: Array = []; for (const queuedRun of prioritizedRuns) { if (claimedRuns.length >= availableSlots) break; const claimed = await claimQueuedRun(queuedRun); if (claimed) claimedRuns.push(claimed); } if (claimedRuns.length === 0) return []; for (const claimedRun of claimedRuns) { void executeRun(claimedRun.id).catch((err) => { logger.error({ err, runId: claimedRun.id }, "queued heartbeat execution failed"); }); } return claimedRuns; }); } async function executeRun(runId: string) { let run = await getRun(runId); if (!run) return; if (run.status !== "queued" && run.status !== "running") return; if (run.status === "queued") { const claimed = await claimQueuedRun(run); if (!claimed) { // claimQueuedRun can also leave the run queued when dependencies are unresolved. return; } run = claimed; } activeRunExecutions.add(run.id); try { const agent = await getAgent(run.agentId); if (!agent) { await setRunStatus(runId, "failed", { error: "Agent not found", errorCode: "agent_not_found", finishedAt: new Date(), }); await setWakeupStatus(run.wakeupRequestId, "failed", { finishedAt: new Date(), error: "Agent not found", }); const failedRun = await getRun(runId); if (failedRun) await releaseIssueExecutionAndPromote(failedRun); return; } const runtime = await ensureRuntimeState(agent); const context = parseObject(run.contextSnapshot); const taskKey = deriveTaskKeyWithHeartbeatFallback(context, null); const sessionCodec = getAdapterSessionCodec(agent.adapterType); const issueId = readNonEmptyString(context.issueId); let issueContext = issueId ? await getIssueExecutionContext(agent.companyId, issueId) : null; const issueDependencyReadiness = issueId ? await issuesSvc.listDependencyReadiness(agent.companyId, [issueId]).then((rows) => rows.get(issueId) ?? null) : null; if ( issueId && issueContext && shouldAutoCheckoutIssueForWake({ contextSnapshot: context, issueStatus: issueContext.status, issueAssigneeAgentId: issueContext.assigneeAgentId, isDependencyReady: issueDependencyReadiness?.isDependencyReady ?? true, agentId: agent.id, }) ) { try { await issuesSvc.checkout(issueId, agent.id, ["todo", "backlog", "blocked"], run.id); context[PAPERCLIP_HARNESS_CHECKOUT_KEY] = true; } catch (error) { if (!isCheckoutConflictError(error)) throw error; context[PAPERCLIP_HARNESS_CHECKOUT_KEY] = false; } issueContext = await getIssueExecutionContext(agent.companyId, issueId); } const issueAssigneeOverrides = issueContext && issueContext.assigneeAgentId === agent.id ? parseIssueAssigneeAdapterOverrides( issueContext.assigneeAdapterOverrides, ) : null; const isolatedWorkspacesEnabled = (await instanceSettings.getExperimental()).enableIsolatedWorkspaces; const issueExecutionWorkspaceSettings = isolatedWorkspacesEnabled ? parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings) : null; const contextProjectId = readNonEmptyString(context.projectId); const executionProjectId = issueContext?.projectId ?? contextProjectId; const projectContext = executionProjectId ? await db .select({ executionWorkspacePolicy: projects.executionWorkspacePolicy, env: projects.env, }) .from(projects) .where(and(eq(projects.id, executionProjectId), eq(projects.companyId, agent.companyId))) .then((rows) => rows[0] ?? null) : null; const projectExecutionWorkspacePolicy = gateProjectExecutionWorkspacePolicy( parseProjectExecutionWorkspacePolicy(projectContext?.executionWorkspacePolicy), isolatedWorkspacesEnabled, ); const taskSession = taskKey ? await getTaskSession(agent.companyId, agent.id, agent.adapterType, taskKey) : null; const resetTaskSession = shouldResetTaskSessionForWake(context); const sessionResetReason = describeSessionResetReason(context); const taskSessionForRun = resetTaskSession ? null : taskSession; const explicitResumeSessionParams = normalizeSessionParams( sessionCodec.deserialize(parseObject(context.resumeSessionParams)), ); const explicitResumeSessionDisplayId = truncateDisplayId( readNonEmptyString(context.resumeSessionDisplayId) ?? (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(explicitResumeSessionParams) : null) ?? readNonEmptyString(explicitResumeSessionParams?.sessionId), ); const previousSessionParams = explicitResumeSessionParams ?? (explicitResumeSessionDisplayId ? { sessionId: explicitResumeSessionDisplayId } : null) ?? normalizeSessionParams(sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null)); const config = parseObject(agent.adapterConfig); const requestedExecutionWorkspaceMode = resolveExecutionWorkspaceMode({ projectPolicy: projectExecutionWorkspacePolicy, issueSettings: issueExecutionWorkspaceSettings, legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null, }); const resolvedWorkspace = await resolveWorkspaceForRun( agent, context, previousSessionParams, { useProjectWorkspace: requestedExecutionWorkspaceMode !== "agent_default" }, ); const issueRef = issueContext ? { id: issueContext.id, identifier: issueContext.identifier, title: issueContext.title, status: issueContext.status, priority: issueContext.priority, projectId: issueContext.projectId, projectWorkspaceId: issueContext.projectWorkspaceId, executionWorkspaceId: issueContext.executionWorkspaceId, executionWorkspacePreference: issueContext.executionWorkspacePreference, } : null; const continuationSummary = issueRef ? await getIssueContinuationSummaryDocument(db, issueRef.id) : null; if (continuationSummary) { context.paperclipContinuationSummary = { key: continuationSummary.key, title: continuationSummary.title, body: continuationSummary.body, updatedAt: continuationSummary.updatedAt.toISOString(), }; } else { delete context.paperclipContinuationSummary; } const paperclipWakePayload = await buildPaperclipWakePayload({ db, companyId: agent.companyId, contextSnapshot: context, continuationSummary, issueSummary: issueRef ? { id: issueRef.id, identifier: issueRef.identifier, title: issueRef.title, status: issueRef.status, priority: issueRef.priority, } : null, }); if (paperclipWakePayload) { context[PAPERCLIP_WAKE_PAYLOAD_KEY] = paperclipWakePayload; } else { delete context[PAPERCLIP_WAKE_PAYLOAD_KEY]; } const existingExecutionWorkspace = issueRef?.executionWorkspaceId ? await executionWorkspacesSvc.getById(issueRef.executionWorkspaceId) : null; const shouldReuseExisting = issueRef?.executionWorkspacePreference === "reuse_existing" && existingExecutionWorkspace && existingExecutionWorkspace.status !== "archived"; const persistedExecutionWorkspaceMode = shouldReuseExisting && existingExecutionWorkspace ? issueExecutionWorkspaceModeForPersistedWorkspace(existingExecutionWorkspace.mode) : null; const effectiveExecutionWorkspaceMode: ReturnType = persistedExecutionWorkspaceMode === "isolated_workspace" || persistedExecutionWorkspaceMode === "operator_branch" || persistedExecutionWorkspaceMode === "agent_default" ? persistedExecutionWorkspaceMode : requestedExecutionWorkspaceMode; const workspaceManagedConfig = shouldReuseExisting ? { ...config } : buildExecutionWorkspaceAdapterConfig({ agentConfig: config, projectPolicy: projectExecutionWorkspacePolicy, issueSettings: issueExecutionWorkspaceSettings, mode: requestedExecutionWorkspaceMode, legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null, }); const persistedWorkspaceManagedConfig = applyPersistedExecutionWorkspaceConfig({ config: workspaceManagedConfig, workspaceConfig: existingExecutionWorkspace?.config ?? null, mode: effectiveExecutionWorkspaceMode, }); const mergedConfig = issueAssigneeOverrides?.adapterConfig ? { ...persistedWorkspaceManagedConfig, ...issueAssigneeOverrides.adapterConfig } : persistedWorkspaceManagedConfig; const defaultEnvironment = await environmentsSvc.ensureLocalEnvironment(agent.companyId); const selectedEnvironmentId = resolveExecutionWorkspaceEnvironmentId({ projectPolicy: projectExecutionWorkspacePolicy, issueSettings: issueExecutionWorkspaceSettings, workspaceConfig: existingExecutionWorkspace?.config ?? null, agentDefaultEnvironmentId: agent.defaultEnvironmentId, defaultEnvironmentId: defaultEnvironment.id, }); const configSnapshot = buildExecutionWorkspaceConfigSnapshot(mergedConfig, selectedEnvironmentId); const executionRunConfig = stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); const { resolvedConfig, secretKeys } = await resolveExecutionRunAdapterConfig({ companyId: agent.companyId, executionRunConfig, projectEnv: projectContext?.env ?? null, secretsSvc, }); const runScopedMentionedSkillKeys = await resolveRunScopedMentionedSkillKeys({ db, companyId: agent.companyId, issueId, }); const effectiveResolvedConfig = applyRunScopedMentionedSkillKeys( resolvedConfig, runScopedMentionedSkillKeys, ); const runtimeSkillEntries = await companySkills.listRuntimeSkillEntries(agent.companyId); const runtimeConfig = { ...effectiveResolvedConfig, paperclipRuntimeSkills: runtimeSkillEntries, }; const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({ companyId: agent.companyId, heartbeatRunId: run.id, executionWorkspaceId: existingExecutionWorkspace?.id ?? null, }); const executionWorkspaceBase = { baseCwd: resolvedWorkspace.cwd, source: resolvedWorkspace.source, projectId: resolvedWorkspace.projectId, workspaceId: resolvedWorkspace.workspaceId, repoUrl: resolvedWorkspace.repoUrl, repoRef: resolvedWorkspace.repoRef, } satisfies ExecutionWorkspaceInput; const reusedExecutionWorkspace = shouldReuseExisting && existingExecutionWorkspace ? buildRealizedExecutionWorkspaceFromPersisted({ base: executionWorkspaceBase, workspace: existingExecutionWorkspace, }) : null; const executionWorkspace = reusedExecutionWorkspace ?? await realizeExecutionWorkspace({ base: executionWorkspaceBase, config: runtimeConfig, issue: issueRef, agent: { id: agent.id, name: agent.name, companyId: agent.companyId, }, recorder: workspaceOperationRecorder, }); const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? executionProjectId ?? null; const resolvedProjectWorkspaceId = issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null; let persistedExecutionWorkspace = null; const nextExecutionWorkspaceMetadataBase = { ...(existingExecutionWorkspace?.metadata ?? {}), source: executionWorkspace.source, createdByRuntime: executionWorkspace.created, } as Record; const nextExecutionWorkspaceMetadata = shouldReuseExisting ? nextExecutionWorkspaceMetadataBase : configSnapshot ? mergeExecutionWorkspaceConfig(nextExecutionWorkspaceMetadataBase, configSnapshot) : nextExecutionWorkspaceMetadataBase; try { persistedExecutionWorkspace = shouldReuseExisting && existingExecutionWorkspace ? await executionWorkspacesSvc.update(existingExecutionWorkspace.id, { cwd: executionWorkspace.cwd, repoUrl: executionWorkspace.repoUrl, baseRef: executionWorkspace.repoRef, branchName: executionWorkspace.branchName, providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs", providerRef: executionWorkspace.worktreePath, status: "active", lastUsedAt: new Date(), metadata: nextExecutionWorkspaceMetadata, }) : resolvedProjectId ? await executionWorkspacesSvc.create({ companyId: agent.companyId, projectId: resolvedProjectId, projectWorkspaceId: resolvedProjectWorkspaceId, sourceIssueId: issueRef?.id ?? null, mode: requestedExecutionWorkspaceMode === "isolated_workspace" ? "isolated_workspace" : requestedExecutionWorkspaceMode === "operator_branch" ? "operator_branch" : requestedExecutionWorkspaceMode === "agent_default" ? "adapter_managed" : "shared_workspace", strategyType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "project_primary", name: executionWorkspace.branchName ?? issueRef?.identifier ?? `workspace-${agent.id.slice(0, 8)}`, status: "active", cwd: executionWorkspace.cwd, repoUrl: executionWorkspace.repoUrl, baseRef: executionWorkspace.repoRef, branchName: executionWorkspace.branchName, providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs", providerRef: executionWorkspace.worktreePath, lastUsedAt: new Date(), openedAt: new Date(), metadata: nextExecutionWorkspaceMetadata, }) : null; } catch (error) { if (executionWorkspace.created) { try { await cleanupExecutionWorkspaceArtifacts({ workspace: { id: existingExecutionWorkspace?.id ?? `transient-${run.id}`, cwd: executionWorkspace.cwd, providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs", providerRef: executionWorkspace.worktreePath, branchName: executionWorkspace.branchName, repoUrl: executionWorkspace.repoUrl, baseRef: executionWorkspace.repoRef, projectId: resolvedProjectId, projectWorkspaceId: resolvedProjectWorkspaceId, sourceIssueId: issueRef?.id ?? null, metadata: { createdByRuntime: true, source: executionWorkspace.source, }, }, projectWorkspace: { cwd: resolvedWorkspace.cwd, cleanupCommand: null, }, cleanupCommand: configSnapshot?.cleanupCommand ?? null, teardownCommand: configSnapshot?.teardownCommand ?? projectExecutionWorkspacePolicy?.workspaceStrategy?.teardownCommand ?? null, recorder: workspaceOperationRecorder, }); } catch (cleanupError) { logger.warn( { runId: run.id, issueId, executionWorkspaceCwd: executionWorkspace.cwd, cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError), }, "Failed to cleanup realized execution workspace after persistence failure", ); } } throw error; } await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace?.id ?? null); if ( existingExecutionWorkspace && persistedExecutionWorkspace && existingExecutionWorkspace.id !== persistedExecutionWorkspace.id && existingExecutionWorkspace.status === "active" ) { await executionWorkspacesSvc.update(existingExecutionWorkspace.id, { status: "idle", cleanupReason: null, }); } if (issueId && persistedExecutionWorkspace) { const nextIssueWorkspaceMode = issueExecutionWorkspaceModeForPersistedWorkspace(persistedExecutionWorkspace.mode); const shouldSwitchIssueToExistingWorkspace = issueRef?.executionWorkspacePreference === "reuse_existing" || requestedExecutionWorkspaceMode === "isolated_workspace" || requestedExecutionWorkspaceMode === "operator_branch"; const nextIssuePatch: Record = {}; if (issueRef?.executionWorkspaceId !== persistedExecutionWorkspace.id) { nextIssuePatch.executionWorkspaceId = persistedExecutionWorkspace.id; } if (resolvedProjectWorkspaceId && issueRef?.projectWorkspaceId !== resolvedProjectWorkspaceId) { nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId; } if (shouldSwitchIssueToExistingWorkspace) { nextIssuePatch.executionWorkspacePreference = "reuse_existing"; nextIssuePatch.executionWorkspaceSettings = { ...(issueExecutionWorkspaceSettings ?? {}), mode: nextIssueWorkspaceMode, }; } if (Object.keys(nextIssuePatch).length > 0) { await issuesSvc.update(issueId, nextIssuePatch); } } if (persistedExecutionWorkspace) { context.executionWorkspaceId = persistedExecutionWorkspace.id; await db .update(heartbeatRuns) .set({ contextSnapshot: context, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.id)); } const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({ agentId: agent.id, previousSessionParams, resolvedWorkspace: { ...resolvedWorkspace, cwd: executionWorkspace.cwd, }, }); const runtimeSessionParams = runtimeSessionResolution.sessionParams; const runtimeWorkspaceWarnings = [ ...resolvedWorkspace.warnings, ...executionWorkspace.warnings, ...(runtimeSessionResolution.warning ? [runtimeSessionResolution.warning] : []), ...(resetTaskSession && sessionResetReason ? [ taskKey ? `Skipping saved session resume for task "${taskKey}" because ${sessionResetReason}.` : `Skipping saved session resume because ${sessionResetReason}.`, ] : []), ]; context.paperclipWorkspace = { cwd: executionWorkspace.cwd, source: executionWorkspace.source, mode: effectiveExecutionWorkspaceMode, strategy: executionWorkspace.strategy, projectId: executionWorkspace.projectId, workspaceId: executionWorkspace.workspaceId, repoUrl: executionWorkspace.repoUrl, repoRef: executionWorkspace.repoRef, branchName: executionWorkspace.branchName, worktreePath: executionWorkspace.worktreePath, agentHome: await (async () => { const home = resolveDefaultAgentWorkspaceDir(agent.id); await fs.mkdir(home, { recursive: true }); return home; })(), }; context.paperclipWorkspaces = resolvedWorkspace.workspaceHints; const selectedEnvironment = selectedEnvironmentId === defaultEnvironment.id ? defaultEnvironment : await environmentsSvc.getById(selectedEnvironmentId); if (!selectedEnvironment || selectedEnvironment.companyId !== agent.companyId) { throw notFound(`Environment "${selectedEnvironmentId}" not found.`); } if (selectedEnvironment.status !== "active") { throw conflict(`Environment "${selectedEnvironment.name}" is not active.`); } if (!isEnvironmentDriverSupportedForAdapter(agent.adapterType, selectedEnvironment.driver)) { throw conflict( `Adapter "${agent.adapterType}" does not support "${selectedEnvironment.driver}" environments.`, ); } const selectedEnvironmentRuntimeConfig = await resolveEnvironmentDriverConfigForRuntime( db, agent.companyId, selectedEnvironment, ); let environmentProvider = selectedEnvironment.driver; let environmentProviderLeaseId: string | null = null; let environmentLeaseMetadata: Record = { driver: selectedEnvironment.driver, executionWorkspaceMode: persistedExecutionWorkspace?.mode ?? effectiveExecutionWorkspaceMode, cwd: executionWorkspace.cwd, }; let executionTarget: AdapterExecutionTarget | null = null; let remoteExecution: SshRemoteExecutionSpec | null = null; if (selectedEnvironmentRuntimeConfig.driver === "ssh") { const { remoteCwd } = await ensureSshWorkspaceReady(selectedEnvironmentRuntimeConfig.config); const paperclipApiUrl = await findReachablePaperclipApiUrlOverSsh({ config: selectedEnvironmentRuntimeConfig.config, candidates: runtimeApiUrlCandidates(), }); remoteExecution = { ...selectedEnvironmentRuntimeConfig.config, remoteCwd, paperclipApiUrl, }; environmentProvider = "ssh"; environmentProviderLeaseId = `ssh://${selectedEnvironmentRuntimeConfig.config.username}@${selectedEnvironmentRuntimeConfig.config.host}:${selectedEnvironmentRuntimeConfig.config.port}${remoteCwd}`; environmentLeaseMetadata = { ...environmentLeaseMetadata, host: selectedEnvironmentRuntimeConfig.config.host, port: selectedEnvironmentRuntimeConfig.config.port, username: selectedEnvironmentRuntimeConfig.config.username, remoteWorkspacePath: selectedEnvironmentRuntimeConfig.config.remoteWorkspacePath, remoteCwd, paperclipApiUrl, }; } const environmentLease = await environmentsSvc.acquireLease({ companyId: agent.companyId, environmentId: selectedEnvironment.id, executionWorkspaceId: persistedExecutionWorkspace?.id ?? null, issueId: issueId ?? null, heartbeatRunId: run.id, leasePolicy: "ephemeral", provider: environmentProvider, providerLeaseId: environmentProviderLeaseId, metadata: environmentLeaseMetadata, }); if (remoteExecution) { executionTarget = { kind: "remote", transport: "ssh", environmentId: selectedEnvironment.id, leaseId: environmentLease.id, remoteCwd: remoteExecution.remoteCwd, paperclipApiUrl: remoteExecution.paperclipApiUrl, spec: remoteExecution, }; } context.paperclipEnvironment = { id: selectedEnvironment.id, name: selectedEnvironment.name, driver: selectedEnvironment.driver, leaseId: environmentLease.id, ...(typeof environmentLease.metadata?.remoteCwd === "string" ? { remoteCwd: environmentLease.metadata.remoteCwd, host: typeof environmentLease.metadata?.host === "string" ? environmentLease.metadata.host : undefined, port: typeof environmentLease.metadata?.port === "number" ? environmentLease.metadata.port : undefined, username: typeof environmentLease.metadata?.username === "string" ? environmentLease.metadata.username : undefined, } : {}), }; await logActivity(db, { companyId: agent.companyId, actorType: "agent", actorId: agent.id, agentId: agent.id, runId: run.id, action: "environment.lease_acquired", entityType: "environment_lease", entityId: environmentLease.id, details: { environmentId: selectedEnvironment.id, driver: selectedEnvironment.driver, leasePolicy: environmentLease.leasePolicy, provider: environmentLease.provider, executionWorkspaceId: environmentLease.executionWorkspaceId, issueId, }, }).catch((err) => { logger.warn({ err, runId: run.id }, "failed to log environment lease acquisition"); }); const runtimeServiceIntents = (() => { const runtimeConfig = parseObject(resolvedConfig.workspaceRuntime); return Array.isArray(runtimeConfig.services) ? runtimeConfig.services.filter( (value): value is Record => typeof value === "object" && value !== null, ) : []; })(); if (runtimeServiceIntents.length > 0) { context.paperclipRuntimeServiceIntents = runtimeServiceIntents; } else { delete context.paperclipRuntimeServiceIntents; } if (executionWorkspace.projectId && !readNonEmptyString(context.projectId)) { context.projectId = executionWorkspace.projectId; } await db .update(heartbeatRuns) .set({ contextSnapshot: context, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.id)); const runtimeSessionFallback = taskKey || resetTaskSession ? null : runtime.sessionId; let previousSessionDisplayId = truncateDisplayId( explicitResumeSessionDisplayId ?? taskSessionForRun?.sessionDisplayId ?? (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(runtimeSessionParams) : null) ?? readNonEmptyString(runtimeSessionParams?.sessionId) ?? runtimeSessionFallback, ); let runtimeSessionIdForAdapter = readNonEmptyString(runtimeSessionParams?.sessionId) ?? runtimeSessionFallback; let runtimeSessionParamsForAdapter = runtimeSessionParams; const sessionCompaction = await evaluateSessionCompaction({ agent, sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter, issueId, continuationSummaryBody: continuationSummary?.body ?? null, }); if (sessionCompaction.rotate) { context.paperclipSessionHandoffMarkdown = sessionCompaction.handoffMarkdown; context.paperclipSessionRotationReason = sessionCompaction.reason; context.paperclipPreviousSessionId = previousSessionDisplayId ?? runtimeSessionIdForAdapter; runtimeSessionIdForAdapter = null; runtimeSessionParamsForAdapter = null; previousSessionDisplayId = null; if (sessionCompaction.reason) { runtimeWorkspaceWarnings.push( `Starting a fresh session because ${sessionCompaction.reason}.`, ); } } else { delete context.paperclipSessionHandoffMarkdown; delete context.paperclipSessionRotationReason; delete context.paperclipPreviousSessionId; } const runtimeForAdapter = { sessionId: runtimeSessionIdForAdapter, sessionParams: runtimeSessionParamsForAdapter, sessionDisplayId: previousSessionDisplayId, taskKey, }; let seq = 1; let handle: RunLogHandle | null = null; let stdoutExcerpt = ""; let stderrExcerpt = ""; try { const startedAt = run.startedAt ?? new Date(); const runningWithSession = await db .update(heartbeatRuns) .set({ startedAt, sessionIdBefore: runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId, contextSnapshot: context, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.id)) .returning() .then((rows) => rows[0] ?? null); if (runningWithSession) run = runningWithSession; const runningAgent = await db .update(agents) .set({ status: "running", updatedAt: new Date() }) .where(eq(agents.id, agent.id)) .returning() .then((rows) => rows[0] ?? null); if (runningAgent) { publishLiveEvent({ companyId: runningAgent.companyId, type: "agent.status", payload: { agentId: runningAgent.id, status: runningAgent.status, outcome: "running", }, }); } const currentRun = run; await appendRunEvent(currentRun, seq++, { eventType: "lifecycle", stream: "system", level: "info", message: "run started", }); handle = await runLogStore.begin({ companyId: run.companyId, agentId: run.agentId, runId, }); await db .update(heartbeatRuns) .set({ logStore: handle.store, logRef: handle.logRef, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); const onLog = async (stream: "stdout" | "stderr", chunk: string) => { const sanitizedChunk = compactRunLogChunk( redactCurrentUserText(chunk, currentUserRedactionOptions), ); if (stream === "stdout") stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk); if (stream === "stderr") stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk); const ts = new Date().toISOString(); if (handle) { await runLogStore.append(handle, { stream, chunk: sanitizedChunk, ts, }); } const payloadChunk = sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES ? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES) : sanitizedChunk; publishLiveEvent({ companyId: run.companyId, type: "heartbeat.run.log", payload: { runId: run.id, agentId: run.agentId, ts, stream, chunk: payloadChunk, truncated: payloadChunk.length !== sanitizedChunk.length, }, }); }; if (runScopedMentionedSkillKeys.length > 0) { await onLog( "stdout", `[paperclip] Enabled run-scoped skills from issue mentions: ${runScopedMentionedSkillKeys.join(", ")}\n`, ); } for (const warning of runtimeWorkspaceWarnings) { const logEntry = formatRuntimeWorkspaceWarningLog(warning); await onLog(logEntry.stream, logEntry.chunk); } const adapterEnv = Object.fromEntries( Object.entries(parseObject(resolvedConfig.env)).filter( (entry): entry is [string, string] => typeof entry[0] === "string" && typeof entry[1] === "string", ), ); const runtimeServices = await ensureRuntimeServicesForRun({ db, runId: run.id, agent: { id: agent.id, name: agent.name, companyId: agent.companyId, }, issue: issueRef, workspace: executionWorkspace, executionWorkspaceId: persistedExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null, config: effectiveResolvedConfig, adapterEnv, onLog, }); if (runtimeServices.length > 0) { context.paperclipRuntimeServices = runtimeServices; context.paperclipRuntimePrimaryUrl = runtimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null; await db .update(heartbeatRuns) .set({ contextSnapshot: context, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.id)); } if (issueId && (executionWorkspace.created || runtimeServices.some((service) => !service.reused))) { try { await issuesSvc.addComment( issueId, buildWorkspaceReadyComment({ workspace: executionWorkspace, runtimeServices, }), { agentId: agent.id, runId: run.id }, ); } catch (err) { await onLog( "stderr", `[paperclip] Failed to post workspace-ready comment: ${err instanceof Error ? err.message : String(err)}\n`, ); } } const onAdapterMeta = async (meta: AdapterInvocationMeta) => { if (meta.env && secretKeys.size > 0) { for (const key of secretKeys) { if (key in meta.env) meta.env[key] = "***REDACTED***"; } } await appendRunEvent(currentRun, seq++, { eventType: "adapter.invoke", stream: "system", level: "info", message: "adapter invocation", payload: meta as unknown as Record, }); }; const adapter = getServerAdapter(agent.adapterType); const authToken = adapter.supportsLocalAgentJwt ? createLocalAgentJwt(agent.id, agent.companyId, agent.adapterType, run.id) : null; if (adapter.supportsLocalAgentJwt && !authToken) { logger.warn( { companyId: agent.companyId, agentId: agent.id, runId: run.id, adapterType: agent.adapterType, }, "local agent jwt secret missing or invalid; running without injected PAPERCLIP_API_KEY", ); } const adapterResult = await adapter.execute({ runId: run.id, agent, runtime: runtimeForAdapter, config: runtimeConfig, context, executionTarget, executionTransport: remoteExecution ? { remoteExecution: remoteExecution as unknown as Record } : undefined, onLog, onMeta: onAdapterMeta, onSpawn: async (meta) => { await persistRunProcessMetadata(run.id, { pid: meta.pid, processGroupId: "processGroupId" in meta && typeof meta.processGroupId === "number" ? meta.processGroupId : null, startedAt: meta.startedAt, }); }, authToken: authToken ?? undefined, }); const adapterManagedRuntimeServices = adapterResult.runtimeServices ? await persistAdapterManagedRuntimeServices({ db, adapterType: agent.adapterType, runId: run.id, agent: { id: agent.id, name: agent.name, companyId: agent.companyId, }, issue: issueRef, workspace: executionWorkspace, reports: adapterResult.runtimeServices, }) : []; if (adapterManagedRuntimeServices.length > 0) { const combinedRuntimeServices = [ ...runtimeServices, ...adapterManagedRuntimeServices, ]; context.paperclipRuntimeServices = combinedRuntimeServices; context.paperclipRuntimePrimaryUrl = combinedRuntimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null; await db .update(heartbeatRuns) .set({ contextSnapshot: context, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.id)); if (issueId) { try { await issuesSvc.addComment( issueId, buildWorkspaceReadyComment({ workspace: executionWorkspace, runtimeServices: adapterManagedRuntimeServices, }), { agentId: agent.id, runId: run.id }, ); } catch (err) { await onLog( "stderr", `[paperclip] Failed to post adapter-managed runtime comment: ${err instanceof Error ? err.message : String(err)}\n`, ); } } } const nextSessionState = resolveNextSessionState({ codec: sessionCodec, adapterResult, previousParams: previousSessionParams, previousDisplayId: runtimeForAdapter.sessionDisplayId, previousLegacySessionId: runtimeForAdapter.sessionId, }); const rawUsage = normalizeUsageTotals(adapterResult.usage); const sessionUsageResolution = await resolveNormalizedUsageForSession({ agentId: agent.id, runId: run.id, sessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId, rawUsage, }); const normalizedUsage = sessionUsageResolution.normalizedUsage; let outcome: "succeeded" | "failed" | "cancelled" | "timed_out"; const latestRun = await getRun(run.id); if (isHeartbeatRunTerminalStatus(latestRun?.status)) { outcome = latestRun.status; } else if (adapterResult.timedOut) { outcome = "timed_out"; } else if ((adapterResult.exitCode ?? 0) === 0 && !adapterResult.errorMessage) { outcome = "succeeded"; } else { outcome = "failed"; } const runErrorMessage = outcome === "cancelled" ? (latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled") : outcome === "succeeded" ? null : redactCurrentUserText( adapterResult.errorMessage ?? (outcome === "timed_out" ? "Timed out" : "Adapter failed"), currentUserRedactionOptions, ); const runErrorCode = outcome === "timed_out" ? "timeout" : outcome === "cancelled" ? (latestRun?.errorCode ?? "cancelled") : outcome === "failed" ? (adapterResult.errorCode ?? "adapter_failed") : null; let logSummary: { bytes: number; sha256?: string; compressed: boolean } | null = null; if (handle) { logSummary = await runLogStore.finalize(handle); } const status = outcome === "succeeded" ? "succeeded" : outcome === "cancelled" ? "cancelled" : outcome === "timed_out" ? "timed_out" : "failed"; const usageJson = normalizedUsage || adapterResult.costUsd != null ? ({ ...(normalizedUsage ?? {}), ...(rawUsage ? { rawInputTokens: rawUsage.inputTokens, rawCachedInputTokens: rawUsage.cachedInputTokens, rawOutputTokens: rawUsage.outputTokens, } : {}), ...(sessionUsageResolution.derivedFromSessionTotals ? { usageSource: "session_delta" } : {}), ...((nextSessionState.displayId ?? nextSessionState.legacySessionId) ? { persistedSessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId } : {}), sessionReused: runtimeForAdapter.sessionId != null || runtimeForAdapter.sessionDisplayId != null, taskSessionReused: taskSessionForRun != null, freshSession: runtimeForAdapter.sessionId == null && runtimeForAdapter.sessionDisplayId == null, sessionRotated: sessionCompaction.rotate, sessionRotationReason: sessionCompaction.reason, provider: readNonEmptyString(adapterResult.provider) ?? "unknown", biller: resolveLedgerBiller(adapterResult), model: readNonEmptyString(adapterResult.model) ?? "unknown", ...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}), billingType: normalizeLedgerBillingType(adapterResult.billingType), } as Record) : null; const persistedResultJson = mergeHeartbeatRunResultJson( mergeRunStopMetadataForAgent(agent, outcome, { resultJson: adapterResult.resultJson ?? null, errorCode: runErrorCode, errorMessage: runErrorMessage, }), adapterResult.summary ?? null, ); let persistedRun = await setRunStatus(run.id, status, { finishedAt: new Date(), error: runErrorMessage, errorCode: runErrorCode, exitCode: adapterResult.exitCode, signal: adapterResult.signal, usageJson, resultJson: persistedResultJson, sessionIdAfter: nextSessionState.displayId ?? nextSessionState.legacySessionId, stdoutExcerpt, stderrExcerpt, logBytes: logSummary?.bytes, logSha256: logSummary?.sha256, logCompressed: logSummary?.compressed ?? false, }); if (persistedRun) { persistedRun = await classifyAndPersistRunLiveness(persistedRun, persistedResultJson) ?? persistedRun; } await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, { finishedAt: new Date(), error: runErrorMessage, }); const finalizedRun = persistedRun ?? (await getRun(run.id)); if (finalizedRun) { await appendRunEvent(finalizedRun, seq++, { eventType: "lifecycle", stream: "system", level: outcome === "succeeded" ? "info" : "error", message: `run ${outcome}`, payload: { status, exitCode: adapterResult.exitCode, }, }); const livenessRun = finalizedRun; await refreshContinuationSummaryForRun(livenessRun, agent); if (issueId && outcome === "succeeded") { try { const existingRunComment = await findRunIssueComment(livenessRun.id, livenessRun.companyId, issueId); if (!existingRunComment) { const issueComment = buildHeartbeatRunIssueComment(persistedResultJson); if (issueComment) { await issuesSvc.addComment(issueId, issueComment, { agentId: agent.id, runId: livenessRun.id }); } } } catch (err) { await onLog( "stderr", `[paperclip] Failed to post run summary comment: ${err instanceof Error ? err.message : String(err)}\n`, ); } } if (outcome === "failed" && livenessRun.errorCode === "codex_transient_upstream") { await scheduleBoundedRetryForRun(livenessRun, agent); } await finalizeIssueCommentPolicy(livenessRun, agent); await releaseIssueExecutionAndPromote(livenessRun); await handleRunLivenessContinuation(livenessRun); } if (finalizedRun) { await updateRuntimeState(agent, finalizedRun, adapterResult, { legacySessionId: nextSessionState.legacySessionId, }, normalizedUsage); if (taskKey) { if (adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId)) { await clearTaskSessions(agent.companyId, agent.id, { taskKey, adapterType: agent.adapterType, }); } else { await upsertTaskSession({ companyId: agent.companyId, agentId: agent.id, adapterType: agent.adapterType, taskKey, sessionParamsJson: nextSessionState.params, sessionDisplayId: nextSessionState.displayId, lastRunId: finalizedRun.id, lastError: outcome === "succeeded" ? null : (adapterResult.errorMessage ?? "run_failed"), }); } } } await finalizeAgentStatus(agent.id, outcome); } catch (err) { const message = redactCurrentUserText( err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions(), ); logger.error({ err, runId }, "heartbeat execution failed"); let logSummary: { bytes: number; sha256?: string; compressed: boolean } | null = null; if (handle) { try { logSummary = await runLogStore.finalize(handle); } catch (finalizeErr) { logger.warn({ err: finalizeErr, runId }, "failed to finalize run log after error"); } } const failedRun = await setRunStatus(run.id, "failed", { error: message, errorCode: "adapter_failed", finishedAt: new Date(), resultJson: mergeRunStopMetadataForAgent(agent, "failed", { errorCode: "adapter_failed", errorMessage: message, }), stdoutExcerpt, stderrExcerpt, logBytes: logSummary?.bytes, logSha256: logSummary?.sha256, logCompressed: logSummary?.compressed ?? false, }); await setWakeupStatus(run.wakeupRequestId, "failed", { finishedAt: new Date(), error: message, }); if (failedRun) { await appendRunEvent(failedRun, seq++, { eventType: "error", stream: "system", level: "error", message, }); const livenessRun = await classifyAndPersistRunLiveness(failedRun) ?? failedRun; await refreshContinuationSummaryForRun(livenessRun, agent); await finalizeIssueCommentPolicy(livenessRun, agent); await releaseIssueExecutionAndPromote(livenessRun); await updateRuntimeState(agent, livenessRun, { exitCode: null, signal: null, timedOut: false, errorMessage: message, }, { legacySessionId: runtimeForAdapter.sessionId, }); if (taskKey && (previousSessionParams || previousSessionDisplayId || taskSession)) { await upsertTaskSession({ companyId: agent.companyId, agentId: agent.id, adapterType: agent.adapterType, taskKey, sessionParamsJson: previousSessionParams, sessionDisplayId: previousSessionDisplayId, lastRunId: failedRun.id, lastError: message, }); } } await finalizeAgentStatus(agent.id, "failed"); } } catch (outerErr) { // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun). // The inner catch did not fire, so we must record the failure here. const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure"; logger.error({ err: outerErr, runId }, "heartbeat execution setup failed"); const setupFailureAgent = await getAgent(run.agentId).catch(() => null); await setRunStatus(runId, "failed", { error: message, errorCode: "adapter_failed", finishedAt: new Date(), ...(setupFailureAgent ? { resultJson: mergeRunStopMetadataForAgent(setupFailureAgent, "failed", { errorCode: "adapter_failed", errorMessage: message, }), } : {}), }).catch(() => undefined); await setWakeupStatus(run.wakeupRequestId, "failed", { finishedAt: new Date(), error: message, }).catch(() => undefined); const failedRun = await getRun(runId).catch(() => null); if (failedRun) { // Emit a run-log event so the failure is visible in the run timeline, // consistent with what the inner catch block does for adapter failures. await appendRunEvent(failedRun, 1, { eventType: "error", stream: "system", level: "error", message, }).catch(() => undefined); const livenessRun = await classifyAndPersistRunLiveness(failedRun).catch(() => failedRun); const failedAgent = setupFailureAgent ?? await getAgent(run.agentId).catch(() => null); if (failedAgent) { await refreshContinuationSummaryForRun(livenessRun, failedAgent).catch(() => undefined); await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined); } await releaseIssueExecutionAndPromote(livenessRun).catch(() => undefined); } // Ensure the agent is not left stuck in "running" if the inner catch handler's // DB calls threw (e.g. a transient DB error in finalizeAgentStatus). await finalizeAgentStatus(run.agentId, "failed").catch(() => undefined); } finally { const latestRun = await getRun(run.id).catch(() => null); const releasedLeases = await environmentsSvc .releaseLeasesForRun(run.id, leaseReleaseStatusForRunStatus(latestRun?.status)) .catch((err) => { logger.warn({ err, runId: run.id }, "failed to release environment leases for heartbeat run"); return []; }); for (const lease of releasedLeases) { await logActivity(db, { companyId: run.companyId, actorType: "agent", actorId: run.agentId, agentId: run.agentId, runId: run.id, action: "environment.lease_released", entityType: "environment_lease", entityId: lease.id, details: { environmentId: lease.environmentId, driver: lease.metadata?.driver ?? "local", leasePolicy: lease.leasePolicy, provider: lease.provider, executionWorkspaceId: lease.executionWorkspaceId, issueId: lease.issueId, status: lease.status, }, }).catch(() => undefined); } await releaseRuntimeServicesForRun(run.id).catch(() => undefined); activeRunExecutions.delete(run.id); await startNextQueuedRunForAgent(run.agentId); } } function buildImmediateExecutionPathRecoveryComment(input: { status: "todo" | "in_progress"; latestRun: Pick | null | undefined; }) { const failureSummary = summarizeRunFailureForIssueComment(input.latestRun); if (input.status === "todo") { return ( "Paperclip automatically retried dispatch for this assigned `todo` issue during terminal run recovery, " + `but it still has no live execution path.${failureSummary ?? ""} ` + "Moving it to `blocked` so it is visible for intervention." ); } return ( "Paperclip automatically retried continuation for this assigned `in_progress` issue during terminal run " + `recovery, but it still has no live execution path.${failureSummary ?? ""} ` + "Moving it to `blocked` so it is visible for intervention." ); } async function releaseIssueExecutionAndPromote(run: typeof heartbeatRuns.$inferSelect) { const runContext = parseObject(run.contextSnapshot); const contextIssueId = readNonEmptyString(runContext.issueId); const taskKey = deriveTaskKeyWithHeartbeatFallback(runContext, null); const recoveryAgent = await getAgent(run.agentId); const recoveryAgentInvokable = recoveryAgent && recoveryAgent.status !== "paused" && recoveryAgent.status !== "terminated" && recoveryAgent.status !== "pending_approval"; const recoverySessionBefore = recoveryAgentInvokable ? await resolveSessionBeforeForWakeup(recoveryAgent, taskKey) : null; const recoveryAgentNameKey = normalizeAgentNameKey(recoveryAgent?.name); const promotionResult = await db.transaction(async (tx) => { if (contextIssueId) { await tx.execute( sql`select id from issues where company_id = ${run.companyId} and id = ${contextIssueId} for update`, ); } else { await tx.execute( sql`select id from issues where company_id = ${run.companyId} and execution_run_id = ${run.id} for update`, ); } let issue = await tx .select({ id: issues.id, companyId: issues.companyId, identifier: issues.identifier, status: issues.status, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, executionRunId: issues.executionRunId, }) .from(issues) .where( and( eq(issues.companyId, run.companyId), contextIssueId ? eq(issues.id, contextIssueId) : eq(issues.executionRunId, run.id), ), ) .then((rows) => rows[0] ?? null); if (!issue) return null; if (issue.executionRunId && issue.executionRunId !== run.id) return null; if (issue.executionRunId === run.id) { await tx .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: new Date(), }) .where(eq(issues.id, issue.id)); } while (true) { const deferred = await tx .select() .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, issue.companyId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`, ), ) .orderBy(asc(agentWakeupRequests.requestedAt)) .limit(1) .then((rows) => rows[0] ?? null); if (!deferred) break; const deferredAgent = await tx .select() .from(agents) .where(eq(agents.id, deferred.agentId)) .then((rows) => rows[0] ?? null); if ( !deferredAgent || deferredAgent.companyId !== issue.companyId || deferredAgent.status === "paused" || deferredAgent.status === "terminated" || deferredAgent.status === "pending_approval" ) { await tx .update(agentWakeupRequests) .set({ status: "failed", finishedAt: new Date(), error: "Deferred wake could not be promoted: agent is not invokable", updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, deferred.id)); continue; } const deferredPayload = parseObject(deferred.payload); const deferredContextSeed = parseObject(deferredPayload[DEFERRED_WAKE_CONTEXT_KEY]); const activePauseHold = await treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id); const treeHoldInteractionWake = activePauseHold && allowsIssueInteractionWake(deferredContextSeed); if (activePauseHold && !treeHoldInteractionWake) { await tx .update(agentWakeupRequests) .set({ status: "cancelled", finishedAt: new Date(), error: "Deferred wake suppressed by active subtree pause hold", updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, deferred.id)); continue; } const promotedContextSeed: Record = { ...deferredContextSeed }; if (activePauseHold) { promotedContextSeed.treeHoldInteraction = true; promotedContextSeed.activeTreeHold = { holdId: activePauseHold.holdId, rootIssueId: activePauseHold.rootIssueId, mode: activePauseHold.mode, reason: activePauseHold.reason, releasePolicy: activePauseHold.releasePolicy, interaction: true, }; } const deferredCommentIds = extractWakeCommentIds(deferredContextSeed); const shouldReopenDeferredCommentWake = deferredCommentIds.length > 0 && (issue.status === "done" || issue.status === "cancelled"); let reopenedActivity: LogActivityInput | null = null; if (shouldReopenDeferredCommentWake) { const reopenedFromStatus = issue.status; const reopenedIssue = await issuesSvc.update( issue.id, { status: "todo", executionState: null, }, tx, ); if (reopenedIssue) { issue = { ...issue, identifier: reopenedIssue.identifier, status: reopenedIssue.status, executionRunId: reopenedIssue.executionRunId, }; if (!readNonEmptyString(promotedContextSeed.reopenedFrom)) { promotedContextSeed.reopenedFrom = reopenedFromStatus; } reopenedActivity = { companyId: issue.companyId, actorType: "system", actorId: "heartbeat", agentId: deferred.agentId, runId: run.id, action: "issue.updated", entityType: "issue", entityId: issue.id, details: { status: "todo", reopened: true, reopenedFrom: reopenedFromStatus, source: "deferred_comment_wake", identifier: issue.identifier, }, }; } } const promotedReason = readNonEmptyString(deferred.reason) ?? "issue_execution_promoted"; const promotedSource = (readNonEmptyString(deferred.source) as WakeupOptions["source"]) ?? "automation"; const promotedTriggerDetail = (readNonEmptyString(deferred.triggerDetail) as WakeupOptions["triggerDetail"]) ?? null; const promotedPayload = deferredPayload; delete promotedPayload[DEFERRED_WAKE_CONTEXT_KEY]; const { contextSnapshot: promotedContextSnapshot, taskKey: promotedTaskKey, } = enrichWakeContextSnapshot({ contextSnapshot: promotedContextSeed, reason: promotedReason, source: promotedSource, triggerDetail: promotedTriggerDetail, payload: promotedPayload, }); const sessionBefore = readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ?? await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey); const promotedContinuationAttempt = readContinuationAttempt( promotedContextSnapshot.livenessContinuationAttempt, ); const now = new Date(); const newRun = await tx .insert(heartbeatRuns) .values({ companyId: deferredAgent.companyId, agentId: deferredAgent.id, invocationSource: promotedSource, triggerDetail: promotedTriggerDetail, status: "queued", wakeupRequestId: deferred.id, contextSnapshot: promotedContextSnapshot, sessionIdBefore: sessionBefore, continuationAttempt: promotedContinuationAttempt, }) .returning() .then((rows) => rows[0]); await tx .update(agentWakeupRequests) .set({ status: "queued", reason: "issue_execution_promoted", runId: newRun.id, claimedAt: null, finishedAt: null, error: null, updatedAt: now, }) .where(eq(agentWakeupRequests.id, deferred.id)); await tx .update(issues) .set({ executionRunId: newRun.id, executionAgentNameKey: normalizeAgentNameKey(deferredAgent.name), executionLockedAt: now, updatedAt: now, }) .where(eq(issues.id, issue.id)); return { kind: "promoted" as const, run: newRun, reopenedActivity, }; } const issueNeedsImmediateRecovery = (issue.status === "todo" || issue.status === "in_progress") && !issue.assigneeUserId && issue.assigneeAgentId === run.agentId && (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled"); if (!issueNeedsImmediateRecovery) { return { kind: "released" as const }; } const existingExecutionPath = await tx .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, issue.companyId), inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, sql`${heartbeatRuns.id} <> ${run.id}`, ), ) .limit(1) .then((rows) => rows[0] ?? null); if (existingExecutionPath) { return { kind: "released" as const }; } const shouldBlockImmediately = !recoveryAgentInvokable || !recoveryAgent || didAutomaticRecoveryFail(run, issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed"); if (shouldBlockImmediately) { const comment = buildImmediateExecutionPathRecoveryComment({ status: issue.status as "todo" | "in_progress", latestRun: run, }); await tx .update(issues) .set({ status: "blocked", updatedAt: new Date(), }) .where(eq(issues.id, issue.id)); return { kind: "blocked" as const, issueId: issue.id, issueIdentifier: issue.identifier, previousStatus: issue.status, comment, }; } const retryReason = issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed"; const recoveryReason = issue.status === "todo" ? "issue_assignment_recovery" : "issue_continuation_needed"; const recoverySource = issue.status === "todo" ? "issue.assignment_recovery" : "issue.continuation_recovery"; const now = new Date(); const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ companyId: issue.companyId, agentId: recoveryAgent.id, source: "automation", triggerDetail: "system", reason: recoveryReason, payload: { issueId: issue.id, retryOfRunId: run.id, }, status: "queued", requestedByActorType: "system", requestedByActorId: null, updatedAt: now, }) .returning() .then((rows) => rows[0]); const queuedRun = await tx .insert(heartbeatRuns) .values({ companyId: issue.companyId, agentId: recoveryAgent.id, invocationSource: "automation", triggerDetail: "system", status: "queued", wakeupRequestId: wakeupRequest.id, contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: recoveryReason, retryReason, source: recoverySource, retryOfRunId: run.id, }, sessionIdBefore: recoverySessionBefore, retryOfRunId: run.id, updatedAt: now, }) .returning() .then((rows) => rows[0]); await tx .update(agentWakeupRequests) .set({ runId: queuedRun.id, updatedAt: now, }) .where(eq(agentWakeupRequests.id, wakeupRequest.id)); await tx .update(issues) .set({ executionRunId: queuedRun.id, executionAgentNameKey: recoveryAgentNameKey, executionLockedAt: now, updatedAt: now, }) .where(eq(issues.id, issue.id)); return { kind: "queued_recovery" as const, run: queuedRun, }; }); if (promotionResult?.kind === "blocked") { await issuesSvc.addComment(promotionResult.issueId, promotionResult.comment, {}); await logActivity(db, { companyId: run.companyId, actorType: "system", actorId: "system", agentId: null, runId: run.id, action: "issue.updated", entityType: "issue", entityId: promotionResult.issueId, details: { identifier: promotionResult.issueIdentifier, status: "blocked", previousStatus: promotionResult.previousStatus, source: "heartbeat.release_issue_execution_and_promote", latestRunId: run.id, latestRunStatus: run.status, latestRunErrorCode: run.errorCode ?? null, }, }); return; } const promotedRun = promotionResult?.run ?? null; if (!promotedRun) return; if (promotionResult?.kind === "promoted" && promotionResult.reopenedActivity) { await logActivity(db, promotionResult.reopenedActivity); } publishLiveEvent({ companyId: promotedRun.companyId, type: "heartbeat.run.queued", payload: { runId: promotedRun.id, agentId: promotedRun.agentId, invocationSource: promotedRun.invocationSource, triggerDetail: promotedRun.triggerDetail, wakeupRequestId: promotedRun.wakeupRequestId, }, }); await startNextQueuedRunForAgent(promotedRun.agentId); } async function enqueueWakeup(agentId: string, opts: WakeupOptions = {}) { const source = opts.source ?? "on_demand"; const triggerDetail = opts.triggerDetail ?? null; const contextSnapshot: Record = { ...(opts.contextSnapshot ?? {}) }; const reason = opts.reason ?? null; const payload = opts.payload ?? null; const { contextSnapshot: enrichedContextSnapshot, issueIdFromPayload, taskKey, wakeCommentId, } = enrichWakeContextSnapshot({ contextSnapshot, reason, source, triggerDetail, payload, }); let issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload; const agent = await getAgent(agentId); if (!agent) throw notFound("Agent not found"); const explicitResumeSession = await resolveExplicitResumeSessionOverride(agent, payload, taskKey); if (explicitResumeSession) { enrichedContextSnapshot.resumeFromRunId = explicitResumeSession.resumeFromRunId; enrichedContextSnapshot.resumeSessionDisplayId = explicitResumeSession.sessionDisplayId; enrichedContextSnapshot.resumeSessionParams = explicitResumeSession.sessionParams; if (!readNonEmptyString(enrichedContextSnapshot.issueId) && explicitResumeSession.issueId) { enrichedContextSnapshot.issueId = explicitResumeSession.issueId; } if (!readNonEmptyString(enrichedContextSnapshot.taskId) && explicitResumeSession.taskId) { enrichedContextSnapshot.taskId = explicitResumeSession.taskId; } if (!readNonEmptyString(enrichedContextSnapshot.taskKey) && explicitResumeSession.taskKey) { enrichedContextSnapshot.taskKey = explicitResumeSession.taskKey; } issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueId; } const effectiveTaskKey = readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey; const sessionBefore = explicitResumeSession?.sessionDisplayId ?? await resolveSessionBeforeForWakeup(agent, effectiveTaskKey); const continuationAttempt = readContinuationAttempt(enrichedContextSnapshot.livenessContinuationAttempt); const writeSkippedRequest = async (skipReason: string) => { await db.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, source, triggerDetail, reason: skipReason, payload, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), }); }; let projectId = readNonEmptyString(enrichedContextSnapshot.projectId); if (!projectId && issueId) { projectId = await db .select({ projectId: issues.projectId }) .from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId))) .then((rows) => rows[0]?.projectId ?? null); } const budgetBlock = await budgets.getInvocationBlock(agent.companyId, agentId, { issueId, projectId, }); if (budgetBlock) { await writeSkippedRequest("budget.blocked"); throw conflict(budgetBlock.reason, { scopeType: budgetBlock.scopeType, scopeId: budgetBlock.scopeId, }); } if ( agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval" ) { throw conflict("Agent is not invokable in its current state", { status: agent.status }); } const policy = parseHeartbeatPolicy(agent); if (source === "timer" && !policy.enabled) { await writeSkippedRequest("heartbeat.disabled"); return null; } if (source !== "timer" && !policy.wakeOnDemand) { await writeSkippedRequest("heartbeat.wakeOnDemand.disabled"); return null; } if (issueId) { const activePauseHold = await treeControlSvc.getActivePauseHoldGate(agent.companyId, issueId); if (activePauseHold) { const treeHoldInteractionWake = allowsIssueInteractionWake(enrichedContextSnapshot); if (!treeHoldInteractionWake) { await writeSkippedRequest("issue_tree_hold_active"); await logActivity(db, { companyId: agent.companyId, actorType: "system", actorId: "system", agentId, runId: null, action: "issue.tree_hold_wakeup_deferred", entityType: "issue", entityId: issueId, details: { holdId: activePauseHold.holdId, rootIssueId: activePauseHold.rootIssueId, requestedReason: reason, source, triggerDetail, securityPrinciples: ["Complete Mediation", "Fail Securely", "Secure Defaults"], }, }); return null; } enrichedContextSnapshot.treeHoldInteraction = true; enrichedContextSnapshot.activeTreeHold = { holdId: activePauseHold.holdId, rootIssueId: activePauseHold.rootIssueId, mode: activePauseHold.mode, reason: activePauseHold.reason, releasePolicy: activePauseHold.releasePolicy, interaction: true, }; } } if (issueId) { // Mention-triggered wakes can request input from another agent, but they must // still respect the issue execution lock so a second agent cannot start on the // same issue workspace while the assignee already has a live run. const agentNameKey = normalizeAgentNameKey(agent.name); const outcome = await db.transaction(async (tx) => { await tx.execute( sql`select id from issues where id = ${issueId} and company_id = ${agent.companyId} for update`, ); const issue = await tx .select({ id: issues.id, companyId: issues.companyId, executionRunId: issues.executionRunId, executionAgentNameKey: issues.executionAgentNameKey, }) .from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId))) .then((rows) => rows[0] ?? null); if (!issue) { await tx.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, source, triggerDetail, reason: "issue_execution_issue_not_found", payload, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), }); return { kind: "skipped" as const }; } let activeExecutionRun = issue.executionRunId ? await tx .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, issue.executionRunId)) .then((rows) => rows[0] ?? null) : null; if ( activeExecutionRun && !EXECUTION_PATH_HEARTBEAT_RUN_STATUSES.includes( activeExecutionRun.status as (typeof EXECUTION_PATH_HEARTBEAT_RUN_STATUSES)[number], ) ) { activeExecutionRun = null; } if (!activeExecutionRun && issue.executionRunId) { await tx .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: new Date(), }) .where(eq(issues.id, issue.id)); } if (!activeExecutionRun) { const legacyRun = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, issue.companyId), inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, ), ) .orderBy( sql`case when ${heartbeatRuns.status} = 'running' then 0 else 1 end`, asc(heartbeatRuns.createdAt), ) .limit(1) .then((rows) => rows[0] ?? null); if (legacyRun) { activeExecutionRun = legacyRun; const legacyAgent = await tx .select({ name: agents.name }) .from(agents) .where(eq(agents.id, legacyRun.agentId)) .then((rows) => rows[0] ?? null); await tx .update(issues) .set({ executionRunId: legacyRun.id, executionAgentNameKey: normalizeAgentNameKey(legacyAgent?.name), executionLockedAt: new Date(), updatedAt: new Date(), }) .where(eq(issues.id, issue.id)); } } const dependencyReadiness = await issuesSvc.listDependencyReadiness( issue.companyId, [issue.id], tx, ).then((rows) => rows.get(issue.id) ?? null); // Blocked descendants should stay idle until the final blocker resolves. // Human comment/mention wakes are the exception: they may run in a // bounded interaction mode so the assignee can answer or triage. const blockedInteractionWake = dependencyReadiness && !dependencyReadiness.isDependencyReady && allowsIssueInteractionWake(enrichedContextSnapshot); if (blockedInteractionWake) { enrichedContextSnapshot.dependencyBlockedInteraction = true; enrichedContextSnapshot.unresolvedBlockerIssueIds = dependencyReadiness.unresolvedBlockerIssueIds; enrichedContextSnapshot.unresolvedBlockerCount = dependencyReadiness.unresolvedBlockerCount; enrichedContextSnapshot.unresolvedBlockerSummaries = await listUnresolvedBlockerSummaries( tx, issue.companyId, issue.id, dependencyReadiness.unresolvedBlockerIssueIds, ); } if (!activeExecutionRun && dependencyReadiness && !dependencyReadiness.isDependencyReady && !blockedInteractionWake) { await tx.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, source, triggerDetail, reason: "issue_dependencies_blocked", payload: { ...(payload ?? {}), issueId, unresolvedBlockerIssueIds: dependencyReadiness.unresolvedBlockerIssueIds, }, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), }); return { kind: "skipped" as const }; } if (activeExecutionRun) { const executionAgent = await tx .select({ name: agents.name }) .from(agents) .where(eq(agents.id, activeExecutionRun.agentId)) .then((rows) => rows[0] ?? null); const executionAgentNameKey = normalizeAgentNameKey(issue.executionAgentNameKey) ?? normalizeAgentNameKey(executionAgent?.name); const isSameExecutionAgent = Boolean(executionAgentNameKey) && executionAgentNameKey === agentNameKey; const shouldQueueFollowupForRunningWake = shouldQueueFollowupForRunningIssueWake({ contextSnapshot: enrichedContextSnapshot, wakeCommentId }) && activeExecutionRun.status === "running" && isSameExecutionAgent; if (isSameExecutionAgent && !shouldQueueFollowupForRunningWake) { const mergedContextSnapshot = mergeCoalescedContextSnapshot( activeExecutionRun.contextSnapshot, enrichedContextSnapshot, ); const mergedRun = await tx .update(heartbeatRuns) .set({ contextSnapshot: mergedContextSnapshot, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, activeExecutionRun.id)) .returning() .then((rows) => rows[0] ?? activeExecutionRun); await tx.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, source, triggerDetail, reason: "issue_execution_same_name", payload, status: "coalesced", coalescedCount: 1, requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, runId: mergedRun.id, finishedAt: new Date(), }); return { kind: "coalesced" as const, run: mergedRun }; } const deferredPayload = { ...(payload ?? {}), issueId, [DEFERRED_WAKE_CONTEXT_KEY]: enrichedContextSnapshot, }; const existingDeferred = await tx .select() .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`, ), ) .orderBy(asc(agentWakeupRequests.requestedAt)) .limit(1) .then((rows) => rows[0] ?? null); if (existingDeferred) { const existingDeferredPayload = parseObject(existingDeferred.payload); const existingDeferredContext = parseObject(existingDeferredPayload[DEFERRED_WAKE_CONTEXT_KEY]); const mergedDeferredContext = mergeCoalescedContextSnapshot( existingDeferredContext, enrichedContextSnapshot, ); const mergedDeferredPayload = { ...existingDeferredPayload, ...(payload ?? {}), issueId, [DEFERRED_WAKE_CONTEXT_KEY]: mergedDeferredContext, }; await tx .update(agentWakeupRequests) .set({ payload: mergedDeferredPayload, coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1, updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, existingDeferred.id)); return { kind: "deferred" as const }; } await tx.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, source, triggerDetail, reason: "issue_execution_deferred", payload: deferredPayload, status: "deferred_issue_execution", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, }); return { kind: "deferred" as const }; } const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ companyId: agent.companyId, agentId, source, triggerDetail, reason, payload, status: "queued", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, }) .returning() .then((rows) => rows[0]); const newRun = await tx .insert(heartbeatRuns) .values({ companyId: agent.companyId, agentId, invocationSource: source, triggerDetail, status: "queued", wakeupRequestId: wakeupRequest.id, contextSnapshot: enrichedContextSnapshot, sessionIdBefore: sessionBefore, continuationAttempt, }) .returning() .then((rows) => rows[0]); await tx .update(agentWakeupRequests) .set({ runId: newRun.id, updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, wakeupRequest.id)); // executionRunId is NOT stamped here (enqueueWakeup queues the run but // doesn't start it). It will be stamped in claimQueuedRun() once the run // transitions to "running" — Fix A (lazy locking). return { kind: "queued" as const, run: newRun }; }); if (outcome.kind === "deferred" || outcome.kind === "skipped") return null; if (outcome.kind === "coalesced") { await startNextQueuedRunForAgent(agent.id); return outcome.run; } const newRun = outcome.run; publishLiveEvent({ companyId: newRun.companyId, type: "heartbeat.run.queued", payload: { runId: newRun.id, agentId: newRun.agentId, invocationSource: newRun.invocationSource, triggerDetail: newRun.triggerDetail, wakeupRequestId: newRun.wakeupRequestId, }, }); await startNextQueuedRunForAgent(agent.id); return newRun; } const activeRuns = await db .select() .from(heartbeatRuns) .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]))) .orderBy(desc(heartbeatRuns.createdAt)); const sameScopeQueuedRun = activeRuns.find( (candidate) => candidate.status === "queued" && isSameTaskScope(runTaskKey(candidate), taskKey), ); const sameScopeScheduledRetryRun = activeRuns.find( (candidate) => candidate.status === "scheduled_retry" && isSameTaskScope(runTaskKey(candidate), taskKey), ); const sameScopeRunningRun = activeRuns.find( (candidate) => candidate.status === "running" && isSameTaskScope(runTaskKey(candidate), taskKey), ); const shouldQueueFollowupForRunningWake = Boolean(sameScopeRunningRun) && !sameScopeQueuedRun && shouldQueueFollowupForRunningIssueWake({ contextSnapshot: enrichedContextSnapshot, wakeCommentId }); const coalescedTargetRun = sameScopeQueuedRun ?? sameScopeScheduledRetryRun ?? (shouldQueueFollowupForRunningWake ? null : sameScopeRunningRun ?? null); if (coalescedTargetRun) { const mergedContextSnapshot = mergeCoalescedContextSnapshot( coalescedTargetRun.contextSnapshot, contextSnapshot, ); const mergedRun = await db .update(heartbeatRuns) .set({ contextSnapshot: mergedContextSnapshot, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, coalescedTargetRun.id)) .returning() .then((rows) => rows[0] ?? coalescedTargetRun); await db.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, source, triggerDetail, reason, payload, status: "coalesced", coalescedCount: 1, requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, runId: mergedRun.id, finishedAt: new Date(), }); return mergedRun; } const wakeupRequest = await db .insert(agentWakeupRequests) .values({ companyId: agent.companyId, agentId, source, triggerDetail, reason, payload, status: "queued", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, }) .returning() .then((rows) => rows[0]); const newRun = await db .insert(heartbeatRuns) .values({ companyId: agent.companyId, agentId, invocationSource: source, triggerDetail, status: "queued", wakeupRequestId: wakeupRequest.id, contextSnapshot: enrichedContextSnapshot, sessionIdBefore: sessionBefore, continuationAttempt, }) .returning() .then((rows) => rows[0]); await db .update(agentWakeupRequests) .set({ runId: newRun.id, updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, wakeupRequest.id)); publishLiveEvent({ companyId: newRun.companyId, type: "heartbeat.run.queued", payload: { runId: newRun.id, agentId: newRun.agentId, invocationSource: newRun.invocationSource, triggerDetail: newRun.triggerDetail, wakeupRequestId: newRun.wakeupRequestId, }, }); await startNextQueuedRunForAgent(agent.id); return newRun; } async function listProjectScopedRunIds(companyId: string, projectId: string) { const runIssueId = sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; const effectiveProjectId = sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`; const rows = await db .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id }) .from(heartbeatRuns) .leftJoin( issues, and( eq(issues.companyId, companyId), sql`${issues.id}::text = ${runIssueId}`, ), ) .where( and( eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), sql`${effectiveProjectId} = ${projectId}`, ), ); return rows.map((row) => row.id); } async function listProjectScopedWakeupIds(companyId: string, projectId: string) { const wakeIssueId = sql`${agentWakeupRequests.payload} ->> 'issueId'`; const effectiveProjectId = sql`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`; const rows = await db .selectDistinctOn([agentWakeupRequests.id], { id: agentWakeupRequests.id }) .from(agentWakeupRequests) .leftJoin( issues, and( eq(issues.companyId, companyId), sql`${issues.id}::text = ${wakeIssueId}`, ), ) .where( and( eq(agentWakeupRequests.companyId, companyId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql`${agentWakeupRequests.runId} is null`, sql`${effectiveProjectId} = ${projectId}`, ), ); return rows.map((row) => row.id); } async function cancelPendingWakeupsForBudgetScope(scope: BudgetEnforcementScope) { const now = new Date(); let wakeupIds: string[] = []; if (scope.scopeType === "company") { wakeupIds = await db .select({ id: agentWakeupRequests.id }) .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, scope.companyId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql`${agentWakeupRequests.runId} is null`, ), ) .then((rows) => rows.map((row) => row.id)); } else if (scope.scopeType === "agent") { wakeupIds = await db .select({ id: agentWakeupRequests.id }) .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.agentId, scope.scopeId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql`${agentWakeupRequests.runId} is null`, ), ) .then((rows) => rows.map((row) => row.id)); } else { wakeupIds = await listProjectScopedWakeupIds(scope.companyId, scope.scopeId); } if (wakeupIds.length === 0) return 0; await db .update(agentWakeupRequests) .set({ status: "cancelled", finishedAt: now, error: "Cancelled due to budget pause", updatedAt: now, }) .where(inArray(agentWakeupRequests.id, wakeupIds)); return wakeupIds.length; } async function cancelRunInternal(runId: string, reason = "Cancelled by control plane") { const run = await getRun(runId); if (!run) throw notFound("Heartbeat run not found"); if (!CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes(run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number])) return run; const agent = await getAgent(run.agentId); const running = runningProcesses.get(run.id); if (running) { await terminateHeartbeatRunProcess({ pid: running.child.pid ?? run.processPid, processGroupId: running.processGroupId ?? run.processGroupId, graceMs: Math.max(1, running.graceSec) * 1000, }); } else if (run.processPid || run.processGroupId) { await terminateHeartbeatRunProcess({ pid: run.processPid, processGroupId: run.processGroupId, }); } const cancelled = await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: reason, errorCode: "cancelled", ...(agent ? { resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", { resultJson: parseObject(run.resultJson), errorCode: "cancelled", errorMessage: reason, }), } : {}), }); await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date(), error: reason, }); if (cancelled) { await appendRunEvent(cancelled, 1, { eventType: "lifecycle", stream: "system", level: "warn", message: "run cancelled", }); await releaseIssueExecutionAndPromote(cancelled); } runningProcesses.delete(run.id); await finalizeAgentStatus(run.agentId, "cancelled"); await startNextQueuedRunForAgent(run.agentId); return cancelled; } async function cancelActiveForAgentInternal(agentId: string, reason = "Cancelled due to agent pause") { const agent = await getAgent(agentId); const runs = await db .select() .from(heartbeatRuns) .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]))); for (const run of runs) { await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: reason, errorCode: "cancelled", ...(agent ? { resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", { resultJson: parseObject(run.resultJson), errorCode: "cancelled", errorMessage: reason, }), } : {}), }); await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date(), error: reason, }); const running = runningProcesses.get(run.id); if (running) { await terminateHeartbeatRunProcess({ pid: running.child.pid ?? run.processPid, processGroupId: running.processGroupId ?? run.processGroupId, graceMs: Math.max(1, running.graceSec) * 1000, }); runningProcesses.delete(run.id); } else if (run.processPid || run.processGroupId) { await terminateHeartbeatRunProcess({ pid: run.processPid, processGroupId: run.processGroupId, }); } await releaseIssueExecutionAndPromote(run); } return runs.length; } async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) { if (scope.scopeType === "agent") { await cancelActiveForAgentInternal(scope.scopeId, "Cancelled due to budget pause"); await cancelPendingWakeupsForBudgetScope(scope); return; } const runIds = scope.scopeType === "company" ? await db .select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, scope.companyId), inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), ), ) .then((rows) => rows.map((row) => row.id)) : await listProjectScopedRunIds(scope.companyId, scope.scopeId); for (const runId of runIds) { await cancelRunInternal(runId, "Cancelled due to budget pause"); } await cancelPendingWakeupsForBudgetScope(scope); } return { list: async (companyId: string, agentId?: string, limit?: number) => { const safeForLegacyEncoding = await hasUnsafeTextProjectionDatabase(); const query = db .select( safeForLegacyEncoding ? { ...heartbeatRunListColumns, error: sql`NULL`.as("error"), ...heartbeatRunListContextColumns, } : { ...heartbeatRunListColumns, ...heartbeatRunListContextColumns, ...heartbeatRunListResultColumns, }, ) .from(heartbeatRuns) .where( agentId ? and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, agentId)) : eq(heartbeatRuns.companyId, companyId), ) .orderBy(desc(heartbeatRuns.createdAt)); const rows = limit ? await query.limit(limit) : await query; return rows.map((row) => { const { contextIssueId, contextTaskId, contextTaskKey, contextCommentId, contextWakeCommentId, contextWakeReason, contextWakeSource, contextWakeTriggerDetail, resultSummary, resultResult, resultMessage, resultError, resultTotalCostUsd, resultCostUsd, resultCostUsdCamel, ...rest } = row as typeof row & { resultSummary?: string | null; resultResult?: string | null; resultMessage?: string | null; resultError?: string | null; resultTotalCostUsd?: string | null; resultCostUsd?: string | null; resultCostUsdCamel?: string | null; }; return { ...rest, contextSnapshot: summarizeHeartbeatRunContextSnapshot({ issueId: contextIssueId, taskId: contextTaskId, taskKey: contextTaskKey, commentId: contextCommentId, wakeCommentId: contextWakeCommentId, wakeReason: contextWakeReason, wakeSource: contextWakeSource, wakeTriggerDetail: contextWakeTriggerDetail, }), resultJson: safeForLegacyEncoding ? null : summarizeHeartbeatRunListResultJson({ summary: resultSummary, result: resultResult, message: resultMessage, error: resultError, totalCostUsd: resultTotalCostUsd, costUsd: resultCostUsd, costUsdCamel: resultCostUsdCamel, }), }; }); }, getRun, getRunLogAccess, getRuntimeState: async (agentId: string) => { const state = await getRuntimeState(agentId); const agent = await getAgent(agentId); if (!agent) return null; const ensured = state ?? (await ensureRuntimeState(agent)); const latestTaskSession = await db .select() .from(agentTaskSessions) .where(and(eq(agentTaskSessions.companyId, agent.companyId), eq(agentTaskSessions.agentId, agent.id))) .orderBy(desc(agentTaskSessions.updatedAt)) .limit(1) .then((rows) => rows[0] ?? null); return { ...ensured, sessionDisplayId: latestTaskSession?.sessionDisplayId ?? ensured.sessionId, sessionParamsJson: latestTaskSession?.sessionParamsJson ?? null, }; }, listTaskSessions: async (agentId: string) => { const agent = await getAgent(agentId); if (!agent) throw notFound("Agent not found"); return db .select() .from(agentTaskSessions) .where(and(eq(agentTaskSessions.companyId, agent.companyId), eq(agentTaskSessions.agentId, agentId))) .orderBy(desc(agentTaskSessions.updatedAt), desc(agentTaskSessions.createdAt)); }, resetRuntimeSession: async (agentId: string, opts?: { taskKey?: string | null }) => { const agent = await getAgent(agentId); if (!agent) throw notFound("Agent not found"); await ensureRuntimeState(agent); const taskKey = readNonEmptyString(opts?.taskKey); const clearedTaskSessions = await clearTaskSessions( agent.companyId, agent.id, taskKey ? { taskKey, adapterType: agent.adapterType } : undefined, ); const runtimePatch: Partial = { sessionId: null, lastError: null, updatedAt: new Date(), }; if (!taskKey) { runtimePatch.stateJson = {}; } const updated = await db .update(agentRuntimeState) .set(runtimePatch) .where(eq(agentRuntimeState.agentId, agentId)) .returning() .then((rows) => rows[0] ?? null); if (!updated) return null; return { ...updated, sessionDisplayId: null, sessionParamsJson: null, clearedTaskSessions, }; }, listEvents: (runId: string, afterSeq = 0, limit = 200) => db .select() .from(heartbeatRunEvents) .where(and(eq(heartbeatRunEvents.runId, runId), gt(heartbeatRunEvents.seq, afterSeq))) .orderBy(asc(heartbeatRunEvents.seq)) .limit(Math.max(1, Math.min(limit, 1000))), getRetryExhaustedReason: async (runId: string) => { const row = await db .select({ message: heartbeatRunEvents.message, }) .from(heartbeatRunEvents) .where( and( eq(heartbeatRunEvents.runId, runId), eq(heartbeatRunEvents.eventType, "lifecycle"), sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, ), ) .orderBy(desc(heartbeatRunEvents.id)) .limit(1) .then((rows) => rows[0] ?? null); return row?.message ?? null; }, readLog: async ( runOrLookup: string | { id: string; companyId: string; logStore: string | null; logRef: string | null; }, opts?: { offset?: number; limitBytes?: number }, ) => { const run = typeof runOrLookup === "string" ? await getRunLogAccess(runOrLookup) : runOrLookup; const runId = typeof runOrLookup === "string" ? runOrLookup : runOrLookup.id; if (!run) throw notFound("Heartbeat run not found"); if (!run.logStore || !run.logRef) throw notFound("Run log not found"); const result = await runLogStore.read( { store: run.logStore as "local_file", logRef: run.logRef, }, opts, ); return { runId, store: run.logStore, logRef: run.logRef, ...result, // Run-log chunks are already redacted before they are appended to the store. // Rewriting the full chunk again on every poll creates avoidable string copies. content: result.content, }; }, invoke: async ( agentId: string, source: "timer" | "assignment" | "on_demand" | "automation" = "on_demand", contextSnapshot: Record = {}, triggerDetail: "manual" | "ping" | "callback" | "system" = "manual", actor?: { actorType?: "user" | "agent" | "system"; actorId?: string | null }, ) => enqueueWakeup(agentId, { source, triggerDetail, contextSnapshot, requestedByActorType: actor?.actorType, requestedByActorId: actor?.actorId ?? null, }), wakeup: enqueueWakeup, reportRunActivity: clearDetachedRunWarning, reapOrphanedRuns, promoteDueScheduledRetries, resumeQueuedRuns, scheduleBoundedRetry: async ( runId: string, opts?: { now?: Date; random?: () => number; retryReason?: string; wakeReason?: string; }, ) => { const run = await getRun(runId, { unsafeFullResultJson: true }); if (!run) return { outcome: "missing_run" as const }; const agent = await getAgent(run.agentId); if (!agent) return { outcome: "missing_agent" as const }; return scheduleBoundedRetryForRun(run, agent, opts); }, reconcileStrandedAssignedIssues, reconcileIssueGraphLiveness, tickTimers: async (now = new Date()) => { const allAgents = await db.select().from(agents); let checked = 0; let enqueued = 0; let skipped = 0; for (const agent of allAgents) { if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") continue; const policy = parseHeartbeatPolicy(agent); if (!policy.enabled || policy.intervalSec <= 0) continue; checked += 1; const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime(); const elapsedMs = now.getTime() - baseline; if (elapsedMs < policy.intervalSec * 1000) continue; const run = await enqueueWakeup(agent.id, { source: "timer", triggerDetail: "system", reason: "heartbeat_timer", requestedByActorType: "system", requestedByActorId: "heartbeat_scheduler", contextSnapshot: { source: "scheduler", reason: "interval_elapsed", now: now.toISOString(), }, }); if (run) enqueued += 1; else skipped += 1; } return { checked, enqueued, skipped }; }, cancelRun: (runId: string) => cancelRunInternal(runId), cancelActiveForAgent: (agentId: string) => cancelActiveForAgentInternal(agentId), cancelBudgetScopeWork, getRunIssueSummary: async (runId: string) => { const [run] = await db .select(heartbeatRunIssueSummaryColumns) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)) .limit(1); return run ?? null; }, getActiveRunForAgent: async (agentId: string) => { const [run] = await db .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running"), ), ) .orderBy(desc(heartbeatRuns.startedAt)) .limit(1); return run ?? null; }, getActiveRunIssueSummaryForAgent: async (agentId: string) => { const [run] = await db .select(heartbeatRunIssueSummaryColumns) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running"), ), ) .orderBy(desc(heartbeatRuns.startedAt)) .limit(1); return run ?? null; }, }; }