fix: harden heartbeat and adapter runtime workflows

This commit is contained in:
Dotta 2026-04-10 22:26:21 -05:00
parent 548721248e
commit c566a9236c
48 changed files with 14922 additions and 600 deletions

View file

@ -2,6 +2,24 @@ import { randomUUID } from "node:crypto";
import { describe, expect, it } from "vitest";
import { runChildProcess } from "./server-utils.js";
function isPidAlive(pid: number) {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
async function waitForPidExit(pid: number, timeoutMs = 2_000) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (!isPidAlive(pid)) return true;
await new Promise((resolve) => setTimeout(resolve, 50));
}
return !isPidAlive(pid);
}
describe("runChildProcess", () => {
it("waits for onSpawn before sending stdin to the child", async () => {
const spawnDelayMs = 150;
@ -35,4 +53,36 @@ describe("runChildProcess", () => {
expect(onSpawnCompletedAt).toBeGreaterThanOrEqual(startedAt + spawnDelayMs);
expect(finishedAt - startedAt).toBeGreaterThanOrEqual(spawnDelayMs);
});
it.skipIf(process.platform === "win32")("kills descendant processes on timeout via the process group", async () => {
let descendantPid: number | null = null;
const result = await runChildProcess(
randomUUID(),
process.execPath,
[
"-e",
[
"const { spawn } = require('node:child_process');",
"const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });",
"process.stdout.write(String(child.pid));",
"setInterval(() => {}, 1000);",
].join(" "),
],
{
cwd: process.cwd(),
env: {},
timeoutSec: 1,
graceSec: 1,
onLog: async () => {},
onSpawn: async () => {},
},
);
descendantPid = Number.parseInt(result.stdout.trim(), 10);
expect(result.timedOut).toBe(true);
expect(Number.isInteger(descendantPid) && descendantPid > 0).toBe(true);
expect(await waitForPidExit(descendantPid!, 2_000)).toBe(true);
});
});

View file

@ -19,6 +19,7 @@ export interface RunProcessResult {
interface RunningProcess {
child: ChildProcess;
graceSec: number;
processGroupId: number | null;
}
interface SpawnTarget {
@ -34,6 +35,28 @@ type ChildProcessWithEvents = ChildProcess & {
): ChildProcess;
};
function resolveProcessGroupId(child: ChildProcess) {
if (process.platform === "win32") return null;
return typeof child.pid === "number" && child.pid > 0 ? child.pid : null;
}
function signalRunningProcess(
running: Pick<RunningProcess, "child" | "processGroupId">,
signal: NodeJS.Signals,
) {
if (process.platform !== "win32" && running.processGroupId && running.processGroupId > 0) {
try {
process.kill(-running.processGroupId, signal);
return;
} catch {
// Fall back to the direct child signal if group signaling fails.
}
}
if (!running.child.killed) {
running.child.kill(signal);
}
}
export const runningProcesses = new Map<string, RunningProcess>();
export const MAX_CAPTURE_BYTES = 4 * 1024 * 1024;
export const MAX_EXCERPT_BYTES = 32 * 1024;
@ -1034,7 +1057,7 @@ export async function runChildProcess(
graceSec: number;
onLog: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
onLogError?: (err: unknown, runId: string, message: string) => void;
onSpawn?: (meta: { pid: number; startedAt: string }) => Promise<void>;
onSpawn?: (meta: { pid: number; processGroupId: number | null; startedAt: string }) => Promise<void>;
stdin?: string;
},
): Promise<RunProcessResult> {
@ -1064,19 +1087,21 @@ export async function runChildProcess(
const child = spawn(target.command, target.args, {
cwd: opts.cwd,
env: mergedEnv,
detached: process.platform !== "win32",
shell: false,
stdio: [opts.stdin != null ? "pipe" : "ignore", "pipe", "pipe"],
}) as ChildProcessWithEvents;
const startedAt = new Date().toISOString();
const processGroupId = resolveProcessGroupId(child);
const spawnPersistPromise =
typeof child.pid === "number" && child.pid > 0 && opts.onSpawn
? opts.onSpawn({ pid: child.pid, startedAt }).catch((err) => {
? opts.onSpawn({ pid: child.pid, processGroupId, startedAt }).catch((err) => {
onLogError(err, runId, "failed to record child process metadata");
})
: Promise.resolve();
runningProcesses.set(runId, { child, graceSec: opts.graceSec });
runningProcesses.set(runId, { child, graceSec: opts.graceSec, processGroupId });
let timedOut = false;
let stdout = "";
@ -1087,11 +1112,9 @@ export async function runChildProcess(
opts.timeoutSec > 0
? setTimeout(() => {
timedOut = true;
child.kill("SIGTERM");
signalRunningProcess({ child, processGroupId }, "SIGTERM");
setTimeout(() => {
if (!child.killed) {
child.kill("SIGKILL");
}
signalRunningProcess({ child, processGroupId }, "SIGKILL");
}, Math.max(1, opts.graceSec) * 1000);
}, opts.timeoutSec * 1000)
: null;

View file

@ -120,7 +120,7 @@ export interface AdapterExecutionContext {
context: Record<string, unknown>;
onLog: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
onMeta?: (meta: AdapterInvocationMeta) => Promise<void>;
onSpawn?: (meta: { pid: number; startedAt: string }) => Promise<void>;
onSpawn?: (meta: { pid: number; processGroupId: number | null; startedAt: string }) => Promise<void>;
authToken?: string;
}

View file

@ -1,5 +1,4 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import type { AdapterExecutionContext, AdapterExecutionResult } from "@paperclipai/adapter-utils";
@ -33,35 +32,10 @@ import {
} from "./parse.js";
import { resolveClaudeDesiredSkillNames } from "./skills.js";
import { isBedrockModelId } from "./models.js";
import { prepareClaudePromptBundle } from "./prompt-cache.js";
const __moduleDir = path.dirname(fileURLToPath(import.meta.url));
/**
* Create a tmpdir with `.claude/skills/` containing symlinks to skills from
* the repo's `skills/` directory, so `--add-dir` makes Claude Code discover
* them as proper registered skills.
*/
async function buildSkillsDir(config: Record<string, unknown>): Promise<string> {
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-skills-"));
const target = path.join(tmp, ".claude", "skills");
await fs.mkdir(target, { recursive: true });
const availableEntries = await readPaperclipRuntimeSkillEntries(config, __moduleDir);
const desiredNames = new Set(
resolveClaudeDesiredSkillNames(
config,
availableEntries,
),
);
for (const entry of availableEntries) {
if (!desiredNames.has(entry.key)) continue;
await fs.symlink(
entry.source,
path.join(target, entry.runtimeName),
);
}
return tmp;
}
interface ClaudeExecutionInput {
runId: string;
agent: AdapterExecutionContext["agent"];
@ -361,30 +335,13 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
),
);
const billingType = resolveClaudeBillingType(effectiveEnv);
const skillsDir = await buildSkillsDir(config);
const runtimeSessionParams = parseObject(runtime.sessionParams);
const runtimeSessionId = asString(runtimeSessionParams.sessionId, runtime.sessionId ?? "");
const runtimeSessionCwd = asString(runtimeSessionParams.cwd, "");
const canResumeSession =
runtimeSessionId.length > 0 &&
(runtimeSessionCwd.length === 0 || path.resolve(runtimeSessionCwd) === path.resolve(cwd));
const sessionId = canResumeSession ? runtimeSessionId : null;
if (runtimeSessionId && !canResumeSession) {
await onLog(
"stdout",
`[paperclip] Claude session "${runtimeSessionId}" was saved for cwd "${runtimeSessionCwd}" and will not be resumed in "${cwd}".\n`,
);
}
let effectiveInstructionsFilePath: string | undefined;
let preparedInstructionsFile = false;
const ensureEffectiveInstructionsFilePath = async (resumeSessionId: string | null) => {
if (resumeSessionId || !instructionsFilePath) return undefined;
if (preparedInstructionsFile) return effectiveInstructionsFilePath;
preparedInstructionsFile = true;
const claudeSkillEntries = await readPaperclipRuntimeSkillEntries(config, __moduleDir);
const desiredSkillNames = new Set(resolveClaudeDesiredSkillNames(config, claudeSkillEntries));
// When instructionsFilePath is configured, build a stable content-addressed
// file that includes both the file content and the path directive, so we only
// need --append-system-prompt-file (Claude CLI forbids using both flags together).
let combinedInstructionsContents: string | null = null;
if (instructionsFilePath) {
try {
const instructionsContent = await fs.readFile(instructionsFilePath, "utf-8");
const pathDirective =
@ -392,20 +349,50 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
`Resolve any relative file references from ${instructionsFileDir}. ` +
`This base directory is authoritative for sibling instruction files such as ` +
`./HEARTBEAT.md, ./SOUL.md, and ./TOOLS.md; do not resolve those from the parent agent directory.`;
const combinedPath = path.join(skillsDir, "agent-instructions.md");
await fs.writeFile(combinedPath, instructionsContent + pathDirective, "utf-8");
effectiveInstructionsFilePath = combinedPath;
combinedInstructionsContents = instructionsContent + pathDirective;
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
await onLog(
"stderr",
`[paperclip] Warning: could not read agent instructions file "${instructionsFilePath}": ${reason}\n`,
);
effectiveInstructionsFilePath = undefined;
}
}
const promptBundle = await prepareClaudePromptBundle({
companyId: agent.companyId,
skills: claudeSkillEntries.filter((entry) => desiredSkillNames.has(entry.key)),
instructionsContents: combinedInstructionsContents,
onLog,
});
const effectiveInstructionsFilePath = promptBundle.instructionsFilePath ?? undefined;
return effectiveInstructionsFilePath;
};
const runtimeSessionParams = parseObject(runtime.sessionParams);
const runtimeSessionId = asString(runtimeSessionParams.sessionId, runtime.sessionId ?? "");
const runtimeSessionCwd = asString(runtimeSessionParams.cwd, "");
const runtimePromptBundleKey = asString(runtimeSessionParams.promptBundleKey, "");
const hasMatchingPromptBundle =
runtimePromptBundleKey.length === 0 || runtimePromptBundleKey === promptBundle.bundleKey;
const canResumeSession =
runtimeSessionId.length > 0 &&
hasMatchingPromptBundle &&
(runtimeSessionCwd.length === 0 || path.resolve(runtimeSessionCwd) === path.resolve(cwd));
const sessionId = canResumeSession ? runtimeSessionId : null;
if (
runtimeSessionId &&
runtimeSessionCwd.length > 0 &&
path.resolve(runtimeSessionCwd) !== path.resolve(cwd)
) {
await onLog(
"stdout",
`[paperclip] Claude session "${runtimeSessionId}" was saved for cwd "${runtimeSessionCwd}" and will not be resumed in "${cwd}".\n`,
);
}
if (runtimeSessionId && runtimePromptBundleKey.length > 0 && runtimePromptBundleKey !== promptBundle.bundleKey) {
await onLog(
"stdout",
`[paperclip] Claude session "${runtimeSessionId}" was saved for prompt bundle "${runtimePromptBundleKey}" and will not be resumed with "${promptBundle.bundleKey}".\n`,
);
}
const bootstrapPromptTemplate = asString(config.bootstrapPromptTemplate, "");
const templateData = {
agentId: agent.id,
@ -460,7 +447,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
if (attemptInstructionsFilePath && !resumeSessionId) {
args.push("--append-system-prompt-file", attemptInstructionsFilePath);
}
args.push("--add-dir", skillsDir);
args.push("--add-dir", promptBundle.addDir);
if (extraArgs.length > 0) args.push(...extraArgs);
return args;
};
@ -482,14 +469,17 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
};
const runAttempt = async (resumeSessionId: string | null) => {
const attemptInstructionsFilePath = await ensureEffectiveInstructionsFilePath(resumeSessionId);
const attemptInstructionsFilePath = resumeSessionId ? undefined : effectiveInstructionsFilePath;
const args = buildClaudeArgs(resumeSessionId, attemptInstructionsFilePath);
const commandNotes =
attemptInstructionsFilePath && !resumeSessionId
? [
`Injected agent instructions via --append-system-prompt-file ${instructionsFilePath} (with path directive appended)`,
]
: [];
const commandNotes: string[] = [];
if (!resumeSessionId) {
commandNotes.push(`Using stable Claude prompt bundle ${promptBundle.bundleKey}.`);
}
if (attemptInstructionsFilePath && !resumeSessionId) {
commandNotes.push(
`Injected agent instructions via --append-system-prompt-file ${instructionsFilePath} (with path directive appended)`,
);
}
if (onMeta) {
await onMeta({
adapterType: "claude_local",
@ -586,6 +576,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
? ({
sessionId: resolvedSessionId,
cwd,
promptBundleKey: promptBundle.bundleKey,
...(workspaceId ? { workspaceId } : {}),
...(workspaceRepoUrl ? { repoUrl: workspaceRepoUrl } : {}),
...(workspaceRepoRef ? { repoRef: workspaceRepoRef } : {}),
@ -618,25 +609,21 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
};
};
try {
const initial = await runAttempt(sessionId ?? null);
if (
sessionId &&
!initial.proc.timedOut &&
(initial.proc.exitCode ?? 0) !== 0 &&
initial.parsed &&
isClaudeUnknownSessionError(initial.parsed)
) {
await onLog(
"stdout",
`[paperclip] Claude resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`,
);
const retry = await runAttempt(null);
return toAdapterResult(retry, { fallbackSessionId: null, clearSessionOnMissingSession: true });
}
return toAdapterResult(initial, { fallbackSessionId: runtimeSessionId || runtime.sessionId });
} finally {
fs.rm(skillsDir, { recursive: true, force: true }).catch(() => {});
const initial = await runAttempt(sessionId ?? null);
if (
sessionId &&
!initial.proc.timedOut &&
(initial.proc.exitCode ?? 0) !== 0 &&
initial.parsed &&
isClaudeUnknownSessionError(initial.parsed)
) {
await onLog(
"stdout",
`[paperclip] Claude resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`,
);
const retry = await runAttempt(null);
return toAdapterResult(retry, { fallbackSessionId: null, clearSessionOnMissingSession: true });
}
return toAdapterResult(initial, { fallbackSessionId: runtimeSessionId || runtime.sessionId });
}

View file

@ -36,12 +36,16 @@ export const sessionCodec: AdapterSessionCodec = {
readNonEmptyString(record.cwd) ??
readNonEmptyString(record.workdir) ??
readNonEmptyString(record.folder);
const promptBundleKey =
readNonEmptyString(record.promptBundleKey) ??
readNonEmptyString(record.prompt_bundle_key);
const workspaceId = readNonEmptyString(record.workspaceId) ?? readNonEmptyString(record.workspace_id);
const repoUrl = readNonEmptyString(record.repoUrl) ?? readNonEmptyString(record.repo_url);
const repoRef = readNonEmptyString(record.repoRef) ?? readNonEmptyString(record.repo_ref);
return {
sessionId,
...(cwd ? { cwd } : {}),
...(promptBundleKey ? { promptBundleKey } : {}),
...(workspaceId ? { workspaceId } : {}),
...(repoUrl ? { repoUrl } : {}),
...(repoRef ? { repoRef } : {}),
@ -55,12 +59,16 @@ export const sessionCodec: AdapterSessionCodec = {
readNonEmptyString(params.cwd) ??
readNonEmptyString(params.workdir) ??
readNonEmptyString(params.folder);
const promptBundleKey =
readNonEmptyString(params.promptBundleKey) ??
readNonEmptyString(params.prompt_bundle_key);
const workspaceId = readNonEmptyString(params.workspaceId) ?? readNonEmptyString(params.workspace_id);
const repoUrl = readNonEmptyString(params.repoUrl) ?? readNonEmptyString(params.repo_url);
const repoRef = readNonEmptyString(params.repoRef) ?? readNonEmptyString(params.repo_ref);
return {
sessionId,
...(cwd ? { cwd } : {}),
...(promptBundleKey ? { promptBundleKey } : {}),
...(workspaceId ? { workspaceId } : {}),
...(repoUrl ? { repoUrl } : {}),
...(repoRef ? { repoRef } : {}),

View file

@ -0,0 +1,172 @@
import { constants as fsConstants } from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { createHash, type Hash } from "node:crypto";
import type { AdapterExecutionContext } from "@paperclipai/adapter-utils";
import { ensurePaperclipSkillSymlink, type PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils";
const DEFAULT_PAPERCLIP_INSTANCE_ID = "default";
type SkillEntry = PaperclipSkillEntry;
export interface ClaudePromptBundle {
bundleKey: string;
rootDir: string;
addDir: string;
instructionsFilePath: string | null;
}
function nonEmpty(value: string | undefined): string | null {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function resolveManagedClaudePromptCacheRoot(
env: NodeJS.ProcessEnv,
companyId: string,
): string {
const paperclipHome = nonEmpty(env.PAPERCLIP_HOME) ?? path.resolve(os.homedir(), ".paperclip");
const instanceId = nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? DEFAULT_PAPERCLIP_INSTANCE_ID;
return path.resolve(
paperclipHome,
"instances",
instanceId,
"companies",
companyId,
"claude-prompt-cache",
);
}
async function hashPathContents(
candidate: string,
hash: Hash,
relativePath: string,
seenDirectories: Set<string>,
): Promise<void> {
const stat = await fs.lstat(candidate);
if (stat.isSymbolicLink()) {
hash.update(`symlink:${relativePath}\n`);
const resolved = await fs.realpath(candidate).catch(() => null);
if (!resolved) {
hash.update("missing\n");
return;
}
await hashPathContents(resolved, hash, relativePath, seenDirectories);
return;
}
if (stat.isDirectory()) {
const realDir = await fs.realpath(candidate).catch(() => candidate);
hash.update(`dir:${relativePath}\n`);
if (seenDirectories.has(realDir)) {
hash.update("loop\n");
return;
}
seenDirectories.add(realDir);
const entries = await fs.readdir(candidate, { withFileTypes: true });
entries.sort((left, right) => left.name.localeCompare(right.name));
for (const entry of entries) {
const childRelativePath = relativePath.length > 0 ? `${relativePath}/${entry.name}` : entry.name;
await hashPathContents(path.join(candidate, entry.name), hash, childRelativePath, seenDirectories);
}
return;
}
if (stat.isFile()) {
hash.update(`file:${relativePath}\n`);
hash.update(await fs.readFile(candidate));
hash.update("\n");
return;
}
hash.update(`other:${relativePath}:${stat.mode}\n`);
}
async function buildClaudePromptBundleKey(input: {
skills: SkillEntry[];
instructionsContents: string | null;
}): Promise<string> {
const hash = createHash("sha256");
hash.update("paperclip-claude-prompt-bundle:v1\n");
if (input.instructionsContents) {
hash.update("instructions\n");
hash.update(input.instructionsContents);
hash.update("\n");
} else {
hash.update("instructions:none\n");
}
const sortedSkills = [...input.skills].sort((left, right) => left.runtimeName.localeCompare(right.runtimeName));
for (const entry of sortedSkills) {
hash.update(`skill:${entry.key}:${entry.runtimeName}\n`);
await hashPathContents(entry.source, hash, entry.runtimeName, new Set<string>());
}
return hash.digest("hex");
}
async function ensureReadableFile(targetPath: string, contents: string): Promise<void> {
try {
await fs.access(targetPath, fsConstants.R_OK);
return;
} catch {
// Fall through and materialize the file.
}
await fs.mkdir(path.dirname(targetPath), { recursive: true });
const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
try {
await fs.writeFile(tempPath, contents, "utf8");
await fs.rename(tempPath, targetPath);
} catch (err) {
const targetReadable = await fs.access(targetPath, fsConstants.R_OK).then(() => true).catch(() => false);
if (!targetReadable) {
throw err;
}
} finally {
await fs.rm(tempPath, { force: true }).catch(() => {});
}
}
export async function prepareClaudePromptBundle(input: {
companyId: string;
skills: SkillEntry[];
instructionsContents: string | null;
onLog: AdapterExecutionContext["onLog"];
}): Promise<ClaudePromptBundle> {
const { companyId, skills, instructionsContents, onLog } = input;
const bundleKey = await buildClaudePromptBundleKey({
skills,
instructionsContents,
});
const rootDir = path.join(resolveManagedClaudePromptCacheRoot(process.env, companyId), bundleKey);
const skillsHome = path.join(rootDir, ".claude", "skills");
await fs.mkdir(skillsHome, { recursive: true });
for (const entry of skills) {
const target = path.join(skillsHome, entry.runtimeName);
try {
await ensurePaperclipSkillSymlink(entry.source, target);
} catch (err) {
await onLog(
"stderr",
`[paperclip] Failed to materialize Claude skill "${entry.key}" into ${skillsHome}: ${err instanceof Error ? err.message : String(err)}\n`,
);
}
}
const instructionsFilePath = instructionsContents
? path.join(rootDir, "agent-instructions.md")
: null;
if (instructionsFilePath && instructionsContents) {
await ensureReadableFile(instructionsFilePath, instructionsContents);
}
return {
bundleKey,
rootDir,
addDir: rootDir,
instructionsFilePath,
};
}

View file

@ -47,7 +47,7 @@ async function buildClaudeSkillSnapshot(config: Record<string, unknown>): Promis
sourcePath: entry.source,
targetPath: null,
detail: desiredSet.has(entry.key)
? "Will be mounted into the ephemeral Claude skill directory on the next run."
? "Will be materialized into the stable Paperclip-managed Claude prompt bundle on the next run."
: null,
required: Boolean(entry.required),
requiredReason: entry.requiredReason ?? null,

View file

@ -27,6 +27,39 @@ describe("parseCodexJsonl", () => {
errorMessage: "resume failed",
});
});
it("uses the last agent message as the summary when commentary updates precede the final answer", () => {
const stdout = [
JSON.stringify({ type: "thread.started", thread_id: "thread_123" }),
JSON.stringify({
type: "item.completed",
item: { type: "reasoning", text: "Checking the heartbeat procedure" },
}),
JSON.stringify({
type: "item.completed",
item: { type: "agent_message", text: "Im checking out the issue and reading the docs now." },
}),
JSON.stringify({
type: "item.completed",
item: { type: "agent_message", text: "Fixed the issue and verified the targeted tests pass." },
}),
JSON.stringify({
type: "turn.completed",
usage: { input_tokens: 10, cached_input_tokens: 2, output_tokens: 4 },
}),
].join("\n");
expect(parseCodexJsonl(stdout)).toEqual({
sessionId: "thread_123",
summary: "Fixed the issue and verified the targeted tests pass.",
usage: {
inputTokens: 10,
cachedInputTokens: 2,
outputTokens: 4,
},
errorMessage: null,
});
});
});
describe("isCodexUnknownSessionError", () => {

View file

@ -2,7 +2,7 @@ import { asString, asNumber, parseObject, parseJson } from "@paperclipai/adapter
export function parseCodexJsonl(stdout: string) {
let sessionId: string | null = null;
const messages: string[] = [];
let finalMessage: string | null = null;
let errorMessage: string | null = null;
const usage = {
inputTokens: 0,
@ -33,7 +33,7 @@ export function parseCodexJsonl(stdout: string) {
const item = parseObject(event.item);
if (asString(item.type, "") === "agent_message") {
const text = asString(item.text, "");
if (text) messages.push(text);
if (text) finalMessage = text;
}
continue;
}
@ -55,7 +55,7 @@ export function parseCodexJsonl(stdout: string) {
return {
sessionId,
summary: messages.join("\n\n").trim(),
summary: finalMessage?.trim() ?? "",
usage,
errorMessage,
};

View file

@ -0,0 +1 @@
ALTER TABLE "heartbeat_runs" ADD COLUMN "process_group_id" integer;--> statement-breakpoint

File diff suppressed because it is too large Load diff

View file

@ -386,6 +386,13 @@
"when": 1775750400000,
"tag": "0054_draft_routines",
"breakpoints": true
},
{
"idx": 55,
"version": "7",
"when": 1775825256196,
"tag": "0055_kind_weapon_omega",
"breakpoints": true
}
]
}
}

View file

@ -32,6 +32,7 @@ export const heartbeatRuns = pgTable(
errorCode: text("error_code"),
externalRunId: text("external_run_id"),
processPid: integer("process_pid"),
processGroupId: integer("process_group_id"),
processStartedAt: timestamp("process_started_at", { withTimezone: true }),
retryOfRunId: uuid("retry_of_run_id").references((): AnyPgColumn => heartbeatRuns.id, {
onDelete: "set null",

View file

@ -6,7 +6,10 @@ import type {
TelemetryState,
} from "./types.js";
const DEFAULT_ENDPOINT = "https://telemetry.paperclip.ing/ingest";
const DEFAULT_ENDPOINTS = [
"https://telemetry.paperclip.ing/ingest",
"https://rusqrrg391.execute-api.us-east-1.amazonaws.com/ingest",
] as const;
const BATCH_SIZE = 50;
const SEND_TIMEOUT_MS = 5_000;
@ -44,29 +47,35 @@ export class TelemetryClient {
const events = this.queue.splice(0);
const state = this.getState();
const endpoint = this.config.endpoint ?? DEFAULT_ENDPOINT;
const endpoints = this.resolveEndpoints();
const app = this.config.app ?? "paperclip";
const schemaVersion = this.config.schemaVersion ?? "1";
const body = JSON.stringify({
app,
schemaVersion,
installId: state.installId,
version: this.version,
events,
});
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), SEND_TIMEOUT_MS);
try {
await fetch(endpoint, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
app,
schemaVersion,
installId: state.installId,
version: this.version,
events,
}),
signal: controller.signal,
});
} catch {
// Fire-and-forget: silent failure, no retries
} finally {
clearTimeout(timer);
for (const endpoint of endpoints) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), SEND_TIMEOUT_MS);
try {
const response = await fetch(endpoint, {
method: "POST",
headers: { "Content-Type": "application/json" },
body,
signal: controller.signal,
});
if (response.ok) {
return;
}
} catch {
// Try the next built-in endpoint before dropping the batch.
} finally {
clearTimeout(timer);
}
}
}
@ -102,4 +111,9 @@ export class TelemetryClient {
}
return this.state;
}
private resolveEndpoints(): readonly string[] {
const configured = this.config.endpoint?.trim();
return configured ? [configured] : DEFAULT_ENDPOINTS;
}
}

View file

@ -34,6 +34,7 @@ export interface HeartbeatRun {
errorCode: string | null;
externalRunId: string | null;
processPid: number | null;
processGroupId: number | null;
processStartedAt: Date | null;
retryOfRunId: string | null;
processLossRetryCount: number;