mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-15 10:30:37 +09:00
[codex] Harden heartbeat scheduling and runtime controls (#4223)
## Thinking Path > - Paperclip orchestrates AI agents through issue checkout, heartbeat runs, routines, and auditable control-plane state > - The runtime path has to recover from lost local processes, transient adapter failures, blocked dependencies, and routine coalescing without stranding work > - The existing branch carried several reliability fixes across heartbeat scheduling, issue runtime controls, routine dispatch, and operator-facing run state > - These changes belong together because they share backend contracts, migrations, and runtime status semantics > - This pull request groups the control-plane/runtime slice so it can merge independently from board UI polish and adapter sandbox work > - The benefit is safer heartbeat recovery, clearer runtime controls, and more predictable recurring execution behavior ## What Changed - Adds bounded heartbeat retry scheduling, scheduled retry state, and Codex transient failure recovery handling. - Tightens heartbeat process recovery, blocker wake behavior, issue comment wake handling, routine dispatch coalescing, and activity/dashboard bounds. - Adds runtime-control MCP tools and Paperclip skill docs for issue workspace runtime management. - Adds migrations `0061_lively_thor_girl.sql` and `0062_routine_run_dispatch_fingerprint.sql`. - Surfaces retry state in run ledger/agent UI and keeps related shared types synchronized. ## Verification - `pnpm exec vitest run server/src/__tests__/heartbeat-retry-scheduling.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/routines-service.test.ts` - `pnpm exec vitest run src/tools.test.ts` from `packages/mcp-server` ## Risks - Medium risk: this touches heartbeat recovery and routine dispatch, which are central execution paths. - Migration order matters if split branches land out of order: merge this PR before branches that assume the new runtime/routine fields. - Runtime retry behavior should be watched in CI and in local operator smoke tests because it changes how transient failures are resumed. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5-based coding agent runtime, shell/git tool use enabled. Exact hosted model build and context window are not exposed in this Paperclip heartbeat environment. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge
This commit is contained in:
Dotta
GitHub
parent
ab9051b595
commit
09d0678840
61 changed files with 17622 additions and 456 deletions
|
|
@ -29,6 +29,15 @@ console.log(JSON.stringify({ type: "turn.completed", usage: { input_tokens: 1, c
|
|||
await fs.chmod(commandPath, 0o755);
|
||||
}
|
||||
|
||||
async function writeFailingCodexCommand(commandPath: string, errorMessage: string): Promise<void> {
|
||||
const script = `#!/usr/bin/env node
|
||||
console.log(JSON.stringify({ type: "error", message: ${JSON.stringify(errorMessage)} }));
|
||||
process.exit(1);
|
||||
`;
|
||||
await fs.writeFile(commandPath, script, "utf8");
|
||||
await fs.chmod(commandPath, 0o755);
|
||||
}
|
||||
|
||||
type CapturePayload = {
|
||||
argv: string[];
|
||||
prompt: string;
|
||||
|
|
@ -369,6 +378,131 @@ describe("codex execute", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("classifies remote-compaction high-demand failures as retryable transient upstream errors", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-transient-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
const commandPath = path.join(root, "codex");
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
await writeFailingCodexCommand(
|
||||
commandPath,
|
||||
"Error running remote compact task: We're currently experiencing high demand, which may cause temporary errors.",
|
||||
);
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
process.env.HOME = root;
|
||||
|
||||
try {
|
||||
const result = await execute({
|
||||
runId: "run-transient-error",
|
||||
agent: {
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
name: "Codex Coder",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
},
|
||||
runtime: {
|
||||
sessionId: null,
|
||||
sessionParams: null,
|
||||
sessionDisplayId: null,
|
||||
taskKey: null,
|
||||
},
|
||||
config: {
|
||||
command: commandPath,
|
||||
cwd: workspace,
|
||||
promptTemplate: "Follow the paperclip heartbeat.",
|
||||
},
|
||||
context: {},
|
||||
authToken: "run-jwt-token",
|
||||
onLog: async () => {},
|
||||
});
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("codex_transient_upstream");
|
||||
expect(result.errorMessage).toContain("high demand");
|
||||
} finally {
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("uses safer invocation settings and a fresh-session handoff for codex transient fallback retries", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-fallback-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
const commandPath = path.join(root, "codex");
|
||||
const capturePath = path.join(root, "capture.json");
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
await writeFakeCodexCommand(commandPath);
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
process.env.HOME = root;
|
||||
|
||||
let commandNotes: string[] = [];
|
||||
try {
|
||||
const result = await execute({
|
||||
runId: "run-fallback",
|
||||
agent: {
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
name: "Codex Coder",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
},
|
||||
runtime: {
|
||||
sessionId: null,
|
||||
sessionParams: {
|
||||
sessionId: "codex-session-stale",
|
||||
cwd: workspace,
|
||||
},
|
||||
sessionDisplayId: "codex-session-stale",
|
||||
taskKey: null,
|
||||
},
|
||||
config: {
|
||||
command: commandPath,
|
||||
cwd: workspace,
|
||||
fastMode: true,
|
||||
model: "gpt-5.4",
|
||||
env: {
|
||||
PAPERCLIP_TEST_CAPTURE_PATH: capturePath,
|
||||
},
|
||||
promptTemplate: "Follow the paperclip heartbeat.",
|
||||
},
|
||||
context: {
|
||||
codexTransientFallbackMode: "fresh_session_safer_invocation",
|
||||
paperclipContinuationSummary: {
|
||||
key: "continuation-summary",
|
||||
title: "Continuation Summary",
|
||||
body: "Issue continuation summary for the next fresh session.",
|
||||
updatedAt: "2026-04-21T01:00:00.000Z",
|
||||
},
|
||||
},
|
||||
authToken: "run-jwt-token",
|
||||
onLog: async () => {},
|
||||
onMeta: async (meta) => {
|
||||
commandNotes = meta.commandNotes ?? [];
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.errorMessage).toBeNull();
|
||||
|
||||
const capture = JSON.parse(await fs.readFile(capturePath, "utf8")) as CapturePayload;
|
||||
expect(capture.argv).toEqual(expect.arrayContaining(["exec", "--json", "-"]));
|
||||
expect(capture.argv).not.toContain("resume");
|
||||
expect(capture.argv).not.toContain('service_tier="fast"');
|
||||
expect(capture.argv).not.toContain("features.fast_mode=true");
|
||||
expect(capture.prompt).toContain("Paperclip session handoff:");
|
||||
expect(capture.prompt).toContain("Issue continuation summary for the next fresh session.");
|
||||
expect(commandNotes).toContain("Codex transient fallback requested safer invocation settings for this retry.");
|
||||
expect(commandNotes).toContain("Codex transient fallback forced a fresh session with a continuation handoff.");
|
||||
} finally {
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("renders execution-stage wake instructions for reviewer and executor roles", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-stage-wake-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue