2026-03-08 16:43:34 +05:30
|
|
|
import { describe, expect, it, vi } from "vitest";
|
[codex] Retry max-turn exhausted heartbeats (#5096)
## Thinking Path
> - Paperclip orchestrates AI agents for autonomous companies, and
heartbeat execution is the control-plane loop that keeps assigned work
moving.
> - Max-turn exhaustion is a recoverable local-adapter stop condition
for Claude and Gemini agents when a run needs another heartbeat to
continue safely.
> - The previous behavior could leave max-turn continuation details hard
to inspect, and duplicate/stale continuation wakes could keep running
after issue state changed.
> - The adapter layer also needed to avoid trusting arbitrary
stdout/stderr text as scheduler control metadata.
> - This pull request adds bounded max-turn continuation scheduling,
visible retry state, structured stop metadata handling, and
stale/duplicate continuation guards.
> - The benefit is safer automatic continuation after max-turn stops,
clearer operator visibility, and fewer duplicate or stale agent runs.
## What Changed
- Replaces closed PR #4952, whose head repository was deleted.
- Rebases the recovered max-turn continuation branch onto current
`paperclipai/paperclip:master`.
- Adds max-turn continuation scheduling and retry-state plumbing for
heartbeat runs.
- Adds stale/duplicate continuation suppression when issue status,
ownership, or execution locks change.
- Normalizes Claude/Gemini max-turn detection around structured stop
metadata instead of unstructured stdout/stderr text.
- Surfaces max-turn continuation settings and retry visibility in the
board UI.
- Adds focused server, adapter, and UI tests for max-turn stop metadata,
retry scheduling, stale queued-run invalidation, adapter
parsing/execution, run ledger display, and agent config patching.
## Verification
- `pnpm install --no-frozen-lockfile` to refresh local dependencies
after rebasing onto current `master`.
- `pnpm run preflight:workspace-links && pnpm exec vitest run
server/src/__tests__/claude-local-adapter.test.ts
server/src/__tests__/claude-local-execute.test.ts
server/src/__tests__/gemini-local-adapter.test.ts
server/src/__tests__/gemini-local-execute.test.ts
server/src/__tests__/heartbeat-retry-scheduling.test.ts
server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
server/src/services/heartbeat-stop-metadata.test.ts
ui/src/components/IssueRunLedger.test.tsx
ui/src/lib/agent-config-patch.test.ts ui/src/lib/runRetryState.test.ts
--testTimeout=20000`
- `pnpm --filter @paperclipai/adapter-claude-local typecheck && pnpm
--filter @paperclipai/adapter-gemini-local typecheck && pnpm --filter
@paperclipai/server typecheck && pnpm --filter @paperclipai/ui
typecheck`
- UI screenshot note: the UI changes are limited to config/ledger state
rendering rather than layout changes; component/unit coverage above
verifies the rendered behavior.
## Risks
- Medium behavior risk: heartbeat retry gating now suppresses max-turn
continuations when issue state or execution locks drift, so any callers
that relied on stale continuations running will now see cancellation
instead.
- Low adapter risk: Claude/Gemini unstructured text no longer triggers
max-turn scheduler metadata, so only structured stop signals and Gemini
exit code 53 are trusted.
- No database migrations.
> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.
## Model Used
- OpenAI Codex coding agent, GPT-5-class model, tool-enabled local
repository editing and command execution.
## Checklist
- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] If this change affects the UI, I have included before/after
screenshots (not applicable: state/default rendering only; covered by
component/unit tests)
- [x] I have updated relevant documentation to reflect my changes (not
applicable: no user-facing command or docs contract changed)
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge
---------
Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-05-03 11:30:48 -05:00
|
|
|
import {
|
|
|
|
|
isGeminiTurnLimitResult,
|
|
|
|
|
isGeminiUnknownSessionError,
|
|
|
|
|
parseGeminiJsonl,
|
|
|
|
|
} from "@paperclipai/adapter-gemini-local/server";
|
2026-03-08 16:43:34 +05:30
|
|
|
import { parseGeminiStdoutLine } from "@paperclipai/adapter-gemini-local/ui";
|
|
|
|
|
import { printGeminiStreamEvent } from "@paperclipai/adapter-gemini-local/cli";
|
|
|
|
|
|
|
|
|
|
describe("gemini_local parser", () => {
|
|
|
|
|
it("extracts session, summary, usage, cost, and terminal error message", () => {
|
|
|
|
|
const stdout = [
|
|
|
|
|
JSON.stringify({ type: "system", subtype: "init", session_id: "gemini-session-1", model: "gemini-2.5-pro" }),
|
|
|
|
|
JSON.stringify({
|
|
|
|
|
type: "assistant",
|
|
|
|
|
message: {
|
|
|
|
|
content: [{ type: "output_text", text: "hello" }],
|
|
|
|
|
},
|
|
|
|
|
}),
|
|
|
|
|
JSON.stringify({
|
|
|
|
|
type: "result",
|
|
|
|
|
subtype: "success",
|
|
|
|
|
session_id: "gemini-session-1",
|
|
|
|
|
usage: {
|
|
|
|
|
promptTokenCount: 12,
|
|
|
|
|
cachedContentTokenCount: 3,
|
|
|
|
|
candidatesTokenCount: 7,
|
|
|
|
|
},
|
|
|
|
|
total_cost_usd: 0.00123,
|
|
|
|
|
result: "done",
|
|
|
|
|
}),
|
|
|
|
|
JSON.stringify({ type: "error", message: "model access denied" }),
|
|
|
|
|
].join("\n");
|
|
|
|
|
|
|
|
|
|
const parsed = parseGeminiJsonl(stdout);
|
|
|
|
|
expect(parsed.sessionId).toBe("gemini-session-1");
|
|
|
|
|
expect(parsed.summary).toBe("hello");
|
|
|
|
|
expect(parsed.usage).toEqual({
|
|
|
|
|
inputTokens: 12,
|
|
|
|
|
cachedInputTokens: 3,
|
|
|
|
|
outputTokens: 7,
|
|
|
|
|
});
|
|
|
|
|
expect(parsed.costUsd).toBeCloseTo(0.00123, 6);
|
|
|
|
|
expect(parsed.errorMessage).toBe("model access denied");
|
|
|
|
|
});
|
2026-03-12 01:34:00 +00:00
|
|
|
|
|
|
|
|
it("extracts structured questions", () => {
|
|
|
|
|
const stdout = [
|
|
|
|
|
JSON.stringify({
|
|
|
|
|
type: "assistant",
|
|
|
|
|
message: {
|
|
|
|
|
content: [
|
|
|
|
|
{ type: "output_text", text: "I have a question." },
|
|
|
|
|
{
|
|
|
|
|
type: "question",
|
|
|
|
|
prompt: "Which model?",
|
|
|
|
|
choices: [
|
|
|
|
|
{ key: "pro", label: "Gemini Pro", description: "Better" },
|
|
|
|
|
{ key: "flash", label: "Gemini Flash" },
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
}),
|
|
|
|
|
].join("\n");
|
|
|
|
|
|
|
|
|
|
const parsed = parseGeminiJsonl(stdout);
|
|
|
|
|
expect(parsed.summary).toBe("I have a question.");
|
|
|
|
|
expect(parsed.question).toEqual({
|
|
|
|
|
prompt: "Which model?",
|
|
|
|
|
choices: [
|
|
|
|
|
{ key: "pro", label: "Gemini Pro", description: "Better" },
|
|
|
|
|
{ key: "flash", label: "Gemini Flash", description: undefined },
|
|
|
|
|
],
|
|
|
|
|
});
|
|
|
|
|
});
|
2026-03-08 16:43:34 +05:30
|
|
|
});
|
|
|
|
|
|
|
|
|
|
describe("gemini_local stale session detection", () => {
|
|
|
|
|
it("treats missing session messages as an unknown session error", () => {
|
|
|
|
|
expect(isGeminiUnknownSessionError("", "unknown session id abc")).toBe(true);
|
|
|
|
|
expect(isGeminiUnknownSessionError("", "checkpoint latest not found")).toBe(true);
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
[codex] Retry max-turn exhausted heartbeats (#5096)
## Thinking Path
> - Paperclip orchestrates AI agents for autonomous companies, and
heartbeat execution is the control-plane loop that keeps assigned work
moving.
> - Max-turn exhaustion is a recoverable local-adapter stop condition
for Claude and Gemini agents when a run needs another heartbeat to
continue safely.
> - The previous behavior could leave max-turn continuation details hard
to inspect, and duplicate/stale continuation wakes could keep running
after issue state changed.
> - The adapter layer also needed to avoid trusting arbitrary
stdout/stderr text as scheduler control metadata.
> - This pull request adds bounded max-turn continuation scheduling,
visible retry state, structured stop metadata handling, and
stale/duplicate continuation guards.
> - The benefit is safer automatic continuation after max-turn stops,
clearer operator visibility, and fewer duplicate or stale agent runs.
## What Changed
- Replaces closed PR #4952, whose head repository was deleted.
- Rebases the recovered max-turn continuation branch onto current
`paperclipai/paperclip:master`.
- Adds max-turn continuation scheduling and retry-state plumbing for
heartbeat runs.
- Adds stale/duplicate continuation suppression when issue status,
ownership, or execution locks change.
- Normalizes Claude/Gemini max-turn detection around structured stop
metadata instead of unstructured stdout/stderr text.
- Surfaces max-turn continuation settings and retry visibility in the
board UI.
- Adds focused server, adapter, and UI tests for max-turn stop metadata,
retry scheduling, stale queued-run invalidation, adapter
parsing/execution, run ledger display, and agent config patching.
## Verification
- `pnpm install --no-frozen-lockfile` to refresh local dependencies
after rebasing onto current `master`.
- `pnpm run preflight:workspace-links && pnpm exec vitest run
server/src/__tests__/claude-local-adapter.test.ts
server/src/__tests__/claude-local-execute.test.ts
server/src/__tests__/gemini-local-adapter.test.ts
server/src/__tests__/gemini-local-execute.test.ts
server/src/__tests__/heartbeat-retry-scheduling.test.ts
server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
server/src/services/heartbeat-stop-metadata.test.ts
ui/src/components/IssueRunLedger.test.tsx
ui/src/lib/agent-config-patch.test.ts ui/src/lib/runRetryState.test.ts
--testTimeout=20000`
- `pnpm --filter @paperclipai/adapter-claude-local typecheck && pnpm
--filter @paperclipai/adapter-gemini-local typecheck && pnpm --filter
@paperclipai/server typecheck && pnpm --filter @paperclipai/ui
typecheck`
- UI screenshot note: the UI changes are limited to config/ledger state
rendering rather than layout changes; component/unit coverage above
verifies the rendered behavior.
## Risks
- Medium behavior risk: heartbeat retry gating now suppresses max-turn
continuations when issue state or execution locks drift, so any callers
that relied on stale continuations running will now see cancellation
instead.
- Low adapter risk: Claude/Gemini unstructured text no longer triggers
max-turn scheduler metadata, so only structured stop signals and Gemini
exit code 53 are trusted.
- No database migrations.
> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.
## Model Used
- OpenAI Codex coding agent, GPT-5-class model, tool-enabled local
repository editing and command execution.
## Checklist
- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] If this change affects the UI, I have included before/after
screenshots (not applicable: state/default rendering only; covered by
component/unit tests)
- [x] I have updated relevant documentation to reflect my changes (not
applicable: no user-facing command or docs contract changed)
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge
---------
Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-05-03 11:30:48 -05:00
|
|
|
describe("gemini_local turn-limit detection", () => {
|
|
|
|
|
it("detects structured turn-limit signals and exit code 53", () => {
|
|
|
|
|
expect(isGeminiTurnLimitResult({ status: "turn_limit" })).toBe(true);
|
|
|
|
|
expect(isGeminiTurnLimitResult({ stopReason: "max_turns_exhausted" })).toBe(true);
|
|
|
|
|
expect(isGeminiTurnLimitResult(null, 53)).toBe(true);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("checks every structured stop field for turn-limit exhaustion", () => {
|
|
|
|
|
expect(
|
|
|
|
|
isGeminiTurnLimitResult({
|
|
|
|
|
status: "success",
|
|
|
|
|
stopReason: "turn_limit_exhausted",
|
|
|
|
|
}),
|
|
|
|
|
).toBe(true);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("does not detect turn-limit exhaustion from unstructured error text", () => {
|
|
|
|
|
expect(isGeminiTurnLimitResult({ error: "max_turns reached" })).toBe(false);
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-03-08 16:43:34 +05:30
|
|
|
describe("gemini_local ui stdout parser", () => {
|
|
|
|
|
it("parses assistant, thinking, and result events", () => {
|
|
|
|
|
const ts = "2026-03-08T00:00:00.000Z";
|
|
|
|
|
|
|
|
|
|
expect(
|
|
|
|
|
parseGeminiStdoutLine(
|
|
|
|
|
JSON.stringify({
|
|
|
|
|
type: "assistant",
|
|
|
|
|
message: {
|
|
|
|
|
content: [
|
|
|
|
|
{ type: "output_text", text: "I checked the repo." },
|
|
|
|
|
{ type: "thinking", text: "Reviewing adapter registry" },
|
|
|
|
|
{ type: "tool_call", name: "shell", input: { command: "ls -1" } },
|
|
|
|
|
{ type: "tool_result", tool_use_id: "tool_1", output: "AGENTS.md\n", status: "ok" },
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
}),
|
|
|
|
|
ts,
|
|
|
|
|
),
|
|
|
|
|
).toEqual([
|
|
|
|
|
{ kind: "assistant", ts, text: "I checked the repo." },
|
|
|
|
|
{ kind: "thinking", ts, text: "Reviewing adapter registry" },
|
|
|
|
|
{ kind: "tool_call", ts, name: "shell", input: { command: "ls -1" } },
|
|
|
|
|
{ kind: "tool_result", ts, toolUseId: "tool_1", content: "AGENTS.md\n", isError: false },
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
expect(
|
|
|
|
|
parseGeminiStdoutLine(
|
|
|
|
|
JSON.stringify({
|
|
|
|
|
type: "result",
|
|
|
|
|
subtype: "success",
|
|
|
|
|
result: "Done",
|
|
|
|
|
usage: {
|
|
|
|
|
promptTokenCount: 10,
|
|
|
|
|
candidatesTokenCount: 5,
|
|
|
|
|
cachedContentTokenCount: 2,
|
|
|
|
|
},
|
|
|
|
|
total_cost_usd: 0.00042,
|
|
|
|
|
is_error: false,
|
|
|
|
|
}),
|
|
|
|
|
ts,
|
|
|
|
|
),
|
|
|
|
|
).toEqual([
|
|
|
|
|
{
|
|
|
|
|
kind: "result",
|
|
|
|
|
ts,
|
|
|
|
|
text: "Done",
|
|
|
|
|
inputTokens: 10,
|
|
|
|
|
outputTokens: 5,
|
|
|
|
|
cachedTokens: 2,
|
|
|
|
|
costUsd: 0.00042,
|
|
|
|
|
subtype: "success",
|
|
|
|
|
isError: false,
|
|
|
|
|
errors: [],
|
|
|
|
|
},
|
|
|
|
|
]);
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
function stripAnsi(value: string): string {
|
|
|
|
|
return value.replace(/\x1b\[[0-9;]*m/g, "");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
describe("gemini_local cli formatter", () => {
|
|
|
|
|
it("prints init, assistant, result, and error events", () => {
|
|
|
|
|
const spy = vi.spyOn(console, "log").mockImplementation(() => {});
|
|
|
|
|
let joined = "";
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
printGeminiStreamEvent(
|
|
|
|
|
JSON.stringify({ type: "system", subtype: "init", session_id: "gemini-session-1", model: "gemini-2.5-pro" }),
|
|
|
|
|
false,
|
|
|
|
|
);
|
|
|
|
|
printGeminiStreamEvent(
|
|
|
|
|
JSON.stringify({
|
|
|
|
|
type: "assistant",
|
|
|
|
|
message: { content: [{ type: "output_text", text: "hello" }] },
|
|
|
|
|
}),
|
|
|
|
|
false,
|
|
|
|
|
);
|
|
|
|
|
printGeminiStreamEvent(
|
|
|
|
|
JSON.stringify({
|
|
|
|
|
type: "result",
|
|
|
|
|
subtype: "success",
|
|
|
|
|
usage: {
|
|
|
|
|
promptTokenCount: 10,
|
|
|
|
|
candidatesTokenCount: 5,
|
|
|
|
|
cachedContentTokenCount: 2,
|
|
|
|
|
},
|
|
|
|
|
total_cost_usd: 0.00042,
|
|
|
|
|
}),
|
|
|
|
|
false,
|
|
|
|
|
);
|
|
|
|
|
printGeminiStreamEvent(
|
|
|
|
|
JSON.stringify({ type: "error", message: "boom" }),
|
|
|
|
|
false,
|
|
|
|
|
);
|
|
|
|
|
joined = spy.mock.calls.map((call) => stripAnsi(call.join(" "))).join("\n");
|
|
|
|
|
} finally {
|
|
|
|
|
spy.mockRestore();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
expect(joined).toContain("Gemini init");
|
|
|
|
|
expect(joined).toContain("assistant: hello");
|
|
|
|
|
expect(joined).toContain("tokens: in=10 out=5 cached=2 cost=$0.000420");
|
|
|
|
|
expect(joined).toContain("error: boom");
|
|
|
|
|
});
|
|
|
|
|
});
|