mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-16 10:50:38 +09:00
[codex] Retry max-turn exhausted heartbeats (#5096)
## Thinking Path > - Paperclip orchestrates AI agents for autonomous companies, and heartbeat execution is the control-plane loop that keeps assigned work moving. > - Max-turn exhaustion is a recoverable local-adapter stop condition for Claude and Gemini agents when a run needs another heartbeat to continue safely. > - The previous behavior could leave max-turn continuation details hard to inspect, and duplicate/stale continuation wakes could keep running after issue state changed. > - The adapter layer also needed to avoid trusting arbitrary stdout/stderr text as scheduler control metadata. > - This pull request adds bounded max-turn continuation scheduling, visible retry state, structured stop metadata handling, and stale/duplicate continuation guards. > - The benefit is safer automatic continuation after max-turn stops, clearer operator visibility, and fewer duplicate or stale agent runs. ## What Changed - Replaces closed PR #4952, whose head repository was deleted. - Rebases the recovered max-turn continuation branch onto current `paperclipai/paperclip:master`. - Adds max-turn continuation scheduling and retry-state plumbing for heartbeat runs. - Adds stale/duplicate continuation suppression when issue status, ownership, or execution locks change. - Normalizes Claude/Gemini max-turn detection around structured stop metadata instead of unstructured stdout/stderr text. - Surfaces max-turn continuation settings and retry visibility in the board UI. - Adds focused server, adapter, and UI tests for max-turn stop metadata, retry scheduling, stale queued-run invalidation, adapter parsing/execution, run ledger display, and agent config patching. ## Verification - `pnpm install --no-frozen-lockfile` to refresh local dependencies after rebasing onto current `master`. - `pnpm run preflight:workspace-links && pnpm exec vitest run server/src/__tests__/claude-local-adapter.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/gemini-local-adapter.test.ts server/src/__tests__/gemini-local-execute.test.ts server/src/__tests__/heartbeat-retry-scheduling.test.ts server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts server/src/services/heartbeat-stop-metadata.test.ts ui/src/components/IssueRunLedger.test.tsx ui/src/lib/agent-config-patch.test.ts ui/src/lib/runRetryState.test.ts --testTimeout=20000` - `pnpm --filter @paperclipai/adapter-claude-local typecheck && pnpm --filter @paperclipai/adapter-gemini-local typecheck && pnpm --filter @paperclipai/server typecheck && pnpm --filter @paperclipai/ui typecheck` - UI screenshot note: the UI changes are limited to config/ledger state rendering rather than layout changes; component/unit coverage above verifies the rendered behavior. ## Risks - Medium behavior risk: heartbeat retry gating now suppresses max-turn continuations when issue state or execution locks drift, so any callers that relied on stale continuations running will now see cancellation instead. - Low adapter risk: Claude/Gemini unstructured text no longer triggers max-turn scheduler metadata, so only structured stop signals and Gemini exit code 53 are trusted. - No database migrations. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex coding agent, GPT-5-class model, tool-enabled local repository editing and command execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots (not applicable: state/default rendering only; covered by component/unit tests) - [x] I have updated relevant documentation to reflect my changes (not applicable: no user-facing command or docs contract changed) - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
57229d0f24
commit
15eac43b43
21 changed files with 1915 additions and 120 deletions
|
|
@ -27,7 +27,7 @@ import {
|
|||
} from "@/components/ui/popover";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { FolderOpen, Heart, ChevronDown, X } from "lucide-react";
|
||||
import { cn } from "../lib/utils";
|
||||
import { asBoolean, asFiniteNumber, asObject, cn } from "../lib/utils";
|
||||
import { extractModelName, extractProviderId } from "../lib/model-utils";
|
||||
import { queryKeys } from "../lib/queryKeys";
|
||||
import { useCompany } from "../context/CompanyContext";
|
||||
|
|
@ -175,6 +175,19 @@ const claudeThinkingEffortOptions = [
|
|||
{ id: "high", label: "High" },
|
||||
] as const;
|
||||
|
||||
const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;
|
||||
const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10;
|
||||
const MAX_TURN_CONTINUATION_DEFAULT_DELAY_SEC = 1;
|
||||
const MAX_TURN_CONTINUATION_MAX_DELAY_SEC = 300;
|
||||
|
||||
function clampInteger(value: number, min: number, max: number) {
|
||||
return Math.max(min, Math.min(max, Math.floor(value)));
|
||||
}
|
||||
|
||||
function clampDelayMsFromSeconds(value: number) {
|
||||
return clampInteger(value, 0, MAX_TURN_CONTINUATION_MAX_DELAY_SEC) * 1000;
|
||||
}
|
||||
|
||||
|
||||
/* ---- Form ---- */
|
||||
|
||||
|
|
@ -628,6 +641,27 @@ export function AgentConfigForm(props: AgentConfigFormProps) {
|
|||
const currentDefaultEnvironmentId = isCreate
|
||||
? val!.defaultEnvironmentId ?? ""
|
||||
: eff("identity", "defaultEnvironmentId", props.agent.defaultEnvironmentId ?? "");
|
||||
const effectiveHeartbeat = asObject(effectiveRuntimeConfig.heartbeat);
|
||||
const maxTurnContinuation = asObject(effectiveHeartbeat.maxTurnContinuation);
|
||||
const maxTurnContinuationEnabled = asBoolean(maxTurnContinuation.enabled, true);
|
||||
const maxTurnContinuationMaxAttempts = clampInteger(
|
||||
asFiniteNumber(maxTurnContinuation.maxAttempts, MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS),
|
||||
0,
|
||||
MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP,
|
||||
);
|
||||
const maxTurnContinuationDelaySec = clampInteger(
|
||||
asFiniteNumber(maxTurnContinuation.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_SEC * 1000) / 1000,
|
||||
0,
|
||||
MAX_TURN_CONTINUATION_MAX_DELAY_SEC,
|
||||
);
|
||||
|
||||
function updateMaxTurnContinuation(patch: Record<string, unknown>) {
|
||||
mark("heartbeat", "maxTurnContinuation", {
|
||||
...maxTurnContinuation,
|
||||
...patch,
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<div className={cn("relative", cards && "space-y-6")}>
|
||||
{/* ---- Floating Save button (edit mode, when dirty) ---- */}
|
||||
|
|
@ -1182,6 +1216,40 @@ export function AgentConfigForm(props: AgentConfigFormProps) {
|
|||
className={inputClass}
|
||||
/>
|
||||
</Field>
|
||||
<div className="rounded-md border border-border/70 px-3 py-2">
|
||||
<ToggleField
|
||||
label="Continue after max-turn stop"
|
||||
hint={help.maxTurnContinuationEnabled}
|
||||
checked={maxTurnContinuationEnabled}
|
||||
onChange={(v) => updateMaxTurnContinuation({ enabled: v })}
|
||||
/>
|
||||
{maxTurnContinuationEnabled ? (
|
||||
<div className="mt-3 grid gap-3 sm:grid-cols-2">
|
||||
<Field label="Continuation attempts" hint={help.maxTurnContinuationMaxAttempts}>
|
||||
<DraftNumberInput
|
||||
value={maxTurnContinuationMaxAttempts}
|
||||
onCommit={(v) =>
|
||||
updateMaxTurnContinuation({
|
||||
maxAttempts: clampInteger(v, 0, MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP),
|
||||
})}
|
||||
immediate
|
||||
className={inputClass}
|
||||
/>
|
||||
</Field>
|
||||
<Field label="Continuation delay (sec)" hint={help.maxTurnContinuationDelaySec}>
|
||||
<DraftNumberInput
|
||||
value={maxTurnContinuationDelaySec}
|
||||
onCommit={(v) =>
|
||||
updateMaxTurnContinuation({
|
||||
delayMs: clampDelayMsFromSeconds(v),
|
||||
})}
|
||||
immediate
|
||||
className={inputClass}
|
||||
/>
|
||||
</Field>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
</CollapsibleSection>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -317,6 +317,44 @@ describe("IssueRunLedger", () => {
|
|||
expect(container.textContent).toContain("Manual intervention required");
|
||||
});
|
||||
|
||||
it("labels max-turn stops and continuation retries without confusing them with per-run turns", () => {
|
||||
renderLedger({
|
||||
runs: [
|
||||
createRun({
|
||||
runId: "run-scheduled-continuation",
|
||||
status: "scheduled_retry",
|
||||
finishedAt: null,
|
||||
livenessState: null,
|
||||
livenessReason: null,
|
||||
retryOfRunId: "run-max-turns",
|
||||
scheduledRetryAt: "2026-04-18T20:15:00.000Z",
|
||||
scheduledRetryAttempt: 1,
|
||||
scheduledRetryReason: "max_turns_continuation",
|
||||
}),
|
||||
createRun({
|
||||
runId: "run-max-turns",
|
||||
resultJson: { stopReason: "max_turns_exhausted" },
|
||||
createdAt: "2026-04-18T19:57:00.000Z",
|
||||
}),
|
||||
createRun({
|
||||
runId: "run-continuation-exhausted",
|
||||
status: "failed",
|
||||
createdAt: "2026-04-18T19:56:00.000Z",
|
||||
retryOfRunId: "run-max-turns",
|
||||
scheduledRetryAttempt: 3,
|
||||
scheduledRetryReason: "max_turns_continuation",
|
||||
retryExhaustedReason: "Bounded retry exhausted after 3 scheduled attempts; no further automatic retry will be queued",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(container.textContent).toContain("Continuation scheduled");
|
||||
expect(container.textContent).toContain("Max-turn continuation");
|
||||
expect(container.textContent).toContain("Next continuation");
|
||||
expect(container.textContent).toContain("Stop max turns exhausted");
|
||||
expect(container.textContent).toContain("Continuation exhausted");
|
||||
});
|
||||
|
||||
it("shows timeout, cancel, and budget stop reasons without raw logs", () => {
|
||||
renderLedger({
|
||||
runs: [
|
||||
|
|
|
|||
|
|
@ -311,6 +311,7 @@ function stopReasonLabel(run: RunForIssue) {
|
|||
if (timeoutFired || stopReason === "timeout") {
|
||||
return timeoutText ? `timeout (${timeoutText})` : "timeout";
|
||||
}
|
||||
if (stopReason === "max_turns_exhausted" || stopReason === "turn_limit_exhausted") return "max turns exhausted";
|
||||
if (stopReason === "budget_paused") return "budget paused";
|
||||
if (stopReason === "cancelled") return "cancelled";
|
||||
if (stopReason === "paused") return "paused by board";
|
||||
|
|
|
|||
|
|
@ -56,6 +56,9 @@ export const help: Record<string, string> = {
|
|||
wakeOnDemand: "Allow this agent to be woken by assignments, API calls, UI actions, or automated systems.",
|
||||
cooldownSec: "Minimum seconds between consecutive heartbeat runs.",
|
||||
maxConcurrentRuns: "Maximum number of heartbeat runs that can execute simultaneously for this agent.",
|
||||
maxTurnContinuationEnabled: "Automatically queue bounded continuation runs when an adapter stops because its per-run turn cap was exhausted.",
|
||||
maxTurnContinuationMaxAttempts: "Maximum automatic continuations after one max-turn stop. This is separate from max turns per run.",
|
||||
maxTurnContinuationDelaySec: "Seconds to wait before starting each max-turn continuation.",
|
||||
budgetMonthlyCents: "Monthly spending limit in cents. 0 means no limit.",
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue