mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-19 12:10:37 +09:00
## Thinking Path > - Paperclip orchestrates AI agents for autonomous companies, and heartbeat execution is the control-plane loop that keeps assigned work moving. > - Max-turn exhaustion is a recoverable local-adapter stop condition for Claude and Gemini agents when a run needs another heartbeat to continue safely. > - The previous behavior could leave max-turn continuation details hard to inspect, and duplicate/stale continuation wakes could keep running after issue state changed. > - The adapter layer also needed to avoid trusting arbitrary stdout/stderr text as scheduler control metadata. > - This pull request adds bounded max-turn continuation scheduling, visible retry state, structured stop metadata handling, and stale/duplicate continuation guards. > - The benefit is safer automatic continuation after max-turn stops, clearer operator visibility, and fewer duplicate or stale agent runs. ## What Changed - Replaces closed PR #4952, whose head repository was deleted. - Rebases the recovered max-turn continuation branch onto current `paperclipai/paperclip:master`. - Adds max-turn continuation scheduling and retry-state plumbing for heartbeat runs. - Adds stale/duplicate continuation suppression when issue status, ownership, or execution locks change. - Normalizes Claude/Gemini max-turn detection around structured stop metadata instead of unstructured stdout/stderr text. - Surfaces max-turn continuation settings and retry visibility in the board UI. - Adds focused server, adapter, and UI tests for max-turn stop metadata, retry scheduling, stale queued-run invalidation, adapter parsing/execution, run ledger display, and agent config patching. ## Verification - `pnpm install --no-frozen-lockfile` to refresh local dependencies after rebasing onto current `master`. - `pnpm run preflight:workspace-links && pnpm exec vitest run server/src/__tests__/claude-local-adapter.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/gemini-local-adapter.test.ts server/src/__tests__/gemini-local-execute.test.ts server/src/__tests__/heartbeat-retry-scheduling.test.ts server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts server/src/services/heartbeat-stop-metadata.test.ts ui/src/components/IssueRunLedger.test.tsx ui/src/lib/agent-config-patch.test.ts ui/src/lib/runRetryState.test.ts --testTimeout=20000` - `pnpm --filter @paperclipai/adapter-claude-local typecheck && pnpm --filter @paperclipai/adapter-gemini-local typecheck && pnpm --filter @paperclipai/server typecheck && pnpm --filter @paperclipai/ui typecheck` - UI screenshot note: the UI changes are limited to config/ledger state rendering rather than layout changes; component/unit coverage above verifies the rendered behavior. ## Risks - Medium behavior risk: heartbeat retry gating now suppresses max-turn continuations when issue state or execution locks drift, so any callers that relied on stale continuations running will now see cancellation instead. - Low adapter risk: Claude/Gemini unstructured text no longer triggers max-turn scheduler metadata, so only structured stop signals and Gemini exit code 53 are trusted. - No database migrations. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex coding agent, GPT-5-class model, tool-enabled local repository editing and command execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots (not applicable: state/default rendering only; covered by component/unit tests) - [x] I have updated relevant documentation to reflect my changes (not applicable: no user-facing command or docs contract changed) - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
391 lines
12 KiB
TypeScript
391 lines
12 KiB
TypeScript
import type { UsageSummary } from "@paperclipai/adapter-utils";
|
|
import {
|
|
asString,
|
|
asNumber,
|
|
parseObject,
|
|
parseJson,
|
|
} from "@paperclipai/adapter-utils/server-utils";
|
|
|
|
const CLAUDE_AUTH_REQUIRED_RE = /(?:not\s+logged\s+in|please\s+log\s+in|please\s+run\s+`?claude\s+login`?|login\s+required|requires\s+login|unauthorized|authentication\s+required)/i;
|
|
const URL_RE = /(https?:\/\/[^\s'"`<>()[\]{};,!?]+[^\s'"`<>()[\]{};,!.?:]+)/gi;
|
|
|
|
const CLAUDE_TRANSIENT_UPSTREAM_RE =
|
|
/(?:rate[-\s]?limit(?:ed)?|rate_limit_error|too\s+many\s+requests|\b429\b|overloaded(?:_error)?|server\s+overloaded|service\s+unavailable|\b503\b|\b529\b|high\s+demand|try\s+again\s+later|temporarily\s+unavailable|throttl(?:ed|ing)|throttlingexception|servicequotaexceededexception|out\s+of\s+extra\s+usage|extra\s+usage\b|claude\s+usage\s+limit\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|usage\s+limit\s+reached|usage\s+cap\s+reached)/i;
|
|
const CLAUDE_EXTRA_USAGE_RESET_RE =
|
|
/(?:out\s+of\s+extra\s+usage|extra\s+usage|usage\s+limit\s+reached|usage\s+cap\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|claude\s+usage\s+limit\s+reached)[\s\S]{0,80}?\bresets?\s+(?:at\s+)?([^\n()]+?)(?:\s*\(([^)]+)\))?(?:[.!]|\n|$)/i;
|
|
|
|
export function parseClaudeStreamJson(stdout: string) {
|
|
let sessionId: string | null = null;
|
|
let model = "";
|
|
let finalResult: Record<string, unknown> | null = null;
|
|
const assistantTexts: string[] = [];
|
|
|
|
for (const rawLine of stdout.split(/\r?\n/)) {
|
|
const line = rawLine.trim();
|
|
if (!line) continue;
|
|
const event = parseJson(line);
|
|
if (!event) continue;
|
|
|
|
const type = asString(event.type, "");
|
|
if (type === "system" && asString(event.subtype, "") === "init") {
|
|
sessionId = asString(event.session_id, sessionId ?? "") || sessionId;
|
|
model = asString(event.model, model);
|
|
continue;
|
|
}
|
|
|
|
if (type === "assistant") {
|
|
sessionId = asString(event.session_id, sessionId ?? "") || sessionId;
|
|
const message = parseObject(event.message);
|
|
const content = Array.isArray(message.content) ? message.content : [];
|
|
for (const entry of content) {
|
|
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) continue;
|
|
const block = entry as Record<string, unknown>;
|
|
if (asString(block.type, "") === "text") {
|
|
const text = asString(block.text, "");
|
|
if (text) assistantTexts.push(text);
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (type === "result") {
|
|
finalResult = event;
|
|
sessionId = asString(event.session_id, sessionId ?? "") || sessionId;
|
|
}
|
|
}
|
|
|
|
if (!finalResult) {
|
|
return {
|
|
sessionId,
|
|
model,
|
|
costUsd: null as number | null,
|
|
usage: null as UsageSummary | null,
|
|
summary: assistantTexts.join("\n\n").trim(),
|
|
resultJson: null as Record<string, unknown> | null,
|
|
};
|
|
}
|
|
|
|
const usageObj = parseObject(finalResult.usage);
|
|
const usage: UsageSummary = {
|
|
inputTokens: asNumber(usageObj.input_tokens, 0),
|
|
cachedInputTokens: asNumber(usageObj.cache_read_input_tokens, 0),
|
|
outputTokens: asNumber(usageObj.output_tokens, 0),
|
|
};
|
|
const costRaw = finalResult.total_cost_usd;
|
|
const costUsd = typeof costRaw === "number" && Number.isFinite(costRaw) ? costRaw : null;
|
|
const summary = asString(finalResult.result, assistantTexts.join("\n\n")).trim();
|
|
|
|
return {
|
|
sessionId,
|
|
model,
|
|
costUsd,
|
|
usage,
|
|
summary,
|
|
resultJson: finalResult,
|
|
};
|
|
}
|
|
|
|
function extractClaudeErrorMessages(parsed: Record<string, unknown>): string[] {
|
|
const raw = Array.isArray(parsed.errors) ? parsed.errors : [];
|
|
const messages: string[] = [];
|
|
|
|
for (const entry of raw) {
|
|
if (typeof entry === "string") {
|
|
const msg = entry.trim();
|
|
if (msg) messages.push(msg);
|
|
continue;
|
|
}
|
|
|
|
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) {
|
|
continue;
|
|
}
|
|
|
|
const obj = entry as Record<string, unknown>;
|
|
const msg = asString(obj.message, "") || asString(obj.error, "") || asString(obj.code, "");
|
|
if (msg) {
|
|
messages.push(msg);
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
messages.push(JSON.stringify(obj));
|
|
} catch {
|
|
// skip non-serializable entry
|
|
}
|
|
}
|
|
|
|
return messages;
|
|
}
|
|
|
|
export function extractClaudeLoginUrl(text: string): string | null {
|
|
const match = text.match(URL_RE);
|
|
if (!match || match.length === 0) return null;
|
|
for (const rawUrl of match) {
|
|
const cleaned = rawUrl.replace(/[\])}.!,?;:'\"]+$/g, "");
|
|
if (cleaned.includes("claude") || cleaned.includes("anthropic") || cleaned.includes("auth")) {
|
|
return cleaned;
|
|
}
|
|
}
|
|
return match[0]?.replace(/[\])}.!,?;:'\"]+$/g, "") ?? null;
|
|
}
|
|
|
|
export function detectClaudeLoginRequired(input: {
|
|
parsed: Record<string, unknown> | null;
|
|
stdout: string;
|
|
stderr: string;
|
|
}): { requiresLogin: boolean; loginUrl: string | null } {
|
|
const resultText = asString(input.parsed?.result, "").trim();
|
|
const messages = [resultText, ...extractClaudeErrorMessages(input.parsed ?? {}), input.stdout, input.stderr]
|
|
.join("\n")
|
|
.split(/\r?\n/)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
|
|
const requiresLogin = messages.some((line) => CLAUDE_AUTH_REQUIRED_RE.test(line));
|
|
return {
|
|
requiresLogin,
|
|
loginUrl: extractClaudeLoginUrl([input.stdout, input.stderr].join("\n")),
|
|
};
|
|
}
|
|
|
|
export function describeClaudeFailure(parsed: Record<string, unknown>): string | null {
|
|
const subtype = asString(parsed.subtype, "");
|
|
const resultText = asString(parsed.result, "").trim();
|
|
const errors = extractClaudeErrorMessages(parsed);
|
|
|
|
let detail = resultText;
|
|
if (!detail && errors.length > 0) {
|
|
detail = errors[0] ?? "";
|
|
}
|
|
|
|
const parts = ["Claude run failed"];
|
|
if (subtype) parts.push(`subtype=${subtype}`);
|
|
if (detail) parts.push(detail);
|
|
return parts.length > 1 ? parts.join(": ") : null;
|
|
}
|
|
|
|
export function isClaudeMaxTurnsResult(parsed: Record<string, unknown> | null | undefined): boolean {
|
|
if (!parsed) return false;
|
|
|
|
const subtype = asString(parsed.subtype, "").trim().toLowerCase();
|
|
if (subtype === "error_max_turns") return true;
|
|
|
|
const structuredStopReasons = [
|
|
parsed.stop_reason,
|
|
parsed.stopReason,
|
|
parsed.error_code,
|
|
parsed.errorCode,
|
|
].map((value) => asString(value, "").trim().toLowerCase());
|
|
|
|
return structuredStopReasons.some((reason) =>
|
|
reason === "max_turns" ||
|
|
reason === "max_turns_exhausted" ||
|
|
reason === "turn_limit" ||
|
|
reason === "turn_limit_exhausted",
|
|
);
|
|
}
|
|
|
|
export function isClaudeUnknownSessionError(parsed: Record<string, unknown>): boolean {
|
|
const resultText = asString(parsed.result, "").trim();
|
|
const allMessages = [resultText, ...extractClaudeErrorMessages(parsed)]
|
|
.map((msg) => msg.trim())
|
|
.filter(Boolean);
|
|
|
|
return allMessages.some((msg) =>
|
|
/no conversation found with session id|unknown session|session .* not found/i.test(msg),
|
|
);
|
|
}
|
|
|
|
function buildClaudeTransientHaystack(input: {
|
|
parsed?: Record<string, unknown> | null;
|
|
stdout?: string | null;
|
|
stderr?: string | null;
|
|
errorMessage?: string | null;
|
|
}): string {
|
|
const parsed = input.parsed ?? null;
|
|
const resultText = parsed ? asString(parsed.result, "") : "";
|
|
const parsedErrors = parsed ? extractClaudeErrorMessages(parsed) : [];
|
|
return [
|
|
input.errorMessage ?? "",
|
|
resultText,
|
|
...parsedErrors,
|
|
input.stdout ?? "",
|
|
input.stderr ?? "",
|
|
]
|
|
.join("\n")
|
|
.split(/\r?\n/)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.join("\n");
|
|
}
|
|
|
|
function readTimeZoneParts(date: Date, timeZone: string) {
|
|
const values = new Map(
|
|
new Intl.DateTimeFormat("en-US", {
|
|
timeZone,
|
|
hourCycle: "h23",
|
|
year: "numeric",
|
|
month: "2-digit",
|
|
day: "2-digit",
|
|
hour: "2-digit",
|
|
minute: "2-digit",
|
|
}).formatToParts(date).map((part) => [part.type, part.value]),
|
|
);
|
|
return {
|
|
year: Number.parseInt(values.get("year") ?? "", 10),
|
|
month: Number.parseInt(values.get("month") ?? "", 10),
|
|
day: Number.parseInt(values.get("day") ?? "", 10),
|
|
hour: Number.parseInt(values.get("hour") ?? "", 10),
|
|
minute: Number.parseInt(values.get("minute") ?? "", 10),
|
|
};
|
|
}
|
|
|
|
function normalizeResetTimeZone(timeZoneHint: string | null | undefined): string | null {
|
|
const normalized = timeZoneHint?.trim();
|
|
if (!normalized) return null;
|
|
if (/^(?:utc|gmt)$/i.test(normalized)) return "UTC";
|
|
|
|
try {
|
|
new Intl.DateTimeFormat("en-US", { timeZone: normalized }).format(new Date(0));
|
|
return normalized;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function dateFromTimeZoneWallClock(input: {
|
|
year: number;
|
|
month: number;
|
|
day: number;
|
|
hour: number;
|
|
minute: number;
|
|
timeZone: string;
|
|
}): Date | null {
|
|
let candidate = new Date(Date.UTC(input.year, input.month - 1, input.day, input.hour, input.minute, 0, 0));
|
|
const targetUtc = Date.UTC(input.year, input.month - 1, input.day, input.hour, input.minute, 0, 0);
|
|
|
|
for (let attempt = 0; attempt < 4; attempt += 1) {
|
|
const actual = readTimeZoneParts(candidate, input.timeZone);
|
|
const actualUtc = Date.UTC(actual.year, actual.month - 1, actual.day, actual.hour, actual.minute, 0, 0);
|
|
const offsetMs = targetUtc - actualUtc;
|
|
if (offsetMs === 0) break;
|
|
candidate = new Date(candidate.getTime() + offsetMs);
|
|
}
|
|
|
|
const verified = readTimeZoneParts(candidate, input.timeZone);
|
|
if (
|
|
verified.year !== input.year ||
|
|
verified.month !== input.month ||
|
|
verified.day !== input.day ||
|
|
verified.hour !== input.hour ||
|
|
verified.minute !== input.minute
|
|
) {
|
|
return null;
|
|
}
|
|
|
|
return candidate;
|
|
}
|
|
|
|
function nextClockTimeInTimeZone(input: {
|
|
now: Date;
|
|
hour: number;
|
|
minute: number;
|
|
timeZoneHint: string;
|
|
}): Date | null {
|
|
const timeZone = normalizeResetTimeZone(input.timeZoneHint);
|
|
if (!timeZone) return null;
|
|
|
|
const nowParts = readTimeZoneParts(input.now, timeZone);
|
|
let retryAt = dateFromTimeZoneWallClock({
|
|
year: nowParts.year,
|
|
month: nowParts.month,
|
|
day: nowParts.day,
|
|
hour: input.hour,
|
|
minute: input.minute,
|
|
timeZone,
|
|
});
|
|
if (!retryAt) return null;
|
|
|
|
if (retryAt.getTime() <= input.now.getTime()) {
|
|
const nextDay = new Date(Date.UTC(nowParts.year, nowParts.month - 1, nowParts.day + 1, 0, 0, 0, 0));
|
|
retryAt = dateFromTimeZoneWallClock({
|
|
year: nextDay.getUTCFullYear(),
|
|
month: nextDay.getUTCMonth() + 1,
|
|
day: nextDay.getUTCDate(),
|
|
hour: input.hour,
|
|
minute: input.minute,
|
|
timeZone,
|
|
});
|
|
}
|
|
|
|
return retryAt;
|
|
}
|
|
|
|
function parseClaudeResetClockTime(clockText: string, now: Date, timeZoneHint?: string | null): Date | null {
|
|
const normalized = clockText.trim().replace(/\s+/g, " ");
|
|
const match = normalized.match(/^(\d{1,2})(?::(\d{2}))?\s*([ap])\.?\s*m\.?/i);
|
|
if (!match) return null;
|
|
|
|
const hour12 = Number.parseInt(match[1] ?? "", 10);
|
|
const minute = Number.parseInt(match[2] ?? "0", 10);
|
|
if (!Number.isInteger(hour12) || hour12 < 1 || hour12 > 12) return null;
|
|
if (!Number.isInteger(minute) || minute < 0 || minute > 59) return null;
|
|
|
|
let hour24 = hour12 % 12;
|
|
if ((match[3] ?? "").toLowerCase() === "p") hour24 += 12;
|
|
|
|
if (timeZoneHint) {
|
|
const explicitRetryAt = nextClockTimeInTimeZone({
|
|
now,
|
|
hour: hour24,
|
|
minute,
|
|
timeZoneHint,
|
|
});
|
|
if (explicitRetryAt) return explicitRetryAt;
|
|
}
|
|
|
|
const retryAt = new Date(now);
|
|
retryAt.setHours(hour24, minute, 0, 0);
|
|
if (retryAt.getTime() <= now.getTime()) {
|
|
retryAt.setDate(retryAt.getDate() + 1);
|
|
}
|
|
return retryAt;
|
|
}
|
|
|
|
export function extractClaudeRetryNotBefore(
|
|
input: {
|
|
parsed?: Record<string, unknown> | null;
|
|
stdout?: string | null;
|
|
stderr?: string | null;
|
|
errorMessage?: string | null;
|
|
},
|
|
now = new Date(),
|
|
): Date | null {
|
|
const haystack = buildClaudeTransientHaystack(input);
|
|
const match = haystack.match(CLAUDE_EXTRA_USAGE_RESET_RE);
|
|
if (!match) return null;
|
|
return parseClaudeResetClockTime(match[1] ?? "", now, match[2]);
|
|
}
|
|
|
|
export function isClaudeTransientUpstreamError(input: {
|
|
parsed?: Record<string, unknown> | null;
|
|
stdout?: string | null;
|
|
stderr?: string | null;
|
|
errorMessage?: string | null;
|
|
}): boolean {
|
|
const parsed = input.parsed ?? null;
|
|
// Deterministic failures are handled by their own classifiers.
|
|
if (parsed && (isClaudeMaxTurnsResult(parsed) || isClaudeUnknownSessionError(parsed))) {
|
|
return false;
|
|
}
|
|
const loginMeta = detectClaudeLoginRequired({
|
|
parsed,
|
|
stdout: input.stdout ?? "",
|
|
stderr: input.stderr ?? "",
|
|
});
|
|
if (loginMeta.requiresLogin) return false;
|
|
|
|
const haystack = buildClaudeTransientHaystack(input);
|
|
if (!haystack) return false;
|
|
return CLAUDE_TRANSIENT_UPSTREAM_RE.test(haystack);
|
|
}
|