mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-14 01:50:39 +09:00
[codex] Improve transient recovery and Codex model refresh (#4383)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies > - Adapter execution and retry classification decide whether agent work pauses, retries, or recovers automatically > - Transient provider failures need to be classified precisely so Paperclip does not convert retryable upstream conditions into false hard failures > - At the same time, operators need an up-to-date model list for Codex-backed agents and prompts should nudge agents toward targeted verification instead of repo-wide sweeps > - This pull request tightens transient recovery classification for Claude and Codex, updates the agent prompt guidance, and adds Codex model refresh support end-to-end > - The benefit is better automatic retry behavior plus fresher operator-facing model configuration ## What Changed - added Codex usage-limit retry-window parsing and Claude extra-usage transient classification - normalized the heartbeat transient-recovery contract across adapter executions and heartbeat scheduling - documented that deferred comment wakes only reopen completed issues for human/comment-reopen interactions, while system follow-ups leave closed work closed - updated adapter-utils prompt guidance to prefer targeted verification - added Codex model refresh support in the server route, registry, shared types, and agent config form - added adapter/server tests covering the new parsing, retry scheduling, and model-refresh behavior ## Verification - `pnpm exec vitest run --project @paperclipai/adapter-utils packages/adapter-utils/src/server-utils.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-claude-local packages/adapters/claude-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-codex-local packages/adapters/codex-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/adapter-model-refresh-routes.test.ts server/src/__tests__/adapter-models.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/codex-local-execute.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/heartbeat-retry-scheduling.test.ts` ## Risks - Moderate behavior risk: retry classification affects whether runs auto-recover or block, so mistakes here could either suppress needed retries or over-retry real failures - Low workflow risk: deferred comment wake reopening is intentionally scoped to human/comment-reopen interactions so system follow-ups do not revive completed issues unexpectedly > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex GPT-5-based coding agent with tool use and code execution in the Codex CLI environment ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
4fdbbeced3
commit
8f1cd0474f
25 changed files with 1455 additions and 48 deletions
|
|
@ -254,6 +254,7 @@ describe("renderPaperclipWakePrompt", () => {
|
|||
it("keeps the default local-agent prompt action-oriented", () => {
|
||||
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("Start actionable work in this heartbeat");
|
||||
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("do not stop at a plan");
|
||||
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("Prefer the smallest verification that proves the change");
|
||||
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("Use child issues");
|
||||
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("instead of polling agents, sessions, or processes");
|
||||
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("Create child issues directly when you know what needs to be done");
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ export const DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE = [
|
|||
"Execution contract:",
|
||||
"- Start actionable work in this heartbeat; do not stop at a plan unless the issue asks for planning.",
|
||||
"- Leave durable progress in comments, documents, or work products with a clear next action.",
|
||||
"- Prefer the smallest verification that proves the change; do not default to full workspace typecheck/build/test on every heartbeat unless the task scope warrants it.",
|
||||
"- Use child issues for parallel or long delegated work instead of polling agents, sessions, or processes.",
|
||||
"- If woken by a human comment on a dependency-blocked issue, respond or triage the comment without treating the blocked deliverable work as unblocked.",
|
||||
"- Create child issues directly when you know what needs to be done; use issue-thread interactions when the board/user must choose suggested tasks, answer structured questions, or confirm a proposal.",
|
||||
|
|
|
|||
|
|
@ -64,12 +64,16 @@ export interface AdapterRuntimeServiceReport {
|
|||
healthStatus?: "unknown" | "healthy" | "unhealthy";
|
||||
}
|
||||
|
||||
export type AdapterExecutionErrorFamily = "transient_upstream";
|
||||
|
||||
export interface AdapterExecutionResult {
|
||||
exitCode: number | null;
|
||||
signal: string | null;
|
||||
timedOut: boolean;
|
||||
errorMessage?: string | null;
|
||||
errorCode?: string | null;
|
||||
errorFamily?: AdapterExecutionErrorFamily | null;
|
||||
retryNotBefore?: string | null;
|
||||
errorMeta?: Record<string, unknown>;
|
||||
usage?: UsageSummary;
|
||||
/**
|
||||
|
|
@ -311,6 +315,13 @@ export interface ServerAdapterModule {
|
|||
supportsLocalAgentJwt?: boolean;
|
||||
models?: AdapterModel[];
|
||||
listModels?: () => Promise<AdapterModel[]>;
|
||||
/**
|
||||
* Optional explicit refresh hook for model discovery.
|
||||
* Use this when the adapter caches discovered models and needs a bypass path
|
||||
* so the UI can fetch newly released models without waiting for cache expiry
|
||||
* or a Paperclip code update.
|
||||
*/
|
||||
refreshModels?: () => Promise<AdapterModel[]>;
|
||||
agentConfigurationDoc?: string;
|
||||
/**
|
||||
* Optional lifecycle hook when an agent is approved/hired (join-request or hire_agent approval).
|
||||
|
|
|
|||
|
|
@ -39,7 +39,9 @@ import {
|
|||
parseClaudeStreamJson,
|
||||
describeClaudeFailure,
|
||||
detectClaudeLoginRequired,
|
||||
extractClaudeRetryNotBefore,
|
||||
isClaudeMaxTurnsResult,
|
||||
isClaudeTransientUpstreamError,
|
||||
isClaudeUnknownSessionError,
|
||||
} from "./parse.js";
|
||||
import { resolveClaudeDesiredSkillNames } from "./skills.js";
|
||||
|
|
@ -625,16 +627,48 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
|||
}
|
||||
|
||||
if (!parsed) {
|
||||
const fallbackErrorMessage = parseFallbackErrorMessage(proc);
|
||||
const transientUpstream =
|
||||
!loginMeta.requiresLogin &&
|
||||
(proc.exitCode ?? 0) !== 0 &&
|
||||
isClaudeTransientUpstreamError({
|
||||
parsed: null,
|
||||
stdout: proc.stdout,
|
||||
stderr: proc.stderr,
|
||||
errorMessage: fallbackErrorMessage,
|
||||
});
|
||||
const transientRetryNotBefore = transientUpstream
|
||||
? extractClaudeRetryNotBefore({
|
||||
parsed: null,
|
||||
stdout: proc.stdout,
|
||||
stderr: proc.stderr,
|
||||
errorMessage: fallbackErrorMessage,
|
||||
})
|
||||
: null;
|
||||
const errorCode = loginMeta.requiresLogin
|
||||
? "claude_auth_required"
|
||||
: transientUpstream
|
||||
? "claude_transient_upstream"
|
||||
: null;
|
||||
return {
|
||||
exitCode: proc.exitCode,
|
||||
signal: proc.signal,
|
||||
timedOut: false,
|
||||
errorMessage: parseFallbackErrorMessage(proc),
|
||||
errorCode: loginMeta.requiresLogin ? "claude_auth_required" : null,
|
||||
errorMessage: fallbackErrorMessage,
|
||||
errorCode,
|
||||
errorFamily: transientUpstream ? "transient_upstream" : null,
|
||||
retryNotBefore: transientRetryNotBefore ? transientRetryNotBefore.toISOString() : null,
|
||||
errorMeta,
|
||||
resultJson: {
|
||||
stdout: proc.stdout,
|
||||
stderr: proc.stderr,
|
||||
...(transientUpstream ? { errorFamily: "transient_upstream" } : {}),
|
||||
...(transientRetryNotBefore
|
||||
? { retryNotBefore: transientRetryNotBefore.toISOString() }
|
||||
: {}),
|
||||
...(transientRetryNotBefore
|
||||
? { transientRetryNotBefore: transientRetryNotBefore.toISOString() }
|
||||
: {}),
|
||||
},
|
||||
clearSession: Boolean(opts.clearSessionOnMissingSession),
|
||||
};
|
||||
|
|
@ -670,16 +704,48 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
|||
} as Record<string, unknown>)
|
||||
: null;
|
||||
const clearSessionForMaxTurns = isClaudeMaxTurnsResult(parsed);
|
||||
const parsedIsError = asBoolean(parsed.is_error, false);
|
||||
const failed = (proc.exitCode ?? 0) !== 0 || parsedIsError;
|
||||
const errorMessage = failed
|
||||
? describeClaudeFailure(parsed) ?? `Claude exited with code ${proc.exitCode ?? -1}`
|
||||
: null;
|
||||
const transientUpstream =
|
||||
failed &&
|
||||
!loginMeta.requiresLogin &&
|
||||
isClaudeTransientUpstreamError({
|
||||
parsed,
|
||||
stdout: proc.stdout,
|
||||
stderr: proc.stderr,
|
||||
errorMessage,
|
||||
});
|
||||
const transientRetryNotBefore = transientUpstream
|
||||
? extractClaudeRetryNotBefore({
|
||||
parsed,
|
||||
stdout: proc.stdout,
|
||||
stderr: proc.stderr,
|
||||
errorMessage,
|
||||
})
|
||||
: null;
|
||||
const resolvedErrorCode = loginMeta.requiresLogin
|
||||
? "claude_auth_required"
|
||||
: transientUpstream
|
||||
? "claude_transient_upstream"
|
||||
: null;
|
||||
const mergedResultJson: Record<string, unknown> = {
|
||||
...parsed,
|
||||
...(transientUpstream ? { errorFamily: "transient_upstream" } : {}),
|
||||
...(transientRetryNotBefore ? { retryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
||||
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
||||
};
|
||||
|
||||
return {
|
||||
exitCode: proc.exitCode,
|
||||
signal: proc.signal,
|
||||
timedOut: false,
|
||||
errorMessage:
|
||||
(proc.exitCode ?? 0) === 0
|
||||
? null
|
||||
: describeClaudeFailure(parsed) ?? `Claude exited with code ${proc.exitCode ?? -1}`,
|
||||
errorCode: loginMeta.requiresLogin ? "claude_auth_required" : null,
|
||||
errorMessage,
|
||||
errorCode: resolvedErrorCode,
|
||||
errorFamily: transientUpstream ? "transient_upstream" : null,
|
||||
retryNotBefore: transientRetryNotBefore ? transientRetryNotBefore.toISOString() : null,
|
||||
errorMeta,
|
||||
usage,
|
||||
sessionId: resolvedSessionId,
|
||||
|
|
@ -690,7 +756,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
|||
model: parsedStream.model || asString(parsed.model, model),
|
||||
billingType,
|
||||
costUsd: parsedStream.costUsd ?? asNumber(parsed.total_cost_usd, 0),
|
||||
resultJson: parsed,
|
||||
resultJson: mergedResultJson,
|
||||
summary: parsedStream.summary || asString(parsed.result, ""),
|
||||
clearSession: clearSessionForMaxTurns || Boolean(opts.clearSessionOnMissingSession && !resolvedSessionId),
|
||||
};
|
||||
|
|
|
|||
123
packages/adapters/claude-local/src/server/parse.test.ts
Normal file
123
packages/adapters/claude-local/src/server/parse.test.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
extractClaudeRetryNotBefore,
|
||||
isClaudeTransientUpstreamError,
|
||||
} from "./parse.js";
|
||||
|
||||
describe("isClaudeTransientUpstreamError", () => {
|
||||
it("classifies the 'out of extra usage' subscription window failure as transient", () => {
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
errorMessage: "You're out of extra usage · resets 4pm (America/Chicago)",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
parsed: {
|
||||
is_error: true,
|
||||
result: "You're out of extra usage. Resets at 4pm (America/Chicago).",
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("classifies Anthropic API rate_limit_error and overloaded_error as transient", () => {
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
parsed: {
|
||||
is_error: true,
|
||||
errors: [{ type: "rate_limit_error", message: "Rate limit reached for requests." }],
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
parsed: {
|
||||
is_error: true,
|
||||
errors: [{ type: "overloaded_error", message: "Overloaded" }],
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
stderr: "HTTP 429: Too Many Requests",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
stderr: "Bedrock ThrottlingException: slow down",
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("classifies the subscription 5-hour / weekly limit wording", () => {
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
errorMessage: "Claude usage limit reached — weekly limit reached. Try again in 2 days.",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
errorMessage: "5-hour limit reached.",
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not classify login/auth failures as transient", () => {
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
stderr: "Please log in. Run `claude login` first.",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does not classify max-turns or unknown-session as transient", () => {
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
parsed: { subtype: "error_max_turns", result: "Maximum turns reached." },
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
parsed: {
|
||||
result: "No conversation found with session id abc-123",
|
||||
errors: [{ message: "No conversation found with session id abc-123" }],
|
||||
},
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does not classify deterministic validation errors as transient", () => {
|
||||
expect(
|
||||
isClaudeTransientUpstreamError({
|
||||
errorMessage: "Invalid request_error: Unknown parameter 'foo'.",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractClaudeRetryNotBefore", () => {
|
||||
it("parses the 'resets 4pm' hint in its explicit timezone", () => {
|
||||
const now = new Date("2026-04-22T15:15:00.000Z");
|
||||
const extracted = extractClaudeRetryNotBefore(
|
||||
{ errorMessage: "You're out of extra usage · resets 4pm (America/Chicago)" },
|
||||
now,
|
||||
);
|
||||
expect(extracted?.toISOString()).toBe("2026-04-22T21:00:00.000Z");
|
||||
});
|
||||
|
||||
it("rolls forward past midnight when the reset time has already passed today", () => {
|
||||
const now = new Date("2026-04-22T23:30:00.000Z");
|
||||
const extracted = extractClaudeRetryNotBefore(
|
||||
{ errorMessage: "Usage limit reached. Resets at 3:15 AM (UTC)." },
|
||||
now,
|
||||
);
|
||||
expect(extracted?.toISOString()).toBe("2026-04-23T03:15:00.000Z");
|
||||
});
|
||||
|
||||
it("returns null when no reset hint is present", () => {
|
||||
expect(
|
||||
extractClaudeRetryNotBefore({ errorMessage: "Overloaded. Try again later." }, new Date()),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,9 +1,19 @@
|
|||
import type { UsageSummary } from "@paperclipai/adapter-utils";
|
||||
import { asString, asNumber, parseObject, parseJson } from "@paperclipai/adapter-utils/server-utils";
|
||||
import {
|
||||
asString,
|
||||
asNumber,
|
||||
parseObject,
|
||||
parseJson,
|
||||
} from "@paperclipai/adapter-utils/server-utils";
|
||||
|
||||
const CLAUDE_AUTH_REQUIRED_RE = /(?:not\s+logged\s+in|please\s+log\s+in|please\s+run\s+`?claude\s+login`?|login\s+required|requires\s+login|unauthorized|authentication\s+required)/i;
|
||||
const URL_RE = /(https?:\/\/[^\s'"`<>()[\]{};,!?]+[^\s'"`<>()[\]{};,!.?:]+)/gi;
|
||||
|
||||
const CLAUDE_TRANSIENT_UPSTREAM_RE =
|
||||
/(?:rate[-\s]?limit(?:ed)?|rate_limit_error|too\s+many\s+requests|\b429\b|overloaded(?:_error)?|server\s+overloaded|service\s+unavailable|\b503\b|\b529\b|high\s+demand|try\s+again\s+later|temporarily\s+unavailable|throttl(?:ed|ing)|throttlingexception|servicequotaexceededexception|out\s+of\s+extra\s+usage|extra\s+usage\b|claude\s+usage\s+limit\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|usage\s+limit\s+reached|usage\s+cap\s+reached)/i;
|
||||
const CLAUDE_EXTRA_USAGE_RESET_RE =
|
||||
/(?:out\s+of\s+extra\s+usage|extra\s+usage|usage\s+limit\s+reached|usage\s+cap\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|claude\s+usage\s+limit\s+reached)[\s\S]{0,80}?\bresets?\s+(?:at\s+)?([^\n()]+?)(?:\s*\(([^)]+)\))?(?:[.!]|\n|$)/i;
|
||||
|
||||
export function parseClaudeStreamJson(stdout: string) {
|
||||
let sessionId: string | null = null;
|
||||
let model = "";
|
||||
|
|
@ -177,3 +187,197 @@ export function isClaudeUnknownSessionError(parsed: Record<string, unknown>): bo
|
|||
/no conversation found with session id|unknown session|session .* not found/i.test(msg),
|
||||
);
|
||||
}
|
||||
|
||||
function buildClaudeTransientHaystack(input: {
|
||||
parsed?: Record<string, unknown> | null;
|
||||
stdout?: string | null;
|
||||
stderr?: string | null;
|
||||
errorMessage?: string | null;
|
||||
}): string {
|
||||
const parsed = input.parsed ?? null;
|
||||
const resultText = parsed ? asString(parsed.result, "") : "";
|
||||
const parsedErrors = parsed ? extractClaudeErrorMessages(parsed) : [];
|
||||
return [
|
||||
input.errorMessage ?? "",
|
||||
resultText,
|
||||
...parsedErrors,
|
||||
input.stdout ?? "",
|
||||
input.stderr ?? "",
|
||||
]
|
||||
.join("\n")
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function readTimeZoneParts(date: Date, timeZone: string) {
|
||||
const values = new Map(
|
||||
new Intl.DateTimeFormat("en-US", {
|
||||
timeZone,
|
||||
hourCycle: "h23",
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
}).formatToParts(date).map((part) => [part.type, part.value]),
|
||||
);
|
||||
return {
|
||||
year: Number.parseInt(values.get("year") ?? "", 10),
|
||||
month: Number.parseInt(values.get("month") ?? "", 10),
|
||||
day: Number.parseInt(values.get("day") ?? "", 10),
|
||||
hour: Number.parseInt(values.get("hour") ?? "", 10),
|
||||
minute: Number.parseInt(values.get("minute") ?? "", 10),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeResetTimeZone(timeZoneHint: string | null | undefined): string | null {
|
||||
const normalized = timeZoneHint?.trim();
|
||||
if (!normalized) return null;
|
||||
if (/^(?:utc|gmt)$/i.test(normalized)) return "UTC";
|
||||
|
||||
try {
|
||||
new Intl.DateTimeFormat("en-US", { timeZone: normalized }).format(new Date(0));
|
||||
return normalized;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function dateFromTimeZoneWallClock(input: {
|
||||
year: number;
|
||||
month: number;
|
||||
day: number;
|
||||
hour: number;
|
||||
minute: number;
|
||||
timeZone: string;
|
||||
}): Date | null {
|
||||
let candidate = new Date(Date.UTC(input.year, input.month - 1, input.day, input.hour, input.minute, 0, 0));
|
||||
const targetUtc = Date.UTC(input.year, input.month - 1, input.day, input.hour, input.minute, 0, 0);
|
||||
|
||||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||||
const actual = readTimeZoneParts(candidate, input.timeZone);
|
||||
const actualUtc = Date.UTC(actual.year, actual.month - 1, actual.day, actual.hour, actual.minute, 0, 0);
|
||||
const offsetMs = targetUtc - actualUtc;
|
||||
if (offsetMs === 0) break;
|
||||
candidate = new Date(candidate.getTime() + offsetMs);
|
||||
}
|
||||
|
||||
const verified = readTimeZoneParts(candidate, input.timeZone);
|
||||
if (
|
||||
verified.year !== input.year ||
|
||||
verified.month !== input.month ||
|
||||
verified.day !== input.day ||
|
||||
verified.hour !== input.hour ||
|
||||
verified.minute !== input.minute
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function nextClockTimeInTimeZone(input: {
|
||||
now: Date;
|
||||
hour: number;
|
||||
minute: number;
|
||||
timeZoneHint: string;
|
||||
}): Date | null {
|
||||
const timeZone = normalizeResetTimeZone(input.timeZoneHint);
|
||||
if (!timeZone) return null;
|
||||
|
||||
const nowParts = readTimeZoneParts(input.now, timeZone);
|
||||
let retryAt = dateFromTimeZoneWallClock({
|
||||
year: nowParts.year,
|
||||
month: nowParts.month,
|
||||
day: nowParts.day,
|
||||
hour: input.hour,
|
||||
minute: input.minute,
|
||||
timeZone,
|
||||
});
|
||||
if (!retryAt) return null;
|
||||
|
||||
if (retryAt.getTime() <= input.now.getTime()) {
|
||||
const nextDay = new Date(Date.UTC(nowParts.year, nowParts.month - 1, nowParts.day + 1, 0, 0, 0, 0));
|
||||
retryAt = dateFromTimeZoneWallClock({
|
||||
year: nextDay.getUTCFullYear(),
|
||||
month: nextDay.getUTCMonth() + 1,
|
||||
day: nextDay.getUTCDate(),
|
||||
hour: input.hour,
|
||||
minute: input.minute,
|
||||
timeZone,
|
||||
});
|
||||
}
|
||||
|
||||
return retryAt;
|
||||
}
|
||||
|
||||
function parseClaudeResetClockTime(clockText: string, now: Date, timeZoneHint?: string | null): Date | null {
|
||||
const normalized = clockText.trim().replace(/\s+/g, " ");
|
||||
const match = normalized.match(/^(\d{1,2})(?::(\d{2}))?\s*([ap])\.?\s*m\.?/i);
|
||||
if (!match) return null;
|
||||
|
||||
const hour12 = Number.parseInt(match[1] ?? "", 10);
|
||||
const minute = Number.parseInt(match[2] ?? "0", 10);
|
||||
if (!Number.isInteger(hour12) || hour12 < 1 || hour12 > 12) return null;
|
||||
if (!Number.isInteger(minute) || minute < 0 || minute > 59) return null;
|
||||
|
||||
let hour24 = hour12 % 12;
|
||||
if ((match[3] ?? "").toLowerCase() === "p") hour24 += 12;
|
||||
|
||||
if (timeZoneHint) {
|
||||
const explicitRetryAt = nextClockTimeInTimeZone({
|
||||
now,
|
||||
hour: hour24,
|
||||
minute,
|
||||
timeZoneHint,
|
||||
});
|
||||
if (explicitRetryAt) return explicitRetryAt;
|
||||
}
|
||||
|
||||
const retryAt = new Date(now);
|
||||
retryAt.setHours(hour24, minute, 0, 0);
|
||||
if (retryAt.getTime() <= now.getTime()) {
|
||||
retryAt.setDate(retryAt.getDate() + 1);
|
||||
}
|
||||
return retryAt;
|
||||
}
|
||||
|
||||
export function extractClaudeRetryNotBefore(
|
||||
input: {
|
||||
parsed?: Record<string, unknown> | null;
|
||||
stdout?: string | null;
|
||||
stderr?: string | null;
|
||||
errorMessage?: string | null;
|
||||
},
|
||||
now = new Date(),
|
||||
): Date | null {
|
||||
const haystack = buildClaudeTransientHaystack(input);
|
||||
const match = haystack.match(CLAUDE_EXTRA_USAGE_RESET_RE);
|
||||
if (!match) return null;
|
||||
return parseClaudeResetClockTime(match[1] ?? "", now, match[2]);
|
||||
}
|
||||
|
||||
export function isClaudeTransientUpstreamError(input: {
|
||||
parsed?: Record<string, unknown> | null;
|
||||
stdout?: string | null;
|
||||
stderr?: string | null;
|
||||
errorMessage?: string | null;
|
||||
}): boolean {
|
||||
const parsed = input.parsed ?? null;
|
||||
// Deterministic failures are handled by their own classifiers.
|
||||
if (parsed && (isClaudeMaxTurnsResult(parsed) || isClaudeUnknownSessionError(parsed))) {
|
||||
return false;
|
||||
}
|
||||
const loginMeta = detectClaudeLoginRequired({
|
||||
parsed,
|
||||
stdout: input.stdout ?? "",
|
||||
stderr: input.stderr ?? "",
|
||||
});
|
||||
if (loginMeta.requiresLogin) return false;
|
||||
|
||||
const haystack = buildClaudeTransientHaystack(input);
|
||||
if (!haystack) return false;
|
||||
return CLAUDE_TRANSIENT_UPSTREAM_RE.test(haystack);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import {
|
|||
} from "@paperclipai/adapter-utils/server-utils";
|
||||
import {
|
||||
parseCodexJsonl,
|
||||
extractCodexRetryNotBefore,
|
||||
isCodexTransientUpstreamError,
|
||||
isCodexUnknownSessionError,
|
||||
} from "./parse.js";
|
||||
|
|
@ -725,6 +726,21 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
|||
parsedError ||
|
||||
stderrLine ||
|
||||
`Codex exited with code ${attempt.proc.exitCode ?? -1}`;
|
||||
const transientRetryNotBefore =
|
||||
(attempt.proc.exitCode ?? 0) !== 0
|
||||
? extractCodexRetryNotBefore({
|
||||
stdout: attempt.proc.stdout,
|
||||
stderr: attempt.proc.stderr,
|
||||
errorMessage: fallbackErrorMessage,
|
||||
})
|
||||
: null;
|
||||
const transientUpstream =
|
||||
(attempt.proc.exitCode ?? 0) !== 0 &&
|
||||
isCodexTransientUpstreamError({
|
||||
stdout: attempt.proc.stdout,
|
||||
stderr: attempt.proc.stderr,
|
||||
errorMessage: fallbackErrorMessage,
|
||||
});
|
||||
|
||||
return {
|
||||
exitCode: attempt.proc.exitCode,
|
||||
|
|
@ -735,14 +751,11 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
|||
? null
|
||||
: fallbackErrorMessage,
|
||||
errorCode:
|
||||
(attempt.proc.exitCode ?? 0) !== 0 &&
|
||||
isCodexTransientUpstreamError({
|
||||
stdout: attempt.proc.stdout,
|
||||
stderr: attempt.proc.stderr,
|
||||
errorMessage: fallbackErrorMessage,
|
||||
})
|
||||
transientUpstream
|
||||
? "codex_transient_upstream"
|
||||
: null,
|
||||
errorFamily: transientUpstream ? "transient_upstream" : null,
|
||||
retryNotBefore: transientRetryNotBefore ? transientRetryNotBefore.toISOString() : null,
|
||||
usage: attempt.parsed.usage,
|
||||
sessionId: resolvedSessionId,
|
||||
sessionParams: resolvedSessionParams,
|
||||
|
|
@ -755,6 +768,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
|||
resultJson: {
|
||||
stdout: attempt.proc.stdout,
|
||||
stderr: attempt.proc.stderr,
|
||||
...(transientUpstream ? { errorFamily: "transient_upstream" } : {}),
|
||||
...(transientRetryNotBefore ? { retryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
||||
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
||||
},
|
||||
summary: attempt.parsed.summary,
|
||||
clearSession: Boolean((clearSessionOnMissingSession || forceFreshSession) && !resolvedSessionId),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
extractCodexRetryNotBefore,
|
||||
isCodexTransientUpstreamError,
|
||||
isCodexUnknownSessionError,
|
||||
parseCodexJsonl,
|
||||
|
|
@ -101,6 +102,25 @@ describe("isCodexTransientUpstreamError", () => {
|
|||
).toBe(true);
|
||||
});
|
||||
|
||||
it("classifies usage-limit windows as transient and extracts the retry time", () => {
|
||||
const errorMessage = "You've hit your usage limit for GPT-5.3-Codex-Spark. Switch to another model now, or try again at 11:31 PM.";
|
||||
const now = new Date(2026, 3, 22, 22, 29, 2);
|
||||
|
||||
expect(isCodexTransientUpstreamError({ errorMessage })).toBe(true);
|
||||
expect(extractCodexRetryNotBefore({ errorMessage }, now)?.getTime()).toBe(
|
||||
new Date(2026, 3, 22, 23, 31, 0, 0).getTime(),
|
||||
);
|
||||
});
|
||||
|
||||
it("parses explicit timezone hints on usage-limit retry windows", () => {
|
||||
const errorMessage = "You've hit your usage limit for GPT-5.3-Codex-Spark. Switch to another model now, or try again at 11:31 PM (America/Chicago).";
|
||||
const now = new Date("2026-04-23T03:29:02.000Z");
|
||||
|
||||
expect(extractCodexRetryNotBefore({ errorMessage }, now)?.toISOString()).toBe(
|
||||
"2026-04-23T04:31:00.000Z",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not classify deterministic compaction errors as transient", () => {
|
||||
expect(
|
||||
isCodexTransientUpstreamError({
|
||||
|
|
|
|||
|
|
@ -1,8 +1,15 @@
|
|||
import { asString, asNumber, parseObject, parseJson } from "@paperclipai/adapter-utils/server-utils";
|
||||
import {
|
||||
asString,
|
||||
asNumber,
|
||||
parseObject,
|
||||
parseJson,
|
||||
} from "@paperclipai/adapter-utils/server-utils";
|
||||
|
||||
const CODEX_TRANSIENT_UPSTREAM_RE =
|
||||
/(?:we(?:'|’)re\s+currently\s+experiencing\s+high\s+demand|temporary\s+errors|rate[-\s]?limit(?:ed)?|too\s+many\s+requests|\b429\b|server\s+overloaded|service\s+unavailable|try\s+again\s+later)/i;
|
||||
const CODEX_REMOTE_COMPACTION_RE = /remote\s+compact\s+task/i;
|
||||
const CODEX_USAGE_LIMIT_RE =
|
||||
/you(?:'|’)ve hit your usage limit for .+\.\s+switch to another model now,\s+or try again at\s+([^.!\n]+)(?:[.!]|\n|$)/i;
|
||||
|
||||
export function parseCodexJsonl(stdout: string) {
|
||||
let sessionId: string | null = null;
|
||||
|
|
@ -76,12 +83,12 @@ export function isCodexUnknownSessionError(stdout: string, stderr: string): bool
|
|||
);
|
||||
}
|
||||
|
||||
export function isCodexTransientUpstreamError(input: {
|
||||
function buildCodexErrorHaystack(input: {
|
||||
stdout?: string | null;
|
||||
stderr?: string | null;
|
||||
errorMessage?: string | null;
|
||||
}): boolean {
|
||||
const haystack = [
|
||||
}): string {
|
||||
return [
|
||||
input.errorMessage ?? "",
|
||||
input.stdout ?? "",
|
||||
input.stderr ?? "",
|
||||
|
|
@ -91,9 +98,164 @@ export function isCodexTransientUpstreamError(input: {
|
|||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function readTimeZoneParts(date: Date, timeZone: string) {
|
||||
const values = new Map(
|
||||
new Intl.DateTimeFormat("en-US", {
|
||||
timeZone,
|
||||
hourCycle: "h23",
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
}).formatToParts(date).map((part) => [part.type, part.value]),
|
||||
);
|
||||
return {
|
||||
year: Number.parseInt(values.get("year") ?? "", 10),
|
||||
month: Number.parseInt(values.get("month") ?? "", 10),
|
||||
day: Number.parseInt(values.get("day") ?? "", 10),
|
||||
hour: Number.parseInt(values.get("hour") ?? "", 10),
|
||||
minute: Number.parseInt(values.get("minute") ?? "", 10),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeResetTimeZone(timeZoneHint: string | null | undefined): string | null {
|
||||
const normalized = timeZoneHint?.trim();
|
||||
if (!normalized) return null;
|
||||
if (/^(?:utc|gmt)$/i.test(normalized)) return "UTC";
|
||||
|
||||
try {
|
||||
new Intl.DateTimeFormat("en-US", { timeZone: normalized }).format(new Date(0));
|
||||
return normalized;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function dateFromTimeZoneWallClock(input: {
|
||||
year: number;
|
||||
month: number;
|
||||
day: number;
|
||||
hour: number;
|
||||
minute: number;
|
||||
timeZone: string;
|
||||
}): Date | null {
|
||||
let candidate = new Date(Date.UTC(input.year, input.month - 1, input.day, input.hour, input.minute, 0, 0));
|
||||
const targetUtc = Date.UTC(input.year, input.month - 1, input.day, input.hour, input.minute, 0, 0);
|
||||
|
||||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||||
const actual = readTimeZoneParts(candidate, input.timeZone);
|
||||
const actualUtc = Date.UTC(actual.year, actual.month - 1, actual.day, actual.hour, actual.minute, 0, 0);
|
||||
const offsetMs = targetUtc - actualUtc;
|
||||
if (offsetMs === 0) break;
|
||||
candidate = new Date(candidate.getTime() + offsetMs);
|
||||
}
|
||||
|
||||
const verified = readTimeZoneParts(candidate, input.timeZone);
|
||||
if (
|
||||
verified.year !== input.year ||
|
||||
verified.month !== input.month ||
|
||||
verified.day !== input.day ||
|
||||
verified.hour !== input.hour ||
|
||||
verified.minute !== input.minute
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function nextClockTimeInTimeZone(input: {
|
||||
now: Date;
|
||||
hour: number;
|
||||
minute: number;
|
||||
timeZoneHint: string;
|
||||
}): Date | null {
|
||||
const timeZone = normalizeResetTimeZone(input.timeZoneHint);
|
||||
if (!timeZone) return null;
|
||||
|
||||
const nowParts = readTimeZoneParts(input.now, timeZone);
|
||||
let retryAt = dateFromTimeZoneWallClock({
|
||||
year: nowParts.year,
|
||||
month: nowParts.month,
|
||||
day: nowParts.day,
|
||||
hour: input.hour,
|
||||
minute: input.minute,
|
||||
timeZone,
|
||||
});
|
||||
if (!retryAt) return null;
|
||||
|
||||
if (retryAt.getTime() <= input.now.getTime()) {
|
||||
const nextDay = new Date(Date.UTC(nowParts.year, nowParts.month - 1, nowParts.day + 1, 0, 0, 0, 0));
|
||||
retryAt = dateFromTimeZoneWallClock({
|
||||
year: nextDay.getUTCFullYear(),
|
||||
month: nextDay.getUTCMonth() + 1,
|
||||
day: nextDay.getUTCDate(),
|
||||
hour: input.hour,
|
||||
minute: input.minute,
|
||||
timeZone,
|
||||
});
|
||||
}
|
||||
|
||||
return retryAt;
|
||||
}
|
||||
|
||||
function parseLocalClockTime(clockText: string, now: Date): Date | null {
|
||||
const normalized = clockText.trim();
|
||||
const match = normalized.match(/^(\d{1,2})(?::(\d{2}))?\s*([ap])\.?\s*m\.?(?:\s*\(([^)]+)\)|\s+([A-Z]{2,5}))?$/i);
|
||||
if (!match) return null;
|
||||
|
||||
const hour12 = Number.parseInt(match[1] ?? "", 10);
|
||||
const minute = Number.parseInt(match[2] ?? "0", 10);
|
||||
if (!Number.isInteger(hour12) || hour12 < 1 || hour12 > 12) return null;
|
||||
if (!Number.isInteger(minute) || minute < 0 || minute > 59) return null;
|
||||
|
||||
let hour24 = hour12 % 12;
|
||||
if ((match[3] ?? "").toLowerCase() === "p") hour24 += 12;
|
||||
|
||||
const timeZoneHint = match[4] ?? match[5];
|
||||
if (timeZoneHint) {
|
||||
const explicitRetryAt = nextClockTimeInTimeZone({
|
||||
now,
|
||||
hour: hour24,
|
||||
minute,
|
||||
timeZoneHint,
|
||||
});
|
||||
if (explicitRetryAt) return explicitRetryAt;
|
||||
}
|
||||
|
||||
const retryAt = new Date(now);
|
||||
retryAt.setHours(hour24, minute, 0, 0);
|
||||
if (retryAt.getTime() <= now.getTime()) {
|
||||
retryAt.setDate(retryAt.getDate() + 1);
|
||||
}
|
||||
return retryAt;
|
||||
}
|
||||
|
||||
export function extractCodexRetryNotBefore(input: {
|
||||
stdout?: string | null;
|
||||
stderr?: string | null;
|
||||
errorMessage?: string | null;
|
||||
}, now = new Date()): Date | null {
|
||||
const haystack = buildCodexErrorHaystack(input);
|
||||
const usageLimitMatch = haystack.match(CODEX_USAGE_LIMIT_RE);
|
||||
if (!usageLimitMatch) return null;
|
||||
return parseLocalClockTime(usageLimitMatch[1] ?? "", now);
|
||||
}
|
||||
|
||||
export function isCodexTransientUpstreamError(input: {
|
||||
stdout?: string | null;
|
||||
stderr?: string | null;
|
||||
errorMessage?: string | null;
|
||||
}): boolean {
|
||||
const haystack = buildCodexErrorHaystack(input);
|
||||
|
||||
if (extractCodexRetryNotBefore(input) != null) return true;
|
||||
if (!CODEX_TRANSIENT_UPSTREAM_RE.test(haystack)) return false;
|
||||
// Keep automatic retries scoped to the observed remote-compaction/high-demand
|
||||
// failure shape; broader 429s may be caused by user or account limits.
|
||||
// failure shape, plus explicit usage-limit windows that tell us when retrying
|
||||
// becomes safe again.
|
||||
return CODEX_REMOTE_COMPACTION_RE.test(haystack) || /high\s+demand|temporary\s+errors/i.test(haystack);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue