mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-15 10:30:37 +09:00
[codex] Improve transient recovery and Codex model refresh (#4383)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies > - Adapter execution and retry classification decide whether agent work pauses, retries, or recovers automatically > - Transient provider failures need to be classified precisely so Paperclip does not convert retryable upstream conditions into false hard failures > - At the same time, operators need an up-to-date model list for Codex-backed agents and prompts should nudge agents toward targeted verification instead of repo-wide sweeps > - This pull request tightens transient recovery classification for Claude and Codex, updates the agent prompt guidance, and adds Codex model refresh support end-to-end > - The benefit is better automatic retry behavior plus fresher operator-facing model configuration ## What Changed - added Codex usage-limit retry-window parsing and Claude extra-usage transient classification - normalized the heartbeat transient-recovery contract across adapter executions and heartbeat scheduling - documented that deferred comment wakes only reopen completed issues for human/comment-reopen interactions, while system follow-ups leave closed work closed - updated adapter-utils prompt guidance to prefer targeted verification - added Codex model refresh support in the server route, registry, shared types, and agent config form - added adapter/server tests covering the new parsing, retry scheduling, and model-refresh behavior ## Verification - `pnpm exec vitest run --project @paperclipai/adapter-utils packages/adapter-utils/src/server-utils.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-claude-local packages/adapters/claude-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-codex-local packages/adapters/codex-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/adapter-model-refresh-routes.test.ts server/src/__tests__/adapter-models.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/codex-local-execute.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/heartbeat-retry-scheduling.test.ts` ## Risks - Moderate behavior risk: retry classification affects whether runs auto-recover or block, so mistakes here could either suppress needed retries or over-retry real failures - Low workflow risk: deferred comment wake reopening is intentionally scoped to human/comment-reopen interactions so system follow-ups do not revive completed issues unexpectedly > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex GPT-5-based coding agent with tool use and code execution in the Codex CLI environment ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
4fdbbeced3
commit
8f1cd0474f
25 changed files with 1455 additions and 48 deletions
|
|
@ -27,6 +27,7 @@ import {
|
|||
principalPermissionGrants,
|
||||
companyMemberships,
|
||||
companySkills,
|
||||
documents,
|
||||
} from "@paperclipai/db";
|
||||
import { notFound, unprocessable } from "../errors.js";
|
||||
|
||||
|
|
@ -279,6 +280,7 @@ export function companyService(db: Db) {
|
|||
await tx.delete(companyMemberships).where(eq(companyMemberships.companyId, id));
|
||||
await tx.delete(companySkills).where(eq(companySkills.companyId, id));
|
||||
await tx.delete(issueReadStates).where(eq(issueReadStates.companyId, id));
|
||||
await tx.delete(documents).where(eq(documents.companyId, id));
|
||||
await tx.delete(issues).where(eq(issues.companyId, id));
|
||||
await tx.delete(companyLogos).where(eq(companyLogos.companyId, id));
|
||||
await tx.delete(assets).where(eq(assets.companyId, id));
|
||||
|
|
|
|||
|
|
@ -179,6 +179,61 @@ function resolveCodexTransientFallbackMode(attempt: number): CodexTransientFallb
|
|||
if (attempt === 3) return "fresh_session";
|
||||
return "fresh_session_safer_invocation";
|
||||
}
|
||||
|
||||
function readHeartbeatRunErrorFamily(
|
||||
run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode" | "resultJson">,
|
||||
) {
|
||||
const resultJson = parseObject(run.resultJson);
|
||||
const persistedFamily = readNonEmptyString(resultJson.errorFamily);
|
||||
if (persistedFamily) return persistedFamily;
|
||||
|
||||
if (run.errorCode === "codex_transient_upstream" || run.errorCode === "claude_transient_upstream") {
|
||||
return "transient_upstream";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function readTransientRetryNotBeforeFromRun(run: Pick<typeof heartbeatRuns.$inferSelect, "resultJson">) {
|
||||
const resultJson = parseObject(run.resultJson);
|
||||
const value = resultJson.retryNotBefore ?? resultJson.transientRetryNotBefore;
|
||||
if (!(typeof value === "string" || typeof value === "number" || value instanceof Date)) {
|
||||
return null;
|
||||
}
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||
}
|
||||
|
||||
function readTransientRecoveryContractFromRun(
|
||||
run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode" | "resultJson">,
|
||||
) {
|
||||
return readHeartbeatRunErrorFamily(run) === "transient_upstream"
|
||||
? {
|
||||
errorFamily: "transient_upstream" as const,
|
||||
retryNotBefore: readTransientRetryNotBeforeFromRun(run),
|
||||
}
|
||||
: null;
|
||||
}
|
||||
|
||||
function mergeAdapterRecoveryMetadata(input: {
|
||||
resultJson: Record<string, unknown> | null | undefined;
|
||||
errorFamily?: string | null;
|
||||
retryNotBefore?: string | null;
|
||||
}) {
|
||||
const errorFamily = readNonEmptyString(input.errorFamily);
|
||||
const retryNotBefore = readNonEmptyString(input.retryNotBefore);
|
||||
if (!input.resultJson && !errorFamily && !retryNotBefore) return input.resultJson ?? null;
|
||||
|
||||
return {
|
||||
...(input.resultJson ?? {}),
|
||||
...(errorFamily ? { errorFamily } : {}),
|
||||
...(retryNotBefore
|
||||
? {
|
||||
retryNotBefore,
|
||||
transientRetryNotBefore: retryNotBefore,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set(["approval_approved"]);
|
||||
const SESSIONED_LOCAL_ADAPTERS = new Set([
|
||||
"claude_local",
|
||||
|
|
@ -3267,13 +3322,18 @@ export function heartbeatService(db: Db) {
|
|||
const retryReason = opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON;
|
||||
const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON;
|
||||
const nextAttempt = (run.scheduledRetryAttempt ?? 0) + 1;
|
||||
const schedule = computeBoundedTransientHeartbeatRetrySchedule(nextAttempt, now, opts?.random);
|
||||
const baseSchedule = computeBoundedTransientHeartbeatRetrySchedule(nextAttempt, now, opts?.random);
|
||||
const transientRecovery =
|
||||
retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON
|
||||
? readTransientRecoveryContractFromRun(run)
|
||||
: null;
|
||||
const codexTransientFallbackMode =
|
||||
agent.adapterType === "codex_local" && retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON && run.errorCode === "codex_transient_upstream"
|
||||
agent.adapterType === "codex_local" && transientRecovery
|
||||
? resolveCodexTransientFallbackMode(nextAttempt)
|
||||
: null;
|
||||
const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null;
|
||||
|
||||
if (!schedule) {
|
||||
if (!baseSchedule) {
|
||||
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
||||
eventType: "lifecycle",
|
||||
stream: "system",
|
||||
|
|
@ -3291,6 +3351,14 @@ export function heartbeatService(db: Db) {
|
|||
maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
|
||||
};
|
||||
}
|
||||
const schedule =
|
||||
transientRetryNotBefore && transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime()
|
||||
? {
|
||||
...baseSchedule,
|
||||
dueAt: transientRetryNotBefore,
|
||||
delayMs: Math.max(0, transientRetryNotBefore.getTime() - now.getTime()),
|
||||
}
|
||||
: baseSchedule;
|
||||
|
||||
const contextSnapshot = parseObject(run.contextSnapshot);
|
||||
const issueId = readNonEmptyString(contextSnapshot.issueId);
|
||||
|
|
@ -3301,8 +3369,10 @@ export function heartbeatService(db: Db) {
|
|||
retryOfRunId: run.id,
|
||||
wakeReason,
|
||||
retryReason,
|
||||
...(transientRecovery ? { errorFamily: transientRecovery.errorFamily } : {}),
|
||||
scheduledRetryAttempt: schedule.attempt,
|
||||
scheduledRetryAt: schedule.dueAt.toISOString(),
|
||||
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
||||
...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
|
||||
};
|
||||
|
||||
|
|
@ -3319,8 +3389,10 @@ export function heartbeatService(db: Db) {
|
|||
...(issueId ? { issueId } : {}),
|
||||
retryOfRunId: run.id,
|
||||
retryReason,
|
||||
...(transientRecovery ? { errorFamily: transientRecovery.errorFamily } : {}),
|
||||
scheduledRetryAttempt: schedule.attempt,
|
||||
scheduledRetryAt: schedule.dueAt.toISOString(),
|
||||
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
||||
...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
|
||||
},
|
||||
status: "queued",
|
||||
|
|
@ -3383,10 +3455,12 @@ export function heartbeatService(db: Db) {
|
|||
payload: {
|
||||
retryRunId: retryRun.id,
|
||||
retryReason,
|
||||
...(transientRecovery ? { errorFamily: transientRecovery.errorFamily } : {}),
|
||||
scheduledRetryAttempt: schedule.attempt,
|
||||
scheduledRetryAt: schedule.dueAt.toISOString(),
|
||||
baseDelayMs: schedule.baseDelayMs,
|
||||
delayMs: schedule.delayMs,
|
||||
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
||||
...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
|
||||
},
|
||||
});
|
||||
|
|
@ -5872,7 +5946,11 @@ export function heartbeatService(db: Db) {
|
|||
|
||||
const persistedResultJson = mergeHeartbeatRunResultJson(
|
||||
mergeRunStopMetadataForAgent(agent, outcome, {
|
||||
resultJson: adapterResult.resultJson ?? null,
|
||||
resultJson: mergeAdapterRecoveryMetadata({
|
||||
resultJson: adapterResult.resultJson ?? null,
|
||||
errorFamily: adapterResult.errorFamily ?? null,
|
||||
retryNotBefore: adapterResult.retryNotBefore ?? null,
|
||||
}),
|
||||
errorCode: runErrorCode,
|
||||
errorMessage: runErrorMessage,
|
||||
}),
|
||||
|
|
@ -5933,7 +6011,7 @@ export function heartbeatService(db: Db) {
|
|||
);
|
||||
}
|
||||
}
|
||||
if (outcome === "failed" && livenessRun.errorCode === "codex_transient_upstream") {
|
||||
if (outcome === "failed" && readTransientRecoveryContractFromRun(livenessRun)) {
|
||||
await scheduleBoundedRetryForRun(livenessRun, agent);
|
||||
}
|
||||
await finalizeIssueCommentPolicy(livenessRun, agent);
|
||||
|
|
@ -6267,8 +6345,16 @@ export function heartbeatService(db: Db) {
|
|||
};
|
||||
}
|
||||
const deferredCommentIds = extractWakeCommentIds(deferredContextSeed);
|
||||
const deferredWakeReason = readNonEmptyString(deferredContextSeed.wakeReason);
|
||||
// Only human/comment-reopen interactions should revive completed issues;
|
||||
// system follow-ups such as retry or cleanup wakes must not reopen closed work.
|
||||
const shouldReopenDeferredCommentWake =
|
||||
deferredCommentIds.length > 0 && (issue.status === "done" || issue.status === "cancelled");
|
||||
deferredCommentIds.length > 0 &&
|
||||
(issue.status === "done" || issue.status === "cancelled") &&
|
||||
(
|
||||
deferred.requestedByActorType === "user" ||
|
||||
deferredWakeReason === "issue_reopened_via_comment"
|
||||
);
|
||||
let reopenedActivity: LogActivityInput | null = null;
|
||||
|
||||
if (shouldReopenDeferredCommentWake) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue