mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-15 10:30:37 +09:00
[codex] Improve transient recovery and Codex model refresh (#4383)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies > - Adapter execution and retry classification decide whether agent work pauses, retries, or recovers automatically > - Transient provider failures need to be classified precisely so Paperclip does not convert retryable upstream conditions into false hard failures > - At the same time, operators need an up-to-date model list for Codex-backed agents and prompts should nudge agents toward targeted verification instead of repo-wide sweeps > - This pull request tightens transient recovery classification for Claude and Codex, updates the agent prompt guidance, and adds Codex model refresh support end-to-end > - The benefit is better automatic retry behavior plus fresher operator-facing model configuration ## What Changed - added Codex usage-limit retry-window parsing and Claude extra-usage transient classification - normalized the heartbeat transient-recovery contract across adapter executions and heartbeat scheduling - documented that deferred comment wakes only reopen completed issues for human/comment-reopen interactions, while system follow-ups leave closed work closed - updated adapter-utils prompt guidance to prefer targeted verification - added Codex model refresh support in the server route, registry, shared types, and agent config form - added adapter/server tests covering the new parsing, retry scheduling, and model-refresh behavior ## Verification - `pnpm exec vitest run --project @paperclipai/adapter-utils packages/adapter-utils/src/server-utils.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-claude-local packages/adapters/claude-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/adapter-codex-local packages/adapters/codex-local/src/server/parse.test.ts` - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/adapter-model-refresh-routes.test.ts server/src/__tests__/adapter-models.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/codex-local-execute.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/heartbeat-retry-scheduling.test.ts` ## Risks - Moderate behavior risk: retry classification affects whether runs auto-recover or block, so mistakes here could either suppress needed retries or over-retry real failures - Low workflow risk: deferred comment wake reopening is intentionally scoped to human/comment-reopen interactions so system follow-ups do not revive completed issues unexpectedly > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex GPT-5-based coding agent with tool use and code execution in the Codex CLI environment ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
4fdbbeced3
commit
8f1cd0474f
25 changed files with 1455 additions and 48 deletions
185
server/src/__tests__/adapter-model-refresh-routes.test.ts
Normal file
185
server/src/__tests__/adapter-model-refresh-routes.test.ts
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ServerAdapterModule } from "../adapters/index.js";
|
||||
|
||||
const mockAccessService = vi.hoisted(() => ({
|
||||
canUser: vi.fn(),
|
||||
hasPermission: vi.fn(),
|
||||
ensureMembership: vi.fn(),
|
||||
setPrincipalPermission: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockCompanySkillService = vi.hoisted(() => ({
|
||||
listRuntimeSkillEntries: vi.fn(),
|
||||
resolveRequestedSkillKeys: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockSecretService = vi.hoisted(() => ({
|
||||
normalizeAdapterConfigForPersistence: vi.fn(async (_companyId: string, config: Record<string, unknown>) => config),
|
||||
resolveAdapterConfigForRuntime: vi.fn(async (_companyId: string, config: Record<string, unknown>) => ({ config })),
|
||||
}));
|
||||
|
||||
const mockAgentInstructionsService = vi.hoisted(() => ({
|
||||
materializeManagedBundle: vi.fn(),
|
||||
getBundle: vi.fn(),
|
||||
readFile: vi.fn(),
|
||||
updateBundle: vi.fn(),
|
||||
writeFile: vi.fn(),
|
||||
deleteFile: vi.fn(),
|
||||
exportFiles: vi.fn(),
|
||||
ensureManagedBundle: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockBudgetService = vi.hoisted(() => ({
|
||||
upsertPolicy: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockHeartbeatService = vi.hoisted(() => ({
|
||||
cancelActiveForAgent: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockIssueApprovalService = vi.hoisted(() => ({
|
||||
linkManyForApproval: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockApprovalService = vi.hoisted(() => ({
|
||||
create: vi.fn(),
|
||||
getById: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockInstanceSettingsService = vi.hoisted(() => ({
|
||||
getGeneral: vi.fn(async () => ({ censorUsernameInLogs: false })),
|
||||
}));
|
||||
|
||||
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||
|
||||
function registerModuleMocks() {
|
||||
vi.doMock("../services/index.js", () => ({
|
||||
agentService: () => ({}),
|
||||
agentInstructionsService: () => mockAgentInstructionsService,
|
||||
accessService: () => mockAccessService,
|
||||
approvalService: () => mockApprovalService,
|
||||
companySkillService: () => mockCompanySkillService,
|
||||
budgetService: () => mockBudgetService,
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
issueApprovalService: () => mockIssueApprovalService,
|
||||
issueService: () => ({}),
|
||||
logActivity: mockLogActivity,
|
||||
secretService: () => mockSecretService,
|
||||
syncInstructionsBundleConfigFromFilePath: vi.fn((_agent, config) => config),
|
||||
workspaceOperationService: () => ({}),
|
||||
}));
|
||||
|
||||
vi.doMock("../services/instance-settings.js", () => ({
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
}));
|
||||
}
|
||||
|
||||
const refreshableAdapterType = "refreshable_adapter_route_test";
|
||||
|
||||
async function createApp() {
|
||||
const [{ agentRoutes }, { errorHandler }] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/agents.js")>("../routes/agents.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
]);
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = {
|
||||
type: "board",
|
||||
userId: "local-board",
|
||||
companyIds: ["company-1"],
|
||||
source: "local_implicit",
|
||||
isInstanceAdmin: false,
|
||||
};
|
||||
next();
|
||||
});
|
||||
app.use("/api", agentRoutes({} as any));
|
||||
app.use(errorHandler);
|
||||
return app;
|
||||
}
|
||||
|
||||
async function requestApp(
|
||||
app: express.Express,
|
||||
buildRequest: (baseUrl: string) => request.Test,
|
||||
) {
|
||||
const { createServer } = await vi.importActual<typeof import("node:http")>("node:http");
|
||||
const server = createServer(app);
|
||||
try {
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("Expected HTTP server to listen on a TCP port");
|
||||
}
|
||||
return await buildRequest(`http://127.0.0.1:${address.port}`);
|
||||
} finally {
|
||||
if (server.listening) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function unregisterTestAdapter(type: string) {
|
||||
const { unregisterServerAdapter } = await import("../adapters/index.js");
|
||||
unregisterServerAdapter(type);
|
||||
}
|
||||
|
||||
describe("adapter model refresh route", () => {
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("../routes/agents.js");
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
registerModuleMocks();
|
||||
vi.clearAllMocks();
|
||||
mockCompanySkillService.listRuntimeSkillEntries.mockResolvedValue([]);
|
||||
mockCompanySkillService.resolveRequestedSkillKeys.mockResolvedValue([]);
|
||||
mockAccessService.canUser.mockResolvedValue(true);
|
||||
mockAccessService.hasPermission.mockResolvedValue(true);
|
||||
mockAccessService.ensureMembership.mockResolvedValue(undefined);
|
||||
mockAccessService.setPrincipalPermission.mockResolvedValue(undefined);
|
||||
mockLogActivity.mockResolvedValue(undefined);
|
||||
await unregisterTestAdapter(refreshableAdapterType);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await unregisterTestAdapter(refreshableAdapterType);
|
||||
});
|
||||
|
||||
it("uses refreshModels when refresh=1 is requested", async () => {
|
||||
const listModels = vi.fn(async () => [{ id: "stale-model", label: "stale-model" }]);
|
||||
const refreshModels = vi.fn(async () => [{ id: "fresh-model", label: "fresh-model" }]);
|
||||
const { registerServerAdapter } = await import("../adapters/index.js");
|
||||
const adapter: ServerAdapterModule = {
|
||||
type: refreshableAdapterType,
|
||||
execute: async () => ({ exitCode: 0, signal: null, timedOut: false }),
|
||||
testEnvironment: async () => ({
|
||||
adapterType: refreshableAdapterType,
|
||||
status: "pass",
|
||||
checks: [],
|
||||
testedAt: new Date(0).toISOString(),
|
||||
}),
|
||||
listModels,
|
||||
refreshModels,
|
||||
};
|
||||
registerServerAdapter(adapter);
|
||||
|
||||
const app = await createApp();
|
||||
const res = await requestApp(app, (baseUrl) =>
|
||||
request(baseUrl).get(`/api/companies/company-1/adapters/${refreshableAdapterType}/models?refresh=1`),
|
||||
);
|
||||
|
||||
expect(res.status, JSON.stringify(res.body)).toBe(200);
|
||||
expect(res.body).toEqual([{ id: "fresh-model", label: "fresh-model" }]);
|
||||
expect(refreshModels).toHaveBeenCalledTimes(1);
|
||||
expect(listModels).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -3,7 +3,7 @@ import { models as codexFallbackModels } from "@paperclipai/adapter-codex-local"
|
|||
import { models as cursorFallbackModels } from "@paperclipai/adapter-cursor-local";
|
||||
import { models as opencodeFallbackModels } from "@paperclipai/adapter-opencode-local";
|
||||
import { resetOpenCodeModelsCacheForTests } from "@paperclipai/adapter-opencode-local/server";
|
||||
import { listAdapterModels } from "../adapters/index.js";
|
||||
import { listAdapterModels, refreshAdapterModels } from "../adapters/index.js";
|
||||
import { resetCodexModelsCacheForTests } from "../adapters/codex-models.js";
|
||||
import { resetCursorModelsCacheForTests, setCursorModelsRunnerForTests } from "../adapters/cursor-models.js";
|
||||
|
||||
|
|
@ -52,6 +52,30 @@ describe("adapter model listing", () => {
|
|||
expect(first.some((model) => model.id === "codex-mini-latest")).toBe(true);
|
||||
});
|
||||
|
||||
it("refreshes cached codex models on demand", async () => {
|
||||
process.env.OPENAI_API_KEY = "sk-test";
|
||||
const fetchSpy = vi.spyOn(globalThis, "fetch")
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [{ id: "gpt-5" }],
|
||||
}),
|
||||
} as Response)
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [{ id: "gpt-5.5" }],
|
||||
}),
|
||||
} as Response);
|
||||
|
||||
const initial = await listAdapterModels("codex_local");
|
||||
const refreshed = await refreshAdapterModels("codex_local");
|
||||
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(2);
|
||||
expect(initial.some((model) => model.id === "gpt-5")).toBe(true);
|
||||
expect(refreshed.some((model) => model.id === "gpt-5.5")).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back to static codex models when OpenAI model discovery fails", async () => {
|
||||
process.env.OPENAI_API_KEY = "sk-test";
|
||||
vi.spyOn(globalThis, "fetch").mockResolvedValue({
|
||||
|
|
|
|||
|
|
@ -1,9 +1,23 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execute } from "@paperclipai/adapter-claude-local/server";
|
||||
|
||||
async function writeFailingClaudeCommand(
|
||||
commandPath: string,
|
||||
options: { resultEvent: Record<string, unknown>; exitCode?: number },
|
||||
): Promise<void> {
|
||||
const payload = JSON.stringify(options.resultEvent);
|
||||
const exit = options.exitCode ?? 1;
|
||||
const script = `#!/usr/bin/env node
|
||||
console.log(${JSON.stringify(payload)});
|
||||
process.exit(${exit});
|
||||
`;
|
||||
await fs.writeFile(commandPath, script, "utf8");
|
||||
await fs.chmod(commandPath, 0o755);
|
||||
}
|
||||
|
||||
async function writeFakeClaudeCommand(commandPath: string): Promise<void> {
|
||||
const script = `#!/usr/bin/env node
|
||||
const fs = require("node:fs");
|
||||
|
|
@ -398,8 +412,10 @@ describe("claude execute", () => {
|
|||
|
||||
const previousHome = process.env.HOME;
|
||||
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
|
||||
const previousPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
|
||||
process.env.HOME = root;
|
||||
process.env.PAPERCLIP_HOME = paperclipHome;
|
||||
process.env.PAPERCLIP_INSTANCE_ID = "default";
|
||||
|
||||
try {
|
||||
const first = await execute({
|
||||
|
|
@ -534,6 +550,8 @@ describe("claude execute", () => {
|
|||
else process.env.HOME = previousHome;
|
||||
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
|
||||
if (previousPaperclipInstanceId === undefined) delete process.env.PAPERCLIP_INSTANCE_ID;
|
||||
else process.env.PAPERCLIP_INSTANCE_ID = previousPaperclipInstanceId;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
|
@ -553,8 +571,10 @@ describe("claude execute", () => {
|
|||
|
||||
const previousHome = process.env.HOME;
|
||||
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
|
||||
const previousPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
|
||||
process.env.HOME = root;
|
||||
process.env.PAPERCLIP_HOME = paperclipHome;
|
||||
process.env.PAPERCLIP_INSTANCE_ID = "default";
|
||||
|
||||
try {
|
||||
const first = await execute({
|
||||
|
|
@ -635,7 +655,184 @@ describe("claude execute", () => {
|
|||
else process.env.HOME = previousHome;
|
||||
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
|
||||
if (previousPaperclipInstanceId === undefined) delete process.env.PAPERCLIP_INSTANCE_ID;
|
||||
else process.env.PAPERCLIP_INSTANCE_ID = previousPaperclipInstanceId;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("classifies Claude 'out of extra usage' failures as transient upstream errors", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-transient-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
const commandPath = path.join(root, "claude");
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
await writeFailingClaudeCommand(commandPath, {
|
||||
resultEvent: {
|
||||
type: "result",
|
||||
subtype: "error",
|
||||
session_id: "claude-session-extra",
|
||||
is_error: true,
|
||||
result: "You're out of extra usage · resets 4pm (America/Chicago)",
|
||||
errors: [{ type: "rate_limit_error", message: "You're out of extra usage" }],
|
||||
},
|
||||
});
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
process.env.HOME = root;
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date(2026, 3, 22, 10, 15, 0));
|
||||
|
||||
try {
|
||||
const result = await execute({
|
||||
runId: "run-claude-transient",
|
||||
agent: {
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
name: "Claude Coder",
|
||||
adapterType: "claude_local",
|
||||
adapterConfig: {},
|
||||
},
|
||||
runtime: {
|
||||
sessionId: null,
|
||||
sessionParams: null,
|
||||
sessionDisplayId: null,
|
||||
taskKey: null,
|
||||
},
|
||||
config: {
|
||||
command: commandPath,
|
||||
cwd: workspace,
|
||||
promptTemplate: "Follow the paperclip heartbeat.",
|
||||
},
|
||||
context: {},
|
||||
authToken: "run-jwt-token",
|
||||
onLog: async () => {},
|
||||
});
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("claude_transient_upstream");
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
expect(result.retryNotBefore).toBe("2026-04-22T21:00:00.000Z");
|
||||
expect(result.resultJson?.retryNotBefore).toBe("2026-04-22T21:00:00.000Z");
|
||||
expect(result.errorMessage ?? "").toContain("extra usage");
|
||||
expect(new Date(String(result.resultJson?.transientRetryNotBefore)).getTime()).toBe(
|
||||
new Date("2026-04-22T21:00:00.000Z").getTime(),
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("classifies rate-limit / overloaded failures without reset metadata as transient", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-rate-limit-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
const commandPath = path.join(root, "claude");
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
await writeFailingClaudeCommand(commandPath, {
|
||||
resultEvent: {
|
||||
type: "result",
|
||||
subtype: "error",
|
||||
session_id: "claude-session-overloaded",
|
||||
is_error: true,
|
||||
result: "Overloaded",
|
||||
errors: [{ type: "overloaded_error", message: "Overloaded_error: API is overloaded." }],
|
||||
},
|
||||
});
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
process.env.HOME = root;
|
||||
|
||||
try {
|
||||
const result = await execute({
|
||||
runId: "run-claude-overloaded",
|
||||
agent: {
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
name: "Claude Coder",
|
||||
adapterType: "claude_local",
|
||||
adapterConfig: {},
|
||||
},
|
||||
runtime: {
|
||||
sessionId: null,
|
||||
sessionParams: null,
|
||||
sessionDisplayId: null,
|
||||
taskKey: null,
|
||||
},
|
||||
config: {
|
||||
command: commandPath,
|
||||
cwd: workspace,
|
||||
promptTemplate: "Follow the paperclip heartbeat.",
|
||||
},
|
||||
context: {},
|
||||
authToken: "run-jwt-token",
|
||||
onLog: async () => {},
|
||||
});
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("claude_transient_upstream");
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
expect(result.retryNotBefore ?? null).toBeNull();
|
||||
expect(result.resultJson?.retryNotBefore ?? null).toBeNull();
|
||||
expect(result.resultJson?.transientRetryNotBefore ?? null).toBeNull();
|
||||
} finally {
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not reclassify deterministic Claude failures (auth, max turns) as transient", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-max-turns-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
const commandPath = path.join(root, "claude");
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
await writeFailingClaudeCommand(commandPath, {
|
||||
resultEvent: {
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
session_id: "claude-session-max-turns",
|
||||
is_error: true,
|
||||
result: "Maximum turns reached.",
|
||||
},
|
||||
});
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
process.env.HOME = root;
|
||||
|
||||
try {
|
||||
const result = await execute({
|
||||
runId: "run-claude-max-turns",
|
||||
agent: {
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
name: "Claude Coder",
|
||||
adapterType: "claude_local",
|
||||
adapterConfig: {},
|
||||
},
|
||||
runtime: {
|
||||
sessionId: null,
|
||||
sessionParams: null,
|
||||
sessionDisplayId: null,
|
||||
taskKey: null,
|
||||
},
|
||||
config: {
|
||||
command: commandPath,
|
||||
cwd: workspace,
|
||||
promptTemplate: "Follow the paperclip heartbeat.",
|
||||
},
|
||||
context: {},
|
||||
authToken: "run-jwt-token",
|
||||
onLog: async () => {},
|
||||
});
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).not.toBe("claude_transient_upstream");
|
||||
} finally {
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,8 +7,11 @@ import {
|
|||
companies,
|
||||
companySkills,
|
||||
createDb,
|
||||
documents,
|
||||
documentRevisions,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
issueExecutionDecisions,
|
||||
issueReadStates,
|
||||
issues,
|
||||
|
|
@ -43,6 +46,8 @@ describeEmbeddedPostgres("cleanup removal services", () => {
|
|||
await db.delete(issueReadStates);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issueExecutionDecisions);
|
||||
await db.delete(documentRevisions);
|
||||
await db.delete(documents);
|
||||
await db.delete(companySkills);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(issues);
|
||||
|
|
@ -148,6 +153,8 @@ describeEmbeddedPostgres("cleanup removal services", () => {
|
|||
|
||||
it("removes issue read states and activity rows before deleting the company", async () => {
|
||||
const { companyId, issueId, runId } = await seedFixture();
|
||||
const documentId = randomUUID();
|
||||
const revisionId = randomUUID();
|
||||
|
||||
await db.insert(issueReadStates).values({
|
||||
id: randomUUID(),
|
||||
|
|
@ -177,11 +184,47 @@ describeEmbeddedPostgres("cleanup removal services", () => {
|
|||
details: {},
|
||||
});
|
||||
|
||||
await db.insert(documents).values({
|
||||
id: documentId,
|
||||
companyId,
|
||||
title: "Run summary",
|
||||
latestBody: "body",
|
||||
latestRevisionId: revisionId,
|
||||
latestRevisionNumber: 1,
|
||||
createdByAgentId: null,
|
||||
createdByUserId: "user-1",
|
||||
updatedByAgentId: null,
|
||||
updatedByUserId: "user-1",
|
||||
});
|
||||
|
||||
await db.insert(issueDocuments).values({
|
||||
id: randomUUID(),
|
||||
companyId,
|
||||
issueId,
|
||||
documentId,
|
||||
key: "summary",
|
||||
});
|
||||
|
||||
await db.insert(documentRevisions).values({
|
||||
id: revisionId,
|
||||
companyId,
|
||||
documentId,
|
||||
revisionNumber: 1,
|
||||
title: "Run summary",
|
||||
format: "markdown",
|
||||
body: "body",
|
||||
createdByAgentId: null,
|
||||
createdByUserId: "user-1",
|
||||
createdByRunId: runId,
|
||||
});
|
||||
|
||||
const removed = await companyService(db).remove(companyId);
|
||||
|
||||
expect(removed?.id).toBe(companyId);
|
||||
await expect(db.select().from(companies).where(eq(companies.id, companyId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(documents).where(eq(documents.id, documentId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(documentRevisions).where(eq(documentRevisions.id, revisionId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(issueReadStates).where(eq(issueReadStates.companyId, companyId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(activityLog).where(eq(activityLog.companyId, companyId))).resolves.toHaveLength(0);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
|
@ -419,6 +419,7 @@ describe("codex execute", () => {
|
|||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("codex_transient_upstream");
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
expect(result.errorMessage).toContain("high demand");
|
||||
} finally {
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
|
|
@ -427,6 +428,68 @@ describe("codex execute", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("persists retry-not-before metadata for codex usage-limit failures", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-usage-limit-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
const commandPath = path.join(root, "codex");
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
await writeFailingCodexCommand(
|
||||
commandPath,
|
||||
"You've hit your usage limit for GPT-5.3-Codex-Spark. Switch to another model now, or try again at 11:31 PM.",
|
||||
);
|
||||
|
||||
const previousHome = process.env.HOME;
|
||||
process.env.HOME = root;
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date(2026, 3, 22, 22, 29, 0));
|
||||
|
||||
try {
|
||||
const result = await execute({
|
||||
runId: "run-usage-limit",
|
||||
agent: {
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
name: "Codex Coder",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
},
|
||||
runtime: {
|
||||
sessionId: "codex-session-usage-limit",
|
||||
sessionParams: {
|
||||
sessionId: "codex-session-usage-limit",
|
||||
cwd: workspace,
|
||||
},
|
||||
sessionDisplayId: "codex-session-usage-limit",
|
||||
taskKey: null,
|
||||
},
|
||||
config: {
|
||||
command: commandPath,
|
||||
cwd: workspace,
|
||||
model: "gpt-5.3-codex-spark",
|
||||
promptTemplate: "Follow the paperclip heartbeat.",
|
||||
},
|
||||
context: {},
|
||||
authToken: "run-jwt-token",
|
||||
onLog: async () => {},
|
||||
});
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("codex_transient_upstream");
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
const expectedRetryNotBefore = new Date(2026, 3, 22, 23, 31, 0, 0).toISOString();
|
||||
expect(result.retryNotBefore).toBe(expectedRetryNotBefore);
|
||||
expect(result.resultJson?.retryNotBefore).toBe(expectedRetryNotBefore);
|
||||
expect(new Date(String(result.resultJson?.transientRetryNotBefore)).getTime()).toBe(
|
||||
new Date(2026, 3, 22, 23, 31, 0, 0).getTime(),
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("uses safer invocation settings and a fresh-session handoff for codex transient fallback retries", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-fallback-"));
|
||||
const workspace = path.join(root, "workspace");
|
||||
|
|
|
|||
|
|
@ -765,11 +765,15 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
exitCode: 1,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorCode: "codex_transient_upstream",
|
||||
errorCode: "adapter_failed",
|
||||
errorFamily: "transient_upstream",
|
||||
errorMessage:
|
||||
"Error running remote compact task: We're currently experiencing high demand, which may cause temporary errors.",
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
resultJson: {
|
||||
errorFamily: "transient_upstream",
|
||||
},
|
||||
});
|
||||
|
||||
const { agentId, runId, issueId } = await seedQueuedIssueRunFixture();
|
||||
|
|
@ -790,7 +794,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
const failedRun = runs?.find((row) => row.id === runId);
|
||||
const retryRun = runs?.find((row) => row.id !== runId);
|
||||
expect(failedRun?.status).toBe("failed");
|
||||
expect(failedRun?.errorCode).toBe("codex_transient_upstream");
|
||||
expect(failedRun?.errorCode).toBe("adapter_failed");
|
||||
expect((failedRun?.resultJson as Record<string, unknown> | null)?.errorFamily).toBe("transient_upstream");
|
||||
expect(retryRun?.status).toBe("scheduled_retry");
|
||||
expect(retryRun?.scheduledRetryReason).toBe("transient_failure");
|
||||
expect((retryRun?.contextSnapshot as Record<string, unknown> | null)?.codexTransientFallbackMode).toBe("same_session");
|
||||
|
|
|
|||
|
|
@ -56,8 +56,15 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
|||
agentId: string;
|
||||
now: Date;
|
||||
errorCode: string;
|
||||
errorFamily?: "transient_upstream" | null;
|
||||
retryNotBefore?: string | null;
|
||||
scheduledRetryAttempt?: number;
|
||||
resultJson?: Record<string, unknown> | null;
|
||||
adapterType?: "codex_local" | "claude_local";
|
||||
agentName?: string;
|
||||
}) {
|
||||
const adapterType = input.adapterType ?? "codex_local";
|
||||
const agentName = input.agentName ?? (adapterType === "claude_local" ? "ClaudeCoder" : "CodexCoder");
|
||||
await db.insert(companies).values({
|
||||
id: input.companyId,
|
||||
name: "Paperclip",
|
||||
|
|
@ -68,10 +75,10 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
|||
await db.insert(agents).values({
|
||||
id: input.agentId,
|
||||
companyId: input.companyId,
|
||||
name: "CodexCoder",
|
||||
name: agentName,
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterType,
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
|
|
@ -93,6 +100,15 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
|||
finishedAt: input.now,
|
||||
scheduledRetryAttempt: input.scheduledRetryAttempt ?? 0,
|
||||
scheduledRetryReason: input.scheduledRetryAttempt ? "transient_failure" : null,
|
||||
resultJson: input.resultJson ?? {
|
||||
...(input.errorFamily ? { errorFamily: input.errorFamily } : {}),
|
||||
...(input.retryNotBefore
|
||||
? {
|
||||
retryNotBefore: input.retryNotBefore,
|
||||
transientRetryNotBefore: input.retryNotBefore,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
contextSnapshot: {
|
||||
issueId: randomUUID(),
|
||||
wakeReason: "issue_assigned",
|
||||
|
|
@ -299,7 +315,8 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
|||
companyId,
|
||||
agentId,
|
||||
now,
|
||||
errorCode: "codex_transient_upstream",
|
||||
errorCode: "adapter_failed",
|
||||
errorFamily: "transient_upstream",
|
||||
scheduledRetryAttempt: index,
|
||||
});
|
||||
|
||||
|
|
@ -335,4 +352,110 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
|||
await db.delete(companies);
|
||||
}
|
||||
});
|
||||
|
||||
it("honors codex retry-not-before timestamps when they exceed the default bounded backoff", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const now = new Date(2026, 3, 22, 22, 29, 0);
|
||||
const retryNotBefore = new Date(2026, 3, 22, 23, 31, 0);
|
||||
|
||||
await seedRetryFixture({
|
||||
runId,
|
||||
companyId,
|
||||
agentId,
|
||||
now,
|
||||
errorCode: "adapter_failed",
|
||||
errorFamily: "transient_upstream",
|
||||
retryNotBefore: retryNotBefore.toISOString(),
|
||||
});
|
||||
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(runId, {
|
||||
now,
|
||||
random: () => 0.5,
|
||||
});
|
||||
|
||||
expect(scheduled.outcome).toBe("scheduled");
|
||||
if (scheduled.outcome !== "scheduled") return;
|
||||
expect(scheduled.dueAt.getTime()).toBe(retryNotBefore.getTime());
|
||||
|
||||
const retryRun = await db
|
||||
.select({
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||||
scheduledRetryAt: heartbeatRuns.scheduledRetryAt,
|
||||
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, scheduled.run.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(retryRun?.scheduledRetryAt?.getTime()).toBe(retryNotBefore.getTime());
|
||||
expect((retryRun?.contextSnapshot as Record<string, unknown> | null)?.transientRetryNotBefore).toBe(
|
||||
retryNotBefore.toISOString(),
|
||||
);
|
||||
|
||||
const wakeupRequest = await db
|
||||
.select({ payload: agentWakeupRequests.payload })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, retryRun?.wakeupRequestId ?? ""))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect((wakeupRequest?.payload as Record<string, unknown> | null)?.transientRetryNotBefore).toBe(
|
||||
retryNotBefore.toISOString(),
|
||||
);
|
||||
});
|
||||
|
||||
it("schedules bounded retries for claude_transient_upstream and honors its retry-not-before hint", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const now = new Date(2026, 3, 22, 10, 0, 0);
|
||||
const retryNotBefore = new Date(2026, 3, 22, 16, 0, 0);
|
||||
|
||||
await seedRetryFixture({
|
||||
runId,
|
||||
companyId,
|
||||
agentId,
|
||||
now,
|
||||
errorCode: "adapter_failed",
|
||||
errorFamily: "transient_upstream",
|
||||
adapterType: "claude_local",
|
||||
retryNotBefore: retryNotBefore.toISOString(),
|
||||
});
|
||||
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(runId, {
|
||||
now,
|
||||
random: () => 0.5,
|
||||
});
|
||||
|
||||
expect(scheduled.outcome).toBe("scheduled");
|
||||
if (scheduled.outcome !== "scheduled") return;
|
||||
expect(scheduled.dueAt.getTime()).toBe(retryNotBefore.getTime());
|
||||
|
||||
const retryRun = await db
|
||||
.select({
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||||
scheduledRetryAt: heartbeatRuns.scheduledRetryAt,
|
||||
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, scheduled.run.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(retryRun?.scheduledRetryAt?.getTime()).toBe(retryNotBefore.getTime());
|
||||
const contextSnapshot = (retryRun?.contextSnapshot as Record<string, unknown> | null) ?? {};
|
||||
expect(contextSnapshot.transientRetryNotBefore).toBe(retryNotBefore.toISOString());
|
||||
// Claude does not participate in the Codex fallback-mode ladder.
|
||||
expect(contextSnapshot.codexTransientFallbackMode ?? null).toBeNull();
|
||||
|
||||
const wakeupRequest = await db
|
||||
.select({ payload: agentWakeupRequests.payload })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, retryRun?.wakeupRequestId ?? ""))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect((wakeupRequest?.payload as Record<string, unknown> | null)?.transientRetryNotBefore).toBe(
|
||||
retryNotBefore.toISOString(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue