From b74d94ba1ece18cfa88a0b00f9e8c7472760e022 Mon Sep 17 00:00:00 2001 From: Dawid Piaskowski Date: Mon, 6 Apr 2026 23:29:41 +0200 Subject: [PATCH] Treat Pi quota exhaustion as a failed run (#2305) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path Paperclip orchestrates AI agent runs and reports their success or failure. The Pi adapter spawns a local Pi process and interprets its JSONL output to determine the run outcome. When Pi hits a quota limit (429 RESOURCE_EXHAUSTED), it retries internally and emits an `auto_retry_end` event with `success: false` — but still exits with code 0. The current adapter trusts the exit code, so Paperclip marks the run as succeeded even though it produced no useful work. This PR teaches the parser to detect quota exhaustion and synthesize a failure. Closes #2234 ## Changes - Parse `auto_retry_end` events with `success: false` into `result.errors` - Parse standalone `error` events into `result.errors` - Synthesize exit code 1 when Pi exits 0 but parsed errors exist - Use the parsed error as `errorMessage` so the failure reason is visible in the UI ## Verification ```bash pnpm vitest run pi-local-execute pnpm vitest run --reporter=verbose 2>&1 | grep pi-local ``` - `parse.test.ts`: covers failed retry, successful retry (no error), standalone error events, and empty error messages - `pi-local-execute.test.ts`: end-to-end test with a fake Pi binary that emits `auto_retry_end` + exits 0, asserts the run is marked failed ## Risks - **Low**: Only affects runs where Pi exits 0 with a parsed error — no change to normal successful or already-failing runs - If Pi emits `auto_retry_end { success: false }` but the run actually produced valid output, this would incorrectly mark it as failed. This seems unlikely given the semantics of the event. ## Model Used - Claude Opus 4.6 (Anthropic) — assisted with test additions and PR template ## Checklist - [x] Thinking path documented - [x] Model specified - [x] Tests pass locally - [x] Test coverage for new parse branches (success path, error events, empty messages) - [x] No UI changes - [x] Risk analysis included --------- Co-authored-by: Dawid Piaskowski Co-authored-by: Claude Opus 4.6 --- .../adapters/pi-local/src/server/execute.ts | 8 +- .../pi-local/src/server/parse.test.ts | 51 +++++++++++++ .../adapters/pi-local/src/server/parse.ts | 17 +++++ server/src/__tests__/pi-local-execute.test.ts | 76 +++++++++++++++++++ 4 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 server/src/__tests__/pi-local-execute.test.ts diff --git a/packages/adapters/pi-local/src/server/execute.ts b/packages/adapters/pi-local/src/server/execute.ts index c7eb61a2..22d6d1b9 100644 --- a/packages/adapters/pi-local/src/server/execute.ts +++ b/packages/adapters/pi-local/src/server/execute.ts @@ -451,13 +451,15 @@ export async function execute(ctx: AdapterExecutionContext): Promise error.trim().length > 0) ?? ""; + const effectiveExitCode = (rawExitCode ?? 0) === 0 && parsedError ? 1 : rawExitCode; + const fallbackErrorMessage = parsedError || stderrLine || `Pi exited with code ${rawExitCode ?? -1}`; return { - exitCode: rawExitCode, + exitCode: effectiveExitCode, signal: attempt.proc.signal, timedOut: false, - errorMessage: (rawExitCode ?? 0) === 0 ? null : fallbackErrorMessage, + errorMessage: (effectiveExitCode ?? 0) === 0 ? null : fallbackErrorMessage, usage: { inputTokens: attempt.parsed.usage.inputTokens, outputTokens: attempt.parsed.usage.outputTokens, diff --git a/packages/adapters/pi-local/src/server/parse.test.ts b/packages/adapters/pi-local/src/server/parse.test.ts index 6a3eef4d..34ef8b93 100644 --- a/packages/adapters/pi-local/src/server/parse.test.ts +++ b/packages/adapters/pi-local/src/server/parse.test.ts @@ -209,6 +209,57 @@ describe("parsePiJsonl", () => { expect(parsed.usage.cachedInputTokens).toBe(25); expect(parsed.usage.costUsd).toBe(0.003); }); + + it("surfaces failed auto-retry exhaustion as an error", () => { + const stdout = [ + JSON.stringify({ + type: "auto_retry_end", + success: false, + attempt: 3, + finalError: "Cloud Code Assist API error (429): RESOURCE_EXHAUSTED", + }), + ].join("\n"); + + const parsed = parsePiJsonl(stdout); + expect(parsed.errors).toEqual(["Cloud Code Assist API error (429): RESOURCE_EXHAUSTED"]); + }); + + it("does not treat successful auto-retry as an error", () => { + const stdout = [ + JSON.stringify({ + type: "auto_retry_end", + success: true, + attempt: 2, + }), + ].join("\n"); + + const parsed = parsePiJsonl(stdout); + expect(parsed.errors).toEqual([]); + }); + + it("surfaces standalone error events", () => { + const stdout = [ + JSON.stringify({ + type: "error", + message: "Connection to model provider lost", + }), + ].join("\n"); + + const parsed = parsePiJsonl(stdout); + expect(parsed.errors).toEqual(["Connection to model provider lost"]); + }); + + it("ignores error events with empty messages", () => { + const stdout = [ + JSON.stringify({ + type: "error", + message: "", + }), + ].join("\n"); + + const parsed = parsePiJsonl(stdout); + expect(parsed.errors).toEqual([]); + }); }); describe("isPiUnknownSessionError", () => { diff --git a/packages/adapters/pi-local/src/server/parse.ts b/packages/adapters/pi-local/src/server/parse.ts index 3ba50d8b..5ab0d4cd 100644 --- a/packages/adapters/pi-local/src/server/parse.ts +++ b/packages/adapters/pi-local/src/server/parse.ts @@ -76,6 +76,15 @@ export function parsePiJsonl(stdout: string): ParsedPiOutput { continue; } + if (eventType === "auto_retry_end") { + const succeeded = event.success === true; + if (!succeeded) { + const finalError = asString(event.finalError, "").trim(); + result.errors.push(finalError || "Pi exhausted automatic retries without producing a response."); + } + continue; + } + // Turn lifecycle if (eventType === "turn_start") { continue; @@ -145,6 +154,14 @@ export function parsePiJsonl(stdout: string): ParsedPiOutput { continue; } + if (eventType === "error") { + const message = asString(event.message, "").trim(); + if (message) { + result.errors.push(message); + } + continue; + } + // Tool execution if (eventType === "tool_execution_start") { const toolCallId = asString(event.toolCallId, ""); diff --git a/server/src/__tests__/pi-local-execute.test.ts b/server/src/__tests__/pi-local-execute.test.ts new file mode 100644 index 00000000..e0f49a27 --- /dev/null +++ b/server/src/__tests__/pi-local-execute.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { execute } from "@paperclipai/adapter-pi-local/server"; + +async function writeFakePiCommand(commandPath: string): Promise { + const script = `#!/usr/bin/env node +if (process.argv.includes("--list-models")) { + console.log("provider model"); + console.log("google gemini-3-flash-preview"); + process.exit(0); +} +console.log(JSON.stringify({ type: "agent_start" })); +console.log(JSON.stringify({ type: "turn_start" })); +console.log(JSON.stringify({ type: "turn_end", message: { role: "assistant", content: "" }, toolResults: [] })); +console.log(JSON.stringify({ type: "agent_end", messages: [] })); +console.log(JSON.stringify({ + type: "auto_retry_end", + success: false, + attempt: 3, + finalError: "Cloud Code Assist API error (429): RESOURCE_EXHAUSTED" +})); +process.exit(0); +`; + await fs.writeFile(commandPath, script, "utf8"); + await fs.chmod(commandPath, 0o755); +} + +describe("pi_local execute", () => { + it("fails the run when Pi exhausts automatic retries despite exiting 0", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-pi-execute-")); + const workspace = path.join(root, "workspace"); + const commandPath = path.join(root, "pi"); + await fs.mkdir(workspace, { recursive: true }); + await writeFakePiCommand(commandPath); + + const previousHome = process.env.HOME; + process.env.HOME = root; + + try { + const result = await execute({ + runId: "run-pi-quota-exhausted", + agent: { + id: "agent-1", + companyId: "company-1", + name: "Pi Agent", + adapterType: "pi_local", + adapterConfig: {}, + }, + runtime: { + sessionId: null, + sessionParams: null, + sessionDisplayId: null, + taskKey: null, + }, + config: { + command: commandPath, + cwd: workspace, + model: "google/gemini-3-flash-preview", + promptTemplate: "Keep working.", + }, + context: {}, + authToken: "run-jwt-token", + onLog: async () => {}, + }); + + expect(result.exitCode).toBe(1); + expect(result.errorMessage).toContain("RESOURCE_EXHAUSTED"); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + await fs.rm(root, { recursive: true, force: true }); + } + }); +});