mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-15 10:30:37 +09:00
[codex] Add plugin orchestration host APIs (#4114)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies. > - The plugin system is the extension path for optional capabilities that should not require core product changes for every integration. > - Plugins need scoped host APIs for issue orchestration, documents, wakeups, summaries, activity attribution, and isolated database state. > - Without those host APIs, richer plugins either cannot coordinate Paperclip work safely or need privileged core-side special cases. > - This pull request adds the plugin orchestration host surface, scoped route dispatch, a database namespace layer, and a smoke plugin that exercises the contract. > - The benefit is a broader plugin API that remains company-scoped, auditable, and covered by tests. ## What Changed - Added plugin orchestration host APIs for issue creation, document access, wakeups, summaries, plugin-origin activity, and scoped API route dispatch. - Added plugin database namespace tables, schema exports, migration checks, and idempotent replay coverage under migration `0059_plugin_database_namespaces`. - Added shared plugin route/API types and validators used by server and SDK boundaries. - Expanded plugin SDK types, protocol helpers, worker RPC host behavior, and testing utilities for orchestration flows. - Added the `plugin-orchestration-smoke-example` package to exercise scoped routes, restricted database namespaces, issue orchestration, documents, wakeups, summaries, and UI status surfaces. - Kept the new orchestration smoke fixture out of the root pnpm workspace importer so this PR preserves the repository policy of not committing `pnpm-lock.yaml`. - Updated plugin docs and database docs for the new orchestration and database namespace surfaces. - Rebased the branch onto `public-gh/master`, resolved conflicts, and removed `pnpm-lock.yaml` from the final PR diff. ## Verification - `pnpm install --frozen-lockfile` - `pnpm --filter @paperclipai/db typecheck` - `pnpm exec vitest run packages/db/src/client.test.ts` - `pnpm exec vitest run server/src/__tests__/plugin-database.test.ts server/src/__tests__/plugin-orchestration-apis.test.ts server/src/__tests__/plugin-routes-authz.test.ts server/src/__tests__/plugin-scoped-api-routes.test.ts server/src/__tests__/plugin-sdk-orchestration-contract.test.ts` - From `packages/plugins/examples/plugin-orchestration-smoke-example`: `pnpm exec vitest run --config ./vitest.config.ts` - `pnpm --dir packages/plugins/examples/plugin-orchestration-smoke-example run typecheck` - `pnpm --filter @paperclipai/server typecheck` - PR CI on latest head `293fc67c`: `policy`, `verify`, `e2e`, and `security/snyk` all passed. ## Risks - Medium risk: this expands plugin host authority, so route auth, company scoping, and plugin-origin activity attribution need careful review. - Medium risk: database namespace migration behavior must remain idempotent for environments that may have seen earlier branch versions. - Medium risk: the orchestration smoke fixture is intentionally excluded from the root workspace importer to avoid a `pnpm-lock.yaml` PR diff; direct fixture verification remains listed above. - Low operational risk from the PR setup itself: the branch is rebased onto current `master`, the migration is ordered after upstream `0057`/`0058`, and `pnpm-lock.yaml` is not in the final diff. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. Roadmap checked: this work aligns with the completed Plugin system milestone and extends the plugin surface rather than duplicating an unrelated planned core feature. ## Model Used - OpenAI Codex, GPT-5-based coding agent in a tool-enabled CLI environment. Exact hosted model build and context-window size are not exposed by the runtime; reasoning/tool use were enabled for repository inspection, editing, testing, git operations, and PR creation. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots (N/A: no core UI screen change; example plugin UI contract is covered by tests) - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
16b2b84d84
commit
9c6f551595
53 changed files with 5584 additions and 53 deletions
|
|
@ -23,11 +23,12 @@ import path from "node:path";
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Router } from "express";
|
||||
import type { Request } from "express";
|
||||
import type { Request, Response } from "express";
|
||||
import { and, desc, eq, gte } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { companies, pluginLogs, pluginWebhookDeliveries } from "@paperclipai/db";
|
||||
import type {
|
||||
PluginApiRouteDeclaration,
|
||||
PluginStatus,
|
||||
PaperclipPluginManifestV1,
|
||||
PluginBridgeErrorCode,
|
||||
|
|
@ -41,6 +42,7 @@ import { pluginLifecycleManager } from "../services/plugin-lifecycle.js";
|
|||
import { getPluginUiContributionMetadata, pluginLoader } from "../services/plugin-loader.js";
|
||||
import { logActivity } from "../services/activity-log.js";
|
||||
import { publishGlobalLiveEvent } from "../services/live-events.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import type { PluginJobScheduler } from "../services/plugin-job-scheduler.js";
|
||||
import type { PluginJobStore } from "../services/plugin-job-store.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
|
|
@ -48,8 +50,16 @@ import type { PluginStreamBus } from "../services/plugin-stream-bus.js";
|
|||
import type { PluginToolDispatcher } from "../services/plugin-tool-dispatcher.js";
|
||||
import type { ToolRunContext } from "@paperclipai/plugin-sdk";
|
||||
import { JsonRpcCallError, PLUGIN_RPC_ERROR_CODES } from "@paperclipai/plugin-sdk";
|
||||
import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin, getActorInfo } from "./authz.js";
|
||||
import {
|
||||
assertAuthenticated,
|
||||
assertBoard,
|
||||
assertBoardOrgAccess,
|
||||
assertCompanyAccess,
|
||||
assertInstanceAdmin,
|
||||
getActorInfo,
|
||||
} from "./authz.js";
|
||||
import { validateInstanceConfig } from "../services/plugin-config-validator.js";
|
||||
import { forbidden, notFound, unauthorized, unprocessable } from "../errors.js";
|
||||
|
||||
/** UI slot declaration extracted from plugin manifest */
|
||||
type PluginUiSlotDeclaration = NonNullable<NonNullable<PaperclipPluginManifestV1["ui"]>["slots"]>[number];
|
||||
|
|
@ -112,6 +122,14 @@ interface PluginHealthCheckResult {
|
|||
const UUID_REGEX =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
|
||||
const PLUGIN_API_BODY_LIMIT_BYTES = 1_000_000;
|
||||
const PLUGIN_SCOPED_API_RESPONSE_HEADER_ALLOWLIST = new Set([
|
||||
"cache-control",
|
||||
"etag",
|
||||
"last-modified",
|
||||
"x-request-id",
|
||||
]);
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = path.resolve(__dirname, "../../..");
|
||||
|
||||
|
|
@ -140,6 +158,14 @@ const BUNDLED_PLUGIN_EXAMPLES: AvailablePluginExample[] = [
|
|||
localPath: "packages/plugins/examples/plugin-kitchen-sink-example",
|
||||
tag: "example",
|
||||
},
|
||||
{
|
||||
packageName: "@paperclipai/plugin-orchestration-smoke-example",
|
||||
pluginKey: "paperclipai.plugin-orchestration-smoke-example",
|
||||
displayName: "Orchestration Smoke (Example)",
|
||||
description: "Acceptance fixture for scoped plugin routes, restricted database namespaces, issue orchestration, documents, wakeups, summaries, and UI status surfaces.",
|
||||
localPath: "packages/plugins/examples/plugin-orchestration-smoke-example",
|
||||
tag: "example",
|
||||
},
|
||||
];
|
||||
|
||||
function listBundledPluginExamples(): AvailablePluginExample[] {
|
||||
|
|
@ -246,6 +272,30 @@ export interface PluginRouteBridgeDeps {
|
|||
streamBus?: PluginStreamBus;
|
||||
}
|
||||
|
||||
interface PluginScopedApiRequest {
|
||||
routeKey: string;
|
||||
method: string;
|
||||
path: string;
|
||||
params: Record<string, string>;
|
||||
query: Record<string, string | string[]>;
|
||||
body: unknown;
|
||||
actor: {
|
||||
actorType: "user" | "agent";
|
||||
actorId: string;
|
||||
agentId?: string | null;
|
||||
userId?: string | null;
|
||||
runId?: string | null;
|
||||
};
|
||||
companyId: string;
|
||||
headers: Record<string, string>;
|
||||
}
|
||||
|
||||
interface PluginScopedApiResponse {
|
||||
status?: number;
|
||||
headers?: Record<string, string>;
|
||||
body?: unknown;
|
||||
}
|
||||
|
||||
/** Request body for POST /api/plugins/tools/execute */
|
||||
interface PluginToolExecuteRequest {
|
||||
/** Fully namespaced tool name (e.g., "acme.linear:search-issues"). */
|
||||
|
|
@ -314,6 +364,146 @@ export function pluginRoutes(
|
|||
loader,
|
||||
workerManager: bridgeDeps?.workerManager ?? webhookDeps?.workerManager,
|
||||
});
|
||||
const issuesSvc = issueService(db);
|
||||
|
||||
function matchScopedApiRoute(route: PluginApiRouteDeclaration, method: string, requestPath: string) {
|
||||
if (route.method !== method) return null;
|
||||
const normalize = (value: string) => value.replace(/\/+$/, "") || "/";
|
||||
const routeSegments = normalize(route.path).split("/").filter(Boolean);
|
||||
const requestSegments = normalize(requestPath).split("/").filter(Boolean);
|
||||
if (routeSegments.length !== requestSegments.length) return null;
|
||||
const params: Record<string, string> = {};
|
||||
for (let i = 0; i < routeSegments.length; i += 1) {
|
||||
const routeSegment = routeSegments[i]!;
|
||||
const requestSegment = requestSegments[i]!;
|
||||
if (routeSegment.startsWith(":")) {
|
||||
params[routeSegment.slice(1)] = decodeURIComponent(requestSegment);
|
||||
continue;
|
||||
}
|
||||
if (routeSegment !== requestSegment) return null;
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
function sanitizePluginRequestHeaders(req: Request): Record<string, string> {
|
||||
const safeHeaderNames = new Set([
|
||||
"accept",
|
||||
"content-type",
|
||||
"user-agent",
|
||||
"x-paperclip-run-id",
|
||||
"x-request-id",
|
||||
]);
|
||||
const headers: Record<string, string> = {};
|
||||
for (const [name, value] of Object.entries(req.headers)) {
|
||||
const lower = name.toLowerCase();
|
||||
if (!safeHeaderNames.has(lower)) continue;
|
||||
if (Array.isArray(value)) {
|
||||
headers[lower] = value.join(", ");
|
||||
} else if (typeof value === "string") {
|
||||
headers[lower] = value;
|
||||
}
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
function applyPluginScopedApiResponseHeaders(
|
||||
res: Response,
|
||||
headers: Record<string, string> | undefined,
|
||||
): void {
|
||||
for (const [name, value] of Object.entries(headers ?? {})) {
|
||||
const lower = name.toLowerCase();
|
||||
if (!PLUGIN_SCOPED_API_RESPONSE_HEADER_ALLOWLIST.has(lower)) continue;
|
||||
res.setHeader(lower, value);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeQuery(query: Request["query"]): Record<string, string | string[]> {
|
||||
const normalized: Record<string, string | string[]> = {};
|
||||
for (const [key, value] of Object.entries(query)) {
|
||||
if (typeof value === "string") {
|
||||
normalized[key] = value;
|
||||
} else if (Array.isArray(value)) {
|
||||
normalized[key] = value.map((entry) => String(entry));
|
||||
}
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
async function resolveScopedApiCompanyId(
|
||||
route: PluginApiRouteDeclaration,
|
||||
params: Record<string, string>,
|
||||
req: Request,
|
||||
) {
|
||||
const resolution = route.companyResolution;
|
||||
if (!resolution) {
|
||||
if (req.actor.type === "agent" && req.actor.companyId) return req.actor.companyId;
|
||||
return null;
|
||||
}
|
||||
|
||||
if (resolution.from === "body") {
|
||||
const body = req.body as Record<string, unknown> | undefined;
|
||||
const companyId = body?.[resolution.key ?? ""];
|
||||
return typeof companyId === "string" ? companyId : null;
|
||||
}
|
||||
|
||||
if (resolution.from === "query") {
|
||||
const value = req.query[resolution.key ?? ""];
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
const issueId = params[resolution.param ?? ""];
|
||||
if (!issueId) return null;
|
||||
const issue = await issuesSvc.getById(issueId);
|
||||
return issue?.companyId ?? null;
|
||||
}
|
||||
|
||||
function assertScopedApiAuth(req: Request, route: PluginApiRouteDeclaration) {
|
||||
if (route.auth === "board") {
|
||||
assertBoard(req);
|
||||
return;
|
||||
}
|
||||
if (route.auth === "agent") {
|
||||
assertAuthenticated(req);
|
||||
if (req.actor.type !== "agent") throw forbidden("Agent access required");
|
||||
return;
|
||||
}
|
||||
if (route.auth === "webhook") {
|
||||
throw unprocessable("Webhook-scoped plugin API routes require a signature verifier and are not enabled");
|
||||
}
|
||||
assertAuthenticated(req);
|
||||
if (req.actor.type !== "board" && req.actor.type !== "agent") {
|
||||
throw forbidden("Board or agent access required");
|
||||
}
|
||||
}
|
||||
|
||||
async function enforceScopedApiCheckout(
|
||||
req: Request,
|
||||
route: PluginApiRouteDeclaration,
|
||||
params: Record<string, string>,
|
||||
companyId: string,
|
||||
) {
|
||||
const policy = route.checkoutPolicy ?? "none";
|
||||
if (policy === "none" || req.actor.type !== "agent") return;
|
||||
const issueId = params.issueId;
|
||||
if (!issueId) {
|
||||
throw unprocessable("Checkout-protected plugin API routes require an issueId route parameter");
|
||||
}
|
||||
const issue = await issuesSvc.getById(issueId);
|
||||
if (!issue || issue.companyId !== companyId) {
|
||||
throw notFound("Issue not found");
|
||||
}
|
||||
if (policy === "required-for-agent-in-progress") {
|
||||
if (issue.status !== "in_progress" || issue.assigneeAgentId !== req.actor.agentId) return;
|
||||
}
|
||||
const runId = req.actor.runId?.trim();
|
||||
if (!runId) {
|
||||
throw unauthorized("Agent run id required");
|
||||
}
|
||||
if (!req.actor.agentId) {
|
||||
throw forbidden("Agent authentication required");
|
||||
}
|
||||
await issuesSvc.assertCheckoutOwner(issueId, req.actor.agentId, runId);
|
||||
}
|
||||
|
||||
async function resolvePluginAuditCompanyIds(req: Request): Promise<string[]> {
|
||||
if (typeof (db as { select?: unknown }).select === "function") {
|
||||
|
|
@ -1189,6 +1379,113 @@ export function pluginRoutes(
|
|||
res.on("error", safeUnsubscribe);
|
||||
});
|
||||
|
||||
router.use("/plugins/:pluginId/api", async (req, res) => {
|
||||
if (!bridgeDeps) {
|
||||
res.status(501).json({ error: "Plugin scoped API routes are not enabled" });
|
||||
return;
|
||||
}
|
||||
|
||||
const { pluginId } = req.params;
|
||||
const plugin = await resolvePlugin(registry, pluginId);
|
||||
if (!plugin) {
|
||||
res.status(404).json({ error: "Plugin not found" });
|
||||
return;
|
||||
}
|
||||
if (plugin.status !== "ready") {
|
||||
res.status(503).json({ error: `Plugin is not ready (current status: ${plugin.status})` });
|
||||
return;
|
||||
}
|
||||
const isWorkerRunning = typeof bridgeDeps.workerManager.isRunning === "function"
|
||||
? bridgeDeps.workerManager.isRunning(plugin.id)
|
||||
: true;
|
||||
if (!isWorkerRunning) {
|
||||
res.status(503).json({ error: "Plugin worker is not running" });
|
||||
return;
|
||||
}
|
||||
if (!plugin.manifestJson.capabilities.includes("api.routes.register")) {
|
||||
res.status(404).json({ error: "Plugin does not expose scoped API routes" });
|
||||
return;
|
||||
}
|
||||
|
||||
const requestPath = req.path || "/";
|
||||
const routes = plugin.manifestJson.apiRoutes ?? [];
|
||||
const match = routes
|
||||
.map((route) => ({ route, params: matchScopedApiRoute(route, req.method, requestPath) }))
|
||||
.find((candidate) => candidate.params !== null);
|
||||
if (!match || !match.params) {
|
||||
res.status(404).json({ error: "Plugin API route not found" });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
assertScopedApiAuth(req, match.route);
|
||||
const companyId = await resolveScopedApiCompanyId(match.route, match.params, req);
|
||||
if (!companyId) {
|
||||
res.status(400).json({ error: "Unable to resolve company for plugin API route" });
|
||||
return;
|
||||
}
|
||||
assertCompanyAccess(req, companyId);
|
||||
await enforceScopedApiCheckout(req, match.route, match.params, companyId);
|
||||
if (req.method !== "GET" && req.headers["content-type"] && !req.is("application/json")) {
|
||||
res.status(415).json({ error: "Plugin API routes accept JSON requests only" });
|
||||
return;
|
||||
}
|
||||
const requestBody = req.body ?? null;
|
||||
const bodySize = Buffer.byteLength(JSON.stringify(requestBody));
|
||||
if (bodySize > PLUGIN_API_BODY_LIMIT_BYTES) {
|
||||
res.status(413).json({ error: "Plugin API request body is too large" });
|
||||
return;
|
||||
}
|
||||
|
||||
const actor = getActorInfo(req);
|
||||
const input: PluginScopedApiRequest = {
|
||||
routeKey: match.route.routeKey,
|
||||
method: req.method,
|
||||
path: requestPath,
|
||||
params: match.params,
|
||||
query: normalizeQuery(req.query),
|
||||
body: requestBody,
|
||||
actor: {
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
agentId: actor.agentId,
|
||||
userId: actor.actorType === "user" ? actor.actorId : null,
|
||||
runId: actor.runId,
|
||||
},
|
||||
companyId,
|
||||
headers: sanitizePluginRequestHeaders(req),
|
||||
};
|
||||
|
||||
const result = await bridgeDeps.workerManager.call(
|
||||
plugin.id,
|
||||
"handleApiRequest",
|
||||
input,
|
||||
) as PluginScopedApiResponse;
|
||||
const status = Number.isInteger(result.status) && Number(result.status) >= 200 && Number(result.status) <= 599
|
||||
? Number(result.status)
|
||||
: 200;
|
||||
applyPluginScopedApiResponseHeaders(res, result.headers);
|
||||
if (status === 204) {
|
||||
res.status(status).end();
|
||||
} else {
|
||||
res.status(status).json(result.body ?? null);
|
||||
}
|
||||
} catch (err) {
|
||||
const status = typeof (err as { status?: unknown }).status === "number"
|
||||
? (err as { status: number }).status
|
||||
: err instanceof JsonRpcCallError && err.code === PLUGIN_RPC_ERROR_CODES.CAPABILITY_DENIED
|
||||
? 403
|
||||
: err instanceof JsonRpcCallError && err.code === PLUGIN_RPC_ERROR_CODES.METHOD_NOT_IMPLEMENTED
|
||||
? 501
|
||||
: err instanceof JsonRpcCallError
|
||||
? 502
|
||||
: 500;
|
||||
res.status(status).json({
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/plugins/:pluginId
|
||||
*
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue