[codex] Add plugin orchestration host APIs (#4114)

## Thinking Path

> - Paperclip orchestrates AI agents for zero-human companies.
> - The plugin system is the extension path for optional capabilities
that should not require core product changes for every integration.
> - Plugins need scoped host APIs for issue orchestration, documents,
wakeups, summaries, activity attribution, and isolated database state.
> - Without those host APIs, richer plugins either cannot coordinate
Paperclip work safely or need privileged core-side special cases.
> - This pull request adds the plugin orchestration host surface, scoped
route dispatch, a database namespace layer, and a smoke plugin that
exercises the contract.
> - The benefit is a broader plugin API that remains company-scoped,
auditable, and covered by tests.

## What Changed

- Added plugin orchestration host APIs for issue creation, document
access, wakeups, summaries, plugin-origin activity, and scoped API route
dispatch.
- Added plugin database namespace tables, schema exports, migration
checks, and idempotent replay coverage under migration
`0059_plugin_database_namespaces`.
- Added shared plugin route/API types and validators used by server and
SDK boundaries.
- Expanded plugin SDK types, protocol helpers, worker RPC host behavior,
and testing utilities for orchestration flows.
- Added the `plugin-orchestration-smoke-example` package to exercise
scoped routes, restricted database namespaces, issue orchestration,
documents, wakeups, summaries, and UI status surfaces.
- Kept the new orchestration smoke fixture out of the root pnpm
workspace importer so this PR preserves the repository policy of not
committing `pnpm-lock.yaml`.
- Updated plugin docs and database docs for the new orchestration and
database namespace surfaces.
- Rebased the branch onto `public-gh/master`, resolved conflicts, and
removed `pnpm-lock.yaml` from the final PR diff.

## Verification

- `pnpm install --frozen-lockfile`
- `pnpm --filter @paperclipai/db typecheck`
- `pnpm exec vitest run packages/db/src/client.test.ts`
- `pnpm exec vitest run server/src/__tests__/plugin-database.test.ts
server/src/__tests__/plugin-orchestration-apis.test.ts
server/src/__tests__/plugin-routes-authz.test.ts
server/src/__tests__/plugin-scoped-api-routes.test.ts
server/src/__tests__/plugin-sdk-orchestration-contract.test.ts`
- From `packages/plugins/examples/plugin-orchestration-smoke-example`:
`pnpm exec vitest run --config ./vitest.config.ts`
- `pnpm --dir
packages/plugins/examples/plugin-orchestration-smoke-example run
typecheck`
- `pnpm --filter @paperclipai/server typecheck`
- PR CI on latest head `293fc67c`: `policy`, `verify`, `e2e`, and
`security/snyk` all passed.

## Risks

- Medium risk: this expands plugin host authority, so route auth,
company scoping, and plugin-origin activity attribution need careful
review.
- Medium risk: database namespace migration behavior must remain
idempotent for environments that may have seen earlier branch versions.
- Medium risk: the orchestration smoke fixture is intentionally excluded
from the root workspace importer to avoid a `pnpm-lock.yaml` PR diff;
direct fixture verification remains listed above.
- Low operational risk from the PR setup itself: the branch is rebased
onto current `master`, the migration is ordered after upstream
`0057`/`0058`, and `pnpm-lock.yaml` is not in the final diff.

> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.

Roadmap checked: this work aligns with the completed Plugin system
milestone and extends the plugin surface rather than duplicating an
unrelated planned core feature.

## Model Used

- OpenAI Codex, GPT-5-based coding agent in a tool-enabled CLI
environment. Exact hosted model build and context-window size are not
exposed by the runtime; reasoning/tool use were enabled for repository
inspection, editing, testing, git operations, and PR creation.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] If this change affects the UI, I have included before/after
screenshots (N/A: no core UI screen change; example plugin UI contract
is covered by tests)
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge

---------

Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta 2026-04-20 08:52:51 -05:00 committed by GitHub
parent 16b2b84d84
commit 9c6f551595
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
53 changed files with 5584 additions and 53 deletions

View file

@ -3,10 +3,12 @@ import type {
PaperclipPluginManifestV1,
PluginCapability,
PluginEventType,
PluginIssueOriginKind,
Company,
Project,
Issue,
IssueComment,
IssueDocument,
Agent,
Goal,
} from "@paperclipai/shared";
@ -72,6 +74,8 @@ export interface TestHarness {
activity: Array<{ message: string; entityType?: string; entityId?: string; metadata?: Record<string, unknown> }>;
metrics: Array<{ name: string; value: number; tags?: Record<string, string> }>;
telemetry: Array<{ eventName: string; dimensions?: Record<string, string | number | boolean> }>;
dbQueries: Array<{ sql: string; params?: unknown[] }>;
dbExecutes: Array<{ sql: string; params?: unknown[] }>;
}
type EventRegistration = {
@ -134,6 +138,8 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
const activity: TestHarness["activity"] = [];
const metrics: TestHarness["metrics"] = [];
const telemetry: TestHarness["telemetry"] = [];
const dbQueries: TestHarness["dbQueries"] = [];
const dbExecutes: TestHarness["dbExecutes"] = [];
const state = new Map<string, unknown>();
const entities = new Map<string, PluginEntityRecord>();
@ -141,7 +147,9 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
const companies = new Map<string, Company>();
const projects = new Map<string, Project>();
const issues = new Map<string, Issue>();
const blockedByIssueIds = new Map<string, string[]>();
const issueComments = new Map<string, IssueComment[]>();
const issueDocuments = new Map<string, IssueDocument>();
const agents = new Map<string, Agent>();
const goals = new Map<string, Goal>();
const projectWorkspaces = new Map<string, PluginWorkspace[]>();
@ -156,6 +164,42 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
const actionHandlers = new Map<string, (params: Record<string, unknown>) => Promise<unknown>>();
const toolHandlers = new Map<string, (params: unknown, runCtx: ToolRunContext) => Promise<ToolResult>>();
function issueRelationSummary(issueId: string) {
const issue = issues.get(issueId);
if (!issue) throw new Error(`Issue not found: ${issueId}`);
const summarize = (candidateId: string) => {
const related = issues.get(candidateId);
if (!related || related.companyId !== issue.companyId) return null;
return {
id: related.id,
identifier: related.identifier,
title: related.title,
status: related.status,
priority: related.priority,
assigneeAgentId: related.assigneeAgentId,
assigneeUserId: related.assigneeUserId,
};
};
const blockedBy = (blockedByIssueIds.get(issueId) ?? [])
.map(summarize)
.filter((value): value is NonNullable<typeof value> => value !== null);
const blocks = [...blockedByIssueIds.entries()]
.filter(([, blockers]) => blockers.includes(issueId))
.map(([blockedIssueId]) => summarize(blockedIssueId))
.filter((value): value is NonNullable<typeof value> => value !== null);
return { blockedBy, blocks };
}
const defaultPluginOriginKind: PluginIssueOriginKind = `plugin:${manifest.id}`;
function normalizePluginOriginKind(originKind: unknown = defaultPluginOriginKind): PluginIssueOriginKind {
if (originKind == null || originKind === "") return defaultPluginOriginKind;
if (typeof originKind !== "string") throw new Error("Plugin issue originKind must be a string");
if (originKind === defaultPluginOriginKind || originKind.startsWith(`${defaultPluginOriginKind}:`)) {
return originKind as PluginIssueOriginKind;
}
throw new Error(`Plugin may only use originKind values under ${defaultPluginOriginKind}`);
}
const ctx: PluginContext = {
manifest,
config: {
@ -195,6 +239,19 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
launchers.set(launcher.id, launcher);
},
},
db: {
namespace: manifest.database ? `test_${manifest.id.replace(/[^a-z0-9_]+/g, "_")}` : "",
async query(sql, params) {
requireCapability(manifest, capabilitySet, "database.namespace.read");
dbQueries.push({ sql, params });
return [];
},
async execute(sql, params) {
requireCapability(manifest, capabilitySet, "database.namespace.write");
dbExecutes.push({ sql, params });
return { rowCount: 0 };
},
},
http: {
async fetch(url, init) {
requireCapability(manifest, capabilitySet, "http.outbound");
@ -338,6 +395,11 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
out = out.filter((issue) => issue.companyId === companyId);
if (input?.projectId) out = out.filter((issue) => issue.projectId === input.projectId);
if (input?.assigneeAgentId) out = out.filter((issue) => issue.assigneeAgentId === input.assigneeAgentId);
if (input?.originKind) {
if (input.originKind.startsWith("plugin:")) normalizePluginOriginKind(input.originKind);
out = out.filter((issue) => issue.originKind === input.originKind);
}
if (input?.originId) out = out.filter((issue) => issue.originId === input.originId);
if (input?.status) out = out.filter((issue) => issue.status === input.status);
if (input?.offset) out = out.slice(input.offset);
if (input?.limit) out = out.slice(0, input.limit);
@ -360,10 +422,10 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
parentId: input.parentId ?? null,
title: input.title,
description: input.description ?? null,
status: "todo",
status: input.status ?? "todo",
priority: input.priority ?? "medium",
assigneeAgentId: input.assigneeAgentId ?? null,
assigneeUserId: null,
assigneeUserId: input.assigneeUserId ?? null,
checkoutRunId: null,
executionRunId: null,
executionAgentNameKey: null,
@ -372,12 +434,15 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
createdByUserId: null,
issueNumber: null,
identifier: null,
requestDepth: 0,
billingCode: null,
originKind: normalizePluginOriginKind(input.originKind),
originId: input.originId ?? null,
originRunId: input.originRunId ?? null,
requestDepth: input.requestDepth ?? 0,
billingCode: input.billingCode ?? null,
assigneeAdapterOverrides: null,
executionWorkspaceId: null,
executionWorkspacePreference: null,
executionWorkspaceSettings: null,
executionWorkspaceId: input.executionWorkspaceId ?? null,
executionWorkspacePreference: input.executionWorkspacePreference ?? null,
executionWorkspaceSettings: input.executionWorkspaceSettings ?? null,
startedAt: null,
completedAt: null,
cancelledAt: null,
@ -386,20 +451,75 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
updatedAt: now,
};
issues.set(record.id, record);
if (input.blockedByIssueIds) blockedByIssueIds.set(record.id, [...new Set(input.blockedByIssueIds)]);
return record;
},
async update(issueId, patch, companyId) {
requireCapability(manifest, capabilitySet, "issues.update");
const record = issues.get(issueId);
if (!isInCompany(record, companyId)) throw new Error(`Issue not found: ${issueId}`);
const { blockedByIssueIds: nextBlockedByIssueIds, ...issuePatch } = patch;
if (issuePatch.originKind !== undefined) {
issuePatch.originKind = normalizePluginOriginKind(issuePatch.originKind);
}
const updated: Issue = {
...record,
...patch,
...issuePatch,
updatedAt: new Date(),
};
issues.set(issueId, updated);
if (nextBlockedByIssueIds !== undefined) {
blockedByIssueIds.set(issueId, [...new Set(nextBlockedByIssueIds)]);
}
return updated;
},
async assertCheckoutOwner(input) {
requireCapability(manifest, capabilitySet, "issues.checkout");
const record = issues.get(input.issueId);
if (!isInCompany(record, input.companyId)) throw new Error(`Issue not found: ${input.issueId}`);
if (
record.status !== "in_progress" ||
record.assigneeAgentId !== input.actorAgentId ||
(record.checkoutRunId !== null && record.checkoutRunId !== input.actorRunId)
) {
throw new Error("Issue run ownership conflict");
}
return {
issueId: record.id,
status: record.status,
assigneeAgentId: record.assigneeAgentId,
checkoutRunId: record.checkoutRunId,
adoptedFromRunId: null,
};
},
async requestWakeup(issueId, companyId) {
requireCapability(manifest, capabilitySet, "issues.wakeup");
const record = issues.get(issueId);
if (!isInCompany(record, companyId)) throw new Error(`Issue not found: ${issueId}`);
if (!record.assigneeAgentId) throw new Error("Issue has no assigned agent to wake");
if (["backlog", "done", "cancelled"].includes(record.status)) {
throw new Error(`Issue is not wakeable in status: ${record.status}`);
}
const unresolved = issueRelationSummary(issueId).blockedBy.filter((blocker) => blocker.status !== "done");
if (unresolved.length > 0) throw new Error("Issue is blocked by unresolved blockers");
return { queued: true, runId: randomUUID() };
},
async requestWakeups(issueIds, companyId) {
requireCapability(manifest, capabilitySet, "issues.wakeup");
const results = [];
for (const issueId of issueIds) {
const record = issues.get(issueId);
if (!isInCompany(record, companyId)) throw new Error(`Issue not found: ${issueId}`);
if (!record.assigneeAgentId) throw new Error("Issue has no assigned agent to wake");
if (["backlog", "done", "cancelled"].includes(record.status)) {
throw new Error(`Issue is not wakeable in status: ${record.status}`);
}
const unresolved = issueRelationSummary(issueId).blockedBy.filter((blocker) => blocker.status !== "done");
if (unresolved.length > 0) throw new Error("Issue is blocked by unresolved blockers");
results.push({ issueId, queued: true, runId: randomUUID() });
}
return results;
},
async listComments(issueId, companyId) {
requireCapability(manifest, capabilitySet, "issue.comments.read");
if (!isInCompany(issues.get(issueId), companyId)) return [];
@ -431,12 +551,14 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
async list(issueId, companyId) {
requireCapability(manifest, capabilitySet, "issue.documents.read");
if (!isInCompany(issues.get(issueId), companyId)) return [];
return [];
return [...issueDocuments.values()]
.filter((document) => document.issueId === issueId && document.companyId === companyId)
.map(({ body: _body, ...summary }) => summary);
},
async get(issueId, _key, companyId) {
async get(issueId, key, companyId) {
requireCapability(manifest, capabilitySet, "issue.documents.read");
if (!isInCompany(issues.get(issueId), companyId)) return null;
return null;
return issueDocuments.get(`${issueId}|${key}`) ?? null;
},
async upsert(input) {
requireCapability(manifest, capabilitySet, "issue.documents.write");
@ -444,7 +566,27 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
if (!isInCompany(parentIssue, input.companyId)) {
throw new Error(`Issue not found: ${input.issueId}`);
}
throw new Error("documents.upsert is not implemented in test context");
const now = new Date();
const existing = issueDocuments.get(`${input.issueId}|${input.key}`);
const document: IssueDocument = {
id: existing?.id ?? randomUUID(),
companyId: input.companyId,
issueId: input.issueId,
key: input.key,
title: input.title ?? existing?.title ?? null,
format: "markdown",
latestRevisionId: randomUUID(),
latestRevisionNumber: (existing?.latestRevisionNumber ?? 0) + 1,
createdByAgentId: existing?.createdByAgentId ?? null,
createdByUserId: existing?.createdByUserId ?? null,
updatedByAgentId: null,
updatedByUserId: null,
createdAt: existing?.createdAt ?? now,
updatedAt: now,
body: input.body,
};
issueDocuments.set(`${input.issueId}|${input.key}`, document);
return document;
},
async delete(issueId, _key, companyId) {
requireCapability(manifest, capabilitySet, "issue.documents.write");
@ -452,6 +594,104 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
if (!isInCompany(parentIssue, companyId)) {
throw new Error(`Issue not found: ${issueId}`);
}
issueDocuments.delete(`${issueId}|${_key}`);
},
},
relations: {
async get(issueId, companyId) {
requireCapability(manifest, capabilitySet, "issue.relations.read");
if (!isInCompany(issues.get(issueId), companyId)) throw new Error(`Issue not found: ${issueId}`);
return issueRelationSummary(issueId);
},
async setBlockedBy(issueId, nextBlockedByIssueIds, companyId) {
requireCapability(manifest, capabilitySet, "issue.relations.write");
if (!isInCompany(issues.get(issueId), companyId)) throw new Error(`Issue not found: ${issueId}`);
blockedByIssueIds.set(issueId, [...new Set(nextBlockedByIssueIds)]);
return issueRelationSummary(issueId);
},
async addBlockers(issueId, blockerIssueIds, companyId) {
requireCapability(manifest, capabilitySet, "issue.relations.write");
if (!isInCompany(issues.get(issueId), companyId)) throw new Error(`Issue not found: ${issueId}`);
const next = new Set(blockedByIssueIds.get(issueId) ?? []);
for (const blockerIssueId of blockerIssueIds) next.add(blockerIssueId);
blockedByIssueIds.set(issueId, [...next]);
return issueRelationSummary(issueId);
},
async removeBlockers(issueId, blockerIssueIds, companyId) {
requireCapability(manifest, capabilitySet, "issue.relations.write");
if (!isInCompany(issues.get(issueId), companyId)) throw new Error(`Issue not found: ${issueId}`);
const removals = new Set(blockerIssueIds);
blockedByIssueIds.set(
issueId,
(blockedByIssueIds.get(issueId) ?? []).filter((blockerIssueId) => !removals.has(blockerIssueId)),
);
return issueRelationSummary(issueId);
},
},
async getSubtree(issueId, companyId, options) {
requireCapability(manifest, capabilitySet, "issue.subtree.read");
const root = issues.get(issueId);
if (!isInCompany(root, companyId)) throw new Error(`Issue not found: ${issueId}`);
const includeRoot = options?.includeRoot !== false;
const allIds = [root.id];
let frontier = [root.id];
while (frontier.length > 0) {
const children = [...issues.values()]
.filter((issue) => issue.companyId === companyId && frontier.includes(issue.parentId ?? ""))
.map((issue) => issue.id)
.filter((id) => !allIds.includes(id));
allIds.push(...children);
frontier = children;
}
const issueIds = includeRoot ? allIds : allIds.filter((id) => id !== root.id);
const subtreeIssues = issueIds.map((id) => issues.get(id)).filter((candidate): candidate is Issue => Boolean(candidate));
return {
rootIssueId: root.id,
companyId,
issueIds,
issues: subtreeIssues,
...(options?.includeRelations
? { relations: Object.fromEntries(issueIds.map((id) => [id, issueRelationSummary(id)])) }
: {}),
...(options?.includeDocuments ? { documents: Object.fromEntries(issueIds.map((id) => [id, []])) } : {}),
...(options?.includeActiveRuns ? { activeRuns: Object.fromEntries(issueIds.map((id) => [id, []])) } : {}),
...(options?.includeAssignees ? { assignees: {} } : {}),
};
},
summaries: {
async getOrchestration(input) {
requireCapability(manifest, capabilitySet, "issues.orchestration.read");
const root = issues.get(input.issueId);
if (!isInCompany(root, input.companyId)) throw new Error(`Issue not found: ${input.issueId}`);
const subtreeIssueIds = [root.id];
if (input.includeSubtree) {
let frontier = [root.id];
while (frontier.length > 0) {
const children = [...issues.values()]
.filter((issue) => issue.companyId === input.companyId && frontier.includes(issue.parentId ?? ""))
.map((issue) => issue.id)
.filter((id) => !subtreeIssueIds.includes(id));
subtreeIssueIds.push(...children);
frontier = children;
}
}
return {
issueId: root.id,
companyId: input.companyId,
subtreeIssueIds,
relations: Object.fromEntries(subtreeIssueIds.map((id) => [id, issueRelationSummary(id)])),
approvals: [],
runs: [],
costs: {
costCents: 0,
inputTokens: 0,
cachedInputTokens: 0,
outputTokens: 0,
billingCode: input.billingCode ?? null,
},
openBudgetIncidents: [],
invocationBlocks: [],
};
},
},
},
@ -660,7 +900,12 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
seed(input) {
for (const row of input.companies ?? []) companies.set(row.id, row);
for (const row of input.projects ?? []) projects.set(row.id, row);
for (const row of input.issues ?? []) issues.set(row.id, row);
for (const row of input.issues ?? []) {
issues.set(row.id, row);
if (row.blockedBy) {
blockedByIssueIds.set(row.id, row.blockedBy.map((blocker) => blocker.id));
}
}
for (const row of input.issueComments ?? []) {
const list = issueComments.get(row.issueId) ?? [];
list.push(row);
@ -738,6 +983,8 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {
activity,
metrics,
telemetry,
dbQueries,
dbExecutes,
};
return harness;