paperclip/packages/db/src/runtime-config.ts

237 lines
6.7 KiB
TypeScript
Raw Normal View History

import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
[codex] Add LLM Wiki plugin host support (#5597) ## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies. > - The plugin system needs host contracts and runtime support before large plugins can integrate cleanly. > - The source branch mixed the LLM Wiki package with supporting host/runtime work, managed plugin skills, root-level storage spaces, and a bookmarks reference plugin. > - [PAP-9173](/PAP/issues/PAP-9173) asked for the current branch to be split by file boundary: plugin package separately from everything else. > - [PAP-9188](/PAP/issues/PAP-9188) clarified that LLM Wiki may have plugin-local spaces, but Paperclip core should not reorganize top-level local storage into spaces. > - Follow-up review clarified that the bookmarks example should not ship in this PR either. > - This pull request contains the non-`packages/plugins/plugin-llm-wiki/` host/runtime work, keeps runtime state under the selected Paperclip instance root, and no longer includes the bookmarks example. ## What Changed - Added/updated plugin host contracts, SDK types, worker RPC plumbing, managed plugin skill support, and related server tests. - Removed the bookmarks example plugin package and its bundled-example/workspace references. - Removed the root-level local spaces CLI/migration surface and restored instance-root runtime defaults for config, db, logs, storage, secrets, workspaces, projects, and adapter homes. - Replaced shared root `space-paths` helpers with `home-paths` helpers for core runtime storage. - Tightened stranded recovery unique-conflict detection so concurrent recovery scans reuse the raced recovery issue when Postgres errors are wrapped. - Kept `packages/plugins/plugin-llm-wiki/` out of this PR diff; plugin-local spaces remain in the stacked plugin-only PR. ## Verification - `pnpm exec vitest run cli/src/__tests__/data-dir.test.ts cli/src/__tests__/home-paths.test.ts cli/src/__tests__/onboard.test.ts packages/shared/src/home-paths.test.ts packages/db/src/runtime-config.test.ts server/src/__tests__/agent-instructions-service.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/codex-local-execute.test.ts` - `pnpm exec vitest run packages/db/src/runtime-config.test.ts` - `pnpm exec vitest run server/src/__tests__/plugin-routes-authz.test.ts` - `pnpm --filter @paperclipai/server typecheck` - `pnpm exec vitest run server/src/__tests__/heartbeat-process-recovery.test.ts -t "reuses the raced stranded recovery issue"` skipped locally because embedded Postgres did not initialize on this macOS temp host; the code path was typechecked and is covered by Linux CI. - Boundary check: no core references remain for `PAPERCLIP_SPACE_ID`, `spaces migrate-default`, `@paperclipai/shared/space-paths`, `registerSpacesCommands`, or the removed bookmarks example. - Previous PR head `4f23e034` had green GitHub checks: `verify`, all four serialized server shards, `e2e`, `Canary Dry Run`, `policy`, Snyk, and `Greptile Review`. Current head `582f466d` is re-running checks after the bookmarks deletion. ## Risks - Plugin host changes touch shared runtime paths, so regressions would most likely appear in adapter startup, plugin loading, or local dev path defaults. - Removing the bookmarks example also removes one demonstration of plugin database namespaces plus local-folder persistence; remaining plugin examples still cover bundled example discovery and plugin host flows. - The plugin package itself is intentionally deferred to the stacked plugin-only PR, where LLM Wiki plugin-local spaces live. - Existing installs that tested the transient root-level spaces CLI should stop using it; this PR intentionally removes that unsupported migration surface before merge. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI GPT-5 Codex via Codex CLI, tool use and local code execution enabled; context window not exposed. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass, except where noted above for host-specific embedded Postgres initialization - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge Stacked follow-up: PR #5592 contains only `packages/plugins/plugin-llm-wiki/` and targets this branch. --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-05-10 07:34:12 -05:00
import {
expandHomePrefix,
resolveDefaultEmbeddedPostgresDir,
resolvePaperclipConfigPathForInstance,
resolvePaperclipEnvPathForConfig,
} from "@paperclipai/shared/home-paths";
const CONFIG_BASENAME = "config.json";
type PartialConfig = {
database?: {
mode?: "embedded-postgres" | "postgres";
connectionString?: string;
embeddedPostgresDataDir?: string;
embeddedPostgresPort?: number;
pgliteDataDir?: string;
pglitePort?: number;
};
};
export type ResolvedDatabaseTarget =
| {
mode: "postgres";
connectionString: string;
source: "DATABASE_URL" | "paperclip-env" | "config.database.connectionString";
configPath: string;
envPath: string;
}
| {
mode: "embedded-postgres";
dataDir: string;
port: number;
source: `embedded-postgres@${number}`;
configPath: string;
envPath: string;
};
function resolveHomeAwarePath(value: string): string {
return path.resolve(expandHomePrefix(value));
}
function findConfigFileFromAncestors(startDir: string): string | null {
let currentDir = path.resolve(startDir);
while (true) {
const candidate = path.resolve(currentDir, ".paperclip", CONFIG_BASENAME);
if (existsSync(candidate)) return candidate;
const nextDir = path.resolve(currentDir, "..");
if (nextDir === currentDir) return null;
currentDir = nextDir;
}
}
function resolvePaperclipConfigPath(): string {
if (process.env.PAPERCLIP_CONFIG?.trim()) {
return path.resolve(process.env.PAPERCLIP_CONFIG.trim());
}
[codex] Add LLM Wiki plugin host support (#5597) ## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies. > - The plugin system needs host contracts and runtime support before large plugins can integrate cleanly. > - The source branch mixed the LLM Wiki package with supporting host/runtime work, managed plugin skills, root-level storage spaces, and a bookmarks reference plugin. > - [PAP-9173](/PAP/issues/PAP-9173) asked for the current branch to be split by file boundary: plugin package separately from everything else. > - [PAP-9188](/PAP/issues/PAP-9188) clarified that LLM Wiki may have plugin-local spaces, but Paperclip core should not reorganize top-level local storage into spaces. > - Follow-up review clarified that the bookmarks example should not ship in this PR either. > - This pull request contains the non-`packages/plugins/plugin-llm-wiki/` host/runtime work, keeps runtime state under the selected Paperclip instance root, and no longer includes the bookmarks example. ## What Changed - Added/updated plugin host contracts, SDK types, worker RPC plumbing, managed plugin skill support, and related server tests. - Removed the bookmarks example plugin package and its bundled-example/workspace references. - Removed the root-level local spaces CLI/migration surface and restored instance-root runtime defaults for config, db, logs, storage, secrets, workspaces, projects, and adapter homes. - Replaced shared root `space-paths` helpers with `home-paths` helpers for core runtime storage. - Tightened stranded recovery unique-conflict detection so concurrent recovery scans reuse the raced recovery issue when Postgres errors are wrapped. - Kept `packages/plugins/plugin-llm-wiki/` out of this PR diff; plugin-local spaces remain in the stacked plugin-only PR. ## Verification - `pnpm exec vitest run cli/src/__tests__/data-dir.test.ts cli/src/__tests__/home-paths.test.ts cli/src/__tests__/onboard.test.ts packages/shared/src/home-paths.test.ts packages/db/src/runtime-config.test.ts server/src/__tests__/agent-instructions-service.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/codex-local-execute.test.ts` - `pnpm exec vitest run packages/db/src/runtime-config.test.ts` - `pnpm exec vitest run server/src/__tests__/plugin-routes-authz.test.ts` - `pnpm --filter @paperclipai/server typecheck` - `pnpm exec vitest run server/src/__tests__/heartbeat-process-recovery.test.ts -t "reuses the raced stranded recovery issue"` skipped locally because embedded Postgres did not initialize on this macOS temp host; the code path was typechecked and is covered by Linux CI. - Boundary check: no core references remain for `PAPERCLIP_SPACE_ID`, `spaces migrate-default`, `@paperclipai/shared/space-paths`, `registerSpacesCommands`, or the removed bookmarks example. - Previous PR head `4f23e034` had green GitHub checks: `verify`, all four serialized server shards, `e2e`, `Canary Dry Run`, `policy`, Snyk, and `Greptile Review`. Current head `582f466d` is re-running checks after the bookmarks deletion. ## Risks - Plugin host changes touch shared runtime paths, so regressions would most likely appear in adapter startup, plugin loading, or local dev path defaults. - Removing the bookmarks example also removes one demonstration of plugin database namespaces plus local-folder persistence; remaining plugin examples still cover bundled example discovery and plugin host flows. - The plugin package itself is intentionally deferred to the stacked plugin-only PR, where LLM Wiki plugin-local spaces live. - Existing installs that tested the transient root-level spaces CLI should stop using it; this PR intentionally removes that unsupported migration surface before merge. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI GPT-5 Codex via Codex CLI, tool use and local code execution enabled; context window not exposed. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass, except where noted above for host-specific embedded Postgres initialization - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge Stacked follow-up: PR #5592 contains only `packages/plugins/plugin-llm-wiki/` and targets this branch. --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-05-10 07:34:12 -05:00
return findConfigFileFromAncestors(process.cwd()) ?? resolvePaperclipConfigPathForInstance();
}
function resolvePaperclipEnvPath(configPath: string): string {
[codex] Add LLM Wiki plugin host support (#5597) ## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies. > - The plugin system needs host contracts and runtime support before large plugins can integrate cleanly. > - The source branch mixed the LLM Wiki package with supporting host/runtime work, managed plugin skills, root-level storage spaces, and a bookmarks reference plugin. > - [PAP-9173](/PAP/issues/PAP-9173) asked for the current branch to be split by file boundary: plugin package separately from everything else. > - [PAP-9188](/PAP/issues/PAP-9188) clarified that LLM Wiki may have plugin-local spaces, but Paperclip core should not reorganize top-level local storage into spaces. > - Follow-up review clarified that the bookmarks example should not ship in this PR either. > - This pull request contains the non-`packages/plugins/plugin-llm-wiki/` host/runtime work, keeps runtime state under the selected Paperclip instance root, and no longer includes the bookmarks example. ## What Changed - Added/updated plugin host contracts, SDK types, worker RPC plumbing, managed plugin skill support, and related server tests. - Removed the bookmarks example plugin package and its bundled-example/workspace references. - Removed the root-level local spaces CLI/migration surface and restored instance-root runtime defaults for config, db, logs, storage, secrets, workspaces, projects, and adapter homes. - Replaced shared root `space-paths` helpers with `home-paths` helpers for core runtime storage. - Tightened stranded recovery unique-conflict detection so concurrent recovery scans reuse the raced recovery issue when Postgres errors are wrapped. - Kept `packages/plugins/plugin-llm-wiki/` out of this PR diff; plugin-local spaces remain in the stacked plugin-only PR. ## Verification - `pnpm exec vitest run cli/src/__tests__/data-dir.test.ts cli/src/__tests__/home-paths.test.ts cli/src/__tests__/onboard.test.ts packages/shared/src/home-paths.test.ts packages/db/src/runtime-config.test.ts server/src/__tests__/agent-instructions-service.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/codex-local-execute.test.ts` - `pnpm exec vitest run packages/db/src/runtime-config.test.ts` - `pnpm exec vitest run server/src/__tests__/plugin-routes-authz.test.ts` - `pnpm --filter @paperclipai/server typecheck` - `pnpm exec vitest run server/src/__tests__/heartbeat-process-recovery.test.ts -t "reuses the raced stranded recovery issue"` skipped locally because embedded Postgres did not initialize on this macOS temp host; the code path was typechecked and is covered by Linux CI. - Boundary check: no core references remain for `PAPERCLIP_SPACE_ID`, `spaces migrate-default`, `@paperclipai/shared/space-paths`, `registerSpacesCommands`, or the removed bookmarks example. - Previous PR head `4f23e034` had green GitHub checks: `verify`, all four serialized server shards, `e2e`, `Canary Dry Run`, `policy`, Snyk, and `Greptile Review`. Current head `582f466d` is re-running checks after the bookmarks deletion. ## Risks - Plugin host changes touch shared runtime paths, so regressions would most likely appear in adapter startup, plugin loading, or local dev path defaults. - Removing the bookmarks example also removes one demonstration of plugin database namespaces plus local-folder persistence; remaining plugin examples still cover bundled example discovery and plugin host flows. - The plugin package itself is intentionally deferred to the stacked plugin-only PR, where LLM Wiki plugin-local spaces live. - Existing installs that tested the transient root-level spaces CLI should stop using it; this PR intentionally removes that unsupported migration surface before merge. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI GPT-5 Codex via Codex CLI, tool use and local code execution enabled; context window not exposed. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass, except where noted above for host-specific embedded Postgres initialization - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge Stacked follow-up: PR #5592 contains only `packages/plugins/plugin-llm-wiki/` and targets this branch. --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-05-10 07:34:12 -05:00
return resolvePaperclipEnvPathForConfig(configPath);
}
function parseEnvFile(contents: string): Record<string, string> {
const entries: Record<string, string> = {};
for (const rawLine of contents.split(/\r?\n/)) {
const line = rawLine.trim();
if (!line || line.startsWith("#")) continue;
const match = rawLine.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/);
if (!match) continue;
const [, key, rawValue] = match;
const value = rawValue.trim();
if (!value) {
entries[key] = "";
continue;
}
if (
(value.startsWith("\"") && value.endsWith("\"")) ||
(value.startsWith("'") && value.endsWith("'"))
) {
entries[key] = value.slice(1, -1);
continue;
}
entries[key] = value.replace(/\s+#.*$/, "").trim();
}
return entries;
}
function readEnvEntries(envPath: string): Record<string, string> {
if (!existsSync(envPath)) return {};
return parseEnvFile(readFileSync(envPath, "utf8"));
}
function migrateLegacyConfig(raw: unknown): PartialConfig | null {
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return null;
const config = { ...(raw as Record<string, unknown>) };
const databaseRaw = config.database;
if (typeof databaseRaw !== "object" || databaseRaw === null || Array.isArray(databaseRaw)) {
return config;
}
const database = { ...(databaseRaw as Record<string, unknown>) };
if (database.mode === "pglite") {
database.mode = "embedded-postgres";
if (
typeof database.embeddedPostgresDataDir !== "string" &&
typeof database.pgliteDataDir === "string"
) {
database.embeddedPostgresDataDir = database.pgliteDataDir;
}
if (
typeof database.embeddedPostgresPort !== "number" &&
typeof database.pglitePort === "number" &&
Number.isFinite(database.pglitePort)
) {
database.embeddedPostgresPort = database.pglitePort;
}
}
config.database = database;
return config as PartialConfig;
}
function asPositiveInt(value: unknown): number | null {
if (typeof value !== "number" || !Number.isFinite(value)) return null;
const rounded = Math.trunc(value);
return rounded > 0 ? rounded : null;
}
function readConfig(configPath: string): PartialConfig | null {
if (!existsSync(configPath)) return null;
let parsed: unknown;
try {
parsed = JSON.parse(readFileSync(configPath, "utf8"));
} catch (err) {
throw new Error(
`Failed to parse config at ${configPath}: ${err instanceof Error ? err.message : String(err)}`,
);
}
const migrated = migrateLegacyConfig(parsed);
if (migrated === null || typeof migrated !== "object" || Array.isArray(migrated)) {
throw new Error(`Invalid config at ${configPath}: expected a JSON object`);
}
const database =
typeof migrated.database === "object" &&
migrated.database !== null &&
!Array.isArray(migrated.database)
? migrated.database
: undefined;
return {
database: database
? {
mode: database.mode === "postgres" ? "postgres" : "embedded-postgres",
connectionString:
typeof database.connectionString === "string" ? database.connectionString : undefined,
embeddedPostgresDataDir:
typeof database.embeddedPostgresDataDir === "string"
? database.embeddedPostgresDataDir
: undefined,
embeddedPostgresPort: asPositiveInt(database.embeddedPostgresPort) ?? undefined,
pgliteDataDir: typeof database.pgliteDataDir === "string" ? database.pgliteDataDir : undefined,
pglitePort: asPositiveInt(database.pglitePort) ?? undefined,
}
: undefined,
};
}
export function resolveDatabaseTarget(): ResolvedDatabaseTarget {
const configPath = resolvePaperclipConfigPath();
const envPath = resolvePaperclipEnvPath(configPath);
const envEntries = readEnvEntries(envPath);
const envUrl = process.env.DATABASE_URL?.trim();
if (envUrl) {
return {
mode: "postgres",
connectionString: envUrl,
source: "DATABASE_URL",
configPath,
envPath,
};
}
const fileEnvUrl = envEntries.DATABASE_URL?.trim();
if (fileEnvUrl) {
return {
mode: "postgres",
connectionString: fileEnvUrl,
source: "paperclip-env",
configPath,
envPath,
};
}
const config = readConfig(configPath);
const connectionString = config?.database?.connectionString?.trim();
if (config?.database?.mode === "postgres" && connectionString) {
return {
mode: "postgres",
connectionString,
source: "config.database.connectionString",
configPath,
envPath,
};
}
const port = config?.database?.embeddedPostgresPort ?? 54329;
const dataDir = resolveHomeAwarePath(
config?.database?.embeddedPostgresDataDir ?? resolveDefaultEmbeddedPostgresDir(),
);
return {
mode: "embedded-postgres",
dataDir,
port,
source: `embedded-postgres@${port}`,
configPath,
envPath,
};
}