mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-17 03:10:38 +09:00
Fix LLM Wiki package and migration validation (#6010)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies. > - Plugins extend the control plane with optional capabilities such as LLM Wiki. > - LLM Wiki needs its package assets and plugin-owned database migrations to work when installed from the packaged plugin. > - The bundled spaces migration used validation-hostile dynamic SQL, and the packaged plugin could omit non-dist runtime assets. > - This pull request makes the LLM Wiki package include its required assets and cuts the spaces migration over to explicit, idempotent SQL that passes the production plugin database validator. > - The benefit is a simpler plugin install path that validates and applies the bundled LLM Wiki migrations without adding plugin-specific legacy handling to Paperclip core. ## What Changed - Added the LLM Wiki package asset allowlist so agents, migrations, skills, templates, dist output, and README are included when packaged. - Renamed the bootstrap `.gitignore` template to `gitignore.template` and updated the runtime lookup so package tooling does not drop the hidden template file. - Relaxed plugin migration validation to allow namespace-scoped `INSERT`/`UPDATE` backfills and `CREATE INDEX` statements while continuing to reject destructive or cross-namespace SQL. - Replaced the LLM Wiki spaces migration's dynamic constraint-drop DO block with explicit `DROP CONSTRAINT IF EXISTS` statements. - Replaced fragile regex-source dispatch in SQL reference extraction with explicit capture-group descriptors. - Added regression coverage that applies the bundled LLM Wiki migrations through the production validator and checks the expected constraints. ## Verification - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/plugin-database.test.ts --pool=forks --poolOptions.forks.isolate=true` - `pnpm --filter @paperclipai/plugin-llm-wiki build` - `git diff --check` - Confirmed `pnpm-lock.yaml` is not included in the branch diff. ## Risks - Low migration risk for current users: LLM Wiki spaces are new, so this intentionally cuts over the plugin migration instead of adding legacy handling in core. - Validator behavior is broader than before, but still requires fully qualified plugin namespace targets, blocks deletes/destructive DDL, and keeps public table access read-only and allowlisted. > Checked [`ROADMAP.md`](ROADMAP.md); this is a targeted plugin packaging/migration fix and does not duplicate planned core feature work. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5 based coding agent, tool-enabled local repo access, reasoning mode managed by the Paperclip/Codex runtime. Exact context window was not surfaced in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
dfcebf082b
commit
eb38b226c2
6 changed files with 203 additions and 41 deletions
|
|
@ -30,6 +30,7 @@ import { buildPluginWorkerEnv, pluginLoader } from "../services/plugin-loader.js
|
|||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
const multiMigrationPluginKey = "paperclip.dbfixture";
|
||||
const llmWikiPluginKey = "paperclipai.plugin-llm-wiki";
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
|
|
@ -48,6 +49,63 @@ describe("plugin database SQL validation", () => {
|
|||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("allows qualified index creation and namespace-scoped migration backfills", () => {
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
"CREATE INDEX IF NOT EXISTS rows_issue_idx ON plugin_test.rows (issue_id)",
|
||||
"plugin_test",
|
||||
)
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
`
|
||||
WITH source_rows AS (
|
||||
SELECT id FROM plugin_test.rows
|
||||
)
|
||||
INSERT INTO plugin_test.row_copies (id)
|
||||
SELECT id FROM source_rows
|
||||
ON CONFLICT (id) DO NOTHING
|
||||
`,
|
||||
"plugin_test",
|
||||
)
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
`
|
||||
UPDATE plugin_test.rows r
|
||||
SET copied_from_id = s.id
|
||||
FROM plugin_test.source_rows s
|
||||
WHERE s.id = r.id
|
||||
`,
|
||||
"plugin_test",
|
||||
)
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("keeps migration backfill writes scoped to the plugin namespace", () => {
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
"CREATE TABLE rows (id uuid PRIMARY KEY, issue_id uuid REFERENCES public.issues(id))",
|
||||
"plugin_test",
|
||||
["issues"],
|
||||
)
|
||||
).toThrow(/fully qualified/i);
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
"WITH source_rows AS (SELECT id FROM plugin_test.rows) INSERT INTO public.issues (id) SELECT id FROM source_rows",
|
||||
"plugin_test",
|
||||
["issues"],
|
||||
)
|
||||
).toThrow(/public/i);
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
"UPDATE public.issues SET title = 'bad'",
|
||||
"plugin_test",
|
||||
["issues"],
|
||||
)
|
||||
).toThrow(/public/i);
|
||||
});
|
||||
|
||||
it("rejects migrations that create public objects", () => {
|
||||
expect(() =>
|
||||
validatePluginMigrationStatement(
|
||||
|
|
@ -137,10 +195,11 @@ describeEmbeddedPostgres("plugin database namespaces", () => {
|
|||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
for (const pluginKey of ["paperclip.dbtest", "paperclip.escape", "paperclip.refresh", multiMigrationPluginKey]) {
|
||||
for (const pluginKey of ["paperclip.dbtest", "paperclip.escape", "paperclip.refresh", multiMigrationPluginKey, llmWikiPluginKey]) {
|
||||
const namespace = derivePluginDatabaseNamespace(pluginKey);
|
||||
await db.execute(sql.raw(`DROP SCHEMA IF EXISTS "${namespace}" CASCADE`));
|
||||
}
|
||||
await db.execute(sql.raw(`DROP SCHEMA IF EXISTS "${derivePluginDatabaseNamespace(llmWikiPluginKey, "llm_wiki")}" CASCADE`));
|
||||
await db.delete(pluginMigrations);
|
||||
await db.delete(pluginDatabaseNamespaces);
|
||||
await db.delete(plugins);
|
||||
|
|
@ -164,6 +223,29 @@ describeEmbeddedPostgres("plugin database namespaces", () => {
|
|||
return packageRoot;
|
||||
}
|
||||
|
||||
function llmWikiManifest(): PaperclipPluginManifestV1 {
|
||||
return {
|
||||
id: llmWikiPluginKey,
|
||||
apiVersion: 1,
|
||||
version: "0.1.0",
|
||||
displayName: "LLM Wiki",
|
||||
description: "Local-file LLM Wiki plugin.",
|
||||
author: "Paperclip",
|
||||
categories: ["automation", "ui"],
|
||||
capabilities: [
|
||||
"database.namespace.migrate",
|
||||
"database.namespace.read",
|
||||
"database.namespace.write",
|
||||
],
|
||||
entrypoints: { worker: "./dist/worker.js" },
|
||||
database: {
|
||||
namespaceSlug: "llm_wiki",
|
||||
migrationsDir: "migrations",
|
||||
coreReadTables: ["companies", "issues", "projects", "agents"],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function createInstallablePluginPackage(
|
||||
pluginManifest: PaperclipPluginManifestV1,
|
||||
migrationSql: string,
|
||||
|
|
@ -252,6 +334,61 @@ describeEmbeddedPostgres("plugin database namespaces", () => {
|
|||
expect(migrations).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("applies the bundled LLM Wiki migrations through the production validator", async () => {
|
||||
const pluginManifest = llmWikiManifest();
|
||||
const repoRoot = path.basename(process.cwd()) === "server" ? path.resolve(process.cwd(), "..") : process.cwd();
|
||||
const packageRoot = path.join(repoRoot, "packages", "plugins", "plugin-llm-wiki");
|
||||
const namespace = derivePluginDatabaseNamespace(pluginManifest.id, pluginManifest.database?.namespaceSlug);
|
||||
const pluginId = await installPluginRecord(pluginManifest);
|
||||
|
||||
await pluginDatabaseService(db).applyMigrations(pluginId, pluginManifest, packageRoot);
|
||||
|
||||
const migrations = await db
|
||||
.select()
|
||||
.from(pluginMigrations)
|
||||
.where(and(eq(pluginMigrations.pluginId, pluginId), eq(pluginMigrations.status, "applied")));
|
||||
expect(migrations.map((migration) => migration.migrationKey)).toEqual([
|
||||
"001_llm_wiki.sql",
|
||||
"002_paperclip_distillation.sql",
|
||||
"003_spaces.sql",
|
||||
]);
|
||||
|
||||
const constraintRows = Array.from(
|
||||
await db.execute(
|
||||
sql<{ table_name: string; conname: string; columns: string[] }>`
|
||||
SELECT t.relname AS table_name, c.conname, array_agg(a.attname ORDER BY constraint_columns.ordinality)::text[] AS columns
|
||||
FROM pg_constraint c
|
||||
JOIN pg_class t ON t.oid = c.conrelid
|
||||
JOIN unnest(c.conkey) WITH ORDINALITY AS constraint_columns(attnum, ordinality) ON true
|
||||
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = constraint_columns.attnum
|
||||
WHERE c.connamespace = ${namespace}::regnamespace AND c.contype = 'u'
|
||||
GROUP BY t.relname, c.conname
|
||||
ORDER BY t.relname, c.conname
|
||||
`,
|
||||
) as Iterable<{ table_name: string; conname: string; columns: string[] }>,
|
||||
);
|
||||
const constraints = constraintRows.map((row) => row.conname);
|
||||
const uniqueColumnSets = new Set(
|
||||
constraintRows.map((row) => `${row.table_name}:${row.columns.join(",")}`),
|
||||
);
|
||||
expect(constraints).toEqual(
|
||||
expect.arrayContaining([
|
||||
"wiki_pages_company_wiki_space_path_key",
|
||||
"distillation_cursors_company_wiki_space_scope_key",
|
||||
"distillation_work_items_company_wiki_space_idempotency_key",
|
||||
"page_bindings_company_wiki_space_page_path_key",
|
||||
]),
|
||||
);
|
||||
expect(constraints).not.toContain("wiki_pages_company_id_wiki_id_path_key");
|
||||
expect(constraints).not.toContain("paperclip_distillation_cursor_company_id_wiki_id_source_sco_key");
|
||||
expect(constraints).not.toContain("paperclip_distillation_work_i_company_id_wiki_id_idempotenc_key");
|
||||
expect(constraints).not.toContain("paperclip_page_bindings_company_id_wiki_id_page_path_key");
|
||||
expect(uniqueColumnSets).not.toContain("wiki_pages:company_id,wiki_id,path");
|
||||
expect(uniqueColumnSets).not.toContain("paperclip_distillation_cursors:company_id,wiki_id,source_scope,scope_key,source_kind");
|
||||
expect(uniqueColumnSets).not.toContain("paperclip_distillation_work_items:company_id,wiki_id,idempotency_key");
|
||||
expect(uniqueColumnSets).not.toContain("paperclip_page_bindings:company_id,wiki_id,page_path");
|
||||
});
|
||||
|
||||
it("applies migrations once and allows whitelisted core joins at runtime", async () => {
|
||||
const pluginManifest = manifest();
|
||||
const namespace = derivePluginDatabaseNamespace(pluginManifest.id);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue