mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-15 18:30:39 +09:00
Expand database backups to non-system schemas (#4859)
## Thinking Path > - Paperclip is the control plane for autonomous AI companies. > - Reliable backups are part of operating that control plane safely. > - The previous backup path was public-schema oriented and did not clearly cover plugin-owned schemas or migration history. > - Paperclip now has plugin database namespaces and Drizzle migration state that must survive backup/restore. > - This pull request expands logical database backups to non-system schemas and documents the backup boundary. > - The benefit is safer restore behavior for core and plugin-owned database state without implying full filesystem disaster recovery. ## What Changed - Include non-system database schemas in JavaScript and pg_dump backup paths. - Preserve enum, table, sequence, index, constraint, migration, and plugin-schema objects across backup/restore. - Add restore coverage for plugin-owned schemas and Drizzle migration history. - Clarify docs that DB backups are logical database backups, not full instance filesystem backups. ## Verification - `pnpm install --frozen-lockfile` - `pnpm exec vitest run packages/db/src/backup-lib.test.ts` - Result: 1 test file passed, 4 tests passed. - Confirmed this PR does not include `pnpm-lock.yaml` or `.github/workflows/*` changes. ## Risks - Medium: backup generation touches schema discovery and restore ordering, so unusual database objects may need additional coverage later. - No migrations are included. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5 coding agent, tool use enabled, medium reasoning effort. Exact hosted context-window details are not exposed in this runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge Note: no UI changes are included in this PR, so screenshots are not applicable. --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
c0ce35d1fb
commit
cd606563f6
5 changed files with 210 additions and 48 deletions
|
|
@ -182,7 +182,135 @@ describeEmbeddedPostgres("runDatabaseBackup", () => {
|
|||
);
|
||||
|
||||
it(
|
||||
"restores statements incrementally when backup comments precede the first breakpoint",
|
||||
"backs up and restores non-public database schemas and migration history",
|
||||
async () => {
|
||||
const sourceConnectionString = await createTempDatabase();
|
||||
const restoreConnectionString = await createSiblingDatabase(
|
||||
sourceConnectionString,
|
||||
"paperclip_full_logical_restore_target",
|
||||
);
|
||||
const backupDir = createTempDir("paperclip-db-full-logical-backup-");
|
||||
const sourceSql = postgres(sourceConnectionString, { max: 1, onnotice: () => {} });
|
||||
const restoreSql = postgres(restoreConnectionString, { max: 1, onnotice: () => {} });
|
||||
|
||||
try {
|
||||
await sourceSql.unsafe(`
|
||||
CREATE SCHEMA IF NOT EXISTS "drizzle";
|
||||
CREATE TABLE IF NOT EXISTS "drizzle"."__drizzle_migrations" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"hash" text NOT NULL,
|
||||
"created_at" bigint
|
||||
);
|
||||
INSERT INTO "drizzle"."__drizzle_migrations" ("hash", "created_at")
|
||||
VALUES ('paperclip-migration-history', 1770000000000);
|
||||
`);
|
||||
await sourceSql.unsafe(`
|
||||
CREATE TABLE "public"."backup_parent_records" (
|
||||
"id" uuid PRIMARY KEY,
|
||||
"name" text NOT NULL
|
||||
);
|
||||
INSERT INTO "public"."backup_parent_records" ("id", "name")
|
||||
VALUES ('11111111-1111-4111-8111-111111111111', 'parent');
|
||||
`);
|
||||
await sourceSql.unsafe(`
|
||||
CREATE TABLE "public"."plugin_rows" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"note" text NOT NULL
|
||||
);
|
||||
CREATE TABLE "public"."audit_rows" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"secret_note" text
|
||||
);
|
||||
INSERT INTO "public"."plugin_rows" ("note")
|
||||
VALUES ('public-collision');
|
||||
INSERT INTO "public"."audit_rows" ("secret_note")
|
||||
VALUES ('public-secret');
|
||||
`);
|
||||
await sourceSql.unsafe(`
|
||||
CREATE SCHEMA "plugin_backup_scope";
|
||||
CREATE TYPE "plugin_backup_scope"."plugin_status" AS ENUM ('ready', 'done');
|
||||
CREATE TABLE "plugin_backup_scope"."plugin_rows" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"parent_id" uuid NOT NULL REFERENCES "public"."backup_parent_records"("id") ON DELETE CASCADE,
|
||||
"status" "plugin_backup_scope"."plugin_status" NOT NULL,
|
||||
"note" text NOT NULL
|
||||
);
|
||||
CREATE TABLE "plugin_backup_scope"."audit_rows" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"secret_note" text
|
||||
);
|
||||
CREATE UNIQUE INDEX "plugin_rows_note_uq" ON "plugin_backup_scope"."plugin_rows" ("note");
|
||||
INSERT INTO "plugin_backup_scope"."plugin_rows" ("parent_id", "status", "note")
|
||||
VALUES ('11111111-1111-4111-8111-111111111111', 'ready', 'first');
|
||||
INSERT INTO "plugin_backup_scope"."audit_rows" ("secret_note")
|
||||
VALUES ('plugin-secret');
|
||||
`);
|
||||
|
||||
const result = await runDatabaseBackup({
|
||||
connectionString: sourceConnectionString,
|
||||
backupDir,
|
||||
retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 },
|
||||
filenamePrefix: "paperclip-full-logical-test",
|
||||
backupEngine: "javascript",
|
||||
excludeTables: ["plugin_rows"],
|
||||
nullifyColumns: {
|
||||
audit_rows: ["secret_note"],
|
||||
},
|
||||
});
|
||||
|
||||
await runDatabaseRestore({
|
||||
connectionString: restoreConnectionString,
|
||||
backupFile: result.backupFile,
|
||||
});
|
||||
|
||||
const migrationRows = await restoreSql.unsafe<{ hash: string }[]>(`
|
||||
SELECT "hash"
|
||||
FROM "drizzle"."__drizzle_migrations"
|
||||
WHERE "hash" = 'paperclip-migration-history'
|
||||
`);
|
||||
expect(migrationRows).toEqual([{ hash: "paperclip-migration-history" }]);
|
||||
|
||||
const pluginRows = await restoreSql.unsafe<{ note: string; status: string; parent_name: string }[]>(`
|
||||
SELECT r."note", r."status"::text AS "status", p."name" AS "parent_name"
|
||||
FROM "plugin_backup_scope"."plugin_rows" r
|
||||
JOIN "public"."backup_parent_records" p ON p."id" = r."parent_id"
|
||||
`);
|
||||
expect(pluginRows).toEqual([{ note: "first", status: "ready", parent_name: "parent" }]);
|
||||
|
||||
const publicCollisionRows = await restoreSql.unsafe<{ count: number }[]>(`
|
||||
SELECT count(*)::int AS count
|
||||
FROM "public"."plugin_rows"
|
||||
`);
|
||||
expect(publicCollisionRows[0]?.count).toBe(0);
|
||||
|
||||
const publicAuditRows = await restoreSql.unsafe<{ secret_note: string | null }[]>(`
|
||||
SELECT "secret_note"
|
||||
FROM "public"."audit_rows"
|
||||
`);
|
||||
expect(publicAuditRows).toEqual([{ secret_note: null }]);
|
||||
|
||||
const pluginAuditRows = await restoreSql.unsafe<{ secret_note: string | null }[]>(`
|
||||
SELECT "secret_note"
|
||||
FROM "plugin_backup_scope"."audit_rows"
|
||||
`);
|
||||
expect(pluginAuditRows).toEqual([{ secret_note: "plugin-secret" }]);
|
||||
|
||||
await expect(
|
||||
restoreSql.unsafe(`
|
||||
INSERT INTO "plugin_backup_scope"."plugin_rows" ("parent_id", "status", "note")
|
||||
VALUES ('11111111-1111-4111-8111-111111111111', 'done', 'first')
|
||||
`),
|
||||
).rejects.toThrow();
|
||||
} finally {
|
||||
await sourceSql.end();
|
||||
await restoreSql.end();
|
||||
}
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it(
|
||||
"restores legacy public-only backups without migration history",
|
||||
async () => {
|
||||
const restoreConnectionString = await createTempDatabase();
|
||||
const restoreSql = postgres(restoreConnectionString, { max: 1, onnotice: () => {} });
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue