mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-13 17:40:38 +09:00
[codex] Runtime control-plane fixes (#6380)
## Thinking Path > - Paperclip orchestrates AI agents through a server-side control plane > - That control plane depends on reliable issue state transitions, plugin lifecycle behavior, import limits, and startup/shutdown handling > - Several small runtime fixes had accumulated on the working branch and were mixed with larger feature work > - Keeping them separate makes the correctness fixes reviewable and mergeable without waiting for cloud-sync UI work > - This pull request groups the server/runtime control-plane fixes into one standalone branch > - The benefit is a tighter, safer runtime baseline for retries, imports, plugin migrations, feedback flushing, and trusted cloud import handling ## What Changed - Fixed updated issue list pagination sorting and scheduled retry comment handling. - Re-applied pending plugin migrations during hot reload and fixed plugin-schema worktree seed restore. - Hardened public tenant DB startup, portable import body limits, trusted cloud import errors, and trusted cloud tenant import mutation access. - Expired stale request confirmations after user comments. - Added feedback export shutdown hardening so database-unavailable flush loops stop cleanly. - Guarded plugin worker `error` event emission when no listener is registered. ## Verification - `pnpm install --frozen-lockfile --ignore-scripts` - `pnpm --filter @paperclipai/plugin-sdk build` - `npm run install --prefix node_modules/.pnpm/sqlite3@5.1.7/node_modules/sqlite3` - `pnpm exec vitest run server/src/__tests__/issues-service.test.ts server/src/__tests__/plugin-lifecycle-restart.test.ts server/src/__tests__/server-startup-feedback-export.test.ts server/src/__tests__/issue-comment-reopen-routes.test.ts server/src/__tests__/issue-thread-interactions-service.test.ts server/src/__tests__/issue-thread-interaction-routes.test.ts server/src/__tests__/body-limits.test.ts server/src/__tests__/feedback-flush-controller.test.ts server/src/__tests__/error-handler.test.ts server/src/__tests__/board-mutation-guard.test.ts packages/db/src/backup-lib.test.ts` initially exposed local setup issues and two 5s test timeouts. - Rerun after local prereq build: `pnpm exec vitest run --testTimeout 15000 server/src/__tests__/issue-comment-reopen-routes.test.ts server/src/__tests__/issue-thread-interaction-routes.test.ts server/src/__tests__/feedback-flush-controller.test.ts server/src/__tests__/server-startup-feedback-export.test.ts` passed. - Some embedded Postgres-backed tests skipped on this host because local Postgres init was unavailable. ## Risks - Runtime-touching branch: startup/shutdown and issue interaction behavior should be reviewed carefully. - The feedback export change disables repeated flush attempts only for database connection-refused failures; other upload failures still log normally. - The plugin worker error guard avoids process crashes from unhandled EventEmitter errors but may hide errors from code paths that expected an emitted listener. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5-based coding agent with local shell/git/tool use. Exact hosted model ID and context-window size are not exposed by the local Paperclip adapter runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
f257530537
commit
c91a062326
26 changed files with 1363 additions and 130 deletions
|
|
@ -309,6 +309,107 @@ describeEmbeddedPostgres("runDatabaseBackup", () => {
|
|||
60_000,
|
||||
);
|
||||
|
||||
it(
|
||||
"preserves composite foreign key column order without duplicate referenced columns",
|
||||
async () => {
|
||||
const sourceConnectionString = await createTempDatabase();
|
||||
const restoreConnectionString = await createSiblingDatabase(
|
||||
sourceConnectionString,
|
||||
"paperclip_composite_fk_restore_target",
|
||||
);
|
||||
const backupDir = createTempDir("paperclip-db-composite-fk-backup-");
|
||||
const sourceSql = postgres(sourceConnectionString, { max: 1, onnotice: () => {} });
|
||||
const restoreSql = postgres(restoreConnectionString, { max: 1, onnotice: () => {} });
|
||||
|
||||
try {
|
||||
await sourceSql.unsafe(`
|
||||
CREATE SCHEMA "plugin_composite_fk";
|
||||
CREATE TABLE "plugin_composite_fk"."content_cases" (
|
||||
"id" uuid PRIMARY KEY,
|
||||
"company_id" uuid NOT NULL,
|
||||
"title" text NOT NULL,
|
||||
CONSTRAINT "content_cases_company_case_unique" UNIQUE ("company_id", "id")
|
||||
);
|
||||
CREATE TABLE "plugin_composite_fk"."content_case_signals" (
|
||||
"company_id" uuid NOT NULL,
|
||||
"case_id" uuid NOT NULL,
|
||||
"signal" text NOT NULL,
|
||||
"scopes" text[] NOT NULL,
|
||||
"warnings" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
||||
CONSTRAINT "content_case_signals_company_case"
|
||||
FOREIGN KEY ("company_id", "case_id")
|
||||
REFERENCES "plugin_composite_fk"."content_cases" ("company_id", "id")
|
||||
ON DELETE CASCADE
|
||||
);
|
||||
INSERT INTO "plugin_composite_fk"."content_cases" ("company_id", "id", "title")
|
||||
VALUES (
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
'case'
|
||||
);
|
||||
INSERT INTO "plugin_composite_fk"."content_case_signals" ("company_id", "case_id", "signal", "scopes", "warnings")
|
||||
VALUES (
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
'signal',
|
||||
ARRAY['upstream_import:preview', 'scope with space', 'quoted "scope"', 'NULL', 'null'],
|
||||
jsonb_build_array('json warning', jsonb_build_object('code', 'quoted "value"'))
|
||||
);
|
||||
`);
|
||||
|
||||
const result = await runDatabaseBackup({
|
||||
connectionString: sourceConnectionString,
|
||||
backupDir,
|
||||
retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 },
|
||||
filenamePrefix: "paperclip-composite-fk-test",
|
||||
backupEngine: "javascript",
|
||||
});
|
||||
|
||||
await runDatabaseRestore({
|
||||
connectionString: restoreConnectionString,
|
||||
backupFile: result.backupFile,
|
||||
});
|
||||
|
||||
const rows = await restoreSql.unsafe<{
|
||||
signal: string;
|
||||
title: string;
|
||||
scopes: string[];
|
||||
warnings: Array<string | { code: string }>;
|
||||
}[]>(`
|
||||
SELECT s."signal", c."title", s."scopes", s."warnings"
|
||||
FROM "plugin_composite_fk"."content_case_signals" s
|
||||
JOIN "plugin_composite_fk"."content_cases" c
|
||||
ON c."company_id" = s."company_id"
|
||||
AND c."id" = s."case_id"
|
||||
`);
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
signal: "signal",
|
||||
title: "case",
|
||||
scopes: ["upstream_import:preview", "scope with space", 'quoted "scope"', "NULL", "null"],
|
||||
warnings: ["json warning", { code: 'quoted "value"' }],
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(
|
||||
restoreSql.unsafe(`
|
||||
INSERT INTO "plugin_composite_fk"."content_case_signals" ("company_id", "case_id", "signal", "scopes")
|
||||
VALUES (
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
'33333333-3333-4333-8333-333333333333',
|
||||
'orphan',
|
||||
ARRAY[]::text[]
|
||||
)
|
||||
`),
|
||||
).rejects.toThrow();
|
||||
} finally {
|
||||
await sourceSql.end();
|
||||
await restoreSql.end();
|
||||
}
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it(
|
||||
"restores legacy public-only backups without migration history",
|
||||
async () => {
|
||||
|
|
|
|||
|
|
@ -249,12 +249,39 @@ function hasBackupTransforms(opts: RunDatabaseBackupOptions): boolean {
|
|||
Object.keys(opts.nullifyColumns ?? {}).length > 0;
|
||||
}
|
||||
|
||||
function formatSqlValue(rawValue: unknown, columnName: string | undefined, nullifiedColumns: Set<string>): string {
|
||||
function formatPostgresArrayElement(value: unknown): string {
|
||||
if (value === null || value === undefined) return "NULL";
|
||||
if (Array.isArray(value)) return formatPostgresArrayLiteral(value);
|
||||
const raw = value instanceof Date
|
||||
? value.toISOString()
|
||||
: typeof value === "object"
|
||||
? JSON.stringify(value)
|
||||
: String(value);
|
||||
if (raw.length === 0 || /^null$/i.test(raw) || /[{}\s,"\\]/.test(raw)) {
|
||||
return `"${raw.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
function formatPostgresArrayLiteral(value: unknown[]): string {
|
||||
return `{${value.map(formatPostgresArrayElement).join(",")}}`;
|
||||
}
|
||||
|
||||
function formatSqlValue(
|
||||
rawValue: unknown,
|
||||
columnName: string | undefined,
|
||||
nullifiedColumns: Set<string>,
|
||||
dataType?: string,
|
||||
): string {
|
||||
const val = columnName && nullifiedColumns.has(columnName) ? null : rawValue;
|
||||
if (val === null || val === undefined) return "NULL";
|
||||
if (dataType === "json" || dataType === "jsonb") {
|
||||
return formatSqlLiteral(JSON.stringify(val));
|
||||
}
|
||||
if (typeof val === "boolean") return val ? "true" : "false";
|
||||
if (typeof val === "number") return String(val);
|
||||
if (val instanceof Date) return formatSqlLiteral(val.toISOString());
|
||||
if (Array.isArray(val)) return formatSqlLiteral(formatPostgresArrayLiteral(val));
|
||||
if (typeof val === "object") return formatSqlLiteral(JSON.stringify(val));
|
||||
return formatSqlLiteral(String(val));
|
||||
}
|
||||
|
|
@ -745,58 +772,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
|
|||
emit("");
|
||||
}
|
||||
|
||||
// Foreign keys (after all tables created)
|
||||
const allForeignKeys = await sql<{
|
||||
constraint_name: string;
|
||||
source_schema: string;
|
||||
source_table: string;
|
||||
source_columns: string[];
|
||||
target_schema: string;
|
||||
target_table: string;
|
||||
target_columns: string[];
|
||||
update_rule: string;
|
||||
delete_rule: string;
|
||||
}[]>`
|
||||
SELECT
|
||||
c.conname AS constraint_name,
|
||||
srcn.nspname AS source_schema,
|
||||
src.relname AS source_table,
|
||||
array_agg(sa.attname ORDER BY array_position(c.conkey, sa.attnum)) AS source_columns,
|
||||
tgtn.nspname AS target_schema,
|
||||
tgt.relname AS target_table,
|
||||
array_agg(ta.attname ORDER BY array_position(c.confkey, ta.attnum)) AS target_columns,
|
||||
CASE c.confupdtype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS update_rule,
|
||||
CASE c.confdeltype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS delete_rule
|
||||
FROM pg_constraint c
|
||||
JOIN pg_class src ON src.oid = c.conrelid
|
||||
JOIN pg_namespace srcn ON srcn.oid = src.relnamespace
|
||||
JOIN pg_class tgt ON tgt.oid = c.confrelid
|
||||
JOIN pg_namespace tgtn ON tgtn.oid = tgt.relnamespace
|
||||
JOIN pg_attribute sa ON sa.attrelid = src.oid AND sa.attnum = ANY(c.conkey)
|
||||
JOIN pg_attribute ta ON ta.attrelid = tgt.oid AND ta.attnum = ANY(c.confkey)
|
||||
WHERE c.contype = 'f'
|
||||
AND ${sql.unsafe(nonSystemSchemaPredicate("srcn.nspname"))}
|
||||
GROUP BY c.conname, srcn.nspname, src.relname, tgtn.nspname, tgt.relname, c.confupdtype, c.confdeltype
|
||||
ORDER BY srcn.nspname, src.relname, c.conname
|
||||
`;
|
||||
const fks = allForeignKeys.filter(
|
||||
(fk) => includedTableNames.has(tableKey(fk.source_schema, fk.source_table))
|
||||
&& includedTableNames.has(tableKey(fk.target_schema, fk.target_table)),
|
||||
);
|
||||
|
||||
if (fks.length > 0) {
|
||||
emit("-- Foreign keys");
|
||||
for (const fk of fks) {
|
||||
const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", ");
|
||||
const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", ");
|
||||
emitStatement(
|
||||
`ALTER TABLE ${quoteQualifiedName(fk.source_schema, fk.source_table)} ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES ${quoteQualifiedName(fk.target_schema, fk.target_table)} (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`,
|
||||
);
|
||||
}
|
||||
emit("");
|
||||
}
|
||||
|
||||
// Unique constraints
|
||||
// Unique constraints must exist before foreign keys that reference them.
|
||||
const allUniqueConstraints = await sql<{
|
||||
constraint_name: string;
|
||||
schema_name: string;
|
||||
|
|
@ -827,6 +803,58 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
|
|||
emit("");
|
||||
}
|
||||
|
||||
// Foreign keys (after all tables and referenced unique constraints are created)
|
||||
const allForeignKeys = await sql<{
|
||||
constraint_name: string;
|
||||
source_schema: string;
|
||||
source_table: string;
|
||||
source_columns: string[];
|
||||
target_schema: string;
|
||||
target_table: string;
|
||||
target_columns: string[];
|
||||
update_rule: string;
|
||||
delete_rule: string;
|
||||
}[]>`
|
||||
SELECT
|
||||
c.conname AS constraint_name,
|
||||
srcn.nspname AS source_schema,
|
||||
src.relname AS source_table,
|
||||
array_agg(sa.attname ORDER BY key_columns.ordinal_position) AS source_columns,
|
||||
tgtn.nspname AS target_schema,
|
||||
tgt.relname AS target_table,
|
||||
array_agg(ta.attname ORDER BY key_columns.ordinal_position) AS target_columns,
|
||||
CASE c.confupdtype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS update_rule,
|
||||
CASE c.confdeltype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS delete_rule
|
||||
FROM pg_constraint c
|
||||
JOIN pg_class src ON src.oid = c.conrelid
|
||||
JOIN pg_namespace srcn ON srcn.oid = src.relnamespace
|
||||
JOIN pg_class tgt ON tgt.oid = c.confrelid
|
||||
JOIN pg_namespace tgtn ON tgtn.oid = tgt.relnamespace
|
||||
JOIN LATERAL unnest(c.conkey, c.confkey) WITH ORDINALITY AS key_columns(source_attnum, target_attnum, ordinal_position) ON true
|
||||
JOIN pg_attribute sa ON sa.attrelid = src.oid AND sa.attnum = key_columns.source_attnum
|
||||
JOIN pg_attribute ta ON ta.attrelid = tgt.oid AND ta.attnum = key_columns.target_attnum
|
||||
WHERE c.contype = 'f'
|
||||
AND ${sql.unsafe(nonSystemSchemaPredicate("srcn.nspname"))}
|
||||
GROUP BY c.conname, srcn.nspname, src.relname, tgtn.nspname, tgt.relname, c.confupdtype, c.confdeltype
|
||||
ORDER BY srcn.nspname, src.relname, c.conname
|
||||
`;
|
||||
const fks = allForeignKeys.filter(
|
||||
(fk) => includedTableNames.has(tableKey(fk.source_schema, fk.source_table))
|
||||
&& includedTableNames.has(tableKey(fk.target_schema, fk.target_table)),
|
||||
);
|
||||
|
||||
if (fks.length > 0) {
|
||||
emit("-- Foreign keys");
|
||||
for (const fk of fks) {
|
||||
const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", ");
|
||||
const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", ");
|
||||
emitStatement(
|
||||
`ALTER TABLE ${quoteQualifiedName(fk.source_schema, fk.source_table)} ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES ${quoteQualifiedName(fk.target_schema, fk.target_table)} (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`,
|
||||
);
|
||||
}
|
||||
emit("");
|
||||
}
|
||||
|
||||
// Indexes (non-primary, non-unique-constraint)
|
||||
const allIndexes = await sql<{ schema_name: string; tablename: string; indexdef: string }[]>`
|
||||
SELECT schemaname AS schema_name, tablename, indexdef
|
||||
|
|
@ -895,7 +923,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
|
|||
for await (const rows of rowCursor) {
|
||||
for (const row of rows) {
|
||||
const values = row.map((rawValue, index) =>
|
||||
formatSqlValue(rawValue, cols[index]?.column_name, nullifiedColumns),
|
||||
formatSqlValue(rawValue, cols[index]?.column_name, nullifiedColumns, cols[index]?.data_type),
|
||||
);
|
||||
emitStatement(`INSERT INTO ${qualifiedTableName} (${colNames}) VALUES (${values.join(", ")});`);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue