ake/paperclip
1
0
Fork 0
mirror of https://github.com/alkimake/paperclip.git synced 2026-06-14 01:50:39 +09:00
Code Issues Projects Releases Packages Wiki Activity

feat(backups): gzip compress backups and add retention config to Instance Settings

Browse source 
Compress database backups with gzip (.sql.gz), reducing file size ~83%.
Add backup retention configuration to Instance Settings UI with preset
options (7 days, 2 weeks, 1 month). The backup scheduler now reads
retention from the database on each tick so changes take effect without
restart. Default retention changed from 30 to 7 days.
This commit is contained in:
Aron Prins 2026-04-07 09:41:13 +02:00
parent 316790ea0a
commit cc44d309c0
11 changed files with 107 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

4
packages/db/src/backup-lib.test.ts
View file

@ -129,8 +129,8 @@ describeEmbeddedPostgres("runDatabaseBackup", () => {
filenamePrefix: "paperclip-test",
});
expect(result.backupFile).toMatch(/paperclip-test-.*\.sql$/);
expect(result.sizeBytes).toBeGreaterThan(1024 * 1024);
expect(result.backupFile).toMatch(/paperclip-test-.*\.sql\.gz$/);
expect(result.sizeBytes).toBeGreaterThan(0);
expect(fs.existsSync(result.backupFile)).toBe(true);
await runDatabaseRestore({

24
packages/db/src/backup-lib.ts
View file

@ -1,6 +1,8 @@
import { createReadStream, createWriteStream, existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from "node:fs";
import { basename, resolve } from "node:path";
import { createInterface } from "node:readline";
import { pipeline } from "node:stream/promises";
import { createGunzip, createGzip } from "node:zlib";
import postgres from "postgres";
export type RunDatabaseBackupOptions = {
@ -82,7 +84,8 @@ function pruneOldBackups(backupDir: string, retentionDays: number, filenamePrefi
let pruned = 0;
for (const name of readdirSync(backupDir)) {
if (!name.startsWith(`${filenamePrefix}-`) || !name.endsWith(".sql")) continue;
if (!name.startsWith(`${filenamePrefix}-`)) continue;
if (!name.endsWith(".sql") && !name.endsWith(".sql.gz")) continue;
const fullPath = resolve(backupDir, name);
const stat = statSync(fullPath);
if (stat.mtimeMs < cutoff) {
@ -148,7 +151,9 @@ function tableKey(schemaName: string, tableName: string): string {
}
async function* readRestoreStatements(backupFile: string): AsyncGenerator<string> {
const stream = createReadStream(backupFile, { encoding: "utf8" });
const raw = createReadStream(backupFile);
const stream = backupFile.endsWith(".gz") ? raw.pipe(createGunzip()) : raw;
stream.setEncoding("utf8");
const reader = createInterface({
input: stream,
crlfDelay: Infinity,
@ -180,6 +185,7 @@ async function* readRestoreStatements(backupFile: string): AsyncGenerator<string
} finally {
reader.close();
stream.destroy();
raw.destroy();
}
}
@ -288,8 +294,9 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
const nullifiedColumnsByTable = normalizeNullifyColumnMap(opts.nullifyColumns);
const sql = postgres(opts.connectionString, { max: 1, connect_timeout: connectTimeout });
mkdirSync(opts.backupDir, { recursive: true });
const backupFile = resolve(opts.backupDir, `${filenamePrefix}-${timestamp()}.sql`);
const writer = createBufferedTextFileWriter(backupFile);
const sqlFile = resolve(opts.backupDir, `${filenamePrefix}-${timestamp()}.sql`);
const backupFile = `${sqlFile}.gz`;
const writer = createBufferedTextFileWriter(sqlFile);
try {
await sql`SELECT 1`;
@ -664,6 +671,12 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
await writer.close();
// Compress the SQL file with gzip
const sqlReadStream = createReadStream(sqlFile);
const gzWriteStream = createWriteStream(backupFile);
await pipeline(sqlReadStream, createGzip(), gzWriteStream);
unlinkSync(sqlFile);
const sizeBytes = statSync(backupFile).size;
const prunedCount = pruneOldBackups(opts.backupDir, retentionDays, filenamePrefix);
@ -674,6 +687,9 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
};
} catch (error) {
await writer.abort();
if (existsSync(backupFile)) {
try { unlinkSync(backupFile); } catch { /* ignore */ }
}
throw error;
} finally {
await sql.end();

4
packages/shared/src/config-schema.ts
View file

@ -21,7 +21,7 @@ export const llmConfigSchema = z.object({
export const databaseBackupConfigSchema = z.object({
enabled: z.boolean().default(true),
intervalMinutes: z.number().int().min(1).max(7 * 24 * 60).default(60),
retentionDays: z.number().int().min(1).max(3650).default(30),
retentionDays: z.number().int().min(1).max(3650).default(7),
dir: z.string().default("~/.paperclip/instances/default/data/backups"),
});
@ -33,7 +33,7 @@ export const databaseConfigSchema = z.object({
backup: databaseBackupConfigSchema.default({
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
retentionDays: 7,
dir: "~/.paperclip/instances/default/data/backups",
}),
});

6
packages/shared/src/index.ts
View file

@ -189,6 +189,7 @@ export type {
InstanceExperimentalSettings,
InstanceGeneralSettings,
InstanceSettings,
BackupRetentionDays,
Agent,
AgentAccessState,
AgentChainOfCommandEntry,
@ -369,6 +370,11 @@ export {
DEFAULT_FEEDBACK_DATA_SHARING_TERMS_VERSION,
} from "./types/feedback.js";
export {
BACKUP_RETENTION_PRESETS,
DEFAULT_BACKUP_RETENTION_DAYS,
} from "./types/instance.js";
export {
getClosedIsolatedExecutionWorkspaceMessage,
isClosedIsolatedExecutionWorkspace,

3
packages/shared/src/types/index.ts
View file

@ -11,7 +11,8 @@ export type {
FeedbackTraceBundleFile,
FeedbackTraceBundle,
} from "./feedback.js";
export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings } from "./instance.js";
export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionDays } from "./instance.js";
export { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "./instance.js";
export type {
CompanySkillSourceType,
CompanySkillTrustLevel,

5
packages/shared/src/types/instance.ts
View file

@ -1,9 +1,14 @@
import type { FeedbackDataSharingPreference } from "./feedback.js";
export const BACKUP_RETENTION_PRESETS = [7, 14, 30] as const;
export type BackupRetentionDays = (typeof BACKUP_RETENTION_PRESETS)[number];
export const DEFAULT_BACKUP_RETENTION_DAYS: BackupRetentionDays = 7;
export interface InstanceGeneralSettings {
censorUsernameInLogs: boolean;
keyboardShortcuts: boolean;
feedbackDataSharingPreference: FeedbackDataSharingPreference;
backupRetentionDays: BackupRetentionDays;
}
export interface InstanceExperimentalSettings {

8
packages/shared/src/validators/instance.ts
View file

@ -1,13 +1,21 @@
import { z } from "zod";
import { DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE } from "../types/feedback.js";
import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "../types/instance.js";
import { feedbackDataSharingPreferenceSchema } from "./feedback.js";
export const backupRetentionDaysSchema = z.number().refine(
(v): v is (typeof BACKUP_RETENTION_PRESETS)[number] =>
(BACKUP_RETENTION_PRESETS as readonly number[]).includes(v),
{ message: `Must be one of: ${BACKUP_RETENTION_PRESETS.join(", ")}` },
);
export const instanceGeneralSettingsSchema = z.object({
censorUsernameInLogs: z.boolean().default(false),
keyboardShortcuts: z.boolean().default(false),
feedbackDataSharingPreference: feedbackDataSharingPreferenceSchema.default(
DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
),
backupRetentionDays: backupRetentionDaysSchema.default(DEFAULT_BACKUP_RETENTION_DAYS),
}).strict();
export const patchInstanceGeneralSettingsSchema = instanceGeneralSettingsSchema.partial();

2
server/src/config.ts
View file

@ -216,7 +216,7 @@ export function loadConfig(): Config {
1,
Number(process.env.PAPERCLIP_DB_BACKUP_RETENTION_DAYS) ||
fileDatabaseBackup?.retentionDays ||
30,
7,
);
const databaseBackupDir = resolveHomeAwarePath(
process.env.PAPERCLIP_DB_BACKUP_DIR ??

16
server/src/index.ts
View file

@ -31,6 +31,7 @@ import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js";
import {
feedbackService,
heartbeatService,
instanceSettingsService,
reconcilePersistedRuntimeServicesOnStartup,
routineService,
} from "./services/index.js";
@ -628,20 +629,25 @@ export async function startServer(): Promise<StartedServer> {
if (config.databaseBackupEnabled) {
const backupIntervalMs = config.databaseBackupIntervalMinutes * 60 * 1000;
const settingsSvc = instanceSettingsService(db);
let backupInFlight = false;
const runScheduledBackup = async () => {
if (backupInFlight) {
logger.warn("Skipping scheduled database backup because a previous backup is still running");
return;
}
backupInFlight = true;
try {
// Read retention from Instance Settings (DB) so changes take effect without restart
const generalSettings = await settingsSvc.getGeneral();
const retentionDays = generalSettings.backupRetentionDays;
const result = await runDatabaseBackup({
connectionString: activeDatabaseConnectionString,
backupDir: config.databaseBackupDir,
retentionDays: config.databaseBackupRetentionDays,
retentionDays,
filenamePrefix: "paperclip",
});
logger.info(
@ -650,7 +656,7 @@ export async function startServer(): Promise<StartedServer> {
sizeBytes: result.sizeBytes,
prunedCount: result.prunedCount,
backupDir: config.databaseBackupDir,
retentionDays: config.databaseBackupRetentionDays,
retentionDays,
},
`Automatic database backup complete: ${formatDatabaseBackupResult(result)}`,
);
@ -660,7 +666,7 @@ export async function startServer(): Promise<StartedServer> {
backupInFlight = false;
}
};
logger.info(
{
intervalMinutes: config.databaseBackupIntervalMinutes,

3
server/src/services/instance-settings.ts
View file

@ -2,6 +2,7 @@ import type { Db } from "@paperclipai/db";
import { companies, instanceSettings } from "@paperclipai/db";
import {
DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
DEFAULT_BACKUP_RETENTION_DAYS,
instanceGeneralSettingsSchema,
type InstanceGeneralSettings,
instanceExperimentalSettingsSchema,
@ -22,12 +23,14 @@ function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings {
keyboardShortcuts: parsed.data.keyboardShortcuts ?? false,
feedbackDataSharingPreference:
parsed.data.feedbackDataSharingPreference ?? DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
backupRetentionDays: parsed.data.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS,
};
}
return {
censorUsernameInLogs: false,
keyboardShortcuts: false,
feedbackDataSharingPreference: DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
backupRetentionDays: DEFAULT_BACKUP_RETENTION_DAYS,
};
}

49
ui/src/pages/InstanceGeneralSettings.tsx
View file

@ -1,7 +1,8 @@
import { useEffect, useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import type { PatchInstanceGeneralSettings } from "@paperclipai/shared";
import { LogOut, SlidersHorizontal } from "lucide-react";
import type { PatchInstanceGeneralSettings, BackupRetentionDays } from "@paperclipai/shared";
import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "@paperclipai/shared";
import { Database, LogOut, SlidersHorizontal } from "lucide-react";
import { authApi } from "@/api/auth";
import { instanceSettingsApi } from "@/api/instanceSettings";
import { Button } from "../components/ui/button";
@ -67,6 +68,7 @@ export function InstanceGeneralSettings() {
const censorUsernameInLogs = generalQuery.data?.censorUsernameInLogs === true;
const keyboardShortcuts = generalQuery.data?.keyboardShortcuts === true;
const feedbackDataSharingPreference = generalQuery.data?.feedbackDataSharingPreference ?? "prompt";
const backupRetentionDays: BackupRetentionDays = generalQuery.data?.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS;
return (
<div className="max-w-4xl space-y-6">
@ -123,6 +125,49 @@ export function InstanceGeneralSettings() {
</div>
</section>
<section className="rounded-xl border border-border bg-card p-5">
<div className="space-y-4">
<div className="flex items-center gap-2">
<Database className="h-4 w-4 text-muted-foreground" />
<div className="space-y-1.5">
<h2 className="text-sm font-semibold">Backup retention</h2>
<p className="max-w-2xl text-sm text-muted-foreground">
How long to keep automatic database backups before pruning. Backups are compressed
with gzip to minimize disk usage.
</p>
</div>
</div>
<div className="flex flex-wrap gap-2">
{BACKUP_RETENTION_PRESETS.map((days) => {
const active = backupRetentionDays === days;
const label =
days === 7 ? "7 days" : days === 14 ? "2 weeks" : "1 month";
return (
<button
key={days}
type="button"
disabled={updateGeneralMutation.isPending}
className={cn(
"rounded-lg border px-3 py-2 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-60",
active
? "border-foreground bg-accent text-foreground"
: "border-border bg-background hover:bg-accent/50",
)}
onClick={() =>
updateGeneralMutation.mutate({ backupRetentionDays: days })
}
>
<div className="text-sm font-medium">{label}</div>
<div className="text-xs text-muted-foreground">
Keep backups for {days} days
</div>
</button>
);
})}
</div>
</div>
</section>
<section className="rounded-xl border border-border bg-card p-5">
<div className="space-y-4">
<div className="space-y-1.5">