diff --git a/packages/db/src/backup-lib.test.ts b/packages/db/src/backup-lib.test.ts index dcdc87c5..2367d26d 100644 --- a/packages/db/src/backup-lib.test.ts +++ b/packages/db/src/backup-lib.test.ts @@ -129,8 +129,8 @@ describeEmbeddedPostgres("runDatabaseBackup", () => { filenamePrefix: "paperclip-test", }); - expect(result.backupFile).toMatch(/paperclip-test-.*\.sql$/); - expect(result.sizeBytes).toBeGreaterThan(1024 * 1024); + expect(result.backupFile).toMatch(/paperclip-test-.*\.sql\.gz$/); + expect(result.sizeBytes).toBeGreaterThan(0); expect(fs.existsSync(result.backupFile)).toBe(true); await runDatabaseRestore({ diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.ts index ea76a2b6..36b5ee98 100644 --- a/packages/db/src/backup-lib.ts +++ b/packages/db/src/backup-lib.ts @@ -1,6 +1,8 @@ import { createReadStream, createWriteStream, existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from "node:fs"; import { basename, resolve } from "node:path"; import { createInterface } from "node:readline"; +import { pipeline } from "node:stream/promises"; +import { createGunzip, createGzip } from "node:zlib"; import postgres from "postgres"; export type RunDatabaseBackupOptions = { @@ -82,7 +84,8 @@ function pruneOldBackups(backupDir: string, retentionDays: number, filenamePrefi let pruned = 0; for (const name of readdirSync(backupDir)) { - if (!name.startsWith(`${filenamePrefix}-`) || !name.endsWith(".sql")) continue; + if (!name.startsWith(`${filenamePrefix}-`)) continue; + if (!name.endsWith(".sql") && !name.endsWith(".sql.gz")) continue; const fullPath = resolve(backupDir, name); const stat = statSync(fullPath); if (stat.mtimeMs < cutoff) { @@ -148,7 +151,9 @@ function tableKey(schemaName: string, tableName: string): string { } async function* readRestoreStatements(backupFile: string): AsyncGenerator { - const stream = createReadStream(backupFile, { encoding: "utf8" }); + const raw = createReadStream(backupFile); + const stream = backupFile.endsWith(".gz") ? raw.pipe(createGunzip()) : raw; + stream.setEncoding("utf8"); const reader = createInterface({ input: stream, crlfDelay: Infinity, @@ -180,6 +185,7 @@ async function* readRestoreStatements(backupFile: string): AsyncGenerator + (BACKUP_RETENTION_PRESETS as readonly number[]).includes(v), + { message: `Must be one of: ${BACKUP_RETENTION_PRESETS.join(", ")}` }, +); + export const instanceGeneralSettingsSchema = z.object({ censorUsernameInLogs: z.boolean().default(false), keyboardShortcuts: z.boolean().default(false), feedbackDataSharingPreference: feedbackDataSharingPreferenceSchema.default( DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, ), + backupRetentionDays: backupRetentionDaysSchema.default(DEFAULT_BACKUP_RETENTION_DAYS), }).strict(); export const patchInstanceGeneralSettingsSchema = instanceGeneralSettingsSchema.partial(); diff --git a/server/src/config.ts b/server/src/config.ts index 71084cc0..5ae50400 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -216,7 +216,7 @@ export function loadConfig(): Config { 1, Number(process.env.PAPERCLIP_DB_BACKUP_RETENTION_DAYS) || fileDatabaseBackup?.retentionDays || - 30, + 7, ); const databaseBackupDir = resolveHomeAwarePath( process.env.PAPERCLIP_DB_BACKUP_DIR ?? diff --git a/server/src/index.ts b/server/src/index.ts index a384342f..768d1fd6 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -31,6 +31,7 @@ import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js"; import { feedbackService, heartbeatService, + instanceSettingsService, reconcilePersistedRuntimeServicesOnStartup, routineService, } from "./services/index.js"; @@ -628,20 +629,25 @@ export async function startServer(): Promise { if (config.databaseBackupEnabled) { const backupIntervalMs = config.databaseBackupIntervalMinutes * 60 * 1000; + const settingsSvc = instanceSettingsService(db); let backupInFlight = false; - + const runScheduledBackup = async () => { if (backupInFlight) { logger.warn("Skipping scheduled database backup because a previous backup is still running"); return; } - + backupInFlight = true; try { + // Read retention from Instance Settings (DB) so changes take effect without restart + const generalSettings = await settingsSvc.getGeneral(); + const retentionDays = generalSettings.backupRetentionDays; + const result = await runDatabaseBackup({ connectionString: activeDatabaseConnectionString, backupDir: config.databaseBackupDir, - retentionDays: config.databaseBackupRetentionDays, + retentionDays, filenamePrefix: "paperclip", }); logger.info( @@ -650,7 +656,7 @@ export async function startServer(): Promise { sizeBytes: result.sizeBytes, prunedCount: result.prunedCount, backupDir: config.databaseBackupDir, - retentionDays: config.databaseBackupRetentionDays, + retentionDays, }, `Automatic database backup complete: ${formatDatabaseBackupResult(result)}`, ); @@ -660,7 +666,7 @@ export async function startServer(): Promise { backupInFlight = false; } }; - + logger.info( { intervalMinutes: config.databaseBackupIntervalMinutes, diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 7856591d..5c7db61d 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -2,6 +2,7 @@ import type { Db } from "@paperclipai/db"; import { companies, instanceSettings } from "@paperclipai/db"; import { DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, + DEFAULT_BACKUP_RETENTION_DAYS, instanceGeneralSettingsSchema, type InstanceGeneralSettings, instanceExperimentalSettingsSchema, @@ -22,12 +23,14 @@ function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings { keyboardShortcuts: parsed.data.keyboardShortcuts ?? false, feedbackDataSharingPreference: parsed.data.feedbackDataSharingPreference ?? DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, + backupRetentionDays: parsed.data.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS, }; } return { censorUsernameInLogs: false, keyboardShortcuts: false, feedbackDataSharingPreference: DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, + backupRetentionDays: DEFAULT_BACKUP_RETENTION_DAYS, }; } diff --git a/ui/src/pages/InstanceGeneralSettings.tsx b/ui/src/pages/InstanceGeneralSettings.tsx index 28e00b29..9004e418 100644 --- a/ui/src/pages/InstanceGeneralSettings.tsx +++ b/ui/src/pages/InstanceGeneralSettings.tsx @@ -1,7 +1,8 @@ import { useEffect, useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import type { PatchInstanceGeneralSettings } from "@paperclipai/shared"; -import { LogOut, SlidersHorizontal } from "lucide-react"; +import type { PatchInstanceGeneralSettings, BackupRetentionDays } from "@paperclipai/shared"; +import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "@paperclipai/shared"; +import { Database, LogOut, SlidersHorizontal } from "lucide-react"; import { authApi } from "@/api/auth"; import { instanceSettingsApi } from "@/api/instanceSettings"; import { Button } from "../components/ui/button"; @@ -67,6 +68,7 @@ export function InstanceGeneralSettings() { const censorUsernameInLogs = generalQuery.data?.censorUsernameInLogs === true; const keyboardShortcuts = generalQuery.data?.keyboardShortcuts === true; const feedbackDataSharingPreference = generalQuery.data?.feedbackDataSharingPreference ?? "prompt"; + const backupRetentionDays: BackupRetentionDays = generalQuery.data?.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS; return (
@@ -123,6 +125,49 @@ export function InstanceGeneralSettings() {
+
+
+
+ +
+

Backup retention

+

+ How long to keep automatic database backups before pruning. Backups are compressed + with gzip to minimize disk usage. +

+
+
+
+ {BACKUP_RETENTION_PRESETS.map((days) => { + const active = backupRetentionDays === days; + const label = + days === 7 ? "7 days" : days === 14 ? "2 weeks" : "1 month"; + return ( + + ); + })} +
+
+
+