feat(backups): tiered daily/weekly/monthly retention with UI controls

Replace single retentionDays with a three-tier BackupRetentionPolicy:
- Daily: keep all backups (presets: 3, 7, 14 days; default 7)
- Weekly: keep one per calendar week (presets: 1, 2, 4 weeks; default 4)
- Monthly: keep one per calendar month (presets: 1, 3, 6 months; default 1)

Pruning sorts backups newest-first and applies each tier's cutoff,
keeping only the newest entry per ISO week/month bucket. The Instance
Settings General page now shows three preset selectors (no icon, matches
existing page design). Remove Database icon import.
This commit is contained in:
Aron Prins 2026-04-07 09:54:39 +02:00
parent cc44d309c0
commit fcbae62baf
13 changed files with 243 additions and 79 deletions

View file

@ -125,7 +125,7 @@ describeEmbeddedPostgres("runDatabaseBackup", () => {
const result = await runDatabaseBackup({
connectionString: sourceConnectionString,
backupDir,
retentionDays: 7,
retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 },
filenamePrefix: "paperclip-test",
});

View file

@ -5,10 +5,16 @@ import { pipeline } from "node:stream/promises";
import { createGunzip, createGzip } from "node:zlib";
import postgres from "postgres";
export type BackupRetentionPolicy = {
dailyDays: number;
weeklyWeeks: number;
monthlyMonths: number;
};
export type RunDatabaseBackupOptions = {
connectionString: string;
backupDir: string;
retentionDays: number;
retention: BackupRetentionPolicy;
filenamePrefix?: string;
connectTimeoutSeconds?: number;
includeMigrationJournal?: boolean;
@ -77,24 +83,91 @@ function timestamp(date: Date = new Date()): string {
return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
}
function pruneOldBackups(backupDir: string, retentionDays: number, filenamePrefix: string): number {
/**
* ISO week key for grouping backups by calendar week (ISO 8601).
*/
function isoWeekKey(date: Date): string {
const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
d.setUTCDate(d.getUTCDate() + 4 - (d.getUTCDay() || 7));
const yearStart = new Date(Date.UTC(d.getUTCFullYear(), 0, 1));
const weekNo = Math.ceil(((d.getTime() - yearStart.getTime()) / 86400000 + 1) / 7);
return `${d.getUTCFullYear()}-W${String(weekNo).padStart(2, "0")}`;
}
function monthKey(date: Date): string {
return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}`;
}
/**
* Tiered backup pruning:
* - Daily tier: keep ALL backups from the last `dailyDays` days
* - Weekly tier: keep the NEWEST backup per calendar week for `weeklyWeeks` weeks
* - Monthly tier: keep the NEWEST backup per calendar month for `monthlyMonths` months
* - Everything else is deleted
*/
function pruneOldBackups(backupDir: string, retention: BackupRetentionPolicy, filenamePrefix: string): number {
if (!existsSync(backupDir)) return 0;
const safeRetention = Math.max(1, Math.trunc(retentionDays));
const cutoff = Date.now() - safeRetention * 24 * 60 * 60 * 1000;
let pruned = 0;
const now = Date.now();
const dailyCutoff = now - Math.max(1, retention.dailyDays) * 24 * 60 * 60 * 1000;
const weeklyCutoff = now - Math.max(1, retention.weeklyWeeks) * 7 * 24 * 60 * 60 * 1000;
const monthlyCutoff = now - Math.max(1, retention.monthlyMonths) * 30 * 24 * 60 * 60 * 1000;
type BackupEntry = { name: string; fullPath: string; mtimeMs: number };
const entries: BackupEntry[] = [];
for (const name of readdirSync(backupDir)) {
if (!name.startsWith(`${filenamePrefix}-`)) continue;
if (!name.endsWith(".sql") && !name.endsWith(".sql.gz")) continue;
const fullPath = resolve(backupDir, name);
const stat = statSync(fullPath);
if (stat.mtimeMs < cutoff) {
unlinkSync(fullPath);
pruned++;
}
entries.push({ name, fullPath, mtimeMs: stat.mtimeMs });
}
return pruned;
// Sort newest first so the first entry per week/month bucket is the one we keep
entries.sort((a, b) => b.mtimeMs - a.mtimeMs);
const keepWeekBuckets = new Set<string>();
const keepMonthBuckets = new Set<string>();
const toDelete: string[] = [];
for (const entry of entries) {
// Daily tier — keep everything within dailyDays
if (entry.mtimeMs >= dailyCutoff) continue;
const date = new Date(entry.mtimeMs);
const week = isoWeekKey(date);
const month = monthKey(date);
// Weekly tier — keep newest per calendar week
if (entry.mtimeMs >= weeklyCutoff) {
if (keepWeekBuckets.has(week)) {
toDelete.push(entry.fullPath);
} else {
keepWeekBuckets.add(week);
}
continue;
}
// Monthly tier — keep newest per calendar month
if (entry.mtimeMs >= monthlyCutoff) {
if (keepMonthBuckets.has(month)) {
toDelete.push(entry.fullPath);
} else {
keepMonthBuckets.add(month);
}
continue;
}
// Beyond all retention tiers — delete
toDelete.push(entry.fullPath);
}
for (const filePath of toDelete) {
unlinkSync(filePath);
}
return toDelete.length;
}
function formatBackupSize(sizeBytes: number): string {
@ -287,7 +360,7 @@ export function createBufferedTextFileWriter(filePath: string, maxBufferedBytes
export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise<RunDatabaseBackupResult> {
const filenamePrefix = opts.filenamePrefix ?? "paperclip";
const retentionDays = Math.max(1, Math.trunc(opts.retentionDays));
const retention = opts.retention;
const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5));
const includeMigrationJournal = opts.includeMigrationJournal === true;
const excludedTableNames = normalizeTableNameSet(opts.excludeTables);
@ -678,7 +751,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
unlinkSync(sqlFile);
const sizeBytes = statSync(backupFile).size;
const prunedCount = pruneOldBackups(opts.backupDir, retentionDays, filenamePrefix);
const prunedCount = pruneOldBackups(opts.backupDir, retention, filenamePrefix);
return {
backupFile,

View file

@ -85,7 +85,7 @@ function resolveBackupDir(config: PartialConfig | null): string {
}
function resolveRetentionDays(config: PartialConfig | null): number {
return asPositiveInt(config?.database?.backup?.retentionDays) ?? 30;
return asPositiveInt(config?.database?.backup?.retentionDays) ?? 7;
}
async function main() {
@ -103,7 +103,7 @@ async function main() {
const result = await runDatabaseBackup({
connectionString,
backupDir,
retentionDays,
retention: { dailyDays: retentionDays, weeklyWeeks: 4, monthlyMonths: 1 },
filenamePrefix: "paperclip",
});

View file

@ -21,6 +21,7 @@ export {
runDatabaseBackup,
runDatabaseRestore,
formatDatabaseBackupResult,
type BackupRetentionPolicy,
type RunDatabaseBackupOptions,
type RunDatabaseBackupResult,
type RunDatabaseRestoreOptions,

View file

@ -189,7 +189,7 @@ export type {
InstanceExperimentalSettings,
InstanceGeneralSettings,
InstanceSettings,
BackupRetentionDays,
BackupRetentionPolicy,
Agent,
AgentAccessState,
AgentChainOfCommandEntry,
@ -371,8 +371,10 @@ export {
} from "./types/feedback.js";
export {
BACKUP_RETENTION_PRESETS,
DEFAULT_BACKUP_RETENTION_DAYS,
DAILY_RETENTION_PRESETS,
WEEKLY_RETENTION_PRESETS,
MONTHLY_RETENTION_PRESETS,
DEFAULT_BACKUP_RETENTION,
} from "./types/instance.js";
export {

View file

@ -11,8 +11,8 @@ export type {
FeedbackTraceBundleFile,
FeedbackTraceBundle,
} from "./feedback.js";
export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionDays } from "./instance.js";
export { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "./instance.js";
export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionPolicy } from "./instance.js";
export { DAILY_RETENTION_PRESETS, WEEKLY_RETENTION_PRESETS, MONTHLY_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION } from "./instance.js";
export type {
CompanySkillSourceType,
CompanySkillTrustLevel,

View file

@ -1,14 +1,26 @@
import type { FeedbackDataSharingPreference } from "./feedback.js";
export const BACKUP_RETENTION_PRESETS = [7, 14, 30] as const;
export type BackupRetentionDays = (typeof BACKUP_RETENTION_PRESETS)[number];
export const DEFAULT_BACKUP_RETENTION_DAYS: BackupRetentionDays = 7;
export const DAILY_RETENTION_PRESETS = [3, 7, 14] as const;
export const WEEKLY_RETENTION_PRESETS = [1, 2, 4] as const;
export const MONTHLY_RETENTION_PRESETS = [1, 3, 6] as const;
export interface BackupRetentionPolicy {
dailyDays: (typeof DAILY_RETENTION_PRESETS)[number];
weeklyWeeks: (typeof WEEKLY_RETENTION_PRESETS)[number];
monthlyMonths: (typeof MONTHLY_RETENTION_PRESETS)[number];
}
export const DEFAULT_BACKUP_RETENTION: BackupRetentionPolicy = {
dailyDays: 7,
weeklyWeeks: 4,
monthlyMonths: 1,
};
export interface InstanceGeneralSettings {
censorUsernameInLogs: boolean;
keyboardShortcuts: boolean;
feedbackDataSharingPreference: FeedbackDataSharingPreference;
backupRetentionDays: BackupRetentionDays;
backupRetention: BackupRetentionPolicy;
}
export interface InstanceExperimentalSettings {

View file

@ -1,13 +1,25 @@
import { z } from "zod";
import { DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE } from "../types/feedback.js";
import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "../types/instance.js";
import {
DAILY_RETENTION_PRESETS,
WEEKLY_RETENTION_PRESETS,
MONTHLY_RETENTION_PRESETS,
DEFAULT_BACKUP_RETENTION,
} from "../types/instance.js";
import { feedbackDataSharingPreferenceSchema } from "./feedback.js";
export const backupRetentionDaysSchema = z.number().refine(
(v): v is (typeof BACKUP_RETENTION_PRESETS)[number] =>
(BACKUP_RETENTION_PRESETS as readonly number[]).includes(v),
{ message: `Must be one of: ${BACKUP_RETENTION_PRESETS.join(", ")}` },
);
function presetSchema<T extends readonly number[]>(presets: T, label: string) {
return z.number().refine(
(v): v is T[number] => (presets as readonly number[]).includes(v),
{ message: `${label} must be one of: ${presets.join(", ")}` },
);
}
export const backupRetentionPolicySchema = z.object({
dailyDays: presetSchema(DAILY_RETENTION_PRESETS, "dailyDays").default(DEFAULT_BACKUP_RETENTION.dailyDays),
weeklyWeeks: presetSchema(WEEKLY_RETENTION_PRESETS, "weeklyWeeks").default(DEFAULT_BACKUP_RETENTION.weeklyWeeks),
monthlyMonths: presetSchema(MONTHLY_RETENTION_PRESETS, "monthlyMonths").default(DEFAULT_BACKUP_RETENTION.monthlyMonths),
});
export const instanceGeneralSettingsSchema = z.object({
censorUsernameInLogs: z.boolean().default(false),
@ -15,7 +27,7 @@ export const instanceGeneralSettingsSchema = z.object({
feedbackDataSharingPreference: feedbackDataSharingPreferenceSchema.default(
DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
),
backupRetentionDays: backupRetentionDaysSchema.default(DEFAULT_BACKUP_RETENTION_DAYS),
backupRetention: backupRetentionPolicySchema.default(DEFAULT_BACKUP_RETENTION),
}).strict();
export const patchInstanceGeneralSettingsSchema = instanceGeneralSettingsSchema.partial();