Merge public-gh/master into PAP-881-document-revisions-bulid-it

This commit is contained in:
dotta 2026-03-31 07:31:17 -05:00
commit 41f261eaf5
194 changed files with 29520 additions and 2185 deletions

View file

@ -1,83 +1,24 @@
import { createHash } from "node:crypto";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import postgres from "postgres";
import {
applyPendingMigrations,
ensurePostgresDatabase,
inspectMigrations,
} from "./client.js";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./test-embedded-postgres.js";
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
const tempPaths: string[] = [];
const runningInstances: EmbeddedPostgresInstance[] = [];
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
}
async function getAvailablePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to allocate test port")));
return;
}
const { port } = address;
server.close((error) => {
if (error) reject(error);
else resolve(port);
});
});
});
}
const cleanups: Array<() => Promise<void>> = [];
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
async function createTempDatabase(): Promise<string> {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-db-client-"));
tempPaths.push(dataDir);
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
await instance.initialise();
await instance.start();
runningInstances.push(instance);
const adminUrl = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminUrl, "paperclip");
return `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
const db = await startEmbeddedPostgresTestDatabase("paperclip-db-client-");
cleanups.push(db.cleanup);
return db.connectionString;
}
async function migrationHash(migrationFile: string): Promise<string> {
@ -89,19 +30,19 @@ async function migrationHash(migrationFile: string): Promise<string> {
}
afterEach(async () => {
while (runningInstances.length > 0) {
const instance = runningInstances.pop();
if (!instance) continue;
await instance.stop();
}
while (tempPaths.length > 0) {
const tempPath = tempPaths.pop();
if (!tempPath) continue;
fs.rmSync(tempPath, { recursive: true, force: true });
while (cleanups.length > 0) {
const cleanup = cleanups.pop();
await cleanup?.();
}
});
describe("applyPendingMigrations", () => {
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres migration tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
describeEmbeddedPostgres("applyPendingMigrations", () => {
it(
"applies an inserted earlier migration without replaying later legacy migrations",
async () => {

View file

@ -0,0 +1,28 @@
import { describe, expect, it } from "vitest";
import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
describe("formatEmbeddedPostgresError", () => {
it("adds a shared-memory hint when initdb logs expose the real cause", () => {
const error = formatEmbeddedPostgresError("Postgres init script exited with code 1.", {
fallbackMessage: "Failed to initialize embedded PostgreSQL cluster",
recentLogs: [
"running bootstrap script ...",
"FATAL: could not create shared memory segment: Cannot allocate memory",
"DETAIL: Failed system call was shmget(key=123, size=56, 03600).",
],
});
expect(error.message).toContain("could not allocate shared memory");
expect(error.message).toContain("kern.sysv.shm");
expect(error.message).toContain("could not create shared memory segment");
});
it("keeps only recent non-empty log lines in the collector", () => {
const buffer = createEmbeddedPostgresLogBuffer(2);
buffer.append("line one\n\n");
buffer.append("line two");
buffer.append("line three");
expect(buffer.getRecentLogs()).toEqual(["line two", "line three"]);
});
});

View file

@ -0,0 +1,89 @@
const DEFAULT_RECENT_LOG_LIMIT = 40;
const RECENT_LOG_SUMMARY_LINES = 8;
function toError(error: unknown, fallbackMessage: string): Error {
if (error instanceof Error) return error;
if (error === undefined) return new Error(fallbackMessage);
if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`);
try {
return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`);
} catch {
return new Error(`${fallbackMessage}: ${String(error)}`);
}
}
function summarizeRecentLogs(recentLogs: string[]): string | null {
if (recentLogs.length === 0) return null;
return recentLogs
.slice(-RECENT_LOG_SUMMARY_LINES)
.map((line) => line.trim())
.filter((line) => line.length > 0)
.join(" | ");
}
function detectEmbeddedPostgresHint(recentLogs: string[]): string | null {
const haystack = recentLogs.join("\n").toLowerCase();
if (!haystack.includes("could not create shared memory segment")) {
return null;
}
return (
"Embedded PostgreSQL bootstrap could not allocate shared memory. " +
"On macOS, this usually means the host's kern.sysv.shm* limits are too low for another local PostgreSQL cluster. " +
"Stop other local PostgreSQL servers or raise the shared-memory sysctls, then retry."
);
}
export function createEmbeddedPostgresLogBuffer(limit = DEFAULT_RECENT_LOG_LIMIT): {
append(message: unknown): void;
getRecentLogs(): string[];
} {
const recentLogs: string[] = [];
return {
append(message: unknown) {
const text =
typeof message === "string"
? message
: message instanceof Error
? message.message
: String(message ?? "");
for (const rawLine of text.split(/\r?\n/)) {
const line = rawLine.trim();
if (!line) continue;
recentLogs.push(line);
if (recentLogs.length > limit) {
recentLogs.splice(0, recentLogs.length - limit);
}
}
},
getRecentLogs() {
return [...recentLogs];
},
};
}
export function formatEmbeddedPostgresError(
error: unknown,
input: {
fallbackMessage: string;
recentLogs?: string[];
},
): Error {
const baseError = toError(error, input.fallbackMessage);
const recentLogs = input.recentLogs ?? [];
const parts = [baseError.message];
const hint = detectEmbeddedPostgresHint(recentLogs);
const recentSummary = summarizeRecentLogs(recentLogs);
if (hint) {
parts.push(hint);
}
if (recentSummary) {
parts.push(`Recent embedded Postgres logs: ${recentSummary}`);
}
return new Error(parts.join(" "));
}

View file

@ -11,6 +11,12 @@ export {
type MigrationBootstrapResult,
type Db,
} from "./client.js";
export {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestSupport,
} from "./test-embedded-postgres.js";
export {
runDatabaseBackup,
runDatabaseRestore,
@ -19,4 +25,8 @@ export {
type RunDatabaseBackupResult,
type RunDatabaseRestoreOptions,
} from "./backup-lib.js";
export {
createEmbeddedPostgresLogBuffer,
formatEmbeddedPostgresError,
} from "./embedded-postgres-error.js";
export * from "./schema/index.js";

View file

@ -2,6 +2,7 @@ import { existsSync, readFileSync, rmSync } from "node:fs";
import { createServer } from "node:net";
import path from "node:path";
import { ensurePostgresDatabase, getPostgresDataDirectory } from "./client.js";
import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
import { resolveDatabaseTarget } from "./runtime-config.js";
type EmbeddedPostgresInstance = {
@ -27,18 +28,6 @@ export type MigrationConnection = {
stop: () => Promise<void>;
};
function toError(error: unknown, fallbackMessage: string): Error {
if (error instanceof Error) return error;
if (error === undefined) return new Error(fallbackMessage);
if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`);
try {
return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`);
} catch {
return new Error(`${fallbackMessage}: ${String(error)}`);
}
}
function readRunningPostmasterPid(postmasterPidFile: string): number | null {
if (!existsSync(postmasterPidFile)) return null;
try {
@ -109,6 +98,7 @@ async function ensureEmbeddedPostgresConnection(
const runningPid = readRunningPostmasterPid(postmasterPidFile);
const runningPort = readPidFilePort(postmasterPidFile);
const preferredAdminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${preferredPort}/postgres`;
const logBuffer = createEmbeddedPostgresLogBuffer();
if (!runningPid && existsSync(pgVersionFile)) {
try {
@ -151,18 +141,19 @@ async function ensureEmbeddedPostgresConnection(
port: selectedPort,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
onLog: logBuffer.append,
onError: logBuffer.append,
});
if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) {
try {
await instance.initialise();
} catch (error) {
throw toError(
error,
`Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`,
);
throw formatEmbeddedPostgresError(error, {
fallbackMessage:
`Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`,
recentLogs: logBuffer.getRecentLogs(),
});
}
}
if (existsSync(postmasterPidFile)) {
@ -171,7 +162,10 @@ async function ensureEmbeddedPostgresConnection(
try {
await instance.start();
} catch (error) {
throw toError(error, `Failed to start embedded PostgreSQL on port ${selectedPort}`);
throw formatEmbeddedPostgresError(error, {
fallbackMessage: `Failed to start embedded PostgreSQL on port ${selectedPort}`,
recentLogs: logBuffer.getRecentLogs(),
});
}
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${selectedPort}/postgres`;

View file

@ -0,0 +1,17 @@
CREATE TABLE "issue_inbox_archives" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"company_id" uuid NOT NULL,
"issue_id" uuid NOT NULL,
"user_id" text NOT NULL,
"archived_at" timestamp with time zone DEFAULT now() NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
DROP INDEX "board_api_keys_key_hash_idx";--> statement-breakpoint
ALTER TABLE "issue_inbox_archives" ADD CONSTRAINT "issue_inbox_archives_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "issue_inbox_archives" ADD CONSTRAINT "issue_inbox_archives_issue_id_issues_id_fk" FOREIGN KEY ("issue_id") REFERENCES "public"."issues"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
CREATE INDEX "issue_inbox_archives_company_issue_idx" ON "issue_inbox_archives" USING btree ("company_id","issue_id");--> statement-breakpoint
CREATE INDEX "issue_inbox_archives_company_user_idx" ON "issue_inbox_archives" USING btree ("company_id","user_id");--> statement-breakpoint
CREATE UNIQUE INDEX "issue_inbox_archives_company_issue_user_idx" ON "issue_inbox_archives" USING btree ("company_id","issue_id","user_id");--> statement-breakpoint
CREATE UNIQUE INDEX "board_api_keys_key_hash_idx" ON "board_api_keys" USING btree ("key_hash");

View file

@ -1,5 +1,6 @@
ALTER TABLE "document_revisions" ADD COLUMN "title" text;--> statement-breakpoint
ALTER TABLE "document_revisions" ADD COLUMN "format" text DEFAULT 'markdown' NOT NULL;--> statement-breakpoint
ALTER TABLE "document_revisions" ADD COLUMN "format" text DEFAULT 'markdown' NOT NULL;
--> statement-breakpoint
UPDATE "document_revisions" AS "dr"
SET
"title" = "d"."title",

View file

@ -1,5 +1,5 @@
{
"id": "185d59be-1832-4c34-95ee-131b7553a67a",
"id": "869b0102-2cb8-48e8-a6d8-cab88f0fa7a8",
"prevId": "a7a034eb-984f-4884-b6e1-87c453404b4e",
"version": "7",
"dialect": "postgresql",
@ -4087,19 +4087,6 @@
"primaryKey": false,
"notNull": true
},
"title": {
"name": "title",
"type": "text",
"primaryKey": false,
"notNull": false
},
"format": {
"name": "format",
"type": "text",
"primaryKey": false,
"notNull": true,
"default": "'markdown'"
},
"body": {
"name": "body",
"type": "text",
@ -6781,6 +6768,162 @@
"checkConstraints": {},
"isRLSEnabled": false
},
"public.issue_inbox_archives": {
"name": "issue_inbox_archives",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"company_id": {
"name": "company_id",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"issue_id": {
"name": "issue_id",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"archived_at": {
"name": "archived_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"updated_at": {
"name": "updated_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
}
},
"indexes": {
"issue_inbox_archives_company_issue_idx": {
"name": "issue_inbox_archives_company_issue_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "issue_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"issue_inbox_archives_company_user_idx": {
"name": "issue_inbox_archives_company_user_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "user_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"issue_inbox_archives_company_issue_user_idx": {
"name": "issue_inbox_archives_company_issue_user_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "issue_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "user_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
"issue_inbox_archives_company_id_companies_id_fk": {
"name": "issue_inbox_archives_company_id_companies_id_fk",
"tableFrom": "issue_inbox_archives",
"tableTo": "companies",
"columnsFrom": [
"company_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
},
"issue_inbox_archives_issue_id_issues_id_fk": {
"name": "issue_inbox_archives_issue_id_issues_id_fk",
"tableFrom": "issue_inbox_archives",
"tableTo": "issues",
"columnsFrom": [
"issue_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.issue_labels": {
"name": "issue_labels",
"schema": "",

File diff suppressed because it is too large Load diff

View file

@ -320,8 +320,15 @@
{
"idx": 45,
"version": "7",
"when": 1774531054196,
"tag": "0045_breezy_dexter_bennett",
"when": 1774530504348,
"tag": "0045_workable_shockwave",
"breakpoints": true
},
{
"idx": 46,
"version": "7",
"when": 1774960197878,
"tag": "0046_smooth_sentinels",
"breakpoints": true
}
]

View file

@ -31,6 +31,7 @@ export { labels } from "./labels.js";
export { issueLabels } from "./issue_labels.js";
export { issueApprovals } from "./issue_approvals.js";
export { issueComments } from "./issue_comments.js";
export { issueInboxArchives } from "./issue_inbox_archives.js";
export { issueReadStates } from "./issue_read_states.js";
export { assets } from "./assets.js";
export { issueAttachments } from "./issue_attachments.js";

View file

@ -0,0 +1,25 @@
import { pgTable, uuid, text, timestamp, index, uniqueIndex } from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { issues } from "./issues.js";
export const issueInboxArchives = pgTable(
"issue_inbox_archives",
{
id: uuid("id").primaryKey().defaultRandom(),
companyId: uuid("company_id").notNull().references(() => companies.id),
issueId: uuid("issue_id").notNull().references(() => issues.id),
userId: text("user_id").notNull(),
archivedAt: timestamp("archived_at", { withTimezone: true }).notNull().defaultNow(),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
companyIssueIdx: index("issue_inbox_archives_company_issue_idx").on(table.companyId, table.issueId),
companyUserIdx: index("issue_inbox_archives_company_user_idx").on(table.companyId, table.userId),
companyIssueUserUnique: uniqueIndex("issue_inbox_archives_company_issue_user_idx").on(
table.companyId,
table.issueId,
table.userId,
),
}),
);

View file

@ -0,0 +1,144 @@
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { applyPendingMigrations, ensurePostgresDatabase } from "./client.js";
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
export type EmbeddedPostgresTestSupport = {
supported: boolean;
reason?: string;
};
export type EmbeddedPostgresTestDatabase = {
connectionString: string;
cleanup(): Promise<void>;
};
let embeddedPostgresSupportPromise: Promise<EmbeddedPostgresTestSupport> | null = null;
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
}
async function getAvailablePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to allocate test port")));
return;
}
const { port } = address;
server.close((error) => {
if (error) reject(error);
else resolve(port);
});
});
});
}
function formatEmbeddedPostgresError(error: unknown): string {
if (error instanceof Error && error.message.length > 0) return error.message;
if (typeof error === "string" && error.length > 0) return error;
return "embedded Postgres startup failed";
}
async function probeEmbeddedPostgresSupport(): Promise<EmbeddedPostgresTestSupport> {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-embedded-postgres-probe-"));
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
try {
await instance.initialise();
await instance.start();
return { supported: true };
} catch (error) {
return {
supported: false,
reason: formatEmbeddedPostgresError(error),
};
} finally {
await instance.stop().catch(() => {});
fs.rmSync(dataDir, { recursive: true, force: true });
}
}
export async function getEmbeddedPostgresTestSupport(): Promise<EmbeddedPostgresTestSupport> {
if (!embeddedPostgresSupportPromise) {
embeddedPostgresSupportPromise = probeEmbeddedPostgresSupport();
}
return await embeddedPostgresSupportPromise;
}
export async function startEmbeddedPostgresTestDatabase(
tempDirPrefix: string,
): Promise<EmbeddedPostgresTestDatabase> {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix));
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
try {
await instance.initialise();
await instance.start();
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
await applyPendingMigrations(connectionString);
return {
connectionString,
cleanup: async () => {
await instance.stop().catch(() => {});
fs.rmSync(dataDir, { recursive: true, force: true });
},
};
} catch (error) {
await instance.stop().catch(() => {});
fs.rmSync(dataDir, { recursive: true, force: true });
throw new Error(
`Failed to start embedded PostgreSQL test database: ${formatEmbeddedPostgresError(error)}`,
);
}
}