merge master into pap-1078-qol-fixes

Resolve the keyboard shortcut conflicts after [#2539](https://github.com/paperclipai/paperclip/pull/2539) and [#2540](https://github.com/paperclipai/paperclip/pull/2540), keep the release package rewrite working with cliVersion, and stabilize the provisioning timeout in the full suite.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
dotta 2026-04-02 13:14:20 -05:00
commit fb3b57ab1f
59 changed files with 16794 additions and 375 deletions

View file

@ -35,11 +35,12 @@
"dist"
],
"scripts": {
"build": "tsc && cp -r src/migrations dist/migrations",
"check:migrations": "tsx src/check-migration-numbering.ts",
"build": "pnpm run check:migrations && tsc && cp -r src/migrations dist/migrations",
"clean": "rm -rf dist",
"typecheck": "tsc --noEmit",
"generate": "tsc -p tsconfig.json && drizzle-kit generate",
"migrate": "tsx src/migrate.ts",
"typecheck": "pnpm run check:migrations && tsc --noEmit",
"generate": "pnpm run check:migrations && tsc -p tsconfig.json && drizzle-kit generate",
"migrate": "pnpm run check:migrations && tsx src/migrate.ts",
"seed": "tsx src/seed.ts"
},
"dependencies": {

View file

@ -0,0 +1,89 @@
import { readdir, readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
const migrationsDir = fileURLToPath(new URL("./migrations", import.meta.url));
const journalPath = fileURLToPath(new URL("./migrations/meta/_journal.json", import.meta.url));
type JournalFile = {
entries?: Array<{
idx?: number;
tag?: string;
}>;
};
function migrationNumber(value: string): string | null {
const match = value.match(/^(\d{4})_/);
return match ? match[1] : null;
}
function ensureNoDuplicates(values: string[], label: string) {
const seen = new Map<string, string>();
for (const value of values) {
const number = migrationNumber(value);
if (!number) {
throw new Error(`${label} entry does not start with a 4-digit migration number: ${value}`);
}
const existing = seen.get(number);
if (existing) {
throw new Error(`Duplicate migration number ${number} in ${label}: ${existing}, ${value}`);
}
seen.set(number, value);
}
}
function ensureStrictlyOrdered(values: string[], label: string) {
const sorted = [...values].sort();
for (let index = 0; index < values.length; index += 1) {
if (values[index] !== sorted[index]) {
throw new Error(
`${label} are out of order at position ${index}: expected ${sorted[index]}, found ${values[index]}`,
);
}
}
}
function ensureJournalMatchesFiles(migrationFiles: string[], journalTags: string[]) {
const journalFiles = journalTags.map((tag) => `${tag}.sql`);
if (journalFiles.length !== migrationFiles.length) {
throw new Error(
`Migration journal/file count mismatch: journal has ${journalFiles.length}, files have ${migrationFiles.length}`,
);
}
for (let index = 0; index < migrationFiles.length; index += 1) {
const migrationFile = migrationFiles[index];
const journalFile = journalFiles[index];
if (migrationFile !== journalFile) {
throw new Error(
`Migration journal/file order mismatch at position ${index}: journal has ${journalFile}, files have ${migrationFile}`,
);
}
}
}
async function main() {
const migrationFiles = (await readdir(migrationsDir))
.filter((entry) => entry.endsWith(".sql"))
.sort();
ensureNoDuplicates(migrationFiles, "migration files");
ensureStrictlyOrdered(migrationFiles, "migration files");
const rawJournal = await readFile(journalPath, "utf8");
const journal = JSON.parse(rawJournal) as JournalFile;
const journalTags = (journal.entries ?? [])
.map((entry, index) => {
if (typeof entry.tag !== "string" || entry.tag.length === 0) {
throw new Error(`Migration journal entry ${index} is missing a tag`);
}
return entry.tag;
});
ensureNoDuplicates(journalTags, "migration journal");
ensureStrictlyOrdered(journalTags, "migration journal");
ensureJournalMatchesFiles(migrationFiles, journalTags);
}
await main();

View file

@ -305,6 +305,99 @@ describeEmbeddedPostgres("applyPendingMigrations", () => {
const finalState = await inspectMigrations(connectionString);
expect(finalState.status).toBe("upToDate");
const verifySql = postgres(connectionString, { max: 1, onnotice: () => {} });
try {
const constraints = await verifySql.unsafe<{ conname: string }[]>(
`
SELECT conname
FROM pg_constraint
WHERE conname IN (
'feedback_exports_company_id_companies_id_fk',
'feedback_exports_feedback_vote_id_feedback_votes_id_fk',
'feedback_exports_issue_id_issues_id_fk',
'feedback_votes_company_id_companies_id_fk',
'feedback_votes_issue_id_issues_id_fk'
)
ORDER BY conname
`,
);
expect(constraints.map((row) => row.conname)).toEqual([
"feedback_exports_company_id_companies_id_fk",
"feedback_exports_feedback_vote_id_feedback_votes_id_fk",
"feedback_exports_issue_id_issues_id_fk",
"feedback_votes_company_id_companies_id_fk",
"feedback_votes_issue_id_issues_id_fk",
]);
} finally {
await verifySql.end();
}
},
20_000,
);
it(
"replays migration 0048 safely when routines.variables already exists",
async () => {
const connectionString = await createTempDatabase();
await applyPendingMigrations(connectionString);
const sql = postgres(connectionString, { max: 1, onnotice: () => {} });
try {
const flashyMarrowHash = await migrationHash("0048_flashy_marrow.sql");
await sql.unsafe(
`DELETE FROM "drizzle"."__drizzle_migrations" WHERE hash = '${flashyMarrowHash}'`,
);
const columns = await sql.unsafe<{ column_name: string }[]>(
`
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'routines'
AND column_name = 'variables'
`,
);
expect(columns).toHaveLength(1);
} finally {
await sql.end();
}
const pendingState = await inspectMigrations(connectionString);
expect(pendingState).toMatchObject({
status: "needsMigrations",
pendingMigrations: ["0048_flashy_marrow.sql"],
reason: "pending-migrations",
});
await applyPendingMigrations(connectionString);
const finalState = await inspectMigrations(connectionString);
expect(finalState.status).toBe("upToDate");
const verifySql = postgres(connectionString, { max: 1, onnotice: () => {} });
try {
const columns = await verifySql.unsafe<{ column_name: string; is_nullable: string; data_type: string }[]>(
`
SELECT column_name, is_nullable, data_type
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'routines'
AND column_name = 'variables'
`,
);
expect(columns).toEqual([
expect.objectContaining({
column_name: "variables",
is_nullable: "NO",
data_type: "jsonb",
}),
]);
} finally {
await verifySql.end();
}
},
20_000,
);

View file

@ -0,0 +1 @@
ALTER TABLE "routines" ADD COLUMN IF NOT EXISTS "variables" jsonb DEFAULT '[]'::jsonb NOT NULL;

File diff suppressed because it is too large Load diff

View file

@ -337,6 +337,13 @@
"when": 1775137972687,
"tag": "0047_overjoyed_groot",
"breakpoints": true
},
{
"idx": 48,
"version": "7",
"when": 1775145655557,
"tag": "0048_flashy_marrow",
"breakpoints": true
}
]
}

View file

@ -15,6 +15,7 @@ import { companySecrets } from "./company_secrets.js";
import { issues } from "./issues.js";
import { projects } from "./projects.js";
import { goals } from "./goals.js";
import type { RoutineVariable } from "@paperclipai/shared";
export const routines = pgTable(
"routines",
@ -31,6 +32,7 @@ export const routines = pgTable(
status: text("status").notNull().default("active"),
concurrencyPolicy: text("concurrency_policy").notNull().default("coalesce_if_active"),
catchUpPolicy: text("catch_up_policy").notNull().default("skip_missed"),
variables: jsonb("variables").$type<RoutineVariable[]>().notNull().default([]),
createdByAgentId: uuid("created_by_agent_id").references(() => agents.id, { onDelete: "set null" }),
createdByUserId: text("created_by_user_id"),
updatedByAgentId: uuid("updated_by_agent_id").references(() => agents.id, { onDelete: "set null" }),

View file

@ -166,6 +166,9 @@ export type RoutineTriggerKind = (typeof ROUTINE_TRIGGER_KINDS)[number];
export const ROUTINE_TRIGGER_SIGNING_MODES = ["bearer", "hmac_sha256"] as const;
export type RoutineTriggerSigningMode = (typeof ROUTINE_TRIGGER_SIGNING_MODES)[number];
export const ROUTINE_VARIABLE_TYPES = ["text", "textarea", "number", "boolean", "select"] as const;
export type RoutineVariableType = (typeof ROUTINE_VARIABLE_TYPES)[number];
export const ROUTINE_RUN_STATUSES = [
"received",
"coalesced",

View file

@ -21,6 +21,7 @@ export {
ROUTINE_CATCH_UP_POLICIES,
ROUTINE_TRIGGER_KINDS,
ROUTINE_TRIGGER_SIGNING_MODES,
ROUTINE_VARIABLE_TYPES,
ROUTINE_RUN_STATUSES,
ROUTINE_RUN_SOURCES,
PAUSE_REASONS,
@ -88,6 +89,7 @@ export {
type RoutineCatchUpPolicy,
type RoutineTriggerKind,
type RoutineTriggerSigningMode,
type RoutineVariableType,
type RoutineRunStatus,
type RoutineRunSource,
type PauseReason,
@ -255,6 +257,8 @@ export type {
FinanceSummary,
FinanceByBiller,
FinanceByKind,
AgentWakeupResponse,
AgentWakeupSkipped,
HeartbeatRun,
HeartbeatRunEvent,
AgentRuntimeState,
@ -304,6 +308,8 @@ export type {
CompanySecret,
SecretProviderDescriptor,
Routine,
RoutineVariable,
RoutineVariableDefaultValue,
RoutineTrigger,
RoutineRun,
RoutineTriggerSecretMaterial,
@ -473,6 +479,7 @@ export {
updateRoutineSchema,
createRoutineTriggerSchema,
updateRoutineTriggerSchema,
routineVariableSchema,
runRoutineSchema,
rotateRoutineTriggerSecretSchema,
type CreateSecret,
@ -597,6 +604,14 @@ export {
type ParsedProjectMention,
} from "./project-mentions.js";
export {
extractRoutineVariableNames,
interpolateRoutineTemplate,
isValidRoutineVariableName,
stringifyRoutineVariableValue,
syncRoutineVariablesWithTemplate,
} from "./routine-variables.js";
export {
paperclipConfigSchema,
configMetaSchema,

View file

@ -0,0 +1,34 @@
import { describe, expect, it } from "vitest";
import {
extractRoutineVariableNames,
interpolateRoutineTemplate,
syncRoutineVariablesWithTemplate,
} from "./routine-variables.js";
describe("routine variable helpers", () => {
it("extracts placeholder names in first-appearance order", () => {
expect(
extractRoutineVariableNames("Review {{repo}} and {{priority}} for {{repo}}"),
).toEqual(["repo", "priority"]);
});
it("preserves existing metadata when syncing variables from a template", () => {
expect(
syncRoutineVariablesWithTemplate("Review {{repo}} and {{priority}}", [
{ name: "repo", label: "Repository", type: "text", defaultValue: "paperclip", required: true, options: [] },
]),
).toEqual([
{ name: "repo", label: "Repository", type: "text", defaultValue: "paperclip", required: true, options: [] },
{ name: "priority", label: null, type: "text", defaultValue: null, required: true, options: [] },
]);
});
it("interpolates provided variable values into the routine template", () => {
expect(
interpolateRoutineTemplate("Review {{repo}} for {{priority}}", {
repo: "paperclip",
priority: "high",
}),
).toBe("Review paperclip for high");
});
});

View file

@ -0,0 +1,62 @@
import type { RoutineVariable } from "./types/routine.js";
const ROUTINE_VARIABLE_MATCHER = /\{\{\s*([A-Za-z][A-Za-z0-9_]*)\s*\}\}/g;
export function isValidRoutineVariableName(name: string): boolean {
return /^[A-Za-z][A-Za-z0-9_]*$/.test(name);
}
export function extractRoutineVariableNames(template: string | null | undefined): string[] {
if (!template) return [];
const found = new Set<string>();
for (const match of template.matchAll(ROUTINE_VARIABLE_MATCHER)) {
const name = match[1];
if (name && !found.has(name)) {
found.add(name);
}
}
return [...found];
}
function defaultRoutineVariable(name: string): RoutineVariable {
return {
name,
label: null,
type: "text",
defaultValue: null,
required: true,
options: [],
};
}
export function syncRoutineVariablesWithTemplate(
template: string | null | undefined,
existing: RoutineVariable[] | null | undefined,
): RoutineVariable[] {
const names = extractRoutineVariableNames(template);
const existingByName = new Map((existing ?? []).map((variable) => [variable.name, variable]));
return names.map((name) => existingByName.get(name) ?? defaultRoutineVariable(name));
}
export function stringifyRoutineVariableValue(value: unknown): string {
if (typeof value === "string") return value;
if (typeof value === "number" || typeof value === "boolean") return String(value);
if (value == null) return "";
try {
return JSON.stringify(value);
} catch {
return String(value);
}
}
export function interpolateRoutineTemplate(
template: string | null | undefined,
values: Record<string, unknown> | null | undefined,
): string | null {
if (template == null) return null;
if (!values || Object.keys(values).length === 0) return template;
return template.replace(ROUTINE_VARIABLE_MATCHER, (match, rawName: string) => {
if (!(rawName in values)) return match;
return stringifyRoutineVariableValue(values[rawName]);
});
}

View file

@ -57,6 +57,8 @@ export interface CompanyPortabilityProjectManifestEntry {
metadata: Record<string, unknown> | null;
}
import type { RoutineVariable } from "./routine.js";
export interface CompanyPortabilityProjectWorkspaceManifestEntry {
key: string;
name: string;
@ -84,6 +86,7 @@ export interface CompanyPortabilityIssueRoutineTriggerManifestEntry {
export interface CompanyPortabilityIssueRoutineManifestEntry {
concurrencyPolicy: string | null;
catchUpPolicy: string | null;
variables?: RoutineVariable[] | null;
triggers: CompanyPortabilityIssueRoutineTriggerManifestEntry[];
}

View file

@ -42,6 +42,18 @@ export interface HeartbeatRun {
updatedAt: Date;
}
export interface AgentWakeupSkipped {
status: "skipped";
reason: string;
message: string | null;
issueId: string | null;
executionRunId: string | null;
executionAgentId: string | null;
executionAgentName: string | null;
}
export type AgentWakeupResponse = HeartbeatRun | AgentWakeupSkipped;
export interface HeartbeatRunEvent {
id: number;
companyId: string;

View file

@ -130,6 +130,8 @@ export type {
} from "./secrets.js";
export type {
Routine,
RoutineVariable,
RoutineVariableDefaultValue,
RoutineTrigger,
RoutineRun,
RoutineTriggerSecretMaterial,
@ -141,6 +143,8 @@ export type {
export type { CostEvent, CostSummary, CostByAgent, CostByProviderModel, CostByBiller, CostByAgentModel, CostWindowSpendRow, CostByProject } from "./cost.js";
export type { FinanceEvent, FinanceSummary, FinanceByBiller, FinanceByKind } from "./finance.js";
export type {
AgentWakeupResponse,
AgentWakeupSkipped,
HeartbeatRun,
HeartbeatRunEvent,
AgentRuntimeState,

View file

@ -1,4 +1,4 @@
import type { IssueOriginKind } from "../constants.js";
import type { IssueOriginKind, RoutineVariableType } from "../constants.js";
export interface RoutineProjectSummary {
id: string;
@ -25,6 +25,17 @@ export interface RoutineIssueSummary {
updatedAt: Date;
}
export type RoutineVariableDefaultValue = string | number | boolean | null;
export interface RoutineVariable {
name: string;
label: string | null;
type: RoutineVariableType;
defaultValue: RoutineVariableDefaultValue;
required: boolean;
options: string[];
}
export interface Routine {
id: string;
companyId: string;
@ -38,6 +49,7 @@ export interface Routine {
status: string;
concurrencyPolicy: string;
catchUpPolicy: string;
variables: RoutineVariable[];
createdByAgentId: string | null;
createdByUserId: string | null;
updatedByAgentId: string | null;

View file

@ -1,4 +1,5 @@
import { z } from "zod";
import { routineVariableSchema } from "./routine.js";
export const portabilityIncludeSchema = z
.object({
@ -123,6 +124,7 @@ export const portabilityIssueRoutineTriggerManifestEntrySchema = z.object({
export const portabilityIssueRoutineManifestEntrySchema = z.object({
concurrencyPolicy: z.string().nullable(),
catchUpPolicy: z.string().nullable(),
variables: z.array(routineVariableSchema).nullable().optional(),
triggers: z.array(portabilityIssueRoutineTriggerManifestEntrySchema).default([]),
});

View file

@ -214,6 +214,7 @@ export {
updateRoutineSchema,
createRoutineTriggerSchema,
updateRoutineTriggerSchema,
routineVariableSchema,
runRoutineSchema,
rotateRoutineTriggerSecretSchema,
type CreateRoutine,

View file

@ -1,6 +1,15 @@
import { z } from "zod";
import { ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js";
export const ISSUE_EXECUTION_WORKSPACE_PREFERENCES = [
"inherit",
"shared_workspace",
"isolated_workspace",
"operator_branch",
"reuse_existing",
"agent_default",
] as const;
const executionWorkspaceStrategySchema = z
.object({
type: z.enum(["project_primary", "git_worktree", "adapter_managed", "cloud_sandbox"]).optional(),
@ -14,7 +23,7 @@ const executionWorkspaceStrategySchema = z
export const issueExecutionWorkspaceSettingsSchema = z
.object({
mode: z.enum(["inherit", "shared_workspace", "isolated_workspace", "operator_branch", "reuse_existing", "agent_default"]).optional(),
mode: z.enum(ISSUE_EXECUTION_WORKSPACE_PREFERENCES).optional(),
workspaceStrategy: executionWorkspaceStrategySchema.optional().nullable(),
workspaceRuntime: z.record(z.unknown()).optional().nullable(),
})
@ -43,14 +52,7 @@ export const createIssueSchema = z.object({
billingCode: z.string().optional().nullable(),
assigneeAdapterOverrides: issueAssigneeAdapterOverridesSchema.optional().nullable(),
executionWorkspaceId: z.string().uuid().optional().nullable(),
executionWorkspacePreference: z.enum([
"inherit",
"shared_workspace",
"isolated_workspace",
"operator_branch",
"reuse_existing",
"agent_default",
]).optional().nullable(),
executionWorkspacePreference: z.enum(ISSUE_EXECUTION_WORKSPACE_PREFERENCES).optional().nullable(),
executionWorkspaceSettings: issueExecutionWorkspaceSettingsSchema.optional().nullable(),
labelIds: z.array(z.string().uuid()).optional(),
});

View file

@ -5,7 +5,47 @@ import {
ROUTINE_CONCURRENCY_POLICIES,
ROUTINE_STATUSES,
ROUTINE_TRIGGER_SIGNING_MODES,
ROUTINE_VARIABLE_TYPES,
} from "../constants.js";
import {
ISSUE_EXECUTION_WORKSPACE_PREFERENCES,
issueExecutionWorkspaceSettingsSchema,
} from "./issue.js";
const routineVariableValueSchema = z.union([z.string(), z.number().finite(), z.boolean()]);
export const routineVariableSchema = z.object({
name: z.string().trim().regex(/^[A-Za-z][A-Za-z0-9_]*$/),
label: z.string().trim().max(120).optional().nullable(),
type: z.enum(ROUTINE_VARIABLE_TYPES).optional().default("text"),
defaultValue: routineVariableValueSchema.optional().nullable(),
required: z.boolean().optional().default(true),
options: z.array(z.string().trim().min(1).max(120)).max(50).optional().default([]),
}).superRefine((value, ctx) => {
if (value.type === "select" && value.options.length === 0) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ["options"],
message: "Select variables require at least one option",
});
}
if (value.type !== "select" && value.options.length > 0) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ["options"],
message: "Only select variables can define options",
});
}
if (value.type === "select" && value.defaultValue != null) {
if (typeof value.defaultValue !== "string" || !value.options.includes(value.defaultValue)) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ["defaultValue"],
message: "Select variable defaults must match one of the allowed options",
});
}
}
});
export const createRoutineSchema = z.object({
projectId: z.string().uuid(),
@ -18,6 +58,7 @@ export const createRoutineSchema = z.object({
status: z.enum(ROUTINE_STATUSES).optional().default("active"),
concurrencyPolicy: z.enum(ROUTINE_CONCURRENCY_POLICIES).optional().default("coalesce_if_active"),
catchUpPolicy: z.enum(ROUTINE_CATCH_UP_POLICIES).optional().default("skip_missed"),
variables: z.array(routineVariableSchema).optional().default([]),
});
export type CreateRoutine = z.infer<typeof createRoutineSchema>;
@ -62,8 +103,12 @@ export type UpdateRoutineTrigger = z.infer<typeof updateRoutineTriggerSchema>;
export const runRoutineSchema = z.object({
triggerId: z.string().uuid().optional().nullable(),
payload: z.record(z.unknown()).optional().nullable(),
variables: z.record(routineVariableValueSchema).optional().nullable(),
idempotencyKey: z.string().trim().max(255).optional().nullable(),
source: z.enum(["manual", "api"]).optional().default("manual"),
executionWorkspaceId: z.string().uuid().optional().nullable(),
executionWorkspacePreference: z.enum(ISSUE_EXECUTION_WORKSPACE_PREFERENCES).optional().nullable(),
executionWorkspaceSettings: issueExecutionWorkspaceSettingsSchema.optional().nullable(),
});
export type RunRoutine = z.infer<typeof runRoutineSchema>;