[codex] Add document annotations and comments (#6733)

## Thinking Path

> - Paperclip orchestrates AI-agent companies through issues, documents,
runs, and durable company-scoped state.
> - Issue documents are where agents and operators capture plans,
handoffs, and work products.
> - Before this change, document collaboration could only happen through
whole-document edits and detached issue comments.
> - Inline document annotations need stable anchors, revision-aware
persistence, and UI affordances that do not break existing document
editing.
> - This pull request adds company-scoped document annotation threads,
comments, anchor snapshots, API routes, and board UI.
> - The benefit is that operators and agents can discuss specific
document passages without losing context as documents evolve.

## What Changed

- Added document annotation tables, schema exports, shared types,
validators, anchor hashing, and text-anchor helpers.
- Added server-side document annotation services and issue routes for
listing, creating, commenting, resolving, and reopening annotation
threads.
- Included annotation summaries in relevant issue document reads and
backup/recovery document workspace behavior.
- Added React UI for inline document highlights, comment panels, mobile
sheet behavior, deep-link focus, and resolved/open filtering.
- Added annotation design artifacts, Storybook coverage, screenshots,
and a screenshot helper script.
- Rebased the branch onto current `paperclipai/paperclip` `master` and
renumbered the annotation migration from `0085_old_swarm` to
`0091_old_swarm`; the SQL uses `IF NOT EXISTS` guards so environments
that previously applied the old migration number can safely apply the
new one.
- Adjusted the new annotation UI tests to use a local async flush helper
because this workspace's React 19.2.4 export does not expose
`React.act`.

## Verification

- `pnpm run preflight:workspace-links && pnpm exec vitest run
packages/shared/src/document-anchors.test.ts
server/src/__tests__/document-annotation-routes.test.ts
server/src/__tests__/document-annotations-service.test.ts
ui/src/components/DocumentAnnotationLayer.test.tsx
ui/src/components/IssueDocumentAnnotations.test.tsx
ui/src/lib/document-annotation-hash.test.ts
ui/src/lib/document-annotation-selection.test.ts`
- Confirmed `git diff --check` passes.
- Confirmed no `pnpm-lock.yaml` or `.github/workflows/*` files are
included in the PR diff.

## Risks

- Medium risk: this adds new persisted annotation tables and routes
across db/shared/server/ui.
- Migration risk is reduced by moving the branch migration to
`0091_old_swarm` after upstream `0090_resource_memberships` and keeping
the SQL idempotent for old `0085_old_swarm` adopters.
- UI risk is mostly around text range anchoring and panel positioning
across long documents, folded content, and mobile layouts; the PR
includes focused unit coverage and design screenshots.

> For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and
discuss it in `#dev` before opening the PR. Feature PRs that overlap
with planned core work may need to be redirected — check the roadmap
first. See `CONTRIBUTING.md`.

## Model Used

- OpenAI Codex, GPT-5 coding agent, tool-using software engineering
mode. Context window size is not exposed in this Paperclip runtime.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] If this change affects the UI, I have included before/after
screenshots
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge

---------

Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta 2026-05-26 08:41:23 -05:00 committed by GitHub
parent f0ddd24d61
commit b7545823be
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
55 changed files with 25070 additions and 31 deletions

View file

@ -0,0 +1,413 @@
import { and, asc, desc, eq, inArray, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
documentAnnotationAnchorSnapshots,
documentAnnotationComments,
documentAnnotationThreads,
documents,
issueDocuments,
} from "@paperclipai/db";
import {
anchorSnapshotToSelector,
remapDocumentAnchor,
selectorToAnchorSnapshot,
verifyDocumentAnchorSelector,
type DocumentAnnotationAnchorSnapshot,
type DocumentAnnotationComment,
type DocumentAnnotationThread,
CreateDocumentAnnotationComment,
CreateDocumentAnnotationThread,
UpdateDocumentAnnotationThread,
} from "@paperclipai/shared";
import { conflict, notFound, unprocessable } from "../errors.js";
type ActorInput = {
actorType: "agent" | "user";
actorId: string;
agentId?: string | null;
userId?: string | null;
runId?: string | null;
};
type IssueDocumentRow = {
issueId: string;
companyId: string;
documentId: string;
documentKey: string;
latestBody: string;
latestRevisionId: string | null;
latestRevisionNumber: number;
};
const threadSelect = {
id: documentAnnotationThreads.id,
companyId: documentAnnotationThreads.companyId,
issueId: documentAnnotationThreads.issueId,
documentId: documentAnnotationThreads.documentId,
documentKey: documentAnnotationThreads.documentKey,
status: documentAnnotationThreads.status,
anchorState: documentAnnotationThreads.anchorState,
anchorConfidence: documentAnnotationThreads.anchorConfidence,
originalRevisionId: documentAnnotationThreads.originalRevisionId,
originalRevisionNumber: documentAnnotationThreads.originalRevisionNumber,
currentRevisionId: documentAnnotationThreads.currentRevisionId,
currentRevisionNumber: documentAnnotationThreads.currentRevisionNumber,
selectedText: documentAnnotationThreads.selectedText,
prefixText: documentAnnotationThreads.prefixText,
suffixText: documentAnnotationThreads.suffixText,
normalizedStart: documentAnnotationThreads.normalizedStart,
normalizedEnd: documentAnnotationThreads.normalizedEnd,
markdownStart: documentAnnotationThreads.markdownStart,
markdownEnd: documentAnnotationThreads.markdownEnd,
anchorSelector: documentAnnotationThreads.anchorSelector,
createdByAgentId: documentAnnotationThreads.createdByAgentId,
createdByUserId: documentAnnotationThreads.createdByUserId,
resolvedByAgentId: documentAnnotationThreads.resolvedByAgentId,
resolvedByUserId: documentAnnotationThreads.resolvedByUserId,
resolvedAt: documentAnnotationThreads.resolvedAt,
createdAt: documentAnnotationThreads.createdAt,
updatedAt: documentAnnotationThreads.updatedAt,
};
const commentSelect = {
id: documentAnnotationComments.id,
companyId: documentAnnotationComments.companyId,
threadId: documentAnnotationComments.threadId,
issueId: documentAnnotationComments.issueId,
documentId: documentAnnotationComments.documentId,
body: documentAnnotationComments.body,
authorType: documentAnnotationComments.authorType,
authorAgentId: documentAnnotationComments.authorAgentId,
authorUserId: documentAnnotationComments.authorUserId,
createdByRunId: documentAnnotationComments.createdByRunId,
createdAt: documentAnnotationComments.createdAt,
updatedAt: documentAnnotationComments.updatedAt,
};
function snapshotFromThread(thread: Pick<DocumentAnnotationThread, "selectedText" | "prefixText" | "suffixText" | "normalizedStart" | "normalizedEnd" | "markdownStart" | "markdownEnd">): DocumentAnnotationAnchorSnapshot {
return {
selectedText: thread.selectedText,
prefixText: thread.prefixText,
suffixText: thread.suffixText,
normalizedStart: thread.normalizedStart,
normalizedEnd: thread.normalizedEnd,
markdownStart: thread.markdownStart,
markdownEnd: thread.markdownEnd,
};
}
export function documentAnnotationService(db: Db) {
async function getIssueDocument(issueId: string, key: string, dbOrTx: any = db): Promise<IssueDocumentRow | null> {
return dbOrTx
.select({
issueId: issueDocuments.issueId,
companyId: documents.companyId,
documentId: documents.id,
documentKey: issueDocuments.key,
latestBody: documents.latestBody,
latestRevisionId: documents.latestRevisionId,
latestRevisionNumber: documents.latestRevisionNumber,
})
.from(issueDocuments)
.innerJoin(documents, eq(issueDocuments.documentId, documents.id))
.where(and(eq(issueDocuments.issueId, issueId), eq(issueDocuments.key, key)))
.then((rows: IssueDocumentRow[]) => rows[0] ?? null);
}
async function getThreadForIssue(
issueId: string,
documentKey: string,
threadId: string,
dbOrTx: any = db,
): Promise<DocumentAnnotationThread | null> {
return dbOrTx
.select(threadSelect)
.from(documentAnnotationThreads)
.where(and(
eq(documentAnnotationThreads.id, threadId),
eq(documentAnnotationThreads.issueId, issueId),
eq(documentAnnotationThreads.documentKey, documentKey),
))
.then((rows: DocumentAnnotationThread[]) => rows[0] ?? null);
}
async function commentsForThreads(threadIds: string[], dbOrTx: any = db): Promise<DocumentAnnotationComment[]> {
if (threadIds.length === 0) return [];
return dbOrTx
.select(commentSelect)
.from(documentAnnotationComments)
.where(inArray(documentAnnotationComments.threadId, threadIds))
.orderBy(asc(documentAnnotationComments.createdAt), asc(documentAnnotationComments.id));
}
return {
listThreadsForIssueDocument: async (
issueId: string,
key: string,
options: { status?: "open" | "resolved" | "all"; includeComments?: boolean } = {},
) => {
const doc = await getIssueDocument(issueId, key);
if (!doc) throw notFound("Document not found");
const conditions = [
eq(documentAnnotationThreads.issueId, issueId),
eq(documentAnnotationThreads.documentId, doc.documentId),
];
if (options.status && options.status !== "all") {
conditions.push(eq(documentAnnotationThreads.status, options.status));
}
const threads: DocumentAnnotationThread[] = await db
.select(threadSelect)
.from(documentAnnotationThreads)
.where(and(...conditions))
.orderBy(desc(documentAnnotationThreads.updatedAt), desc(documentAnnotationThreads.id));
if (!options.includeComments) return threads;
const comments = await commentsForThreads(threads.map((thread) => thread.id));
const commentsByThread = new Map<string, DocumentAnnotationComment[]>();
for (const comment of comments) {
const existing = commentsByThread.get(comment.threadId) ?? [];
existing.push(comment);
commentsByThread.set(comment.threadId, existing);
}
return threads.map((thread) => ({
...thread,
comments: commentsByThread.get(thread.id) ?? [],
}));
},
getThreadForIssueDocument: async (issueId: string, key: string, threadId: string) => {
const thread = await getThreadForIssue(issueId, key, threadId);
if (!thread) return null;
const comments = await commentsForThreads([thread.id]);
return { ...thread, comments };
},
createThread: async (
issueId: string,
key: string,
input: CreateDocumentAnnotationThread,
actor: ActorInput,
) => db.transaction(async (tx) => {
await tx.execute(sql`
select ${documents.id}
from ${issueDocuments}
inner join ${documents} on ${issueDocuments.documentId} = ${documents.id}
where ${and(eq(issueDocuments.issueId, issueId), eq(issueDocuments.key, key))}
for update of ${documents}
`);
const doc = await getIssueDocument(issueId, key, tx);
if (!doc) throw notFound("Document not found");
if (
input.baseRevisionId !== doc.latestRevisionId
|| input.baseRevisionNumber !== doc.latestRevisionNumber
) {
throw conflict("Annotation anchor requires the current document revision", {
currentRevisionId: doc.latestRevisionId,
currentRevisionNumber: doc.latestRevisionNumber,
});
}
const verification = verifyDocumentAnchorSelector({
markdown: doc.latestBody,
selector: input.selector,
});
if (!verification.ok || !verification.anchor) {
throw unprocessable("Annotation anchor does not match the current document revision", {
reason: verification.reason,
});
}
const now = new Date();
const [thread] = await tx
.insert(documentAnnotationThreads)
.values({
companyId: doc.companyId,
issueId,
documentId: doc.documentId,
documentKey: doc.documentKey,
status: "open",
anchorState: "active",
anchorConfidence: "exact",
originalRevisionId: doc.latestRevisionId,
originalRevisionNumber: doc.latestRevisionNumber,
currentRevisionId: doc.latestRevisionId,
currentRevisionNumber: doc.latestRevisionNumber,
selectedText: verification.anchor.selectedText,
prefixText: verification.anchor.prefixText,
suffixText: verification.anchor.suffixText,
normalizedStart: verification.anchor.normalizedStart,
normalizedEnd: verification.anchor.normalizedEnd,
markdownStart: verification.anchor.markdownStart,
markdownEnd: verification.anchor.markdownEnd,
anchorSelector: input.selector,
createdByAgentId: actor.agentId ?? null,
createdByUserId: actor.userId ?? null,
createdAt: now,
updatedAt: now,
})
.returning(threadSelect);
const [comment] = await tx
.insert(documentAnnotationComments)
.values({
companyId: doc.companyId,
threadId: thread.id,
issueId,
documentId: doc.documentId,
body: input.body,
authorType: actor.actorType,
authorAgentId: actor.agentId ?? null,
authorUserId: actor.userId ?? null,
createdByRunId: actor.runId ?? null,
createdAt: now,
updatedAt: now,
})
.returning(commentSelect);
return { ...thread, comments: [comment] };
}),
addComment: async (
issueId: string,
key: string,
threadId: string,
input: CreateDocumentAnnotationComment,
actor: ActorInput,
) => db.transaction(async (tx) => {
const thread = await getThreadForIssue(issueId, key, threadId, tx);
if (!thread) throw notFound("Annotation thread not found");
const now = new Date();
const [comment] = await tx
.insert(documentAnnotationComments)
.values({
companyId: thread.companyId,
threadId: thread.id,
issueId: thread.issueId,
documentId: thread.documentId,
body: input.body,
authorType: actor.actorType,
authorAgentId: actor.agentId ?? null,
authorUserId: actor.userId ?? null,
createdByRunId: actor.runId ?? null,
createdAt: now,
updatedAt: now,
})
.returning(commentSelect);
await tx
.update(documentAnnotationThreads)
.set({ updatedAt: now })
.where(eq(documentAnnotationThreads.id, thread.id));
return comment;
}),
updateThread: async (
issueId: string,
key: string,
threadId: string,
input: UpdateDocumentAnnotationThread,
actor: ActorInput,
) => db.transaction(async (tx) => {
const thread = await getThreadForIssue(issueId, key, threadId, tx);
if (!thread) throw notFound("Annotation thread not found");
if (!input.status || input.status === thread.status) return thread;
const now = new Date();
const [updated] = await tx
.update(documentAnnotationThreads)
.set(input.status === "resolved"
? {
status: "resolved",
resolvedByAgentId: actor.agentId ?? null,
resolvedByUserId: actor.userId ?? null,
resolvedAt: now,
updatedAt: now,
}
: {
status: "open",
resolvedByAgentId: null,
resolvedByUserId: null,
resolvedAt: null,
updatedAt: now,
})
.where(eq(documentAnnotationThreads.id, thread.id))
.returning(threadSelect);
return updated;
}),
remapOpenThreadsForDocument: async (input: {
issueId: string;
key: string;
documentId: string;
nextRevisionId: string | null;
nextRevisionNumber: number;
nextBody: string;
}) => db.transaction(async (tx) => {
const threads: DocumentAnnotationThread[] = await tx
.select(threadSelect)
.from(documentAnnotationThreads)
.where(and(
eq(documentAnnotationThreads.issueId, input.issueId),
eq(documentAnnotationThreads.documentId, input.documentId),
eq(documentAnnotationThreads.status, "open"),
));
const changed = [];
const now = new Date();
for (const thread of threads) {
if (thread.currentRevisionId === input.nextRevisionId) continue;
const previousAnchor = snapshotFromThread(thread);
const remap = remapDocumentAnchor({
previousAnchor,
nextMarkdown: input.nextBody,
});
const nextAnchor = remap.anchor;
const nextSelector = nextAnchor ? anchorSnapshotToSelector(nextAnchor) : thread.anchorSelector;
const [updated] = await tx
.update(documentAnnotationThreads)
.set({
currentRevisionId: input.nextRevisionId,
currentRevisionNumber: input.nextRevisionNumber,
anchorState: remap.anchorState,
anchorConfidence: remap.confidence,
...(nextAnchor
? {
selectedText: nextAnchor.selectedText,
prefixText: nextAnchor.prefixText,
suffixText: nextAnchor.suffixText,
normalizedStart: nextAnchor.normalizedStart,
normalizedEnd: nextAnchor.normalizedEnd,
markdownStart: nextAnchor.markdownStart,
markdownEnd: nextAnchor.markdownEnd,
}
: {}),
anchorSelector: nextSelector,
updatedAt: now,
})
.where(eq(documentAnnotationThreads.id, thread.id))
.returning(threadSelect);
const [snapshot] = await tx
.insert(documentAnnotationAnchorSnapshots)
.values({
companyId: thread.companyId,
threadId: thread.id,
documentId: thread.documentId,
fromRevisionId: thread.currentRevisionId,
fromRevisionNumber: thread.currentRevisionNumber,
toRevisionId: input.nextRevisionId,
toRevisionNumber: input.nextRevisionNumber,
previousAnchor,
nextAnchor,
anchorState: remap.anchorState,
anchorConfidence: remap.confidence,
failureReason: remap.anchor ? null : remap.reason,
createdAt: now,
})
.returning();
changed.push({ thread: updated, snapshot });
}
return changed;
}),
selectorToAnchorSnapshot,
};
}

View file

@ -29,6 +29,8 @@ import {
activityLog,
approvals,
companySkills as companySkillsTable,
documentAnnotationComments,
documentAnnotationThreads,
documentRevisions,
issueDocuments,
heartbeatRunEvents,
@ -1981,6 +1983,7 @@ async function buildPaperclipWakePayload(input: {
}) {
const executionStage = parseObject(input.contextSnapshot.executionStage);
const commentIds = extractWakeCommentIds(input.contextSnapshot);
const annotationCommentId = readNonEmptyString(input.contextSnapshot.annotationCommentId);
const issueId = readNonEmptyString(input.contextSnapshot.issueId);
const continuationSummary = input.continuationSummary ?? null;
const issueSummary =
@ -2071,6 +2074,57 @@ async function buildPaperclipWakePayload(input: {
});
}
const annotationDeltas = annotationCommentId
? await input.db
.select({
id: documentAnnotationComments.id,
issueId: documentAnnotationComments.issueId,
threadId: documentAnnotationComments.threadId,
body: documentAnnotationComments.body,
authorType: documentAnnotationComments.authorType,
authorAgentId: documentAnnotationComments.authorAgentId,
authorUserId: documentAnnotationComments.authorUserId,
createdAt: documentAnnotationComments.createdAt,
documentKey: documentAnnotationThreads.documentKey,
status: documentAnnotationThreads.status,
anchorState: documentAnnotationThreads.anchorState,
anchorConfidence: documentAnnotationThreads.anchorConfidence,
currentRevisionNumber: documentAnnotationThreads.currentRevisionNumber,
selectedText: documentAnnotationThreads.selectedText,
prefixText: documentAnnotationThreads.prefixText,
suffixText: documentAnnotationThreads.suffixText,
})
.from(documentAnnotationComments)
.innerJoin(documentAnnotationThreads, eq(documentAnnotationComments.threadId, documentAnnotationThreads.id))
.where(and(
eq(documentAnnotationComments.companyId, input.companyId),
eq(documentAnnotationComments.id, annotationCommentId),
))
.then((rows) => rows.map((row) => ({
id: row.id,
issueId: row.issueId,
threadId: row.threadId,
documentKey: row.documentKey,
revisionNumber: row.currentRevisionNumber,
quote: row.selectedText,
prefix: row.prefixText,
suffix: row.suffixText,
threadStatus: row.status,
anchorState: row.anchorState,
anchorConfidence: row.anchorConfidence,
body: row.body.length > MAX_INLINE_WAKE_COMMENT_BODY_CHARS
? row.body.slice(0, MAX_INLINE_WAKE_COMMENT_BODY_CHARS)
: row.body,
bodyTruncated: row.body.length > MAX_INLINE_WAKE_COMMENT_BODY_CHARS,
createdAt: row.createdAt.toISOString(),
author: row.authorAgentId
? { type: "agent", id: row.authorAgentId }
: row.authorUserId
? { type: "user", id: row.authorUserId }
: { type: row.authorType, id: null },
})))
: [];
return {
reason: readNonEmptyString(input.contextSnapshot.wakeReason),
issue: issueSummary
@ -2128,6 +2182,7 @@ async function buildPaperclipWakePayload(input: {
commentIds,
latestCommentId: commentIds[commentIds.length - 1] ?? null,
comments,
annotationDeltas,
commentWindow: {
requestedCount: commentIds.length,
includedCount: comments.length,
@ -4080,7 +4135,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
continuationAttempt: decision.nextAttempt,
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, continuationRun.id));
.where(eq(heartbeatRuns.id, run.id));
}
}

View file

@ -6,6 +6,7 @@ export { agentService, deduplicateAgentName } from "./agents.js";
export { agentInstructionsService, syncInstructionsBundleConfigFromFilePath } from "./agent-instructions.js";
export { assetService } from "./assets.js";
export { documentService, extractLegacyPlanBody } from "./documents.js";
export { documentAnnotationService } from "./document-annotations.js";
export {
ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY,
buildContinuationSummaryMarkdown,

View file

@ -1,6 +1,13 @@
import { and, asc, eq, inArray, isNull } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { documents, issueComments, issueDocuments, issueReferenceMentions, issues } from "@paperclipai/db";
import {
documentAnnotationComments,
documents,
issueComments,
issueDocuments,
issueReferenceMentions,
issues,
} from "@paperclipai/db";
import type {
IssueReferenceSource,
IssueReferenceSourceKind,
@ -230,6 +237,29 @@ export function issueReferenceService(db: Db) {
}, dbOrTx);
}
async function syncAnnotationComment(commentId: string, dbOrTx: any = db) {
const comment = await dbOrTx
.select({
id: documentAnnotationComments.id,
companyId: documentAnnotationComments.companyId,
issueId: documentAnnotationComments.issueId,
body: documentAnnotationComments.body,
})
.from(documentAnnotationComments)
.where(eq(documentAnnotationComments.id, commentId))
.then((rows: Array<{ id: string; companyId: string; issueId: string; body: string }>) => rows[0] ?? null);
if (!comment) throw notFound("Document annotation comment not found");
await replaceSourceMentions({
companyId: comment.companyId,
sourceIssueId: comment.issueId,
sourceKind: "comment",
sourceRecordId: comment.id,
documentKey: null,
text: comment.body,
}, dbOrTx);
}
async function syncDocument(documentId: string, dbOrTx: any = db) {
const document = await dbOrTx
.select({
@ -396,6 +426,7 @@ export function issueReferenceService(db: Db) {
return {
syncIssue,
syncComment,
syncAnnotationComment,
syncDocument,
deleteDocumentSource,
syncAllForIssue,

View file

@ -3248,7 +3248,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
let escalation: Awaited<ReturnType<typeof issuesSvc.create>>;
try {
escalation = await issuesSvc.create(issue.companyId, {
title: `Unblock liveness incident for ${recoveryIssue.identifier ?? recoveryIssue.title}`,
title: `Unblock liveness incident for ${issue.identifier ?? issue.id}`,
description: buildLivenessEscalationDescription(input.finding),
status: "todo",
priority: "high",