mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-14 01:50:39 +09:00
[codex] Add local Cloud Upstream sync (#6548)
## Thinking Path > - Paperclip is the control plane for AI-agent companies. > - Operators need a path to move local company state toward Paperclip Cloud without losing local-first control. > - The Cloud Upstream flow needs API, persistence, CLI, and board UI surfaces that agree on the same manifest/run model. > - The existing branch had the feature work plus UX and error-handling follow-ups. > - This pull request packages the remaining Cloud Upstream sync work into one standalone branch. > - The benefit is an inspectable local-to-cloud sync workflow with preview, conflicts, activation, and captured UX review states. ## What Changed - Added Cloud Upstream shared types, server routes/services, and persisted run schema/migration. - Added Paperclip Cloud CLI sync helpers and local connection storage. - Added the Cloud Upstream board UI, settings entry points, query keys, and UX lab page. - Added preview/activation checklist behavior, redirect handling, manifest-only preview support, friendly errors, in-flight hints, and entity count summaries. ## Verification - `pnpm --filter @paperclipai/plugin-sdk build` - `NODE_ENV=test pnpm exec vitest run cli/src/__tests__/cloud.test.ts server/src/__tests__/instance-settings-routes.test.ts server/src/__tests__/instance-settings-service.test.ts ui/src/pages/CloudUpstream.test.tsx ui/src/components/CompanySettingsSidebar.test.tsx` - `NODE_ENV=test pnpm exec vitest run server/src/__tests__/cloud-upstreams.test.ts` Worktree setup note: the isolated worktree install skipped native sqlite build scripts, so I copied the already-built local sqlite binding from the main checkout before running `server/src/__tests__/cloud-upstreams.test.ts`. The test then passed. ## Risks - Medium: this adds a database migration and a broad feature path across CLI/server/UI. - Merge order: this is the only PR in this split with a DB migration; merge it before any future Cloud Upstream migration follow-up. - Mitigation: the PR is based directly on current `origin/master`, has targeted route/service/UI tests, and keeps the feature behind existing experimental Cloud Sync settings. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI GPT-5 Codex via `codex_local`, tool-enabled coding session; exact context window not exposed by this runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, screenshot artifacts are intentionally omitted per reviewer request - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge
This commit is contained in:
parent
a1835cfa5e
commit
e43b392a79
36 changed files with 5592 additions and 7 deletions
|
|
@ -226,6 +226,21 @@ By default, agents run on scheduled heartbeats and event-based triggers (task as
|
|||
|
||||
<br/>
|
||||
|
||||
## Paperclip Cloud Sync
|
||||
|
||||
Cloud upstream sync is behind the `Cloud Sync` experimental setting. Enable it in Instance Settings before pushing.
|
||||
|
||||
```bash
|
||||
paperclipai cloud connect https://your-stack.paperclip.app
|
||||
paperclipai cloud connect https://your-stack.paperclip.app --no-browser
|
||||
paperclipai cloud push --company <local-company-id> --dry-run
|
||||
paperclipai cloud push --company <local-company-id>
|
||||
```
|
||||
|
||||
`cloud connect` authorizes the local instance against the target stack and stores the upstream token in the local instance secret store. The default path opens a browser for consent; `--no-browser` uses the device-code flow and prints the verification URL and user code.
|
||||
|
||||
`cloud push --dry-run` exports the selected local company, sends a preview bundle to the connected Cloud stack, and exits with code `2` when conflicts need user resolution. A schema mismatch exits with code `3`. Running without `--dry-run` stages chunks idempotently, applies the run, and prints the final summary and recent progress events.
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
|
|
|
|||
243
cli/src/__tests__/cloud.test.ts
Normal file
243
cli/src/__tests__/cloud.test.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CompanyPortabilityExportResult } from "@paperclipai/shared";
|
||||
import {
|
||||
assertDiscoveryCompatible,
|
||||
buildBundleFromLocalCompany,
|
||||
cloudCommandExitCodes,
|
||||
connectCloud,
|
||||
resolveDeviceCodeExpiresAt,
|
||||
} from "../commands/client/cloud.js";
|
||||
import {
|
||||
LocalUpstreamPushCoordinator,
|
||||
normalizedContentHash,
|
||||
type LocalUpstreamExportBundle,
|
||||
} from "../commands/client/cloud-transfer.js";
|
||||
import { getCloudConnection } from "../commands/client/cloud-store.js";
|
||||
|
||||
const originalEnv = { ...process.env };
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
describe("cloud CLI helpers", () => {
|
||||
let tempHome: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-cloud-cli-"));
|
||||
process.env = { ...originalEnv, PAPERCLIP_HOME: tempHome };
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = { ...originalEnv };
|
||||
globalThis.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("connects with the device-code flow and stores the resulting cloud connection", async () => {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const requestUrl = String(url);
|
||||
if (requestUrl.endsWith("/.well-known/paperclip-upstream")) {
|
||||
return jsonResponse(discovery());
|
||||
}
|
||||
if (requestUrl.endsWith("/api/upstream-sync/device-code")) {
|
||||
expect(JSON.parse(String(init?.body))).toMatchObject({
|
||||
stackId: "stack-1",
|
||||
scopes: ["upstream_import:preview", "upstream_import:write", "upstream_import:read"],
|
||||
});
|
||||
return jsonResponse({
|
||||
deviceCode: "device-1",
|
||||
userCode: "ABCD-EFGH",
|
||||
verificationUri: "https://cloud.example.test/api/upstream-sync/device-code/approve",
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(),
|
||||
intervalSeconds: 0,
|
||||
});
|
||||
}
|
||||
if (requestUrl.endsWith("/api/upstream-sync/token")) {
|
||||
return jsonResponse({
|
||||
accessToken: "upt_test",
|
||||
scopes: ["upstream_import:preview"],
|
||||
token: {
|
||||
id: "token-1",
|
||||
companyStackId: "stack-1",
|
||||
targetOrigin: "https://cloud.example.test",
|
||||
sourceInstanceId: "paperclip-local-default",
|
||||
sourceInstanceFingerprint: "sha256:test",
|
||||
scopes: ["upstream_import:preview"],
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(),
|
||||
},
|
||||
});
|
||||
}
|
||||
return jsonResponse({ error: "not_found" }, 404);
|
||||
}) as typeof fetch;
|
||||
|
||||
const connection = await connectCloud("https://cloud.example.test", { noBrowser: true, json: true });
|
||||
|
||||
expect(connection.accessToken).toBe("upt_test");
|
||||
expect(getCloudConnection("https://cloud.example.test")?.token.id).toBe("token-1");
|
||||
});
|
||||
|
||||
it("hard-blocks incompatible transfer schema versions with the stable schema exit code", () => {
|
||||
expect(() => assertDiscoveryCompatible(discovery({ supportedSchemaMajor: 99 }))).toThrow(/schema mismatch/i);
|
||||
expect(cloudCommandExitCodes.schemaMismatch).toBe(3);
|
||||
});
|
||||
|
||||
it("falls back to a bounded device-code expiry when the cloud omits or malforms expiresAt", () => {
|
||||
const now = Date.UTC(2026, 4, 22, 13, 0, 0);
|
||||
const validExpiry = "2026-05-22T13:05:00.000Z";
|
||||
|
||||
expect(resolveDeviceCodeExpiresAt(validExpiry, now)).toBe(Date.parse(validExpiry));
|
||||
expect(resolveDeviceCodeExpiresAt(undefined, now)).toBe(now + 15 * 60_000);
|
||||
expect(resolveDeviceCodeExpiresAt("not-a-date", now)).toBe(now + 15 * 60_000);
|
||||
});
|
||||
|
||||
it("builds deterministic chunks with validated payload hashes", async () => {
|
||||
const bundle = await buildTestBundle();
|
||||
|
||||
expect(bundle.chunks).toHaveLength(2);
|
||||
expect(bundle.chunks[0]?.sha256).toBe(normalizedContentHash(bundle.chunks[0]?.payload));
|
||||
expect(bundle.manifest.chunks[0]?.manifestHash).toBe(bundle.manifest.manifestHash);
|
||||
expect(bundle.manifest.idempotencyKey).toBe((await buildTestBundle()).manifest.idempotencyKey);
|
||||
});
|
||||
|
||||
it("reuses the same manifest and chunk identity when an interrupted apply is retried", async () => {
|
||||
const bundle = await buildTestBundle();
|
||||
const calls: Array<{ path: string; body: unknown }> = [];
|
||||
const coordinator = new LocalUpstreamPushCoordinator({
|
||||
targetOrigin: "https://cloud.example.test",
|
||||
paperclipCompanyId: "target-company-1",
|
||||
fetch: async (url, init) => {
|
||||
const parsed = new URL(String(url));
|
||||
const body = init?.body ? JSON.parse(String(init.body)) as unknown : {};
|
||||
calls.push({ path: parsed.pathname, body });
|
||||
if (parsed.pathname.endsWith("/runs")) return jsonResponse({ run: { id: "run-1" } });
|
||||
return jsonResponse({ run: { id: "run-1" }, summary: { create: 0, update: 0, adopt: 0, skip: 2, conflict: 0, staleMapping: 0 } });
|
||||
},
|
||||
});
|
||||
|
||||
await coordinator.apply(bundle);
|
||||
await coordinator.apply(bundle);
|
||||
|
||||
const runBodies = calls.filter((call) => call.path.endsWith("/runs")).map((call) => call.body as { manifest: { idempotencyKey: string } });
|
||||
const chunkBodies = calls.filter((call) => call.path.endsWith("/chunks")).map((call) => call.body as { chunkIndex: number; sha256: string });
|
||||
expect(runBodies).toHaveLength(2);
|
||||
expect(runBodies[0]?.manifest.idempotencyKey).toBe(runBodies[1]?.manifest.idempotencyKey);
|
||||
expect(chunkBodies[0]).toEqual(chunkBodies[2]);
|
||||
expect(chunkBodies[1]).toEqual(chunkBodies[3]);
|
||||
});
|
||||
});
|
||||
|
||||
async function buildTestBundle(): Promise<LocalUpstreamExportBundle> {
|
||||
return buildBundleFromLocalCompany({
|
||||
localCompanyId: "local-company-1",
|
||||
connection: {
|
||||
id: "conn-1",
|
||||
remoteUrl: "https://cloud.example.test",
|
||||
targetOrigin: "https://cloud.example.test",
|
||||
targetHost: "cloud.example.test",
|
||||
stackId: "stack-1",
|
||||
targetCompanyId: "target-company-1",
|
||||
accessToken: "upt_test",
|
||||
token: {
|
||||
id: "token-1",
|
||||
companyStackId: "stack-1",
|
||||
targetOrigin: "https://cloud.example.test",
|
||||
sourceInstanceId: "paperclip-local-default",
|
||||
sourceInstanceFingerprint: "sha256:test",
|
||||
scopes: ["upstream_import:preview"],
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(),
|
||||
},
|
||||
privateKeyPem: "unused",
|
||||
sourcePublicKey: "unused",
|
||||
sourceInstanceId: "paperclip-local-default",
|
||||
sourceInstanceFingerprint: "sha256:test",
|
||||
scopes: ["upstream_import:preview"],
|
||||
createdAt: "2026-05-18T00:00:00.000Z",
|
||||
updatedAt: "2026-05-18T00:00:00.000Z",
|
||||
},
|
||||
discovery: discovery(),
|
||||
localApi: {
|
||||
post: async <T>() => portabilityExport() as T,
|
||||
},
|
||||
maxEntitiesPerChunk: 1,
|
||||
mode: "apply",
|
||||
});
|
||||
}
|
||||
|
||||
function discovery(overrides: Partial<{ supportedSchemaMajor: number }> = {}) {
|
||||
return {
|
||||
schema: "paperclip-upstream-discovery-v1",
|
||||
stack: {
|
||||
id: "stack-1",
|
||||
slug: "cloud-test",
|
||||
displayName: "Cloud Test",
|
||||
companyId: "target-company-1",
|
||||
origin: "https://cloud.example.test",
|
||||
},
|
||||
auth: {
|
||||
deviceCode: {
|
||||
deviceCodeUrl: "https://cloud.example.test/api/upstream-sync/device-code",
|
||||
verificationUrl: "https://cloud.example.test/api/upstream-sync/device-code/approve",
|
||||
tokenUrl: "https://cloud.example.test/api/upstream-sync/token",
|
||||
},
|
||||
scopes: ["upstream_import:preview", "upstream_import:write", "upstream_import:read"],
|
||||
},
|
||||
transfer: {
|
||||
supportedSchemaMajor: overrides.supportedSchemaMajor ?? 1,
|
||||
featureFlags: ["cloud_sync"],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function portabilityExport(): CompanyPortabilityExportResult {
|
||||
return {
|
||||
rootPath: ".",
|
||||
paperclipExtensionPath: ".paperclip.yaml",
|
||||
manifest: {
|
||||
schemaVersion: 1,
|
||||
generatedAt: "2026-05-18T00:00:00.000Z",
|
||||
source: {
|
||||
companyId: "local-company-1",
|
||||
companyName: "Local Company",
|
||||
},
|
||||
includes: {
|
||||
company: true,
|
||||
agents: true,
|
||||
projects: true,
|
||||
issues: true,
|
||||
skills: true,
|
||||
},
|
||||
company: {
|
||||
path: "company.json",
|
||||
name: "Local Company",
|
||||
description: null,
|
||||
brandColor: null,
|
||||
logoPath: null,
|
||||
attachmentMaxBytes: null,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
feedbackDataSharingEnabled: false,
|
||||
feedbackDataSharingConsentAt: null,
|
||||
feedbackDataSharingConsentByUserId: null,
|
||||
feedbackDataSharingTermsVersion: null,
|
||||
},
|
||||
sidebar: null,
|
||||
agents: [],
|
||||
skills: [],
|
||||
projects: [],
|
||||
issues: [],
|
||||
envInputs: [],
|
||||
},
|
||||
files: {
|
||||
"README.md": "Local Company",
|
||||
},
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
177
cli/src/commands/client/cloud-store.ts
Normal file
177
cli/src/commands/client/cloud-store.ts
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolvePaperclipInstanceRoot } from "../../config/home.js";
|
||||
|
||||
export interface CloudConnectionTokenRecord {
|
||||
id: string;
|
||||
companyStackId: string;
|
||||
targetOrigin: string;
|
||||
sourceInstanceId: string;
|
||||
sourceInstanceFingerprint: string;
|
||||
scopes: string[];
|
||||
expiresAt: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface CloudConnection {
|
||||
id: string;
|
||||
remoteUrl: string;
|
||||
targetOrigin: string;
|
||||
targetHost: string;
|
||||
stackId: string;
|
||||
stackSlug?: string | null;
|
||||
stackDisplayName?: string | null;
|
||||
targetCompanyId: string;
|
||||
accessToken: string;
|
||||
token: CloudConnectionTokenRecord;
|
||||
privateKeyPem: string;
|
||||
sourcePublicKey: string;
|
||||
sourceInstanceId: string;
|
||||
sourceInstanceFingerprint: string;
|
||||
scopes: string[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
interface CloudConnectionStore {
|
||||
version: 1;
|
||||
connections: Record<string, CloudConnection>;
|
||||
currentConnectionId?: string;
|
||||
}
|
||||
|
||||
function defaultStore(): CloudConnectionStore {
|
||||
return {
|
||||
version: 1,
|
||||
connections: {},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveCloudConnectionStorePath(): string {
|
||||
return path.resolve(resolvePaperclipInstanceRoot(), "secrets", "cloud-upstream-connections.json");
|
||||
}
|
||||
|
||||
export function readCloudConnectionStore(storePath = resolveCloudConnectionStorePath()): CloudConnectionStore {
|
||||
if (!fs.existsSync(storePath)) return defaultStore();
|
||||
const raw = JSON.parse(fs.readFileSync(storePath, "utf8")) as Partial<CloudConnectionStore> | null;
|
||||
const connections: Record<string, CloudConnection> = {};
|
||||
if (raw?.connections && typeof raw.connections === "object") {
|
||||
for (const [id, value] of Object.entries(raw.connections)) {
|
||||
const normalized = normalizeConnection(value);
|
||||
if (normalized) connections[id] = normalized;
|
||||
}
|
||||
}
|
||||
const currentConnectionId =
|
||||
typeof raw?.currentConnectionId === "string" && connections[raw.currentConnectionId]
|
||||
? raw.currentConnectionId
|
||||
: Object.values(connections).sort((left, right) => right.updatedAt.localeCompare(left.updatedAt))[0]?.id;
|
||||
return {
|
||||
version: 1,
|
||||
connections,
|
||||
currentConnectionId,
|
||||
};
|
||||
}
|
||||
|
||||
export function writeCloudConnectionStore(
|
||||
store: CloudConnectionStore,
|
||||
storePath = resolveCloudConnectionStorePath(),
|
||||
): void {
|
||||
fs.mkdirSync(path.dirname(storePath), { recursive: true });
|
||||
fs.writeFileSync(storePath, `${JSON.stringify(store, null, 2)}\n`, { mode: 0o600 });
|
||||
}
|
||||
|
||||
export function upsertCloudConnection(
|
||||
connection: CloudConnection,
|
||||
storePath = resolveCloudConnectionStorePath(),
|
||||
): CloudConnection {
|
||||
const store = readCloudConnectionStore(storePath);
|
||||
const existing = store.connections[connection.id];
|
||||
const now = new Date().toISOString();
|
||||
const next = {
|
||||
...connection,
|
||||
createdAt: existing?.createdAt ?? connection.createdAt ?? now,
|
||||
updatedAt: now,
|
||||
};
|
||||
store.connections[next.id] = next;
|
||||
store.currentConnectionId = next.id;
|
||||
writeCloudConnectionStore(store, storePath);
|
||||
return next;
|
||||
}
|
||||
|
||||
export function getCloudConnection(
|
||||
remoteUrlOrOrigin?: string,
|
||||
storePath = resolveCloudConnectionStorePath(),
|
||||
): CloudConnection | null {
|
||||
const store = readCloudConnectionStore(storePath);
|
||||
if (remoteUrlOrOrigin?.trim()) {
|
||||
const needle = normalizeRemoteLookup(remoteUrlOrOrigin);
|
||||
return Object.values(store.connections).find((connection) =>
|
||||
normalizeRemoteLookup(connection.remoteUrl) === needle ||
|
||||
normalizeRemoteLookup(connection.targetOrigin) === needle
|
||||
) ?? null;
|
||||
}
|
||||
return store.currentConnectionId ? store.connections[store.currentConnectionId] ?? null : null;
|
||||
}
|
||||
|
||||
function normalizeRemoteLookup(value: string): string {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return url.origin.replace(/\/+$/u, "");
|
||||
} catch {
|
||||
return value.trim().replace(/\/+$/u, "");
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeConnection(value: unknown): CloudConnection | null {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
|
||||
const record = value as Record<string, unknown>;
|
||||
const id = stringValue(record.id);
|
||||
const remoteUrl = stringValue(record.remoteUrl);
|
||||
const targetOrigin = stringValue(record.targetOrigin);
|
||||
const targetHost = stringValue(record.targetHost);
|
||||
const stackId = stringValue(record.stackId);
|
||||
const targetCompanyId = stringValue(record.targetCompanyId);
|
||||
const accessToken = stringValue(record.accessToken);
|
||||
const token = typeof record.token === "object" && record.token !== null && !Array.isArray(record.token)
|
||||
? record.token as CloudConnectionTokenRecord
|
||||
: null;
|
||||
const privateKeyPem = stringValue(record.privateKeyPem);
|
||||
const sourcePublicKey = stringValue(record.sourcePublicKey);
|
||||
const sourceInstanceId = stringValue(record.sourceInstanceId);
|
||||
const sourceInstanceFingerprint = stringValue(record.sourceInstanceFingerprint);
|
||||
const createdAt = stringValue(record.createdAt);
|
||||
const updatedAt = stringValue(record.updatedAt);
|
||||
if (
|
||||
!id || !remoteUrl || !targetOrigin || !targetHost || !stackId || !targetCompanyId ||
|
||||
!accessToken || !token || !privateKeyPem || !sourcePublicKey || !sourceInstanceId ||
|
||||
!sourceInstanceFingerprint || !createdAt || !updatedAt
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
id,
|
||||
remoteUrl,
|
||||
targetOrigin,
|
||||
targetHost,
|
||||
stackId,
|
||||
stackSlug: stringValue(record.stackSlug),
|
||||
stackDisplayName: stringValue(record.stackDisplayName),
|
||||
targetCompanyId,
|
||||
accessToken,
|
||||
token,
|
||||
privateKeyPem,
|
||||
sourcePublicKey,
|
||||
sourceInstanceId,
|
||||
sourceInstanceFingerprint,
|
||||
scopes: stringArray(record.scopes),
|
||||
createdAt,
|
||||
updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function stringArray(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === "string") : [];
|
||||
}
|
||||
297
cli/src/commands/client/cloud-transfer.ts
Normal file
297
cli/src/commands/client/cloud-transfer.ts
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
import { createHash } from "node:crypto";
|
||||
|
||||
export const upstreamTransferSchema = {
|
||||
family: "paperclip-upstream-transfer",
|
||||
version: "1.0.0",
|
||||
major: 1,
|
||||
minor: 0,
|
||||
} as const;
|
||||
|
||||
export type NormalizedSha256 = `sha256:${string}`;
|
||||
|
||||
export interface SourceEntityKey {
|
||||
sourceInstanceId: string;
|
||||
sourceCompanyId: string;
|
||||
sourceEntityType: string;
|
||||
sourceEntityId: string;
|
||||
sourceNaturalKey?: string;
|
||||
}
|
||||
|
||||
export interface UpstreamTransferWarning {
|
||||
code: string;
|
||||
severity: "info" | "warning" | "blocker";
|
||||
message: string;
|
||||
entity?: SourceEntityKey;
|
||||
}
|
||||
|
||||
export interface UpstreamTransferEntityRecord {
|
||||
key: SourceEntityKey;
|
||||
contentHash: NormalizedSha256;
|
||||
dependencies: SourceEntityKey[];
|
||||
warnings: UpstreamTransferWarning[];
|
||||
}
|
||||
|
||||
export interface UpstreamTransferManifestSource {
|
||||
sourceInstanceId: string;
|
||||
sourceCompanyId: string;
|
||||
sourceInstanceKeyFingerprint: string;
|
||||
exporterVersion: string;
|
||||
sourceSchemaVersion: string;
|
||||
}
|
||||
|
||||
export interface UpstreamTransferManifestTarget {
|
||||
targetStackId: string;
|
||||
targetCompanyId: string;
|
||||
targetOrigin: string;
|
||||
supportedSchemaMajor: number;
|
||||
}
|
||||
|
||||
export interface UpstreamTransferChunk {
|
||||
chunkIndex: number;
|
||||
totalChunks: number;
|
||||
byteLength: number;
|
||||
sha256: NormalizedSha256;
|
||||
manifestHash: NormalizedSha256;
|
||||
}
|
||||
|
||||
export interface UpstreamTransferManifest {
|
||||
schema: typeof upstreamTransferSchema;
|
||||
source: UpstreamTransferManifestSource;
|
||||
target: UpstreamTransferManifestTarget;
|
||||
runId: string;
|
||||
idempotencyKey: string;
|
||||
generatedAt: string;
|
||||
entityCount: number;
|
||||
entities: UpstreamTransferEntityRecord[];
|
||||
chunks: UpstreamTransferChunk[];
|
||||
warnings: UpstreamTransferWarning[];
|
||||
featureFlags: string[];
|
||||
manifestHash: NormalizedSha256;
|
||||
}
|
||||
|
||||
export interface LocalUpstreamExportEntityInput {
|
||||
key: SourceEntityKey;
|
||||
body: Record<string, unknown>;
|
||||
dependencies?: SourceEntityKey[];
|
||||
warnings?: UpstreamTransferWarning[];
|
||||
conflictKeys?: string[];
|
||||
}
|
||||
|
||||
export interface LocalUpstreamExportEntity {
|
||||
record: UpstreamTransferEntityRecord;
|
||||
body: Record<string, unknown>;
|
||||
conflictKeys?: string[];
|
||||
}
|
||||
|
||||
export interface LocalUpstreamExportChunk {
|
||||
chunkIndex: number;
|
||||
totalChunks: number;
|
||||
byteLength: number;
|
||||
sha256: NormalizedSha256;
|
||||
payload: {
|
||||
entityKeys: SourceEntityKey[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface LocalUpstreamExportBundle {
|
||||
manifest: UpstreamTransferManifest;
|
||||
entities: LocalUpstreamExportEntity[];
|
||||
chunks: LocalUpstreamExportChunk[];
|
||||
}
|
||||
|
||||
export interface BuildLocalUpstreamExportBundleInput {
|
||||
source: UpstreamTransferManifestSource;
|
||||
target: UpstreamTransferManifestTarget;
|
||||
runId: string;
|
||||
idempotencyKey: string;
|
||||
entities: LocalUpstreamExportEntityInput[];
|
||||
warnings?: UpstreamTransferWarning[];
|
||||
featureFlags?: string[];
|
||||
maxEntitiesPerChunk?: number;
|
||||
}
|
||||
|
||||
export interface LocalUpstreamPushCoordinatorOptions {
|
||||
targetOrigin: string;
|
||||
paperclipCompanyId: string;
|
||||
fetch?: typeof fetch;
|
||||
headers?: (input: { method: string; path: string }) => HeadersInit | Promise<HeadersInit>;
|
||||
}
|
||||
|
||||
export class UpstreamImportRequestError extends Error {
|
||||
readonly status: number;
|
||||
readonly body: unknown;
|
||||
|
||||
constructor(status: number, message: string, body: unknown) {
|
||||
super(message);
|
||||
this.status = status;
|
||||
this.body = body;
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalUpstreamPushCoordinator {
|
||||
readonly #targetOrigin: string;
|
||||
readonly #paperclipCompanyId: string;
|
||||
readonly #fetch: typeof fetch;
|
||||
readonly #headers: NonNullable<LocalUpstreamPushCoordinatorOptions["headers"]>;
|
||||
|
||||
constructor(options: LocalUpstreamPushCoordinatorOptions) {
|
||||
this.#targetOrigin = options.targetOrigin.replace(/\/+$/u, "");
|
||||
this.#paperclipCompanyId = options.paperclipCompanyId;
|
||||
this.#fetch = options.fetch ?? fetch;
|
||||
this.#headers = options.headers ?? (() => ({}));
|
||||
}
|
||||
|
||||
async preview(bundle: LocalUpstreamExportBundle): Promise<unknown> {
|
||||
return this.post(`/api/companies/${encodeURIComponent(this.#paperclipCompanyId)}/upstream-imports/preview`, {
|
||||
manifest: bundle.manifest,
|
||||
entities: bundle.entities,
|
||||
});
|
||||
}
|
||||
|
||||
async apply(bundle: LocalUpstreamExportBundle): Promise<unknown> {
|
||||
const run = await this.post(`/api/companies/${encodeURIComponent(this.#paperclipCompanyId)}/upstream-imports/runs`, {
|
||||
mode: "apply",
|
||||
manifest: bundle.manifest,
|
||||
entities: bundle.entities,
|
||||
}) as { run?: { id?: unknown } };
|
||||
const runId = typeof run.run?.id === "string" ? run.run.id : undefined;
|
||||
if (!runId) {
|
||||
throw new Error("Remote upstream importer did not return a run id");
|
||||
}
|
||||
|
||||
for (const chunk of bundle.chunks) {
|
||||
await this.post(`/api/upstream-import-runs/${encodeURIComponent(runId)}/chunks`, chunk);
|
||||
}
|
||||
|
||||
return this.post(`/api/upstream-import-runs/${encodeURIComponent(runId)}/apply`, {});
|
||||
}
|
||||
|
||||
async events(runId: string): Promise<unknown> {
|
||||
return this.get(`/api/upstream-import-runs/${encodeURIComponent(runId)}/events`);
|
||||
}
|
||||
|
||||
private async get(path: string): Promise<unknown> {
|
||||
const response = await this.#fetch(`${this.#targetOrigin}${path}`, {
|
||||
method: "GET",
|
||||
headers: await this.#headers({ method: "GET", path }),
|
||||
});
|
||||
return parseCoordinatorResponse(response);
|
||||
}
|
||||
|
||||
private async post(path: string, body: unknown): Promise<unknown> {
|
||||
const response = await this.#fetch(`${this.#targetOrigin}${path}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(await this.#headers({ method: "POST", path })),
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return parseCoordinatorResponse(response);
|
||||
}
|
||||
}
|
||||
|
||||
export function buildLocalUpstreamExportBundle(
|
||||
input: BuildLocalUpstreamExportBundleInput,
|
||||
): LocalUpstreamExportBundle {
|
||||
const entities = input.entities.map<LocalUpstreamExportEntity>((entity) => ({
|
||||
record: {
|
||||
key: entity.key,
|
||||
contentHash: normalizedContentHash(entity.body),
|
||||
dependencies: entity.dependencies ?? [],
|
||||
warnings: entity.warnings ?? [],
|
||||
},
|
||||
body: entity.body,
|
||||
conflictKeys: entity.conflictKeys,
|
||||
}));
|
||||
const chunks = buildLocalChunks(entities, input.maxEntitiesPerChunk ?? 100);
|
||||
const manifestWithoutHash = {
|
||||
schema: upstreamTransferSchema,
|
||||
source: input.source,
|
||||
target: input.target,
|
||||
runId: input.runId,
|
||||
idempotencyKey: input.idempotencyKey,
|
||||
generatedAt: new Date(0).toISOString(),
|
||||
entityCount: entities.length,
|
||||
entities: entities.map((entity) => entity.record),
|
||||
chunks: chunks.map(({ payload: _payload, ...chunk }) => chunk),
|
||||
warnings: input.warnings ?? [],
|
||||
featureFlags: (input.featureFlags ?? ["cloud_sync"]).slice().sort(),
|
||||
};
|
||||
const manifestHash = normalizedContentHash(manifestWithoutHash);
|
||||
return {
|
||||
manifest: {
|
||||
...manifestWithoutHash,
|
||||
chunks: manifestWithoutHash.chunks.map((chunk) => ({ ...chunk, manifestHash })),
|
||||
manifestHash,
|
||||
},
|
||||
entities,
|
||||
chunks,
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizedContentHash(value: unknown): NormalizedSha256 {
|
||||
return `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`;
|
||||
}
|
||||
|
||||
export function canonicalJson(value: unknown): string {
|
||||
return JSON.stringify(sortJson(value));
|
||||
}
|
||||
|
||||
function buildLocalChunks(
|
||||
entities: LocalUpstreamExportEntity[],
|
||||
maxEntitiesPerChunk: number,
|
||||
): LocalUpstreamExportChunk[] {
|
||||
if (!Number.isInteger(maxEntitiesPerChunk) || maxEntitiesPerChunk < 1) {
|
||||
throw new Error("maxEntitiesPerChunk must be a positive integer");
|
||||
}
|
||||
if (entities.length === 0) return [];
|
||||
|
||||
const groups: LocalUpstreamExportEntity[][] = [];
|
||||
for (let index = 0; index < entities.length; index += maxEntitiesPerChunk) {
|
||||
groups.push(entities.slice(index, index + maxEntitiesPerChunk));
|
||||
}
|
||||
|
||||
return groups.map((group, index) => {
|
||||
const payload = {
|
||||
entityKeys: group.map((entity) => entity.record.key),
|
||||
};
|
||||
return {
|
||||
chunkIndex: index,
|
||||
totalChunks: groups.length,
|
||||
byteLength: Buffer.byteLength(canonicalJson(payload)),
|
||||
sha256: normalizedContentHash(payload),
|
||||
payload,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function sortJson(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(sortJson);
|
||||
if (typeof value !== "object" || value === null) return value;
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, entry]) => [key, sortJson(entry)]),
|
||||
);
|
||||
}
|
||||
|
||||
async function parseCoordinatorResponse(response: Response): Promise<unknown> {
|
||||
const text = await response.text();
|
||||
const parsed = text.trim() ? safeParseJson(text) : {};
|
||||
if (!response.ok) {
|
||||
const message = typeof parsed === "object" && parsed !== null && "error" in parsed
|
||||
? String((parsed as { error: unknown }).error)
|
||||
: `Upstream importer request failed with ${response.status}`;
|
||||
throw new UpstreamImportRequestError(response.status, message, parsed);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function safeParseJson(text: string): unknown {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
721
cli/src/commands/client/cloud.ts
Normal file
721
cli/src/commands/client/cloud.ts
Normal file
|
|
@ -0,0 +1,721 @@
|
|||
import { createHash, generateKeyPairSync, randomBytes, randomUUID, sign } from "node:crypto";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { URL } from "node:url";
|
||||
import { Command } from "commander";
|
||||
import pc from "picocolors";
|
||||
import type {
|
||||
CompanyPortabilityExportResult,
|
||||
CompanyPortabilityFileEntry,
|
||||
InstanceExperimentalSettings,
|
||||
} from "@paperclipai/shared";
|
||||
import { openUrl } from "../../client/board-auth.js";
|
||||
import { resolvePaperclipInstanceId } from "../../config/home.js";
|
||||
import {
|
||||
addCommonClientOptions,
|
||||
handleCommandError,
|
||||
printOutput,
|
||||
resolveCommandContext,
|
||||
type BaseClientOptions,
|
||||
} from "./common.js";
|
||||
import {
|
||||
buildLocalUpstreamExportBundle,
|
||||
LocalUpstreamPushCoordinator,
|
||||
normalizedContentHash,
|
||||
upstreamTransferSchema,
|
||||
UpstreamImportRequestError,
|
||||
type LocalUpstreamExportBundle,
|
||||
type LocalUpstreamExportEntityInput,
|
||||
type SourceEntityKey,
|
||||
type UpstreamTransferManifestSource,
|
||||
type UpstreamTransferManifestTarget,
|
||||
type UpstreamTransferWarning,
|
||||
} from "./cloud-transfer.js";
|
||||
import {
|
||||
getCloudConnection,
|
||||
upsertCloudConnection,
|
||||
type CloudConnection,
|
||||
type CloudConnectionTokenRecord,
|
||||
} from "./cloud-store.js";
|
||||
|
||||
const CLOUD_SYNC_CONFLICT_EXIT_CODE = 2;
|
||||
const CLOUD_SYNC_SCHEMA_MISMATCH_EXIT_CODE = 3;
|
||||
const CLOUD_SYNC_SCOPES = ["upstream_import:preview", "upstream_import:write", "upstream_import:read"];
|
||||
const DEVICE_CODE_FALLBACK_EXPIRES_MS = 15 * 60_000;
|
||||
|
||||
interface CloudConnectOptions extends BaseClientOptions {
|
||||
noBrowser?: boolean;
|
||||
}
|
||||
|
||||
interface CloudPushOptions extends BaseClientOptions {
|
||||
company?: string;
|
||||
remoteUrl?: string;
|
||||
dryRun?: boolean;
|
||||
maxEntitiesPerChunk?: number;
|
||||
}
|
||||
|
||||
interface UpstreamDiscovery {
|
||||
schema: string;
|
||||
stack: {
|
||||
id: string;
|
||||
slug?: string;
|
||||
displayName?: string;
|
||||
companyId: string;
|
||||
origin: string;
|
||||
};
|
||||
auth: {
|
||||
pkce?: {
|
||||
authorizeUrl: string;
|
||||
tokenUrl: string;
|
||||
codeChallengeMethod: string;
|
||||
};
|
||||
deviceCode?: {
|
||||
deviceCodeUrl: string;
|
||||
verificationUrl: string;
|
||||
tokenUrl: string;
|
||||
};
|
||||
scopes?: string[];
|
||||
};
|
||||
transfer: {
|
||||
supportedSchemaMajor: number;
|
||||
featureFlags?: string[];
|
||||
};
|
||||
}
|
||||
|
||||
interface TokenResponse {
|
||||
accessToken: string;
|
||||
token: CloudConnectionTokenRecord;
|
||||
scopes?: string[];
|
||||
expiresAt?: string;
|
||||
}
|
||||
|
||||
class CloudAuthRequestError extends Error {
|
||||
readonly status: number;
|
||||
readonly body: unknown;
|
||||
|
||||
constructor(status: number, message: string, body: unknown) {
|
||||
super(message);
|
||||
this.status = status;
|
||||
this.body = body;
|
||||
}
|
||||
}
|
||||
|
||||
export function registerCloudCommands(program: Command): void {
|
||||
const cloud = program.command("cloud").description("Paperclip Cloud upstream sync commands");
|
||||
|
||||
addCommonClientOptions(
|
||||
cloud
|
||||
.command("connect")
|
||||
.description("Authorize this local instance to push into a Paperclip Cloud stack")
|
||||
.argument("<remote-url>", "Paperclip Cloud stack URL")
|
||||
.option("--no-browser", "Use the device-code flow instead of opening a browser", false)
|
||||
.action(async (remoteUrl: string, opts: CloudConnectOptions) => {
|
||||
try {
|
||||
await connectCloud(remoteUrl, opts);
|
||||
} catch (err) {
|
||||
handleCommandError(err);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
addCommonClientOptions(
|
||||
cloud
|
||||
.command("push")
|
||||
.description("Preview or apply a local company push into the connected Paperclip Cloud stack")
|
||||
.requiredOption("--company <local-company-id>", "Local company ID to export")
|
||||
.option("--remote-url <remote-url>", "Use a specific stored cloud connection")
|
||||
.option("--dry-run", "Preview without applying", false)
|
||||
.option("--max-entities-per-chunk <count>", "Chunk size for upstream uploads", (value) => Number(value), 100)
|
||||
.action(async (opts: CloudPushOptions) => {
|
||||
try {
|
||||
await pushCloud(opts);
|
||||
} catch (err) {
|
||||
if (isSchemaMismatchError(err)) {
|
||||
console.error(pc.red(err instanceof Error ? err.message : String(err)));
|
||||
process.exitCode = CLOUD_SYNC_SCHEMA_MISMATCH_EXIT_CODE;
|
||||
return;
|
||||
}
|
||||
handleCommandError(err);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function connectCloud(remoteUrl: string, opts: CloudConnectOptions = {}): Promise<CloudConnection> {
|
||||
const ctx = resolveCommandContext(opts);
|
||||
const discovery = await discoverUpstream(remoteUrl);
|
||||
assertDiscoveryCompatible(discovery);
|
||||
const source = createSourceIdentity();
|
||||
const token = await authorizeConnection(discovery, source, {
|
||||
noBrowser: Boolean(opts.noBrowser),
|
||||
});
|
||||
const targetOrigin = discovery.stack.origin.replace(/\/+$/u, "");
|
||||
const targetHost = new URL(targetOrigin).host;
|
||||
const now = new Date().toISOString();
|
||||
const connection = upsertCloudConnection({
|
||||
id: connectionId(targetOrigin),
|
||||
remoteUrl,
|
||||
targetOrigin,
|
||||
targetHost,
|
||||
stackId: discovery.stack.id,
|
||||
stackSlug: discovery.stack.slug ?? null,
|
||||
stackDisplayName: discovery.stack.displayName ?? null,
|
||||
targetCompanyId: discovery.stack.companyId,
|
||||
accessToken: token.accessToken,
|
||||
token: token.token,
|
||||
privateKeyPem: source.privateKeyPem,
|
||||
sourcePublicKey: source.sourcePublicKey,
|
||||
sourceInstanceId: source.sourceInstanceId,
|
||||
sourceInstanceFingerprint: source.sourceInstanceFingerprint,
|
||||
scopes: token.scopes ?? token.token.scopes ?? CLOUD_SYNC_SCOPES,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
|
||||
if (ctx.json) {
|
||||
printOutput(redactConnection(connection), { json: true });
|
||||
} else {
|
||||
console.log(pc.bold("Connected to Paperclip Cloud"));
|
||||
console.log(`stack=${connection.stackDisplayName ?? connection.stackSlug ?? connection.stackId}`);
|
||||
console.log(`origin=${connection.targetOrigin}`);
|
||||
console.log(`company=${connection.targetCompanyId}`);
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
export async function pushCloud(opts: CloudPushOptions): Promise<unknown> {
|
||||
const ctx = resolveCommandContext(opts, { requireCompany: false });
|
||||
const localCompanyId = requiredString(opts.company, "--company");
|
||||
await assertCloudSyncEnabled(ctx.api.get<InstanceExperimentalSettings>("/api/instance/settings/experimental"));
|
||||
const connection = getCloudConnection(opts.remoteUrl);
|
||||
if (!connection) {
|
||||
throw new Error("No cloud connection found. Run `paperclipai cloud connect <remote-url>` first.");
|
||||
}
|
||||
|
||||
const discovery = await discoverUpstream(connection.targetOrigin);
|
||||
assertDiscoveryCompatible(discovery);
|
||||
const bundle = await buildBundleFromLocalCompany({
|
||||
localCompanyId,
|
||||
connection,
|
||||
discovery,
|
||||
localApi: ctx.api,
|
||||
maxEntitiesPerChunk: opts.maxEntitiesPerChunk,
|
||||
mode: opts.dryRun ? "preview" : "apply",
|
||||
});
|
||||
const coordinator = new LocalUpstreamPushCoordinator({
|
||||
targetOrigin: connection.targetOrigin,
|
||||
paperclipCompanyId: connection.targetCompanyId,
|
||||
headers: ({ method, path }) => cloudProofHeaders(connection, method, path),
|
||||
});
|
||||
|
||||
const result = opts.dryRun ? await coordinator.preview(bundle) : await coordinator.apply(bundle);
|
||||
const runId = getRunId(result);
|
||||
const events = !opts.dryRun && runId ? await coordinator.events(runId).catch(() => null) : null;
|
||||
const summary = summarizeResult(result);
|
||||
const conflictCount = summary.conflict + summary.staleMapping;
|
||||
|
||||
if (ctx.json) {
|
||||
printOutput({ result, events }, { json: true });
|
||||
} else {
|
||||
console.log(pc.bold(opts.dryRun ? "Cloud Push Preview" : "Cloud Push Applied"));
|
||||
console.log(`run=${runId ?? "-"}`);
|
||||
console.log(`manifest=${bundle.manifest.manifestHash}`);
|
||||
console.log(
|
||||
`create=${summary.create} update=${summary.update} adopt=${summary.adopt} ` +
|
||||
`skip=${summary.skip} conflict=${summary.conflict} staleMapping=${summary.staleMapping}`,
|
||||
);
|
||||
printWarnings(result);
|
||||
printConflicts(result);
|
||||
printEvents(events);
|
||||
}
|
||||
|
||||
if (conflictCount > 0) {
|
||||
process.exitCode = CLOUD_SYNC_CONFLICT_EXIT_CODE;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function discoverUpstream(remoteUrl: string): Promise<UpstreamDiscovery> {
|
||||
const base = new URL(remoteUrl);
|
||||
const discoveryUrl = new URL("/.well-known/paperclip-upstream", base);
|
||||
return requestCloudJson<UpstreamDiscovery>(discoveryUrl.toString(), { method: "GET" });
|
||||
}
|
||||
|
||||
export function assertDiscoveryCompatible(discovery: UpstreamDiscovery): void {
|
||||
if (discovery.schema !== "paperclip-upstream-discovery-v1") {
|
||||
throw new Error("Remote URL is not a Paperclip Cloud upstream target.");
|
||||
}
|
||||
if (discovery.transfer.supportedSchemaMajor !== upstreamTransferSchema.major) {
|
||||
throw new Error(
|
||||
`Cloud upstream schema mismatch: local major ${upstreamTransferSchema.major}, remote supports ${discovery.transfer.supportedSchemaMajor}.`,
|
||||
);
|
||||
}
|
||||
if (!discovery.transfer.featureFlags?.includes("cloud_sync")) {
|
||||
throw new Error("Remote Paperclip Cloud stack does not advertise the cloud_sync transfer flag.");
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveDeviceCodeExpiresAt(expiresAt: string | undefined, nowMs = Date.now()): number {
|
||||
const parsed = typeof expiresAt === "string" ? Date.parse(expiresAt) : NaN;
|
||||
return Number.isFinite(parsed) ? parsed : nowMs + DEVICE_CODE_FALLBACK_EXPIRES_MS;
|
||||
}
|
||||
|
||||
export async function buildBundleFromLocalCompany(input: {
|
||||
localCompanyId: string;
|
||||
connection: CloudConnection;
|
||||
discovery: UpstreamDiscovery;
|
||||
localApi: {
|
||||
post<T>(path: string, body?: unknown): Promise<T | null>;
|
||||
};
|
||||
maxEntitiesPerChunk?: number;
|
||||
mode: "preview" | "apply";
|
||||
}): Promise<LocalUpstreamExportBundle> {
|
||||
const exported = await input.localApi.post<CompanyPortabilityExportResult>(
|
||||
`/api/companies/${input.localCompanyId}/export`,
|
||||
{
|
||||
include: {
|
||||
company: true,
|
||||
agents: true,
|
||||
projects: true,
|
||||
issues: true,
|
||||
skills: true,
|
||||
},
|
||||
expandReferencedSkills: true,
|
||||
},
|
||||
);
|
||||
if (!exported) throw new Error("Local company export returned no data.");
|
||||
|
||||
const sourceHash = normalizedContentHash({
|
||||
manifest: exported.manifest,
|
||||
files: exported.files,
|
||||
});
|
||||
const source: UpstreamTransferManifestSource = {
|
||||
sourceInstanceId: input.connection.sourceInstanceId,
|
||||
sourceCompanyId: input.localCompanyId,
|
||||
sourceInstanceKeyFingerprint: input.connection.sourceInstanceFingerprint,
|
||||
exporterVersion: "paperclipai-cli-cloud-v1",
|
||||
sourceSchemaVersion: "paperclip-local-portability-v1",
|
||||
};
|
||||
const target: UpstreamTransferManifestTarget = {
|
||||
targetStackId: input.discovery.stack.id,
|
||||
targetCompanyId: input.discovery.stack.companyId,
|
||||
targetOrigin: input.discovery.stack.origin,
|
||||
supportedSchemaMajor: input.discovery.transfer.supportedSchemaMajor,
|
||||
};
|
||||
const entities = buildEntitiesFromPortableExport(input.localCompanyId, input.connection.sourceInstanceId, exported);
|
||||
const idempotencyKey = [
|
||||
input.mode,
|
||||
input.connection.sourceInstanceId,
|
||||
input.localCompanyId,
|
||||
input.discovery.stack.id,
|
||||
sourceHash,
|
||||
].join(":");
|
||||
return buildLocalUpstreamExportBundle({
|
||||
source,
|
||||
target,
|
||||
runId: `local-${input.mode}-${shortHash(idempotencyKey)}`,
|
||||
idempotencyKey,
|
||||
entities,
|
||||
warnings: exported.warnings.map((message): UpstreamTransferWarning => ({
|
||||
code: "local_company_export_warning",
|
||||
severity: "warning",
|
||||
message,
|
||||
})),
|
||||
featureFlags: ["cloud_sync"],
|
||||
maxEntitiesPerChunk: input.maxEntitiesPerChunk,
|
||||
});
|
||||
}
|
||||
|
||||
async function authorizeConnection(
|
||||
discovery: UpstreamDiscovery,
|
||||
source: ReturnType<typeof createSourceIdentity>,
|
||||
opts: { noBrowser: boolean },
|
||||
): Promise<TokenResponse> {
|
||||
if (!opts.noBrowser && canOpenBrowser() && discovery.auth.pkce) {
|
||||
try {
|
||||
return await authorizeWithBrowser(discovery, source);
|
||||
} catch (error) {
|
||||
console.error(pc.yellow(`Browser authorization failed; falling back to device-code flow. ${errorMessage(error)}`));
|
||||
}
|
||||
}
|
||||
if (!discovery.auth.deviceCode) {
|
||||
throw new Error("Remote Paperclip Cloud stack does not support device-code authorization.");
|
||||
}
|
||||
return authorizeWithDeviceCode(discovery, source, { openBrowser: !opts.noBrowser && canOpenBrowser() });
|
||||
}
|
||||
|
||||
async function authorizeWithBrowser(
|
||||
discovery: UpstreamDiscovery,
|
||||
source: ReturnType<typeof createSourceIdentity>,
|
||||
): Promise<TokenResponse> {
|
||||
const pkce = discovery.auth.pkce;
|
||||
if (!pkce) throw new Error("Remote did not advertise PKCE authorization.");
|
||||
const callback = await startPkceCallbackServer();
|
||||
const verifier = randomBytes(32).toString("base64url");
|
||||
const challenge = createHash("sha256").update(verifier).digest("base64url");
|
||||
const state = randomUUID();
|
||||
const authorizeUrl = new URL(pkce.authorizeUrl);
|
||||
authorizeUrl.searchParams.set("redirectUri", callback.redirectUri);
|
||||
authorizeUrl.searchParams.set("state", state);
|
||||
authorizeUrl.searchParams.set("codeChallenge", challenge);
|
||||
authorizeUrl.searchParams.set("codeChallengeMethod", "S256");
|
||||
authorizeUrl.searchParams.set("sourceInstanceId", source.sourceInstanceId);
|
||||
authorizeUrl.searchParams.set("sourceInstanceFingerprint", source.sourceInstanceFingerprint);
|
||||
authorizeUrl.searchParams.set("sourcePublicKey", source.sourcePublicKey);
|
||||
authorizeUrl.searchParams.set("scopes", CLOUD_SYNC_SCOPES.join(" "));
|
||||
|
||||
try {
|
||||
console.error(`Open this URL to approve cloud sync:\n${authorizeUrl.toString()}`);
|
||||
if (!openUrl(authorizeUrl.toString())) {
|
||||
throw new Error("Could not open a browser.");
|
||||
}
|
||||
const code = await callback.waitForCode(state);
|
||||
return requestCloudJson<TokenResponse>(pkce.tokenUrl, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
grantType: "authorization_code",
|
||||
code,
|
||||
redirectUri: callback.redirectUri,
|
||||
codeVerifier: verifier,
|
||||
}),
|
||||
});
|
||||
} finally {
|
||||
await callback.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function authorizeWithDeviceCode(
|
||||
discovery: UpstreamDiscovery,
|
||||
source: ReturnType<typeof createSourceIdentity>,
|
||||
opts: { openBrowser: boolean },
|
||||
): Promise<TokenResponse> {
|
||||
const device = discovery.auth.deviceCode;
|
||||
if (!device) throw new Error("Remote did not advertise device-code authorization.");
|
||||
const response = await requestCloudJson<{
|
||||
deviceCode: string;
|
||||
userCode: string;
|
||||
verificationUri: string;
|
||||
expiresAt?: string;
|
||||
intervalSeconds?: number;
|
||||
}>(device.deviceCodeUrl, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
stackId: discovery.stack.id,
|
||||
sourceInstanceId: source.sourceInstanceId,
|
||||
sourceInstanceFingerprint: source.sourceInstanceFingerprint,
|
||||
sourcePublicKey: source.sourcePublicKey,
|
||||
scopes: CLOUD_SYNC_SCOPES,
|
||||
}),
|
||||
});
|
||||
console.error(pc.bold("Cloud device authorization required"));
|
||||
console.error(`Open: ${response.verificationUri}`);
|
||||
console.error(`Code: ${response.userCode}`);
|
||||
if (opts.openBrowser) openUrl(response.verificationUri);
|
||||
|
||||
const expiresAt = resolveDeviceCodeExpiresAt(response.expiresAt);
|
||||
const intervalMs = Math.max(500, (response.intervalSeconds ?? 5) * 1000);
|
||||
while (Date.now() < expiresAt) {
|
||||
await sleep(intervalMs);
|
||||
try {
|
||||
return await requestCloudJson<TokenResponse>(device.tokenUrl, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
grantType: "device_code",
|
||||
deviceCode: response.deviceCode,
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof CloudAuthRequestError && error.body && typeof error.body === "object") {
|
||||
const code = (error.body as { error?: unknown }).error;
|
||||
if (code === "authorization_pending") continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error("Device-code authorization expired before it was approved.");
|
||||
}
|
||||
|
||||
function buildEntitiesFromPortableExport(
|
||||
localCompanyId: string,
|
||||
sourceInstanceId: string,
|
||||
exported: CompanyPortabilityExportResult,
|
||||
): LocalUpstreamExportEntityInput[] {
|
||||
const companyKey: SourceEntityKey = {
|
||||
sourceInstanceId,
|
||||
sourceCompanyId: localCompanyId,
|
||||
sourceEntityType: "company",
|
||||
sourceEntityId: localCompanyId,
|
||||
sourceNaturalKey: exported.manifest.company?.name ?? localCompanyId,
|
||||
};
|
||||
const entities: LocalUpstreamExportEntityInput[] = [
|
||||
{
|
||||
key: companyKey,
|
||||
body: {
|
||||
kind: "paperclip_company_portability_manifest",
|
||||
manifest: exported.manifest,
|
||||
rootPath: exported.rootPath,
|
||||
paperclipExtensionPath: exported.paperclipExtensionPath,
|
||||
fileCount: Object.keys(exported.files).length,
|
||||
},
|
||||
conflictKeys: [`company:${companyKey.sourceNaturalKey ?? localCompanyId}`],
|
||||
},
|
||||
];
|
||||
|
||||
for (const [filePath, entry] of Object.entries(exported.files).sort(([left], [right]) => left.localeCompare(right))) {
|
||||
entities.push({
|
||||
key: {
|
||||
sourceInstanceId,
|
||||
sourceCompanyId: localCompanyId,
|
||||
sourceEntityType: "company_setting",
|
||||
sourceEntityId: shortHash(filePath),
|
||||
sourceNaturalKey: filePath,
|
||||
},
|
||||
body: {
|
||||
kind: "paperclip_portable_file",
|
||||
path: filePath,
|
||||
entry: normalizePortableFileEntry(entry),
|
||||
},
|
||||
dependencies: [companyKey],
|
||||
conflictKeys: [`portable_file:${filePath}`],
|
||||
});
|
||||
}
|
||||
return entities;
|
||||
}
|
||||
|
||||
function normalizePortableFileEntry(entry: CompanyPortabilityFileEntry): Record<string, unknown> {
|
||||
if (typeof entry === "string") {
|
||||
return { encoding: "utf8", data: entry };
|
||||
}
|
||||
return { ...entry };
|
||||
}
|
||||
|
||||
async function assertCloudSyncEnabled(settingsPromise: Promise<InstanceExperimentalSettings | null>): Promise<void> {
|
||||
const settings = await settingsPromise;
|
||||
if (settings?.enableCloudSync !== true) {
|
||||
throw new Error(
|
||||
"Cloud sync is disabled. Enable the cloud sync experimental setting before running `paperclipai cloud push`.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function cloudProofHeaders(connection: CloudConnection, method: string, pathAndSearch: string): Record<string, string> {
|
||||
const timestamp = new Date().toISOString();
|
||||
const nonce = randomUUID();
|
||||
const payload = [
|
||||
method,
|
||||
connection.targetHost.toLowerCase(),
|
||||
pathAndSearch,
|
||||
connection.token.id,
|
||||
connection.sourceInstanceId,
|
||||
timestamp,
|
||||
nonce,
|
||||
].join("\n");
|
||||
return {
|
||||
Authorization: `Bearer ${connection.accessToken}`,
|
||||
"X-Paperclip-Upstream-Source-Instance-Id": connection.sourceInstanceId,
|
||||
"X-Paperclip-Upstream-Proof-Timestamp": timestamp,
|
||||
"X-Paperclip-Upstream-Proof-Nonce": nonce,
|
||||
"X-Paperclip-Upstream-Proof-Signature": sign(
|
||||
null,
|
||||
Buffer.from(payload, "utf8"),
|
||||
connection.privateKeyPem,
|
||||
).toString("base64url"),
|
||||
};
|
||||
}
|
||||
|
||||
async function requestCloudJson<T>(url: string, init: RequestInit): Promise<T> {
|
||||
const headers = new Headers(init.headers);
|
||||
headers.set("accept", "application/json");
|
||||
if (init.body !== undefined && !headers.has("content-type")) {
|
||||
headers.set("content-type", "application/json");
|
||||
}
|
||||
const response = await fetch(url, { ...init, headers });
|
||||
const text = await response.text();
|
||||
const parsed = text.trim() ? JSON.parse(text) as unknown : {};
|
||||
if (!response.ok) {
|
||||
const message = typeof parsed === "object" && parsed !== null && "error" in parsed
|
||||
? String((parsed as { error: unknown }).error)
|
||||
: `Cloud request failed with ${response.status}`;
|
||||
throw new CloudAuthRequestError(response.status, message, parsed);
|
||||
}
|
||||
return parsed as T;
|
||||
}
|
||||
|
||||
function createSourceIdentity() {
|
||||
const { publicKey, privateKey } = generateKeyPairSync("ed25519");
|
||||
const sourcePublicKey = publicKey.export({ type: "spki", format: "pem" }).toString();
|
||||
const sourceInstanceFingerprint = `sha256:${createHash("sha256")
|
||||
.update(publicKey.export({ type: "spki", format: "der" }))
|
||||
.digest("hex")}`;
|
||||
return {
|
||||
sourceInstanceId: `paperclip-local-${resolvePaperclipInstanceId()}`,
|
||||
sourceInstanceFingerprint,
|
||||
sourcePublicKey,
|
||||
privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
|
||||
};
|
||||
}
|
||||
|
||||
async function startPkceCallbackServer(): Promise<{
|
||||
redirectUri: string;
|
||||
waitForCode: (state: string) => Promise<string>;
|
||||
close: () => Promise<void>;
|
||||
}> {
|
||||
let resolveCode: ((code: string) => void) | null = null;
|
||||
let rejectCode: ((error: Error) => void) | null = null;
|
||||
let expectedState = "";
|
||||
const codePromise = new Promise<string>((resolve, reject) => {
|
||||
resolveCode = resolve;
|
||||
rejectCode = reject;
|
||||
});
|
||||
const server = createServer((req, res) => {
|
||||
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
||||
const code = url.searchParams.get("code");
|
||||
const state = url.searchParams.get("state");
|
||||
if (!code || state !== expectedState) {
|
||||
res.writeHead(400, { "Content-Type": "text/plain" });
|
||||
res.end("Paperclip Cloud authorization failed. You can close this tab.");
|
||||
rejectCode?.(new Error("Authorization callback was missing a valid code or state."));
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { "Content-Type": "text/plain" });
|
||||
res.end("Paperclip Cloud authorization complete. You can close this tab.");
|
||||
resolveCode?.(code);
|
||||
});
|
||||
await listenOnLoopback(server);
|
||||
const address = server.address();
|
||||
if (typeof address !== "object" || !address?.port) {
|
||||
throw new Error("Failed to start local authorization callback server.");
|
||||
}
|
||||
return {
|
||||
redirectUri: `http://127.0.0.1:${address.port}/cloud/callback`,
|
||||
waitForCode: (state: string) => {
|
||||
expectedState = state;
|
||||
return codePromise;
|
||||
},
|
||||
close: () => closeServer(server),
|
||||
};
|
||||
}
|
||||
|
||||
function listenOnLoopback(server: Server): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
server.off("error", reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function closeServer(server: Server): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
server.close((error) => error ? reject(error) : resolve());
|
||||
});
|
||||
}
|
||||
|
||||
function canOpenBrowser(): boolean {
|
||||
if (process.platform === "darwin" || process.platform === "win32") return true;
|
||||
return Boolean(process.env.DISPLAY || process.env.WAYLAND_DISPLAY);
|
||||
}
|
||||
|
||||
function summarizeResult(result: unknown): {
|
||||
create: number;
|
||||
update: number;
|
||||
adopt: number;
|
||||
skip: number;
|
||||
conflict: number;
|
||||
staleMapping: number;
|
||||
} {
|
||||
const summary = asRecord(asRecord(result)?.summary);
|
||||
return {
|
||||
create: numberValue(summary?.create),
|
||||
update: numberValue(summary?.update),
|
||||
adopt: numberValue(summary?.adopt),
|
||||
skip: numberValue(summary?.skip),
|
||||
conflict: numberValue(summary?.conflict),
|
||||
staleMapping: numberValue(summary?.staleMapping),
|
||||
};
|
||||
}
|
||||
|
||||
function printWarnings(result: unknown): void {
|
||||
const warnings = Array.isArray(asRecord(result)?.warnings) ? asRecord(result)?.warnings as unknown[] : [];
|
||||
for (const warning of warnings) {
|
||||
const record = asRecord(warning);
|
||||
console.log(pc.yellow(`warning=${record?.code ?? "warning"} ${record?.message ?? ""}`.trim()));
|
||||
}
|
||||
}
|
||||
|
||||
function printConflicts(result: unknown): void {
|
||||
const conflicts = Array.isArray(asRecord(result)?.conflicts) ? asRecord(result)?.conflicts as unknown[] : [];
|
||||
for (const conflict of conflicts.slice(0, 10)) {
|
||||
const record = asRecord(conflict);
|
||||
console.log(pc.red(`conflict=${record?.conflictKind ?? "target_conflict"} target=${record?.targetEntityId ?? "-"}`));
|
||||
}
|
||||
if (conflicts.length > 10) console.log(pc.red(`conflicts_truncated=${conflicts.length - 10}`));
|
||||
}
|
||||
|
||||
function printEvents(events: unknown): void {
|
||||
const rows = Array.isArray(asRecord(events)?.events) ? asRecord(events)?.events as unknown[] : [];
|
||||
for (const row of rows.slice(-10)) {
|
||||
const event = asRecord(row);
|
||||
console.log(pc.dim(`event=${event?.action ?? "-"} target=${event?.targetEntityId ?? "-"}`));
|
||||
}
|
||||
}
|
||||
|
||||
function getRunId(result: unknown): string | null {
|
||||
const run = asRecord(asRecord(result)?.run);
|
||||
return typeof run?.id === "string" ? run.id : null;
|
||||
}
|
||||
|
||||
function redactConnection(connection: CloudConnection): Record<string, unknown> {
|
||||
return {
|
||||
id: connection.id,
|
||||
remoteUrl: connection.remoteUrl,
|
||||
targetOrigin: connection.targetOrigin,
|
||||
stackId: connection.stackId,
|
||||
targetCompanyId: connection.targetCompanyId,
|
||||
scopes: connection.scopes,
|
||||
expiresAt: connection.token.expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
function connectionId(targetOrigin: string): string {
|
||||
return `cloud-${shortHash(targetOrigin)}`;
|
||||
}
|
||||
|
||||
function shortHash(value: string): string {
|
||||
return createHash("sha256").update(value).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
function requiredString(value: unknown, label: string): string {
|
||||
if (typeof value === "string" && value.trim()) return value.trim();
|
||||
throw new Error(`${label} is required.`);
|
||||
}
|
||||
|
||||
function numberValue(value: unknown): number {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: null;
|
||||
}
|
||||
|
||||
function isSchemaMismatchError(error: unknown): boolean {
|
||||
if (error instanceof UpstreamImportRequestError) {
|
||||
return JSON.stringify(error.body).toLowerCase().includes("schema");
|
||||
}
|
||||
return error instanceof Error && error.message.toLowerCase().includes("schema mismatch");
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export const cloudCommandExitCodes = {
|
||||
conflict: CLOUD_SYNC_CONFLICT_EXIT_CODE,
|
||||
schemaMismatch: CLOUD_SYNC_SCHEMA_MISMATCH_EXIT_CODE,
|
||||
} as const;
|
||||
|
|
@ -19,6 +19,7 @@ import { registerDashboardCommands } from "./commands/client/dashboard.js";
|
|||
import { registerRoutineCommands } from "./commands/routines.js";
|
||||
import { registerFeedbackCommands } from "./commands/client/feedback.js";
|
||||
import { registerSecretCommands } from "./commands/client/secrets.js";
|
||||
import { registerCloudCommands } from "./commands/client/cloud.js";
|
||||
import { applyDataDirOverride, type DataDirOptionLike } from "./config/data-dir.js";
|
||||
import { loadPaperclipEnvFile } from "./config/env.js";
|
||||
import { initTelemetryFromConfigFile, flushTelemetry } from "./telemetry.js";
|
||||
|
|
@ -149,6 +150,7 @@ registerDashboardCommands(program);
|
|||
registerRoutineCommands(program);
|
||||
registerFeedbackCommands(program);
|
||||
registerSecretCommands(program);
|
||||
registerCloudCommands(program);
|
||||
registerWorktreeCommands(program);
|
||||
registerEnvLabCommands(program);
|
||||
registerPluginCommands(program);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue