mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-14 01:50:39 +09:00
Harden remote workspace sync and restore flows (#5444)
## Thinking Path
> - Paperclip orchestrates AI agents for zero-human companies
> - When an agent runs against a remote target, Paperclip syncs the
workspace out to the remote at run start and restores changes back to
the local workspace at run end
> - The previous restore flow naïvely overwrote local files with
whatever the remote returned, so files that the remote run never touched
but had timestamp/mode drift could be needlessly rewritten — and a
single static `refs/paperclip/ssh-sync/imported` ref made concurrent SSH
workspace exports race on the same git ref
> - This pull request adds a `workspace-restore-merge` module that diffs
a pre-run snapshot against the post-run remote state and only writes
back files the remote actually changed; SSH workspace exports now use a
per-import unique ref so concurrent runs can't trample each other
> - Every adapter's execute path threads the snapshot through
`prepareAdapterExecutionTargetRuntime` so the merge has the baseline it
needs
> - The benefit is workspace restores no longer churn untouched files,
and concurrent SSH runs no longer collide on the import ref
## What Changed
- `packages/adapter-utils/src/workspace-restore-merge.{ts,test.ts}`: new
module — directory snapshot (kind/mode/sha256/symlink target) plus
snapshot-aware merge that writes only the files the remote changed
- `packages/adapter-utils/src/ssh.ts`: SSH workspace export uses a
per-import unique ref (`refs/paperclip/ssh-sync/imported/<uuid>`);
restore goes through the new merge helper; `ssh-fixture.test.ts` covers
the unique-ref + merge paths
- `packages/adapter-utils/src/sandbox-managed-runtime.ts` +
`remote-managed-runtime.ts`: thread the snapshot/merge through the
sandbox and SSH paths
- `packages/adapter-utils/src/server-utils.{ts,test.ts}` +
`execution-target.ts`: helpers for capturing the pre-run snapshot;
`prepareAdapterExecutionTargetRuntime` gains required `runId` and
optional `workspaceRemoteDir`, and returns the realized
`workspaceRemoteDir`
- Each adapter's `execute.ts` (acpx, claude, codex, cursor, gemini,
opencode, pi) takes the snapshot at run start and passes it through to
the runtime restore
- Remote execute test mocks updated to match the new
`prepareWorkspaceForSshExecution` return shape and the per-run
`${managedRemoteWorkspace}` cwd subdirectory
## Verification
- `pnpm vitest run --no-coverage --project @paperclipai/adapter-utils
--project @paperclipai/adapter-acpx-local --project
@paperclipai/adapter-claude-local --project
@paperclipai/adapter-codex-local --project
@paperclipai/adapter-cursor-local --project
@paperclipai/adapter-gemini-local --project
@paperclipai/adapter-opencode-local --project
@paperclipai/adapter-pi-local` — 196/196 passing
- `pnpm typecheck` clean across the workspace
## Risks
Medium. The restore path now writes a strict subset of what it
previously did — files the remote did not touch are no longer rewritten.
If any flow was relying on a touch-without-content-change being copied
back (timestamp or permission propagation only), that behavior is now
skipped. Snapshot capture adds an O(N-files-in-workspace) hash pass at
run start; the cost is bounded by the existing exclude list. The `runId`
parameter on `prepareAdapterExecutionTargetRuntime` is now required —
every in-tree caller is updated; out-of-tree adapter authors need to
pass it.
## Model Used
Claude Opus 4.7 (1M context)
## Checklist
- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable — new module +
every adapter execute path covered
- [x] If this change affects the UI, I have included before/after
screenshots — N/A (no UI)
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] I will address all Greptile and reviewer comments before
requesting merge
This commit is contained in:
parent
824298f414
commit
12cb7b40fd
23 changed files with 1234 additions and 183 deletions
|
|
@ -67,6 +67,7 @@ export type AdapterManagedRuntimeAsset = RemoteManagedRuntimeAsset;
|
|||
|
||||
export interface PreparedAdapterExecutionTargetRuntime {
|
||||
target: AdapterExecutionTarget;
|
||||
workspaceRemoteDir: string | null;
|
||||
runtimeRootDir: string | null;
|
||||
assetDirs: Record<string, string>;
|
||||
restoreWorkspace(): Promise<void>;
|
||||
|
|
@ -167,6 +168,33 @@ export function adapterExecutionTargetRemoteCwd(
|
|||
return target?.kind === "remote" ? target.remoteCwd : localCwd;
|
||||
}
|
||||
|
||||
export function overrideAdapterExecutionTargetRemoteCwd(
|
||||
target: AdapterExecutionTarget | null | undefined,
|
||||
remoteCwd: string | null | undefined,
|
||||
): AdapterExecutionTarget | null | undefined {
|
||||
const nextRemoteCwd = remoteCwd?.trim();
|
||||
if (!target || target.kind !== "remote" || !nextRemoteCwd) {
|
||||
return target;
|
||||
}
|
||||
if (target.remoteCwd === nextRemoteCwd) {
|
||||
return target;
|
||||
}
|
||||
if (target.transport === "ssh") {
|
||||
return {
|
||||
...target,
|
||||
remoteCwd: nextRemoteCwd,
|
||||
spec: {
|
||||
...target.spec,
|
||||
remoteCwd: nextRemoteCwd,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
...target,
|
||||
remoteCwd: nextRemoteCwd,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveAdapterExecutionTargetCwd(
|
||||
target: AdapterExecutionTarget | null | undefined,
|
||||
configuredCwd: string | null | undefined,
|
||||
|
|
@ -858,9 +886,11 @@ export function readAdapterExecutionTarget(input: {
|
|||
}
|
||||
|
||||
export async function prepareAdapterExecutionTargetRuntime(input: {
|
||||
runId: string;
|
||||
target: AdapterExecutionTarget | null | undefined;
|
||||
adapterKey: string;
|
||||
workspaceLocalDir: string;
|
||||
workspaceRemoteDir?: string;
|
||||
workspaceExclude?: string[];
|
||||
preserveAbsentOnRestore?: string[];
|
||||
assets?: AdapterManagedRuntimeAsset[];
|
||||
|
|
@ -872,6 +902,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
if (target.kind === "local") {
|
||||
return {
|
||||
target,
|
||||
workspaceRemoteDir: null,
|
||||
runtimeRootDir: null,
|
||||
assetDirs: {},
|
||||
restoreWorkspace: async () => {},
|
||||
|
|
@ -881,12 +912,15 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
if (target.transport === "ssh") {
|
||||
const prepared = await prepareRemoteManagedRuntime({
|
||||
spec: target.spec,
|
||||
runId: input.runId,
|
||||
adapterKey: input.adapterKey,
|
||||
workspaceLocalDir: input.workspaceLocalDir,
|
||||
workspaceRemoteDir: input.workspaceRemoteDir,
|
||||
assets: input.assets,
|
||||
});
|
||||
return {
|
||||
target,
|
||||
workspaceRemoteDir: prepared.workspaceRemoteDir,
|
||||
runtimeRootDir: prepared.runtimeRootDir,
|
||||
assetDirs: prepared.assetDirs,
|
||||
restoreWorkspace: prepared.restoreWorkspace,
|
||||
|
|
@ -904,6 +938,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
},
|
||||
adapterKey: input.adapterKey,
|
||||
workspaceLocalDir: input.workspaceLocalDir,
|
||||
workspaceRemoteDir: input.workspaceRemoteDir,
|
||||
workspaceExclude: input.workspaceExclude,
|
||||
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
|
||||
assets: input.assets,
|
||||
|
|
@ -912,6 +947,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
});
|
||||
return {
|
||||
target,
|
||||
workspaceRemoteDir: prepared.workspaceRemoteDir,
|
||||
runtimeRootDir: prepared.runtimeRootDir,
|
||||
assetDirs: prepared.assetDirs,
|
||||
restoreWorkspace: prepared.restoreWorkspace,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
restoreWorkspaceFromSshExecution,
|
||||
syncDirectoryToSsh,
|
||||
} from "./ssh.js";
|
||||
import { captureDirectorySnapshot } from "./workspace-restore-merge.js";
|
||||
|
||||
export interface RemoteManagedRuntimeAsset {
|
||||
key: string;
|
||||
|
|
@ -63,19 +64,31 @@ export function remoteExecutionSessionMatches(saved: unknown, current: SshRemote
|
|||
|
||||
export async function prepareRemoteManagedRuntime(input: {
|
||||
spec: SshRemoteExecutionSpec;
|
||||
runId: string;
|
||||
adapterKey: string;
|
||||
workspaceLocalDir: string;
|
||||
workspaceRemoteDir?: string;
|
||||
assets?: RemoteManagedRuntimeAsset[];
|
||||
}): Promise<PreparedRemoteManagedRuntime> {
|
||||
const workspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd;
|
||||
const baseWorkspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd;
|
||||
const workspaceRemoteDir = path.posix.join(
|
||||
baseWorkspaceRemoteDir,
|
||||
".paperclip-runtime",
|
||||
"runs",
|
||||
input.runId,
|
||||
"workspace",
|
||||
);
|
||||
const runtimeRootDir = path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey);
|
||||
|
||||
await prepareWorkspaceForSshExecution({
|
||||
const preparedWorkspace = await prepareWorkspaceForSshExecution({
|
||||
spec: input.spec,
|
||||
localDir: input.workspaceLocalDir,
|
||||
remoteDir: workspaceRemoteDir,
|
||||
});
|
||||
const restoreExclude = preparedWorkspace.gitBacked ? [".git", ".paperclip-runtime"] : [".paperclip-runtime"];
|
||||
const baselineSnapshot = await captureDirectorySnapshot(input.workspaceLocalDir, {
|
||||
exclude: restoreExclude,
|
||||
});
|
||||
|
||||
const assetDirs: Record<string, string> = {};
|
||||
try {
|
||||
|
|
@ -95,6 +108,8 @@ export async function prepareRemoteManagedRuntime(input: {
|
|||
spec: input.spec,
|
||||
localDir: input.workspaceLocalDir,
|
||||
remoteDir: workspaceRemoteDir,
|
||||
baselineSnapshot,
|
||||
restoreGitHistory: preparedWorkspace.gitBacked,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
|
|
@ -110,6 +125,8 @@ export async function prepareRemoteManagedRuntime(input: {
|
|||
spec: input.spec,
|
||||
localDir: input.workspaceLocalDir,
|
||||
remoteDir: workspaceRemoteDir,
|
||||
baselineSnapshot,
|
||||
restoreGitHistory: preparedWorkspace.gitBacked,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ describe("sandbox managed runtime", () => {
|
|||
|
||||
await expect(readFile(path.join(localWorkspaceDir, "README.md"), "utf8")).resolves.toBe("remote workspace\n");
|
||||
await expect(readFile(path.join(localWorkspaceDir, "remote-only.txt"), "utf8")).resolves.toBe("sync back\n");
|
||||
await expect(readFile(path.join(localWorkspaceDir, "local-stale.txt"), "utf8")).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(readFile(path.join(localWorkspaceDir, "local-stale.txt"), "utf8")).resolves.toBe("remove\n");
|
||||
await expect(readFile(path.join(localWorkspaceDir, ".claude", "settings.json"), "utf8")).resolves.toBe("{\"local\":true}\n");
|
||||
await expect(readFile(path.join(localWorkspaceDir, ".paperclip-runtime", "state.json"), "utf8")).resolves.toBe("{}\n");
|
||||
});
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { constants as fsConstants, promises as fs } from "node:fs";
|
|||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { captureDirectorySnapshot, mergeDirectoryWithBaseline } from "./workspace-restore-merge.js";
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
|
||||
|
|
@ -248,6 +249,9 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
}): Promise<PreparedSandboxManagedRuntime> {
|
||||
const workspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd;
|
||||
const runtimeRootDir = path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey);
|
||||
const baselineSnapshot = await captureDirectorySnapshot(input.workspaceLocalDir, {
|
||||
exclude: [...new Set([".paperclip-runtime", ...(input.preserveAbsentOnRestore ?? []), ...(input.workspaceExclude ?? [])])],
|
||||
});
|
||||
|
||||
await withTempDir("paperclip-sandbox-sync-", async (tempDir) => {
|
||||
const workspaceTarPath = path.join(tempDir, "workspace.tar");
|
||||
|
|
@ -326,8 +330,10 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
archivePath: localArchivePath,
|
||||
localDir: extractedDir,
|
||||
});
|
||||
await mirrorDirectory(extractedDir, input.workspaceLocalDir, {
|
||||
preserveAbsent: [".paperclip-runtime", ...(input.preserveAbsentOnRestore ?? [])],
|
||||
await mergeDirectoryWithBaseline({
|
||||
baseline: baselineSnapshot,
|
||||
sourceDir: extractedDir,
|
||||
targetDir: input.workspaceLocalDir,
|
||||
});
|
||||
});
|
||||
},
|
||||
|
|
|
|||
|
|
@ -9,11 +9,13 @@ import {
|
|||
buildInvocationEnvForLogs,
|
||||
DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE,
|
||||
materializePaperclipSkillCopy,
|
||||
refreshPaperclipWorkspaceEnvForExecution,
|
||||
renderPaperclipWakePrompt,
|
||||
runningProcesses,
|
||||
runChildProcess,
|
||||
sanitizeSshRemoteEnv,
|
||||
shapePaperclipWorkspaceEnvForExecution,
|
||||
rewriteWorkspaceCwdEnvVarsForExecution,
|
||||
stringifyPaperclipWakePayload,
|
||||
} from "./server-utils.js";
|
||||
|
||||
|
|
@ -810,6 +812,99 @@ describe("shapePaperclipWorkspaceEnvForExecution", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("rewriteWorkspaceCwdEnvVarsForExecution", () => {
|
||||
it("rewrites custom *_WORKSPACE_CWD env vars for remote execution", () => {
|
||||
const env = rewriteWorkspaceCwdEnvVarsForExecution({
|
||||
workspaceCwd: "/host/workspace",
|
||||
executionCwd: "/remote/workspace",
|
||||
executionTargetIsRemote: true,
|
||||
env: {
|
||||
QA_PROJECT_WORKSPACE_CWD: "/host/workspace",
|
||||
RANDOM_WORKSPACE_CWD: "/host/workspace",
|
||||
OTHER_ENV: "/host/workspace",
|
||||
},
|
||||
});
|
||||
|
||||
expect(env).toEqual({
|
||||
QA_PROJECT_WORKSPACE_CWD: "/remote/workspace",
|
||||
RANDOM_WORKSPACE_CWD: "/remote/workspace",
|
||||
OTHER_ENV: "/host/workspace",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not rewrite matching values for local execution", () => {
|
||||
const env = rewriteWorkspaceCwdEnvVarsForExecution({
|
||||
workspaceCwd: "/host/workspace",
|
||||
executionCwd: "/remote/workspace",
|
||||
executionTargetIsRemote: false,
|
||||
env: {
|
||||
QA_PROJECT_WORKSPACE_CWD: "/host/workspace",
|
||||
RANDOM_WORKSPACE_CWD_TOKEN: "/host/workspace",
|
||||
},
|
||||
});
|
||||
|
||||
expect(env).toEqual({
|
||||
QA_PROJECT_WORKSPACE_CWD: "/host/workspace",
|
||||
RANDOM_WORKSPACE_CWD_TOKEN: "/host/workspace",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("refreshPaperclipWorkspaceEnvForExecution", () => {
|
||||
it("rewrites Paperclip workspace env to the prepared remote runtime cwd", () => {
|
||||
const env: Record<string, string> = {
|
||||
PAPERCLIP_WORKSPACE_CWD: "/remote/workspace",
|
||||
PAPERCLIP_WORKSPACE_WORKTREE_PATH: "/host/worktree",
|
||||
PAPERCLIP_WORKSPACES_JSON: JSON.stringify([
|
||||
{ workspaceId: "workspace-1", cwd: "/remote/workspace" },
|
||||
{ workspaceId: "workspace-2", cwd: "/tmp/other" },
|
||||
]),
|
||||
QA_PROJECT_WORKSPACE_CWD: "/remote/workspace",
|
||||
};
|
||||
|
||||
const shaped = refreshPaperclipWorkspaceEnvForExecution({
|
||||
env,
|
||||
envConfig: {
|
||||
QA_PROJECT_WORKSPACE_CWD: "/host/workspace",
|
||||
},
|
||||
workspaceCwd: "/host/workspace",
|
||||
workspaceWorktreePath: "/host/worktree",
|
||||
workspaceHints: [
|
||||
{ workspaceId: "workspace-1", cwd: "/host/workspace" },
|
||||
{ workspaceId: "workspace-2", cwd: "/tmp/other" },
|
||||
],
|
||||
executionTargetIsRemote: true,
|
||||
executionCwd: "/remote/workspace/.paperclip-runtime/runs/run-1/workspace",
|
||||
});
|
||||
|
||||
expect(shaped).toEqual({
|
||||
workspaceCwd: "/remote/workspace/.paperclip-runtime/runs/run-1/workspace",
|
||||
workspaceWorktreePath: null,
|
||||
workspaceHints: [
|
||||
{
|
||||
workspaceId: "workspace-1",
|
||||
cwd: "/remote/workspace/.paperclip-runtime/runs/run-1/workspace",
|
||||
},
|
||||
{
|
||||
workspaceId: "workspace-2",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(env.PAPERCLIP_WORKSPACE_CWD).toBe("/remote/workspace/.paperclip-runtime/runs/run-1/workspace");
|
||||
expect(env.PAPERCLIP_WORKSPACE_WORKTREE_PATH).toBeUndefined();
|
||||
expect(env.QA_PROJECT_WORKSPACE_CWD).toBe("/remote/workspace/.paperclip-runtime/runs/run-1/workspace");
|
||||
expect(JSON.parse(env.PAPERCLIP_WORKSPACES_JSON ?? "[]")).toEqual([
|
||||
{
|
||||
workspaceId: "workspace-1",
|
||||
cwd: "/remote/workspace/.paperclip-runtime/runs/run-1/workspace",
|
||||
},
|
||||
{
|
||||
workspaceId: "workspace-2",
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendWithByteCap", () => {
|
||||
it("keeps valid UTF-8 when trimming through multibyte text", () => {
|
||||
const output = appendWithByteCap("prefix ", "hello — world", 7);
|
||||
|
|
|
|||
|
|
@ -999,6 +999,99 @@ export function shapePaperclipWorkspaceEnvForExecution(input: {
|
|||
};
|
||||
}
|
||||
|
||||
export function rewriteWorkspaceCwdEnvVarsForExecution(input: {
|
||||
env: Record<string, unknown>;
|
||||
workspaceCwd?: string | null;
|
||||
executionCwd?: string | null;
|
||||
executionTargetIsRemote?: boolean;
|
||||
}): Record<string, string> {
|
||||
const nextEnv = Object.fromEntries(
|
||||
Object.entries(input.env)
|
||||
.filter((entry): entry is [string, string] => typeof entry[1] === "string"),
|
||||
) as Record<string, string>;
|
||||
const localWorkspaceCwd = typeof input.workspaceCwd === "string" && input.workspaceCwd.trim().length > 0
|
||||
? path.resolve(input.workspaceCwd)
|
||||
: null;
|
||||
const remoteWorkspaceCwd = typeof input.executionCwd === "string" && input.executionCwd.trim().length > 0
|
||||
? path.resolve(input.executionCwd)
|
||||
: null;
|
||||
|
||||
if (!input.executionTargetIsRemote || !localWorkspaceCwd || !remoteWorkspaceCwd) {
|
||||
return nextEnv;
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(nextEnv)) {
|
||||
if (!key.endsWith("_WORKSPACE_CWD")) continue;
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) continue;
|
||||
if (path.resolve(trimmed) !== localWorkspaceCwd) continue;
|
||||
nextEnv[key] = remoteWorkspaceCwd;
|
||||
}
|
||||
|
||||
return nextEnv;
|
||||
}
|
||||
|
||||
export function refreshPaperclipWorkspaceEnvForExecution(input: {
|
||||
env: Record<string, string>;
|
||||
envConfig?: Record<string, unknown>;
|
||||
workspaceCwd?: string | null;
|
||||
workspaceSource?: string | null;
|
||||
workspaceStrategy?: string | null;
|
||||
workspaceId?: string | null;
|
||||
workspaceRepoUrl?: string | null;
|
||||
workspaceRepoRef?: string | null;
|
||||
workspaceBranch?: string | null;
|
||||
workspaceWorktreePath?: string | null;
|
||||
workspaceHints?: Array<Record<string, unknown>>;
|
||||
agentHome?: string | null;
|
||||
executionTargetIsRemote?: boolean;
|
||||
executionCwd?: string | null;
|
||||
}): {
|
||||
workspaceCwd: string | null;
|
||||
workspaceWorktreePath: string | null;
|
||||
workspaceHints: Array<Record<string, unknown>>;
|
||||
} {
|
||||
const shapedWorkspaceEnv = shapePaperclipWorkspaceEnvForExecution({
|
||||
workspaceCwd: input.workspaceCwd,
|
||||
workspaceWorktreePath: input.workspaceWorktreePath,
|
||||
workspaceHints: input.workspaceHints,
|
||||
executionTargetIsRemote: input.executionTargetIsRemote,
|
||||
executionCwd: input.executionCwd,
|
||||
});
|
||||
|
||||
delete input.env.PAPERCLIP_WORKSPACE_CWD;
|
||||
delete input.env.PAPERCLIP_WORKSPACE_WORKTREE_PATH;
|
||||
delete input.env.PAPERCLIP_WORKSPACES_JSON;
|
||||
|
||||
applyPaperclipWorkspaceEnv(input.env, {
|
||||
workspaceCwd: shapedWorkspaceEnv.workspaceCwd,
|
||||
workspaceSource: input.workspaceSource,
|
||||
workspaceStrategy: input.workspaceStrategy,
|
||||
workspaceId: input.workspaceId,
|
||||
workspaceRepoUrl: input.workspaceRepoUrl,
|
||||
workspaceRepoRef: input.workspaceRepoRef,
|
||||
workspaceBranch: input.workspaceBranch,
|
||||
workspaceWorktreePath: shapedWorkspaceEnv.workspaceWorktreePath,
|
||||
agentHome: input.agentHome,
|
||||
});
|
||||
|
||||
if (shapedWorkspaceEnv.workspaceHints.length > 0) {
|
||||
input.env.PAPERCLIP_WORKSPACES_JSON = JSON.stringify(shapedWorkspaceEnv.workspaceHints);
|
||||
}
|
||||
|
||||
const shapedEnvConfig = rewriteWorkspaceCwdEnvVarsForExecution({
|
||||
env: input.envConfig ?? {},
|
||||
workspaceCwd: input.workspaceCwd,
|
||||
executionCwd: shapedWorkspaceEnv.workspaceCwd,
|
||||
executionTargetIsRemote: input.executionTargetIsRemote,
|
||||
});
|
||||
for (const [key, value] of Object.entries(shapedEnvConfig)) {
|
||||
input.env[key] = value;
|
||||
}
|
||||
|
||||
return shapedWorkspaceEnv;
|
||||
}
|
||||
|
||||
export function sanitizeInheritedPaperclipEnv(baseEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
||||
const env: NodeJS.ProcessEnv = { ...baseEnv };
|
||||
for (const key of Object.keys(env)) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
|
@ -15,6 +15,7 @@ import {
|
|||
startSshEnvLabFixture,
|
||||
stopSshEnvLabFixture,
|
||||
} from "./ssh.js";
|
||||
import { prepareRemoteManagedRuntime } from "./remote-managed-runtime.js";
|
||||
|
||||
async function git(cwd: string, args: string[]): Promise<string> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
|
|
@ -308,4 +309,245 @@ describe("ssh env-lab fixture", () => {
|
|||
expect(await git(localRepo, ["status", "--short"])).toContain("M tracked.txt");
|
||||
expect(await git(localRepo, ["status", "--short"])).not.toContain("._tracked.txt");
|
||||
});
|
||||
|
||||
it("preserves both concurrent SSH restores in a shared git workspace", async () => {
|
||||
const support = await getSshEnvLabSupport();
|
||||
if (!support.supported) {
|
||||
console.warn(
|
||||
`Skipping concurrent SSH restore test: ${support.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-fixture-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const statePath = path.join(rootDir, "state.json");
|
||||
const localRepo = path.join(rootDir, "local-workspace");
|
||||
|
||||
await mkdir(localRepo, { recursive: true });
|
||||
await git(localRepo, ["init", "-b", "main"]);
|
||||
await git(localRepo, ["config", "user.name", "Paperclip Test"]);
|
||||
await git(localRepo, ["config", "user.email", "test@paperclip.dev"]);
|
||||
await writeFile(path.join(localRepo, "tracked.txt"), "base\n", "utf8");
|
||||
await git(localRepo, ["add", "tracked.txt"]);
|
||||
await git(localRepo, ["commit", "-m", "initial"]);
|
||||
|
||||
const started = await startSshEnvLabFixture({ statePath });
|
||||
const config = await buildSshEnvLabFixtureConfig(started);
|
||||
const spec = {
|
||||
...config,
|
||||
remoteCwd: started.workspaceDir,
|
||||
} as const;
|
||||
|
||||
const preparedA = await prepareRemoteManagedRuntime({
|
||||
spec,
|
||||
runId: "run-a",
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localRepo,
|
||||
});
|
||||
const preparedB = await prepareRemoteManagedRuntime({
|
||||
spec,
|
||||
runId: "run-b",
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localRepo,
|
||||
});
|
||||
|
||||
expect(preparedA.workspaceRemoteDir).not.toBe(preparedB.workspaceRemoteDir);
|
||||
|
||||
await runSshCommand(
|
||||
config,
|
||||
`sh -lc 'printf "from run a\\n" > ${JSON.stringify(path.posix.join(preparedA.workspaceRemoteDir, "run-a.txt"))}'`,
|
||||
{ timeoutMs: 30_000, maxBuffer: 256 * 1024 },
|
||||
);
|
||||
await runSshCommand(
|
||||
config,
|
||||
`sh -lc 'printf "from run b\\n" > ${JSON.stringify(path.posix.join(preparedB.workspaceRemoteDir, "run-b.txt"))}'`,
|
||||
{ timeoutMs: 30_000, maxBuffer: 256 * 1024 },
|
||||
);
|
||||
|
||||
await Promise.all([
|
||||
preparedA.restoreWorkspace(),
|
||||
preparedB.restoreWorkspace(),
|
||||
]);
|
||||
|
||||
await expect(readFile(path.join(localRepo, "run-a.txt"), "utf8")).resolves.toBe("from run a\n");
|
||||
await expect(readFile(path.join(localRepo, "run-b.txt"), "utf8")).resolves.toBe("from run b\n");
|
||||
});
|
||||
|
||||
it("preserves nested per-run files across sequential SSH restores with stale baselines", async () => {
|
||||
const support = await getSshEnvLabSupport();
|
||||
if (!support.supported) {
|
||||
console.warn(
|
||||
`Skipping sequential nested SSH restore test: ${support.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-fixture-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const statePath = path.join(rootDir, "state.json");
|
||||
const localRepo = path.join(rootDir, "local-workspace");
|
||||
|
||||
await mkdir(localRepo, { recursive: true });
|
||||
await git(localRepo, ["init", "-b", "main"]);
|
||||
await git(localRepo, ["config", "user.name", "Paperclip Test"]);
|
||||
await git(localRepo, ["config", "user.email", "test@paperclip.dev"]);
|
||||
await writeFile(path.join(localRepo, "tracked.txt"), "base\n", "utf8");
|
||||
await git(localRepo, ["add", "tracked.txt"]);
|
||||
await git(localRepo, ["commit", "-m", "initial"]);
|
||||
|
||||
const started = await startSshEnvLabFixture({ statePath });
|
||||
const config = await buildSshEnvLabFixtureConfig(started);
|
||||
const spec = {
|
||||
...config,
|
||||
remoteCwd: started.workspaceDir,
|
||||
} as const;
|
||||
|
||||
const preparedA = await prepareRemoteManagedRuntime({
|
||||
spec,
|
||||
runId: "run-a",
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localRepo,
|
||||
});
|
||||
const preparedB = await prepareRemoteManagedRuntime({
|
||||
spec,
|
||||
runId: "run-b",
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localRepo,
|
||||
});
|
||||
|
||||
await runSshCommand(
|
||||
config,
|
||||
`sh -lc 'mkdir -p ${JSON.stringify(path.posix.join(preparedA.workspaceRemoteDir, "manual-qa/environment-matrix/ssh"))} && printf "from run a\\n" > ${JSON.stringify(path.posix.join(preparedA.workspaceRemoteDir, "manual-qa/environment-matrix/ssh/claude_local.md"))}'`,
|
||||
{ timeoutMs: 30_000, maxBuffer: 256 * 1024 },
|
||||
);
|
||||
await runSshCommand(
|
||||
config,
|
||||
`sh -lc 'mkdir -p ${JSON.stringify(path.posix.join(preparedB.workspaceRemoteDir, "manual-qa/environment-matrix/ssh"))} && printf "from run b\\n" > ${JSON.stringify(path.posix.join(preparedB.workspaceRemoteDir, "manual-qa/environment-matrix/ssh/codex_local.md"))}'`,
|
||||
{ timeoutMs: 30_000, maxBuffer: 256 * 1024 },
|
||||
);
|
||||
|
||||
await preparedA.restoreWorkspace();
|
||||
await preparedB.restoreWorkspace();
|
||||
|
||||
await expect(readFile(path.join(localRepo, "manual-qa/environment-matrix/ssh/claude_local.md"), "utf8")).resolves
|
||||
.toBe("from run a\n");
|
||||
await expect(readFile(path.join(localRepo, "manual-qa/environment-matrix/ssh/codex_local.md"), "utf8")).resolves
|
||||
.toBe("from run b\n");
|
||||
});
|
||||
|
||||
it("round-trips remote git commits through the managed runtime restore path", async () => {
|
||||
const support = await getSshEnvLabSupport();
|
||||
if (!support.supported) {
|
||||
console.warn(
|
||||
`Skipping managed-runtime SSH git round-trip test: ${support.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-fixture-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const statePath = path.join(rootDir, "state.json");
|
||||
const localRepo = path.join(rootDir, "local-workspace");
|
||||
|
||||
await mkdir(localRepo, { recursive: true });
|
||||
await git(localRepo, ["init", "-b", "main"]);
|
||||
await git(localRepo, ["config", "user.name", "Paperclip Test"]);
|
||||
await git(localRepo, ["config", "user.email", "test@paperclip.dev"]);
|
||||
await writeFile(path.join(localRepo, "tracked.txt"), "base\n", "utf8");
|
||||
await git(localRepo, ["add", "tracked.txt"]);
|
||||
await git(localRepo, ["commit", "-m", "initial"]);
|
||||
|
||||
const started = await startSshEnvLabFixture({ statePath });
|
||||
const config = await buildSshEnvLabFixtureConfig(started);
|
||||
const spec = {
|
||||
...config,
|
||||
remoteCwd: started.workspaceDir,
|
||||
} as const;
|
||||
|
||||
const prepared = await prepareRemoteManagedRuntime({
|
||||
spec,
|
||||
runId: "run-commit",
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localRepo,
|
||||
});
|
||||
|
||||
await runSshCommand(
|
||||
config,
|
||||
`sh -lc 'cd ${JSON.stringify(prepared.workspaceRemoteDir)} && git config user.name "Paperclip SSH" && git config user.email "ssh@paperclip.dev" && printf "committed\\n" > tracked.txt && git add tracked.txt && git commit -m "remote update" >/dev/null && printf "dirty remote\\n" > tracked.txt'`,
|
||||
{ timeoutMs: 30_000, maxBuffer: 256 * 1024 },
|
||||
);
|
||||
|
||||
await prepared.restoreWorkspace();
|
||||
|
||||
expect(await git(localRepo, ["log", "-1", "--pretty=%s"])).toBe("remote update");
|
||||
await expect(readFile(path.join(localRepo, "tracked.txt"), "utf8")).resolves.toBe("dirty remote\n");
|
||||
});
|
||||
|
||||
it("merges concurrent remote commits through the managed runtime restore path", async () => {
|
||||
const support = await getSshEnvLabSupport();
|
||||
if (!support.supported) {
|
||||
console.warn(
|
||||
`Skipping concurrent managed-runtime SSH git merge test: ${support.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-fixture-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const statePath = path.join(rootDir, "state.json");
|
||||
const localRepo = path.join(rootDir, "local-workspace");
|
||||
|
||||
await mkdir(localRepo, { recursive: true });
|
||||
await git(localRepo, ["init", "-b", "main"]);
|
||||
await git(localRepo, ["config", "user.name", "Paperclip Test"]);
|
||||
await git(localRepo, ["config", "user.email", "test@paperclip.dev"]);
|
||||
await writeFile(path.join(localRepo, "tracked.txt"), "base\n", "utf8");
|
||||
await git(localRepo, ["add", "tracked.txt"]);
|
||||
await git(localRepo, ["commit", "-m", "initial"]);
|
||||
|
||||
const started = await startSshEnvLabFixture({ statePath });
|
||||
const config = await buildSshEnvLabFixtureConfig(started);
|
||||
const spec = {
|
||||
...config,
|
||||
remoteCwd: started.workspaceDir,
|
||||
} as const;
|
||||
|
||||
const preparedA = await prepareRemoteManagedRuntime({
|
||||
spec,
|
||||
runId: "run-commit-a",
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localRepo,
|
||||
});
|
||||
const preparedB = await prepareRemoteManagedRuntime({
|
||||
spec,
|
||||
runId: "run-commit-b",
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localRepo,
|
||||
});
|
||||
|
||||
await runSshCommand(
|
||||
config,
|
||||
`sh -lc 'cd ${JSON.stringify(preparedA.workspaceRemoteDir)} && git config user.name "Paperclip SSH" && git config user.email "ssh@paperclip.dev" && printf "from run a\\n" > run-a.txt && git add run-a.txt && git commit -m "remote update a" >/dev/null'`,
|
||||
{ timeoutMs: 30_000, maxBuffer: 256 * 1024 },
|
||||
);
|
||||
await runSshCommand(
|
||||
config,
|
||||
`sh -lc 'cd ${JSON.stringify(preparedB.workspaceRemoteDir)} && git config user.name "Paperclip SSH" && git config user.email "ssh@paperclip.dev" && printf "from run b\\n" > run-b.txt && git add run-b.txt && git commit -m "remote update b" >/dev/null'`,
|
||||
{ timeoutMs: 30_000, maxBuffer: 256 * 1024 },
|
||||
);
|
||||
|
||||
await Promise.all([
|
||||
preparedA.restoreWorkspace(),
|
||||
preparedB.restoreWorkspace(),
|
||||
]);
|
||||
|
||||
await expect(readFile(path.join(localRepo, "run-a.txt"), "utf8")).resolves.toBe("from run a\n");
|
||||
await expect(readFile(path.join(localRepo, "run-b.txt"), "utf8")).resolves.toBe("from run b\n");
|
||||
expect(await git(localRepo, ["log", "-1", "--pretty=%s"])).toContain("Paperclip SSH sync merge");
|
||||
|
||||
const recentSubjects = await git(localRepo, ["log", "--pretty=%s", "-3"]);
|
||||
expect(recentSubjects).toContain("remote update a");
|
||||
expect(recentSubjects).toContain("remote update b");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { execFile, spawn } from "node:child_process";
|
||||
import { constants as fsConstants, createReadStream, createWriteStream, promises as fs } from "node:fs";
|
||||
import net from "node:net";
|
||||
|
|
@ -5,6 +6,8 @@ import os from "node:os";
|
|||
import path from "node:path";
|
||||
import type { CommandManagedRuntimeRunner } from "./command-managed-runtime.js";
|
||||
import type { RunProcessResult } from "./server-utils.js";
|
||||
import type { DirectorySnapshot } from "./workspace-restore-merge.js";
|
||||
import { mergeDirectoryWithBaseline } from "./workspace-restore-merge.js";
|
||||
|
||||
export interface SshConnectionConfig {
|
||||
host: string;
|
||||
|
|
@ -596,7 +599,9 @@ async function importGitWorkspaceToSsh(input: {
|
|||
}): Promise<void> {
|
||||
const bundleDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-bundle-"));
|
||||
const bundlePath = path.join(bundleDir, "workspace.bundle");
|
||||
const tempRef = "refs/paperclip/ssh-sync/import";
|
||||
// Per-import unique ref so concurrent imports against the same local repo
|
||||
// can't race on `update-ref` between this run's update and bundle create.
|
||||
const tempRef = `refs/paperclip/ssh-sync/import/${randomUUID()}`;
|
||||
|
||||
try {
|
||||
await runLocalGit(input.localDir, ["update-ref", tempRef, input.snapshot.headCommit], {
|
||||
|
|
@ -621,6 +626,8 @@ async function importGitWorkspaceToSsh(input: {
|
|||
: `git -C ${shellQuote(input.remoteDir)} -c advice.detachedHead=false checkout --force --detach ${shellQuote(input.snapshot.headCommit)} >/dev/null`,
|
||||
`git -C ${shellQuote(input.remoteDir)} reset --hard ${shellQuote(input.snapshot.headCommit)} >/dev/null`,
|
||||
`git -C ${shellQuote(input.remoteDir)} clean -fdx -e .paperclip-runtime >/dev/null`,
|
||||
// Drop the per-import ref on the remote side too so it can't accumulate.
|
||||
`git -C ${shellQuote(input.remoteDir)} update-ref -d ${shellQuote(tempRef)} >/dev/null 2>&1 || true`,
|
||||
].join("\n");
|
||||
|
||||
await streamLocalFileToSsh({
|
||||
|
|
@ -641,10 +648,12 @@ async function exportGitWorkspaceFromSsh(input: {
|
|||
spec: SshRemoteExecutionSpec;
|
||||
remoteDir: string;
|
||||
localDir: string;
|
||||
}): Promise<void> {
|
||||
importedRef?: string;
|
||||
resetLocalWorkspace?: boolean;
|
||||
}): Promise<string> {
|
||||
const bundleDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-bundle-"));
|
||||
const bundlePath = path.join(bundleDir, "workspace.bundle");
|
||||
const importedRef = "refs/paperclip/ssh-sync/imported";
|
||||
const importedRef = input.importedRef ?? `refs/paperclip/ssh-sync/imported/${randomUUID()}`;
|
||||
|
||||
try {
|
||||
const exportScript = [
|
||||
|
|
@ -668,19 +677,97 @@ async function exportGitWorkspaceFromSsh(input: {
|
|||
timeout: 60_000,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
await runLocalGit(input.localDir, ["reset", "--hard", importedRef], {
|
||||
timeout: 60_000,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
} finally {
|
||||
await runLocalGit(input.localDir, ["update-ref", "-d", importedRef], {
|
||||
if (input.resetLocalWorkspace !== false) {
|
||||
await runLocalGit(input.localDir, ["reset", "--hard", importedRef], {
|
||||
timeout: 60_000,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
}
|
||||
const importedHead = await runLocalGit(input.localDir, ["rev-parse", importedRef], {
|
||||
timeout: 10_000,
|
||||
maxBuffer: 16 * 1024,
|
||||
}).catch(() => undefined);
|
||||
});
|
||||
return importedHead.stdout.trim();
|
||||
} finally {
|
||||
if (input.resetLocalWorkspace !== false) {
|
||||
await runLocalGit(input.localDir, ["update-ref", "-d", importedRef], {
|
||||
timeout: 10_000,
|
||||
maxBuffer: 16 * 1024,
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
await fs.rm(bundleDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async function integrateImportedGitHead(input: {
|
||||
localDir: string;
|
||||
importedHead: string;
|
||||
}): Promise<void> {
|
||||
const snapshot = await readLocalGitWorkspaceSnapshot(input.localDir);
|
||||
if (!snapshot) return;
|
||||
|
||||
const currentHead = snapshot.headCommit;
|
||||
if (!currentHead || currentHead === input.importedHead) return;
|
||||
|
||||
const headRef = snapshot.branchName ? `refs/heads/${snapshot.branchName}` : "HEAD";
|
||||
const mergeBase = await runLocalGit(input.localDir, ["merge-base", currentHead, input.importedHead], {
|
||||
timeout: 10_000,
|
||||
maxBuffer: 16 * 1024,
|
||||
}).catch(() => null);
|
||||
const mergeBaseHead = mergeBase?.stdout.trim() ?? "";
|
||||
|
||||
if (mergeBaseHead === input.importedHead) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (mergeBaseHead === currentHead) {
|
||||
await runLocalGit(input.localDir, ["update-ref", headRef, input.importedHead, currentHead], {
|
||||
timeout: 10_000,
|
||||
maxBuffer: 16 * 1024,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let mergedTree;
|
||||
try {
|
||||
mergedTree = await runLocalGit(input.localDir, ["merge-tree", "--write-tree", currentHead, input.importedHead], {
|
||||
timeout: 60_000,
|
||||
maxBuffer: 256 * 1024,
|
||||
});
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(
|
||||
`Failed to merge concurrent SSH git histories for ${currentHead.slice(0, 12)} and ${input.importedHead.slice(0, 12)}: ${reason}`,
|
||||
);
|
||||
}
|
||||
const mergedTreeId = mergedTree.stdout.trim().split("\n")[0]?.trim() ?? "";
|
||||
if (!mergedTreeId) {
|
||||
throw new Error("Failed to compute a merged git tree for SSH workspace restore.");
|
||||
}
|
||||
|
||||
const mergeCommit = await runLocalGit(
|
||||
input.localDir,
|
||||
[
|
||||
"commit-tree",
|
||||
mergedTreeId,
|
||||
"-p",
|
||||
currentHead,
|
||||
"-p",
|
||||
input.importedHead,
|
||||
"-m",
|
||||
`Paperclip SSH sync merge ${input.importedHead.slice(0, 12)}`,
|
||||
],
|
||||
{
|
||||
timeout: 60_000,
|
||||
maxBuffer: 64 * 1024,
|
||||
},
|
||||
);
|
||||
await runLocalGit(input.localDir, ["update-ref", headRef, mergeCommit.stdout.trim(), currentHead], {
|
||||
timeout: 10_000,
|
||||
maxBuffer: 16 * 1024,
|
||||
});
|
||||
}
|
||||
|
||||
async function clearRemoteDirectory(input: {
|
||||
spec: SshConnectionConfig;
|
||||
remoteDir: string;
|
||||
|
|
@ -1117,7 +1204,7 @@ export async function prepareWorkspaceForSshExecution(input: {
|
|||
spec: SshRemoteExecutionSpec;
|
||||
localDir: string;
|
||||
remoteDir?: string;
|
||||
}): Promise<void> {
|
||||
}): Promise<{ gitBacked: boolean }> {
|
||||
const remoteDir = input.remoteDir ?? input.spec.remoteCwd;
|
||||
const gitSnapshot = await readLocalGitWorkspaceSnapshot(input.localDir);
|
||||
|
||||
|
|
@ -1139,7 +1226,7 @@ export async function prepareWorkspaceForSshExecution(input: {
|
|||
remoteDir,
|
||||
deletedPaths: gitSnapshot.deletedPaths,
|
||||
});
|
||||
return;
|
||||
return { gitBacked: true };
|
||||
}
|
||||
|
||||
await clearRemoteDirectory({
|
||||
|
|
@ -1153,14 +1240,64 @@ export async function prepareWorkspaceForSshExecution(input: {
|
|||
remoteDir,
|
||||
exclude: [".paperclip-runtime"],
|
||||
});
|
||||
return { gitBacked: false };
|
||||
}
|
||||
|
||||
export async function restoreWorkspaceFromSshExecution(input: {
|
||||
spec: SshRemoteExecutionSpec;
|
||||
localDir: string;
|
||||
remoteDir?: string;
|
||||
baselineSnapshot?: DirectorySnapshot;
|
||||
restoreGitHistory?: boolean;
|
||||
}): Promise<void> {
|
||||
const remoteDir = input.remoteDir ?? input.spec.remoteCwd;
|
||||
if (input.baselineSnapshot) {
|
||||
const stagingDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-sync-back-"));
|
||||
const importedRef = input.restoreGitHistory
|
||||
? `refs/paperclip/ssh-sync/imported/${randomUUID()}`
|
||||
: null;
|
||||
try {
|
||||
const importedHead = input.restoreGitHistory
|
||||
? await exportGitWorkspaceFromSsh({
|
||||
spec: input.spec,
|
||||
remoteDir,
|
||||
localDir: input.localDir,
|
||||
importedRef: importedRef ?? undefined,
|
||||
resetLocalWorkspace: false,
|
||||
})
|
||||
: null;
|
||||
await syncDirectoryFromSsh({
|
||||
spec: input.spec,
|
||||
remoteDir,
|
||||
localDir: stagingDir,
|
||||
exclude: input.baselineSnapshot.exclude,
|
||||
});
|
||||
await mergeDirectoryWithBaseline({
|
||||
baseline: input.baselineSnapshot,
|
||||
sourceDir: stagingDir,
|
||||
targetDir: input.localDir,
|
||||
// Git history advances via integrateImportedGitHead; the working tree
|
||||
// still comes from the remote file snapshot so dirty remote edits win.
|
||||
beforeApply: importedHead
|
||||
? async () => {
|
||||
await integrateImportedGitHead({
|
||||
localDir: input.localDir,
|
||||
importedHead,
|
||||
});
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
} finally {
|
||||
if (importedRef) {
|
||||
await runLocalGit(input.localDir, ["update-ref", "-d", importedRef], {
|
||||
timeout: 10_000,
|
||||
maxBuffer: 16 * 1024,
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const gitSnapshot = await readLocalGitWorkspaceSnapshot(input.localDir);
|
||||
|
||||
if (gitSnapshot) {
|
||||
|
|
|
|||
61
packages/adapter-utils/src/workspace-restore-merge.test.ts
Normal file
61
packages/adapter-utils/src/workspace-restore-merge.test.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { captureDirectorySnapshot, mergeDirectoryWithBaseline } from "./workspace-restore-merge.js";
|
||||
|
||||
describe("workspace restore merge", () => {
|
||||
const cleanupDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
while (cleanupDirs.length > 0) {
|
||||
const dir = cleanupDirs.pop();
|
||||
if (!dir) continue;
|
||||
await rm(dir, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves sibling files when sequential stale-baseline restores create the same nested directory tree", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-restore-merge-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
|
||||
const targetDir = path.join(rootDir, "target");
|
||||
const sourceADir = path.join(rootDir, "source-a");
|
||||
const sourceBDir = path.join(rootDir, "source-b");
|
||||
await mkdir(targetDir, { recursive: true });
|
||||
await mkdir(path.join(sourceADir, "manual-qa", "environment-matrix", "ssh"), { recursive: true });
|
||||
await mkdir(path.join(sourceBDir, "manual-qa", "environment-matrix", "ssh"), { recursive: true });
|
||||
|
||||
const baseline = await captureDirectorySnapshot(targetDir, { exclude: [] });
|
||||
|
||||
await writeFile(
|
||||
path.join(sourceADir, "manual-qa", "environment-matrix", "ssh", "claude_local.md"),
|
||||
"ssh claude\n",
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(sourceBDir, "manual-qa", "environment-matrix", "ssh", "codex_local.md"),
|
||||
"ssh codex\n",
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await mergeDirectoryWithBaseline({
|
||||
baseline,
|
||||
sourceDir: sourceADir,
|
||||
targetDir,
|
||||
});
|
||||
await mergeDirectoryWithBaseline({
|
||||
baseline,
|
||||
sourceDir: sourceBDir,
|
||||
targetDir,
|
||||
});
|
||||
|
||||
await expect(
|
||||
readFile(path.join(targetDir, "manual-qa", "environment-matrix", "ssh", "claude_local.md"), "utf8"),
|
||||
).resolves.toBe("ssh claude\n");
|
||||
await expect(
|
||||
readFile(path.join(targetDir, "manual-qa", "environment-matrix", "ssh", "codex_local.md"), "utf8"),
|
||||
).resolves.toBe("ssh codex\n");
|
||||
});
|
||||
});
|
||||
257
packages/adapter-utils/src/workspace-restore-merge.ts
Normal file
257
packages/adapter-utils/src/workspace-restore-merge.ts
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { constants as fsConstants, promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
type SnapshotEntry =
|
||||
| { kind: "dir" }
|
||||
| { kind: "file"; mode: number; hash: string }
|
||||
| { kind: "symlink"; target: string };
|
||||
|
||||
export interface DirectorySnapshot {
|
||||
exclude: string[];
|
||||
entries: Map<string, SnapshotEntry>;
|
||||
}
|
||||
|
||||
function isRelativePathOrDescendant(relative: string, candidate: string): boolean {
|
||||
return relative === candidate || relative.startsWith(`${candidate}/`);
|
||||
}
|
||||
|
||||
function shouldExclude(relative: string, exclude: readonly string[]): boolean {
|
||||
return exclude.some((candidate) => isRelativePathOrDescendant(relative, candidate));
|
||||
}
|
||||
|
||||
async function hashFile(filePath: string): Promise<string> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const hash = createHash("sha256");
|
||||
const stream = createReadStream(filePath);
|
||||
stream.on("data", (chunk) => hash.update(chunk));
|
||||
stream.on("error", reject);
|
||||
stream.on("end", () => resolve(hash.digest("hex")));
|
||||
});
|
||||
}
|
||||
|
||||
async function walkDirectory(
|
||||
root: string,
|
||||
exclude: readonly string[],
|
||||
relative = "",
|
||||
out: Map<string, SnapshotEntry> = new Map(),
|
||||
): Promise<Map<string, SnapshotEntry>> {
|
||||
const current = relative ? path.join(root, relative) : root;
|
||||
const entries = await fs.readdir(current, { withFileTypes: true }).catch(() => []);
|
||||
entries.sort((left, right) => left.name.localeCompare(right.name));
|
||||
|
||||
for (const entry of entries) {
|
||||
const nextRelative = relative ? path.posix.join(relative, entry.name) : entry.name;
|
||||
if (shouldExclude(nextRelative, exclude)) continue;
|
||||
|
||||
const fullPath = path.join(root, nextRelative);
|
||||
const stats = await fs.lstat(fullPath);
|
||||
if (stats.isDirectory()) {
|
||||
out.set(nextRelative, { kind: "dir" });
|
||||
await walkDirectory(root, exclude, nextRelative, out);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (stats.isSymbolicLink()) {
|
||||
out.set(nextRelative, {
|
||||
kind: "symlink",
|
||||
target: await fs.readlink(fullPath),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
out.set(nextRelative, {
|
||||
kind: "file",
|
||||
mode: stats.mode,
|
||||
hash: await hashFile(fullPath),
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
async function readSnapshotEntry(root: string, relative: string): Promise<SnapshotEntry | null> {
|
||||
const fullPath = path.join(root, relative);
|
||||
let stats;
|
||||
try {
|
||||
stats = await fs.lstat(fullPath);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (stats.isDirectory()) return { kind: "dir" };
|
||||
if (stats.isSymbolicLink()) {
|
||||
return {
|
||||
kind: "symlink",
|
||||
target: await fs.readlink(fullPath),
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: "file",
|
||||
mode: stats.mode,
|
||||
hash: await hashFile(fullPath),
|
||||
};
|
||||
}
|
||||
|
||||
function entriesMatch(left: SnapshotEntry | null | undefined, right: SnapshotEntry | null | undefined): boolean {
|
||||
if (!left || !right) return false;
|
||||
if (left.kind !== right.kind) return false;
|
||||
if (left.kind === "dir") return true;
|
||||
if (left.kind === "symlink" && right.kind === "symlink") {
|
||||
return left.target === right.target;
|
||||
}
|
||||
if (left.kind === "file" && right.kind === "file") {
|
||||
return left.mode === right.mode && left.hash === right.hash;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function isHolderAlive(lockDir: string): Promise<boolean> {
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(lockDir, "owner.json"), "utf8");
|
||||
const owner = JSON.parse(raw) as { pid?: unknown };
|
||||
const pid = typeof owner.pid === "number" && Number.isFinite(owner.pid) && owner.pid > 0 ? owner.pid : null;
|
||||
if (pid === null) {
|
||||
// Owner record is unparseable / missing pid — treat as stale.
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
// owner.json missing or unreadable — treat as stale.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function acquireDirectoryMergeLock(lockDir: string): Promise<() => Promise<void>> {
|
||||
const deadline = Date.now() + 30_000;
|
||||
while (true) {
|
||||
try {
|
||||
await fs.mkdir(lockDir);
|
||||
await fs.writeFile(
|
||||
path.join(lockDir, "owner.json"),
|
||||
`${JSON.stringify({ pid: process.pid, createdAt: new Date().toISOString() })}\n`,
|
||||
"utf8",
|
||||
);
|
||||
return async () => {
|
||||
await fs.rm(lockDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
};
|
||||
} catch (error) {
|
||||
const code = error && typeof error === "object" ? (error as { code?: unknown }).code : null;
|
||||
if (code !== "EEXIST") throw error;
|
||||
// Stale-lock detection: if the owner PID is dead (SIGKILL / OOM / crash),
|
||||
// the lockDir would otherwise persist forever and stall restores. Mirror
|
||||
// the materializePaperclipSkillCopy lock pattern — remove and retry.
|
||||
if (!(await isHolderAlive(lockDir))) {
|
||||
await fs.rm(lockDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
continue;
|
||||
}
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error(`Timed out waiting for workspace restore lock at ${lockDir}`);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function withDirectoryMergeLock<T>(
|
||||
targetDir: string,
|
||||
fn: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const releaseLock = await acquireDirectoryMergeLock(`${targetDir}.paperclip-restore.lock`);
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
await releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
async function copySnapshotEntry(sourceDir: string, targetDir: string, relative: string, entry: SnapshotEntry): Promise<void> {
|
||||
const sourcePath = path.join(sourceDir, relative);
|
||||
const targetPath = path.join(targetDir, relative);
|
||||
|
||||
if (entry.kind === "dir") {
|
||||
const existing = await fs.lstat(targetPath).catch(() => null);
|
||||
if (existing?.isDirectory()) {
|
||||
return;
|
||||
}
|
||||
if (existing) {
|
||||
await fs.rm(targetPath, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
await fs.mkdir(targetPath, { recursive: true });
|
||||
return;
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(targetPath), { recursive: true });
|
||||
await fs.rm(targetPath, { recursive: true, force: true }).catch(() => undefined);
|
||||
if (entry.kind === "symlink") {
|
||||
await fs.symlink(entry.target, targetPath);
|
||||
return;
|
||||
}
|
||||
|
||||
await fs.copyFile(sourcePath, targetPath, fsConstants.COPYFILE_FICLONE).catch(async () => {
|
||||
await fs.copyFile(sourcePath, targetPath);
|
||||
});
|
||||
await fs.chmod(targetPath, entry.mode);
|
||||
}
|
||||
|
||||
export async function captureDirectorySnapshot(
|
||||
rootDir: string,
|
||||
options: { exclude?: string[] } = {},
|
||||
): Promise<DirectorySnapshot> {
|
||||
const exclude = [...new Set(options.exclude ?? [])];
|
||||
return {
|
||||
exclude,
|
||||
entries: await walkDirectory(rootDir, exclude),
|
||||
};
|
||||
}
|
||||
|
||||
export async function mergeDirectoryWithBaseline(input: {
|
||||
baseline: DirectorySnapshot;
|
||||
sourceDir: string;
|
||||
targetDir: string;
|
||||
beforeApply?: () => Promise<void>;
|
||||
}): Promise<void> {
|
||||
const source = await captureDirectorySnapshot(input.sourceDir, { exclude: input.baseline.exclude });
|
||||
await withDirectoryMergeLock(input.targetDir, async () => {
|
||||
await input.beforeApply?.();
|
||||
const current = await captureDirectorySnapshot(input.targetDir, { exclude: input.baseline.exclude });
|
||||
const deletedLeafEntries = [...input.baseline.entries.entries()]
|
||||
.filter(([relative, entry]) => entry.kind !== "dir" && !source.entries.has(relative))
|
||||
.sort(([left], [right]) => right.length - left.length);
|
||||
|
||||
for (const [relative, baselineEntry] of deletedLeafEntries) {
|
||||
if (!entriesMatch(current.entries.get(relative), baselineEntry)) continue;
|
||||
await fs.rm(path.join(input.targetDir, relative), { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
|
||||
const deletedDirs = [...input.baseline.entries.entries()]
|
||||
.filter(([relative, entry]) => entry.kind === "dir" && !source.entries.has(relative))
|
||||
.sort(([left], [right]) => right.length - left.length);
|
||||
|
||||
for (const [relative] of deletedDirs) {
|
||||
await fs.rmdir(path.join(input.targetDir, relative)).catch(() => undefined);
|
||||
}
|
||||
|
||||
const changedSourceEntries = [...source.entries.entries()]
|
||||
.filter(([relative, entry]) => !entriesMatch(input.baseline.entries.get(relative), entry))
|
||||
.sort(([left], [right]) => left.localeCompare(right));
|
||||
|
||||
for (const [relative, entry] of changedSourceEntries) {
|
||||
await copySnapshotEntry(input.sourceDir, input.targetDir, relative, entry);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function directoryEntryMatchesBaseline(
|
||||
rootDir: string,
|
||||
relative: string,
|
||||
baselineEntry: SnapshotEntry,
|
||||
): Promise<boolean> {
|
||||
return entriesMatch(await readSnapshotEntry(rootDir, relative), baselineEntry);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue