import { randomUUID } from "node:crypto"; import { execFile, spawn } from "node:child_process"; import { constants as fsConstants, createReadStream, createWriteStream, promises as fs } from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; import type { CommandManagedRuntimeRunner } from "./command-managed-runtime.js"; import type { RunProcessResult } from "./server-utils.js"; import type { DirectorySnapshot } from "./workspace-restore-merge.js"; import { mergeDirectoryWithBaseline } from "./workspace-restore-merge.js"; export interface SshConnectionConfig { host: string; port: number; username: string; remoteWorkspacePath: string; privateKey: string | null; knownHosts: string | null; strictHostKeyChecking: boolean; } export interface SshCommandResult { stdout: string; stderr: string; } export interface SshRemoteExecutionSpec extends SshConnectionConfig { remoteCwd: string; } export function createSshCommandManagedRuntimeRunner(input: { spec: SshRemoteExecutionSpec; defaultCwd?: string | null; maxBufferBytes?: number | null; }): CommandManagedRuntimeRunner { const defaultCwd = input.defaultCwd?.trim() || input.spec.remoteCwd; const maxBufferBytes = typeof input.maxBufferBytes === "number" && Number.isFinite(input.maxBufferBytes) && input.maxBufferBytes > 0 ? Math.trunc(input.maxBufferBytes) : 1024 * 1024; return { execute: async (commandInput): Promise => { const startedAt = new Date().toISOString(); const command = commandInput.command.trim(); const args = commandInput.args ?? []; const cwd = commandInput.cwd?.trim() || defaultCwd; const envEntries = Object.entries(commandInput.env ?? {}) .filter((entry): entry is [string, string] => typeof entry[1] === "string"); const envPrefix = envEntries.length > 0 ? `env ${envEntries.map(([key, value]) => `${key}=${shellQuote(value)}`).join(" ")} ` : ""; const exportPrefix = envEntries.length > 0 ? envEntries.map(([key, value]) => `export ${key}=${shellQuote(value)};`).join(" ") + " " : ""; const commandScript = command === "sh" || command === "bash" ? args[0] === "-lc" && typeof args[1] === "string" ? `${exportPrefix}${args[1]}` : `${envPrefix}exec ${[shellQuote(command), ...args.map((arg) => shellQuote(arg))].join(" ")}` : `${envPrefix}exec ${[shellQuote(command), ...args.map((arg) => shellQuote(arg))].join(" ")}`; const remoteCommand = `${command === "bash" ? "bash" : "sh"} -lc ${ shellQuote(`cd ${shellQuote(cwd)} && ${commandScript}`) }`; try { const result = await runSshCommand(input.spec, remoteCommand, { stdin: commandInput.stdin, timeoutMs: commandInput.timeoutMs, maxBuffer: maxBufferBytes, }); if (result.stdout) await commandInput.onLog?.("stdout", result.stdout); if (result.stderr) await commandInput.onLog?.("stderr", result.stderr); return { exitCode: 0, signal: null, timedOut: false, stdout: result.stdout, stderr: result.stderr, pid: null, startedAt, }; } catch (error) { const failure = error as { stdout?: unknown; stderr?: unknown; code?: unknown; signal?: unknown; killed?: unknown; }; const stdout = typeof failure.stdout === "string" ? failure.stdout : ""; const stderr = typeof failure.stderr === "string" ? failure.stderr : error instanceof Error ? error.message : String(error); if (stdout) await commandInput.onLog?.("stdout", stdout); if (stderr) await commandInput.onLog?.("stderr", stderr); return { exitCode: typeof failure.code === "number" ? failure.code : null, signal: typeof failure.signal === "string" ? failure.signal : null, timedOut: failure.killed === true, stdout, stderr, pid: null, startedAt, }; } }, }; } export interface SshEnvLabSupport { supported: boolean; reason: string | null; } export interface SshEnvLabFixtureState { kind: "ssh_openbsd"; bindHost: string; host: string; port: number; username: string; rootDir: string; workspaceDir: string; statePath: string; pid: number; createdAt: string; clientPrivateKeyPath: string; clientPublicKeyPath: string; hostPrivateKeyPath: string; hostPublicKeyPath: string; authorizedKeysPath: string; knownHostsPath: string; sshdConfigPath: string; sshdLogPath: string; } interface LocalGitWorkspaceSnapshot { headCommit: string; branchName: string | null; deletedPaths: string[]; } export function shellQuote(value: string) { return `'${value.replace(/'/g, `'\"'\"'`)}'`; } function isValidShellEnvKey(value: string) { return /^[A-Za-z_][A-Za-z0-9_]*$/.test(value); } export function parseSshRemoteExecutionSpec(value: unknown): SshRemoteExecutionSpec | null { if (!value || typeof value !== "object" || Array.isArray(value)) { return null; } const parsed = value as Record; const host = typeof parsed.host === "string" ? parsed.host.trim() : ""; const username = typeof parsed.username === "string" ? parsed.username.trim() : ""; const remoteCwd = typeof parsed.remoteCwd === "string" ? parsed.remoteCwd.trim() : ""; const portValue = typeof parsed.port === "number" ? parsed.port : Number(parsed.port); if (!host || !username || !remoteCwd || !Number.isInteger(portValue) || portValue < 1 || portValue > 65535) { return null; } return { host, port: portValue, username, remoteCwd, remoteWorkspacePath: typeof parsed.remoteWorkspacePath === "string" && parsed.remoteWorkspacePath.trim().length > 0 ? parsed.remoteWorkspacePath.trim() : remoteCwd, privateKey: typeof parsed.privateKey === "string" && parsed.privateKey.length > 0 ? parsed.privateKey : null, knownHosts: typeof parsed.knownHosts === "string" && parsed.knownHosts.length > 0 ? parsed.knownHosts : null, strictHostKeyChecking: typeof parsed.strictHostKeyChecking === "boolean" ? parsed.strictHostKeyChecking : true, }; } async function execFileText( file: string, args: string[], options: { timeout?: number; maxBuffer?: number; } = {}, ): Promise { return await new Promise((resolve, reject) => { execFile( file, args, { timeout: options.timeout ?? 15_000, maxBuffer: options.maxBuffer ?? 1024 * 128, }, (error, stdout, stderr) => { if (error) { reject(Object.assign(error, { stdout: stdout ?? "", stderr: stderr ?? "" })); return; } resolve({ stdout: stdout ?? "", stderr: stderr ?? "", }); }, ); }); } async function spawnText( file: string, args: string[], options: { stdin?: string; timeout?: number; maxBuffer?: number; } = {}, ): Promise { return await new Promise((resolve, reject) => { const child = spawn(file, args, { stdio: [options.stdin != null ? "pipe" : "ignore", "pipe", "pipe"], }); const maxBuffer = options.maxBuffer ?? 1024 * 128; let stdout = ""; let stderr = ""; let settled = false; let timedOut = false; const finishReject = (error: Error & { stdout?: string; stderr?: string; code?: number | null; killed?: boolean }) => { if (settled) return; settled = true; error.stdout = stdout; error.stderr = stderr; error.killed = timedOut; reject(error); }; const append = ( streamName: "stdout" | "stderr", chunk: unknown, ) => { const text = String(chunk); if (streamName === "stdout") { stdout += text; } else { stderr += text; } if (Buffer.byteLength(stdout, "utf8") > maxBuffer || Buffer.byteLength(stderr, "utf8") > maxBuffer) { child.kill("SIGTERM"); finishReject(Object.assign(new Error(`Process output exceeded maxBuffer of ${maxBuffer} bytes.`), { code: null, })); } }; let killEscalation: NodeJS.Timeout | null = null; const timeout = options.timeout && options.timeout > 0 ? setTimeout(() => { timedOut = true; child.kill("SIGTERM"); // Escalate to SIGKILL after a 5s grace window so a hung remote // command that ignores SIGTERM cannot keep the child alive // indefinitely. killEscalation = setTimeout(() => { try { child.kill("SIGKILL"); } catch { // child may have already exited between the SIGTERM and the // escalation — that's fine. } }, 5_000); killEscalation.unref?.(); }, options.timeout) : null; const clearTimers = () => { if (timeout) clearTimeout(timeout); if (killEscalation) clearTimeout(killEscalation); }; child.stdout?.on("data", (chunk) => { append("stdout", chunk); }); child.stderr?.on("data", (chunk) => { append("stderr", chunk); }); child.on("error", (error) => { clearTimers(); finishReject(Object.assign(error, { code: null })); }); child.on("close", (code, signal) => { clearTimers(); if (settled) return; settled = true; if (code === 0) { resolve({ stdout, stderr }); return; } reject(Object.assign(new Error(stderr.trim() || stdout.trim() || `Process exited with code ${code ?? -1}`), { stdout, stderr, code, signal, killed: timedOut, })); }); if (options.stdin != null && child.stdin) { child.stdin.end(options.stdin); } }); } async function runLocalGit( localDir: string, args: string[], options: { timeout?: number; maxBuffer?: number; } = {}, ): Promise { return await execFileText("git", ["-C", localDir, ...args], options); } async function commandExists(command: string): Promise { return (await resolveCommandPath(command)) !== null; } async function resolveCommandPath(command: string): Promise { try { const result = await execFileText("sh", ["-lc", `command -v ${shellQuote(command)}`], { timeout: 5_000, maxBuffer: 8 * 1024, }); const resolved = result.stdout.trim().split("\n")[0]?.trim() ?? ""; return resolved.length > 0 ? resolved : null; } catch { return null; } } async function withTempFile( prefix: string, contents: string, mode: number, ): Promise<{ path: string; cleanup: () => Promise }> { const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); const filePath = path.join(dir, "payload"); const normalizedContents = contents.endsWith("\n") ? contents : `${contents}\n`; await fs.writeFile(filePath, normalizedContents, { mode, encoding: "utf8" }); return { path: filePath, cleanup: async () => { await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); }, }; } async function createSshAuthArgs( config: Pick, ): Promise<{ args: string[]; cleanup: () => Promise }> { const tempFiles: Array<() => Promise> = []; const sshArgs = [ "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o", `StrictHostKeyChecking=${config.strictHostKeyChecking ? "yes" : "no"}`, ]; if (config.strictHostKeyChecking) { if (config.knownHosts) { const knownHosts = await withTempFile("paperclip-ssh-known-hosts-", config.knownHosts, 0o600); tempFiles.push(knownHosts.cleanup); sshArgs.push("-o", `UserKnownHostsFile=${knownHosts.path}`); } } else { sshArgs.push("-o", "UserKnownHostsFile=/dev/null"); } if (config.privateKey) { const privateKey = await withTempFile("paperclip-ssh-key-", config.privateKey, 0o600); tempFiles.push(privateKey.cleanup); sshArgs.push("-i", privateKey.path); } return { args: sshArgs, cleanup: async () => { await Promise.all(tempFiles.map((cleanup) => cleanup())); }, }; } function tarExcludeArgs(exclude: string[] | undefined): string[] { const combined = ["._*", ...(exclude ?? [])]; return combined.flatMap((entry) => ["--exclude", entry]); } function tarSpawnEnv(): NodeJS.ProcessEnv { return { ...process.env, // Prevent macOS bsdtar from emitting AppleDouble metadata files like ._README.md. COPYFILE_DISABLE: "1", }; } async function runSshScript( config: SshConnectionConfig, script: string, options: { timeoutMs?: number; maxBuffer?: number; } = {}, ): Promise { return await runSshCommand( config, `sh -lc ${shellQuote(script)}`, options, ); } async function clearLocalDirectory( localDir: string, preserveEntries: string[] = [], ): Promise { await fs.mkdir(localDir, { recursive: true }); const preserve = new Set(preserveEntries); const entries = await fs.readdir(localDir); await Promise.all( entries .filter((entry) => !preserve.has(entry)) .map((entry) => fs.rm(path.join(localDir, entry), { recursive: true, force: true })), ); } async function copyDirectoryContents(sourceDir: string, targetDir: string): Promise { await fs.mkdir(targetDir, { recursive: true }); const entries = await fs.readdir(sourceDir); await Promise.all(entries.map(async (entry) => { await fs.cp(path.join(sourceDir, entry), path.join(targetDir, entry), { recursive: true, force: true, preserveTimestamps: true, }); })); } async function readLocalGitWorkspaceSnapshot(localDir: string): Promise { try { const insideWorkTree = await runLocalGit(localDir, ["rev-parse", "--is-inside-work-tree"], { timeout: 10_000, maxBuffer: 16 * 1024, }); if (insideWorkTree.stdout.trim() !== "true") { return null; } const [headCommitResult, branchResult, deletedResult] = await Promise.all([ runLocalGit(localDir, ["rev-parse", "HEAD"], { timeout: 10_000, maxBuffer: 16 * 1024, }), runLocalGit(localDir, ["rev-parse", "--abbrev-ref", "HEAD"], { timeout: 10_000, maxBuffer: 16 * 1024, }), runLocalGit(localDir, ["ls-files", "--deleted", "-z"], { timeout: 10_000, maxBuffer: 256 * 1024, }), ]); const branchName = branchResult.stdout.trim(); return { headCommit: headCommitResult.stdout.trim(), branchName: branchName && branchName !== "HEAD" ? branchName : null, deletedPaths: deletedResult.stdout .split("\0") .map((entry) => entry.trim()) .filter(Boolean), }; } catch { return null; } } async function streamLocalFileToSsh(input: { spec: SshConnectionConfig; localFile: string; remoteScript: string; }): Promise { const auth = await createSshAuthArgs(input.spec); const sshArgs = [ ...auth.args, "-p", String(input.spec.port), `${input.spec.username}@${input.spec.host}`, `sh -lc ${shellQuote(input.remoteScript)}`, ]; await new Promise((resolve, reject) => { const source = createReadStream(input.localFile); const ssh = spawn("ssh", sshArgs, { stdio: ["pipe", "ignore", "pipe"], }); let sshStderr = ""; let settled = false; const fail = (error: Error) => { if (settled) return; settled = true; source.destroy(); ssh.kill("SIGTERM"); reject(error); }; ssh.stderr?.on("data", (chunk) => { sshStderr += String(chunk); }); source.on("error", fail); ssh.on("error", fail); source.pipe(ssh.stdin ?? null); ssh.on("close", (code) => { if (settled) return; settled = true; if ((code ?? 0) !== 0) { reject(new Error(sshStderr.trim() || `ssh exited with code ${code ?? -1}`)); return; } resolve(); }); }).finally(auth.cleanup); } async function streamSshToLocalFile(input: { spec: SshConnectionConfig; remoteScript: string; localFile: string; }): Promise { const auth = await createSshAuthArgs(input.spec); const sshArgs = [ ...auth.args, "-p", String(input.spec.port), `${input.spec.username}@${input.spec.host}`, `sh -lc ${shellQuote(input.remoteScript)}`, ]; await new Promise((resolve, reject) => { const ssh = spawn("ssh", sshArgs, { stdio: ["ignore", "pipe", "pipe"], }); const sink = createWriteStream(input.localFile, { mode: 0o600 }); let sshStderr = ""; let settled = false; const fail = (error: Error) => { if (settled) return; settled = true; ssh.kill("SIGTERM"); sink.destroy(); reject(error); }; ssh.stdout?.pipe(sink); ssh.stderr?.on("data", (chunk) => { sshStderr += String(chunk); }); ssh.on("error", fail); sink.on("error", fail); ssh.on("close", (code) => { sink.end(() => { if (settled) return; settled = true; if ((code ?? 0) !== 0) { reject(new Error(sshStderr.trim() || `ssh exited with code ${code ?? -1}`)); return; } resolve(); }); }); }).finally(auth.cleanup); } async function importGitWorkspaceToSsh(input: { spec: SshRemoteExecutionSpec; localDir: string; remoteDir: string; snapshot: LocalGitWorkspaceSnapshot; }): Promise { const bundleDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-bundle-")); const bundlePath = path.join(bundleDir, "workspace.bundle"); // Per-import unique ref so concurrent imports against the same local repo // can't race on `update-ref` between this run's update and bundle create. const tempRef = `refs/paperclip/ssh-sync/import/${randomUUID()}`; try { await runLocalGit(input.localDir, ["update-ref", tempRef, input.snapshot.headCommit], { timeout: 10_000, maxBuffer: 16 * 1024, }); await runLocalGit(input.localDir, ["bundle", "create", bundlePath, tempRef], { timeout: 60_000, maxBuffer: 1024 * 1024, }); const remoteSetupScript = [ "set -e", `mkdir -p ${shellQuote(path.posix.join(input.remoteDir, ".paperclip-runtime"))}`, `tmp_bundle=$(mktemp ${shellQuote(path.posix.join(input.remoteDir, ".paperclip-runtime", "import-XXXXXX.bundle"))})`, 'trap \'rm -f "$tmp_bundle"\' EXIT', 'cat > "$tmp_bundle"', `if [ ! -d ${shellQuote(path.posix.join(input.remoteDir, ".git"))} ]; then git init ${shellQuote(input.remoteDir)} >/dev/null; fi`, `git -C ${shellQuote(input.remoteDir)} fetch --force "$tmp_bundle" '${tempRef}:${tempRef}' >/dev/null`, input.snapshot.branchName ? `git -C ${shellQuote(input.remoteDir)} checkout --force -B ${shellQuote(input.snapshot.branchName)} ${shellQuote(input.snapshot.headCommit)} >/dev/null` : `git -C ${shellQuote(input.remoteDir)} -c advice.detachedHead=false checkout --force --detach ${shellQuote(input.snapshot.headCommit)} >/dev/null`, `git -C ${shellQuote(input.remoteDir)} reset --hard ${shellQuote(input.snapshot.headCommit)} >/dev/null`, `git -C ${shellQuote(input.remoteDir)} clean -fdx -e .paperclip-runtime >/dev/null`, // Drop the per-import ref on the remote side too so it can't accumulate. `git -C ${shellQuote(input.remoteDir)} update-ref -d ${shellQuote(tempRef)} >/dev/null 2>&1 || true`, ].join("\n"); await streamLocalFileToSsh({ spec: input.spec, localFile: bundlePath, remoteScript: remoteSetupScript, }); } finally { await runLocalGit(input.localDir, ["update-ref", "-d", tempRef], { timeout: 10_000, maxBuffer: 16 * 1024, }).catch(() => undefined); await fs.rm(bundleDir, { recursive: true, force: true }).catch(() => undefined); } } async function exportGitWorkspaceFromSsh(input: { spec: SshRemoteExecutionSpec; remoteDir: string; localDir: string; importedRef?: string; resetLocalWorkspace?: boolean; }): Promise { const bundleDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-bundle-")); const bundlePath = path.join(bundleDir, "workspace.bundle"); const importedRef = input.importedRef ?? `refs/paperclip/ssh-sync/imported/${randomUUID()}`; try { const exportScript = [ "set -e", `git -C ${shellQuote(input.remoteDir)} update-ref refs/paperclip/ssh-sync/export HEAD`, `mkdir -p ${shellQuote(path.posix.join(input.remoteDir, ".paperclip-runtime"))}`, `tmp_bundle=$(mktemp ${shellQuote(path.posix.join(input.remoteDir, ".paperclip-runtime", "export-XXXXXX.bundle"))})`, 'cleanup() { rm -f "$tmp_bundle"; git -C ' + shellQuote(input.remoteDir) + ' update-ref -d refs/paperclip/ssh-sync/export >/dev/null 2>&1 || true; }', 'trap cleanup EXIT', `git -C ${shellQuote(input.remoteDir)} bundle create "$tmp_bundle" refs/paperclip/ssh-sync/export >/dev/null`, 'cat "$tmp_bundle"', ].join("\n"); await streamSshToLocalFile({ spec: input.spec, remoteScript: exportScript, localFile: bundlePath, }); await runLocalGit(input.localDir, ["fetch", "--force", bundlePath, `refs/paperclip/ssh-sync/export:${importedRef}`], { timeout: 60_000, maxBuffer: 1024 * 1024, }); if (input.resetLocalWorkspace !== false) { await runLocalGit(input.localDir, ["reset", "--hard", importedRef], { timeout: 60_000, maxBuffer: 1024 * 1024, }); } const importedHead = await runLocalGit(input.localDir, ["rev-parse", importedRef], { timeout: 10_000, maxBuffer: 16 * 1024, }); return importedHead.stdout.trim(); } finally { if (input.resetLocalWorkspace !== false) { await runLocalGit(input.localDir, ["update-ref", "-d", importedRef], { timeout: 10_000, maxBuffer: 16 * 1024, }).catch(() => undefined); } await fs.rm(bundleDir, { recursive: true, force: true }).catch(() => undefined); } } async function integrateImportedGitHead(input: { localDir: string; importedHead: string; }): Promise { const snapshot = await readLocalGitWorkspaceSnapshot(input.localDir); if (!snapshot) return; const currentHead = snapshot.headCommit; if (!currentHead || currentHead === input.importedHead) return; const headRef = snapshot.branchName ? `refs/heads/${snapshot.branchName}` : "HEAD"; const mergeBase = await runLocalGit(input.localDir, ["merge-base", currentHead, input.importedHead], { timeout: 10_000, maxBuffer: 16 * 1024, }).catch(() => null); const mergeBaseHead = mergeBase?.stdout.trim() ?? ""; if (mergeBaseHead === input.importedHead) { return; } if (mergeBaseHead === currentHead) { await runLocalGit(input.localDir, ["update-ref", headRef, input.importedHead, currentHead], { timeout: 10_000, maxBuffer: 16 * 1024, }); return; } let mergedTree; try { mergedTree = await runLocalGit(input.localDir, ["merge-tree", "--write-tree", currentHead, input.importedHead], { timeout: 60_000, maxBuffer: 256 * 1024, }); } catch (error) { const reason = error instanceof Error ? error.message : String(error); throw new Error( `Failed to merge concurrent SSH git histories for ${currentHead.slice(0, 12)} and ${input.importedHead.slice(0, 12)}: ${reason}`, ); } const mergedTreeId = mergedTree.stdout.trim().split("\n")[0]?.trim() ?? ""; if (!mergedTreeId) { throw new Error("Failed to compute a merged git tree for SSH workspace restore."); } const mergeCommit = await runLocalGit( input.localDir, [ "commit-tree", mergedTreeId, "-p", currentHead, "-p", input.importedHead, "-m", `Paperclip SSH sync merge ${input.importedHead.slice(0, 12)}`, ], { timeout: 60_000, maxBuffer: 64 * 1024, }, ); await runLocalGit(input.localDir, ["update-ref", headRef, mergeCommit.stdout.trim(), currentHead], { timeout: 10_000, maxBuffer: 16 * 1024, }); } async function clearRemoteDirectory(input: { spec: SshConnectionConfig; remoteDir: string; preserveEntries?: string[]; }): Promise { const preservePatterns = (input.preserveEntries ?? []) .map((entry) => `! -name ${shellQuote(entry)}`) .join(" "); const script = [ "set -e", `mkdir -p ${shellQuote(input.remoteDir)}`, `find ${shellQuote(input.remoteDir)} -mindepth 1 -maxdepth 1 ${preservePatterns} -exec rm -rf -- {} +`, ].join("\n"); await runSshScript(input.spec, script, { timeoutMs: 30_000, maxBuffer: 256 * 1024, }); } async function removeDeletedPathsOnSsh(input: { spec: SshConnectionConfig; remoteDir: string; deletedPaths: string[]; }): Promise { if (input.deletedPaths.length === 0) return; const quotedPaths = input.deletedPaths.map((entry) => shellQuote(entry)).join(" "); const script = `cd ${shellQuote(input.remoteDir)} && rm -rf -- ${quotedPaths}`; await runSshScript(input.spec, script, { timeoutMs: 30_000, maxBuffer: 256 * 1024, }); } async function allocateLoopbackPort(host: string): Promise { return await new Promise((resolve, reject) => { const server = net.createServer(); server.once("error", reject); server.listen(0, host, () => { const address = server.address(); if (!address || typeof address === "string") { server.close(() => reject(new Error("Failed to allocate a loopback port."))); return; } const { port } = address; server.close((error) => { if (error) { reject(error); return; } resolve(port); }); }); }); } async function waitForCondition( fn: () => Promise, options: { timeoutMs?: number; intervalMs?: number; } = {}, ): Promise { const timeoutAt = Date.now() + (options.timeoutMs ?? 10_000); const intervalMs = options.intervalMs ?? 200; let lastError: unknown = null; while (Date.now() < timeoutAt) { try { await fn(); return; } catch (error) { lastError = error; await new Promise((resolve) => setTimeout(resolve, intervalMs)); } } throw lastError instanceof Error ? lastError : new Error("Timed out waiting for SSH fixture readiness."); } async function isPidRunning(pid: number): Promise { try { process.kill(pid, 0); return true; } catch { return false; } } async function readProcessCommand(pid: number): Promise { for (const format of ["command=", "args="]) { try { const result = await execFileText("ps", ["-o", format, "-p", String(pid)], { timeout: 5_000, maxBuffer: 16 * 1024, }); const command = result.stdout.trim(); if (command.length > 0) { return command; } } catch { continue; } } return null; } async function isSshEnvLabFixtureProcess(state: Pick): Promise { if (!(await isPidRunning(state.pid))) { return false; } const command = await readProcessCommand(state.pid); if (!command) { return false; } return command.includes(state.sshdConfigPath); } export async function getSshEnvLabSupport(): Promise { for (const command of ["ssh", "sshd", "ssh-keygen"]) { if (!(await commandExists(command))) { return { supported: false, reason: `Missing required command: ${command}`, }; } } return { supported: true, reason: null, }; } export function buildKnownHostsEntry(input: { host: string; port: number; publicKey: string; }): string { return `[${input.host}]:${input.port} ${input.publicKey.trim()}`; } export async function runSshCommand( config: SshConnectionConfig, remoteCommand: string, options: { env?: Record; stdin?: string; timeoutMs?: number; maxBuffer?: number; } = {}, ): Promise { let cleanup: () => Promise = () => Promise.resolve(); try { const auth = await createSshAuthArgs(config); cleanup = auth.cleanup; const sshArgs = [...auth.args]; const envEntries = Object.entries(options.env ?? {}) .filter((entry): entry is [string, string] => typeof entry[1] === "string"); for (const [key] of envEntries) { if (!isValidShellEnvKey(key)) { throw new Error(`Invalid SSH environment variable key: ${key}`); } } // Mirror buildSshSpawnTarget: source login profiles first, then run // `env KEY=VAL cmd` so user-supplied identity overrides win over anything // a profile re-exports. Without this, a remote profile that resets HOME // / NVM_DIR / etc. would silently undo the explicit env passed in here. const envArgs = envEntries.map(([key, value]) => `${key}=${shellQuote(value)}`); const remoteScript = [ 'if [ -f "$HOME/.profile" ]; then . "$HOME/.profile" >/dev/null 2>&1 || true; fi', 'if [ -f "$HOME/.bash_profile" ]; then . "$HOME/.bash_profile" >/dev/null 2>&1 || true; fi', 'if [ -f "$HOME/.zprofile" ]; then . "$HOME/.zprofile" >/dev/null 2>&1 || true; fi', envArgs.length > 0 ? `exec env ${envArgs.join(" ")} sh -c ${shellQuote(remoteCommand)}` : `exec sh -c ${shellQuote(remoteCommand)}`, ].join(" && "); sshArgs.push( "-p", String(config.port), `${config.username}@${config.host}`, `sh -lc ${shellQuote(remoteScript)}`, ); return options.stdin != null ? await spawnText("ssh", sshArgs, { stdin: options.stdin, timeout: options.timeoutMs ?? 15_000, maxBuffer: options.maxBuffer ?? 1024 * 128, }) : await execFileText("ssh", sshArgs, { timeout: options.timeoutMs ?? 15_000, maxBuffer: options.maxBuffer ?? 1024 * 128, }); } finally { await cleanup(); } } export async function buildSshSpawnTarget(input: { spec: SshRemoteExecutionSpec; command: string; args: string[]; env: Record; }): Promise<{ command: string; args: string[]; cleanup: () => Promise; }> { for (const key of Object.keys(input.env)) { if (!isValidShellEnvKey(key)) { throw new Error(`Invalid SSH environment variable key: ${key}`); } } const auth = await createSshAuthArgs(input.spec); const sshArgs = [...auth.args]; const envArgs = Object.entries(input.env) .filter((entry): entry is [string, string] => typeof entry[1] === "string") .map(([key, value]) => `${key}=${shellQuote(value)}`); const remoteCommandParts = [shellQuote(input.command), ...input.args.map((arg) => shellQuote(arg))].join(" "); const remoteScript = [ 'if [ -f "$HOME/.profile" ]; then . "$HOME/.profile" >/dev/null 2>&1 || true; fi', 'if [ -f "$HOME/.bash_profile" ]; then . "$HOME/.bash_profile" >/dev/null 2>&1 || true; fi', 'if [ -f "$HOME/.zprofile" ]; then . "$HOME/.zprofile" >/dev/null 2>&1 || true; fi', 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', '[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" >/dev/null 2>&1 || true', `cd ${shellQuote(input.spec.remoteCwd)}`, envArgs.length > 0 ? `exec env ${envArgs.join(" ")} ${remoteCommandParts}` : `exec ${remoteCommandParts}`, ].join(" && "); sshArgs.push( "-p", String(input.spec.port), `${input.spec.username}@${input.spec.host}`, `sh -lc ${shellQuote(remoteScript)}`, ); return { command: "ssh", args: sshArgs, cleanup: auth.cleanup, }; } export async function syncDirectoryToSsh(input: { spec: SshRemoteExecutionSpec; localDir: string; remoteDir: string; exclude?: string[]; followSymlinks?: boolean; }): Promise { const auth = await createSshAuthArgs(input.spec); const sshArgs = [ ...auth.args, "-p", String(input.spec.port), `${input.spec.username}@${input.spec.host}`, `sh -lc ${shellQuote(`mkdir -p ${shellQuote(input.remoteDir)} && tar -xf - -C ${shellQuote(input.remoteDir)}`)}`, ]; await new Promise((resolve, reject) => { const tarArgs = [ ...(input.followSymlinks ? ["-h"] : []), "-C", input.localDir, ...tarExcludeArgs(input.exclude), "-cf", "-", ".", ]; const tar = spawn("tar", tarArgs, { stdio: ["ignore", "pipe", "pipe"], env: tarSpawnEnv(), }); const ssh = spawn("ssh", sshArgs, { stdio: ["pipe", "ignore", "pipe"], }); let tarStderr = ""; let sshStderr = ""; let settled = false; let tarExited = false; let sshExited = false; let tarExitCode: number | null = null; let sshExitCode: number | null = null; const maybeFinish = () => { if (settled || !tarExited || !sshExited) { return; } settled = true; if ((tarExitCode ?? 0) !== 0) { reject(new Error(tarStderr.trim() || `tar exited with code ${tarExitCode ?? -1}`)); return; } if ((sshExitCode ?? 0) !== 0) { reject(new Error(sshStderr.trim() || `ssh exited with code ${sshExitCode ?? -1}`)); return; } resolve(); }; const fail = (error: Error) => { if (settled) { return; } settled = true; tar.kill("SIGTERM"); ssh.kill("SIGTERM"); reject(error); }; tar.stdout?.pipe(ssh.stdin ?? null); tar.stderr?.on("data", (chunk) => { tarStderr += String(chunk); }); ssh.stderr?.on("data", (chunk) => { sshStderr += String(chunk); }); tar.on("error", fail); ssh.on("error", fail); tar.on("close", (code) => { tarExited = true; tarExitCode = code; maybeFinish(); }); ssh.on("close", (code) => { sshExited = true; sshExitCode = code; maybeFinish(); }); }).finally(auth.cleanup); } export async function syncDirectoryFromSsh(input: { spec: SshRemoteExecutionSpec; remoteDir: string; localDir: string; exclude?: string[]; preserveLocalEntries?: string[]; }): Promise { const auth = await createSshAuthArgs(input.spec); const stagingDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-sync-back-")); const remoteTarScript = [ `cd ${shellQuote(input.remoteDir)}`, `tar ${[...tarExcludeArgs(input.exclude).map(shellQuote), "-cf", "-", "."].join(" ")}`, ].join(" && "); const sshArgs = [ ...auth.args, "-p", String(input.spec.port), `${input.spec.username}@${input.spec.host}`, `sh -lc ${shellQuote(remoteTarScript)}`, ]; try { await new Promise((resolve, reject) => { const ssh = spawn("ssh", sshArgs, { stdio: ["ignore", "pipe", "pipe"], }); const tar = spawn("tar", ["-xf", "-", "-C", stagingDir], { stdio: ["pipe", "ignore", "pipe"], env: tarSpawnEnv(), }); let sshStderr = ""; let tarStderr = ""; let settled = false; let sshExited = false; let tarExited = false; let sshExitCode: number | null = null; let tarExitCode: number | null = null; const maybeFinish = () => { if (settled || !sshExited || !tarExited) return; settled = true; if ((sshExitCode ?? 0) !== 0) { reject(new Error(sshStderr.trim() || `ssh exited with code ${sshExitCode ?? -1}`)); return; } if ((tarExitCode ?? 0) !== 0) { reject(new Error(tarStderr.trim() || `tar exited with code ${tarExitCode ?? -1}`)); return; } resolve(); }; const fail = (error: Error) => { if (settled) return; settled = true; ssh.kill("SIGTERM"); tar.kill("SIGTERM"); reject(error); }; ssh.stdout?.pipe(tar.stdin ?? null); ssh.stderr?.on("data", (chunk) => { sshStderr += String(chunk); }); tar.stderr?.on("data", (chunk) => { tarStderr += String(chunk); }); ssh.on("error", fail); tar.on("error", fail); ssh.on("close", (code) => { sshExited = true; sshExitCode = code; maybeFinish(); }); tar.on("close", (code) => { tarExited = true; tarExitCode = code; maybeFinish(); }); }); await clearLocalDirectory(input.localDir, input.preserveLocalEntries); await copyDirectoryContents(stagingDir, input.localDir); } finally { await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); await auth.cleanup(); } } export async function prepareWorkspaceForSshExecution(input: { spec: SshRemoteExecutionSpec; localDir: string; remoteDir?: string; }): Promise<{ gitBacked: boolean }> { const remoteDir = input.remoteDir ?? input.spec.remoteCwd; const gitSnapshot = await readLocalGitWorkspaceSnapshot(input.localDir); if (gitSnapshot) { await importGitWorkspaceToSsh({ spec: input.spec, localDir: input.localDir, remoteDir, snapshot: gitSnapshot, }); await syncDirectoryToSsh({ spec: input.spec, localDir: input.localDir, remoteDir, exclude: [".git", ".paperclip-runtime"], }); await removeDeletedPathsOnSsh({ spec: input.spec, remoteDir, deletedPaths: gitSnapshot.deletedPaths, }); return { gitBacked: true }; } await clearRemoteDirectory({ spec: input.spec, remoteDir, preserveEntries: [".paperclip-runtime"], }); await syncDirectoryToSsh({ spec: input.spec, localDir: input.localDir, remoteDir, exclude: [".paperclip-runtime"], }); return { gitBacked: false }; } export async function restoreWorkspaceFromSshExecution(input: { spec: SshRemoteExecutionSpec; localDir: string; remoteDir?: string; baselineSnapshot?: DirectorySnapshot; restoreGitHistory?: boolean; }): Promise { const remoteDir = input.remoteDir ?? input.spec.remoteCwd; if (input.baselineSnapshot) { const stagingDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-sync-back-")); const importedRef = input.restoreGitHistory ? `refs/paperclip/ssh-sync/imported/${randomUUID()}` : null; try { const importedHead = input.restoreGitHistory ? await exportGitWorkspaceFromSsh({ spec: input.spec, remoteDir, localDir: input.localDir, importedRef: importedRef ?? undefined, resetLocalWorkspace: false, }) : null; await syncDirectoryFromSsh({ spec: input.spec, remoteDir, localDir: stagingDir, exclude: input.baselineSnapshot.exclude, }); await mergeDirectoryWithBaseline({ baseline: input.baselineSnapshot, sourceDir: stagingDir, targetDir: input.localDir, // Git history advances via integrateImportedGitHead; the working tree // still comes from the remote file snapshot so dirty remote edits win. beforeApply: importedHead ? async () => { await integrateImportedGitHead({ localDir: input.localDir, importedHead, }); } : undefined, }); } finally { if (importedRef) { await runLocalGit(input.localDir, ["update-ref", "-d", importedRef], { timeout: 10_000, maxBuffer: 16 * 1024, }).catch(() => undefined); } await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); } return; } const gitSnapshot = await readLocalGitWorkspaceSnapshot(input.localDir); if (gitSnapshot) { await exportGitWorkspaceFromSsh({ spec: input.spec, remoteDir, localDir: input.localDir, }); await syncDirectoryFromSsh({ spec: input.spec, remoteDir, localDir: input.localDir, exclude: [".git", ".paperclip-runtime"], preserveLocalEntries: [".git"], }); return; } await syncDirectoryFromSsh({ spec: input.spec, remoteDir, localDir: input.localDir, exclude: [".paperclip-runtime"], }); } export async function ensureSshWorkspaceReady( config: SshConnectionConfig, ): Promise<{ remoteCwd: string }> { const result = await runSshCommand( config, `sh -lc ${shellQuote(`mkdir -p ${shellQuote(config.remoteWorkspacePath)} && cd ${shellQuote(config.remoteWorkspacePath)} && pwd`)}`, ); return { remoteCwd: result.stdout.trim(), }; } export async function readSshEnvLabFixtureState( statePath: string, ): Promise { try { const raw = JSON.parse(await fs.readFile(statePath, "utf8")) as SshEnvLabFixtureState; if (!raw || raw.kind !== "ssh_openbsd") return null; return raw; } catch { return null; } } export async function stopSshEnvLabFixture(statePath: string): Promise { const state = await readSshEnvLabFixtureState(statePath); if (!state) return false; if (await isSshEnvLabFixtureProcess(state)) { process.kill(state.pid, "SIGTERM"); await waitForCondition(async () => { if (await isSshEnvLabFixtureProcess(state)) { throw new Error("SSH fixture process is still running."); } }, { timeoutMs: 5_000, intervalMs: 100 }); } await fs.rm(state.rootDir, { recursive: true, force: true }).catch(() => undefined); return true; } export async function startSshEnvLabFixture(input: { statePath: string; bindHost?: string; host?: string; }): Promise { const existing = await readSshEnvLabFixtureState(input.statePath); if (existing && await isSshEnvLabFixtureProcess(existing)) { return existing; } if (existing) { await fs.rm(existing.rootDir, { recursive: true, force: true }).catch(() => undefined); } const support = await getSshEnvLabSupport(); if (!support.supported) { throw new Error(`SSH env-lab fixture is unavailable: ${support.reason}`); } const sshdPath = await resolveCommandPath("sshd"); if (!sshdPath) { throw new Error("SSH env-lab fixture is unavailable: missing required command: sshd"); } const bindHost = input.bindHost ?? "127.0.0.1"; const host = input.host ?? bindHost; const rootDir = path.dirname(input.statePath); await fs.mkdir(rootDir, { recursive: true }); const username = os.userInfo().username; const port = await allocateLoopbackPort(bindHost); const workspaceDir = path.join(rootDir, "workspace"); const clientPrivateKeyPath = path.join(rootDir, "client_key"); const clientPublicKeyPath = `${clientPrivateKeyPath}.pub`; const hostPrivateKeyPath = path.join(rootDir, "host_key"); const hostPublicKeyPath = `${hostPrivateKeyPath}.pub`; const authorizedKeysPath = path.join(rootDir, "authorized_keys"); const knownHostsPath = path.join(rootDir, "known_hosts"); const sshdConfigPath = path.join(rootDir, "sshd_config"); const sshdLogPath = path.join(rootDir, "sshd.log"); const sshdPidPath = path.join(rootDir, "sshd.pid"); await fs.mkdir(workspaceDir, { recursive: true }); await execFileText("ssh-keygen", ["-q", "-t", "ed25519", "-N", "", "-f", clientPrivateKeyPath], { timeout: 15_000, }); await execFileText("ssh-keygen", ["-q", "-t", "ed25519", "-N", "", "-f", hostPrivateKeyPath], { timeout: 15_000, }); await fs.copyFile(clientPublicKeyPath, authorizedKeysPath); const hostPublicKey = (await execFileText("ssh-keygen", ["-y", "-f", hostPrivateKeyPath], { timeout: 15_000, })).stdout.trim(); await fs.writeFile( knownHostsPath, `${buildKnownHostsEntry({ host, port, publicKey: hostPublicKey })}\n`, { mode: 0o600 }, ); await fs.writeFile( sshdConfigPath, [ `Port ${port}`, `ListenAddress ${bindHost}`, `HostKey ${hostPrivateKeyPath}`, `PidFile ${sshdPidPath}`, `AuthorizedKeysFile ${authorizedKeysPath}`, "PasswordAuthentication no", "ChallengeResponseAuthentication no", "KbdInteractiveAuthentication no", "PubkeyAuthentication yes", "PermitRootLogin no", "UsePAM no", "StrictModes no", `AllowUsers ${username}`, "LogLevel VERBOSE", "PrintMotd no", "UseDNS no", "Subsystem sftp internal-sftp", "", ].join("\n"), { mode: 0o600 }, ); const child = spawn(sshdPath, ["-D", "-f", sshdConfigPath, "-E", sshdLogPath], { detached: true, stdio: "ignore", }); child.unref(); const state: SshEnvLabFixtureState = { kind: "ssh_openbsd", bindHost, host, port, username, rootDir, workspaceDir, statePath: input.statePath, pid: child.pid ?? 0, createdAt: new Date().toISOString(), clientPrivateKeyPath, clientPublicKeyPath, hostPrivateKeyPath, hostPublicKeyPath, authorizedKeysPath, knownHostsPath, sshdConfigPath, sshdLogPath, }; if (!state.pid) { throw new Error("Failed to start SSH env-lab fixture."); } try { await waitForCondition(async () => { if (!(await isPidRunning(state.pid))) { const logOutput = await fs.readFile(sshdLogPath, "utf8").catch(() => ""); throw new Error(logOutput || "SSH env-lab fixture exited before becoming ready."); } const config = await buildSshEnvLabFixtureConfig(state); await ensureSshWorkspaceReady(config); }, { timeoutMs: 10_000, intervalMs: 250 }); await fs.writeFile(input.statePath, JSON.stringify(state, null, 2), { mode: 0o600 }); return state; } catch (error) { if (await isPidRunning(state.pid)) { process.kill(state.pid, "SIGTERM"); } await fs.rm(rootDir, { recursive: true, force: true }).catch(() => undefined); throw error; } } export async function buildSshEnvLabFixtureConfig( state: SshEnvLabFixtureState, ): Promise { const [privateKey, knownHosts] = await Promise.all([ fs.readFile(state.clientPrivateKeyPath, "utf8"), fs.readFile(state.knownHostsPath, "utf8"), ]); return { host: state.host, port: state.port, username: state.username, remoteWorkspacePath: state.workspaceDir, privateKey, knownHosts, strictHostKeyChecking: true, }; } export async function readSshEnvLabFixtureStatus(statePath: string): Promise<{ running: boolean; state: SshEnvLabFixtureState | null; }> { const state = await readSshEnvLabFixtureState(statePath); if (!state) { return { running: false, state: null }; } return { running: await isSshEnvLabFixtureProcess(state), state, }; } export async function fileExists(filePath: string): Promise { try { await fs.access(filePath, fsConstants.F_OK); return true; } catch { return false; } }