mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-15 10:30:37 +09:00
Fix company zip imports
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
dotta
parent
9786ebb7ba
commit
dcead97650
5 changed files with 420 additions and 7 deletions
|
|
@ -1,5 +1,5 @@
|
|||
import { execFile, spawn } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import net from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
|
@ -182,6 +182,107 @@ function createCliEnv() {
|
|||
return env;
|
||||
}
|
||||
|
||||
function writeUint16(target: Uint8Array, offset: number, value: number) {
|
||||
target[offset] = value & 0xff;
|
||||
target[offset + 1] = (value >>> 8) & 0xff;
|
||||
}
|
||||
|
||||
function writeUint32(target: Uint8Array, offset: number, value: number) {
|
||||
target[offset] = value & 0xff;
|
||||
target[offset + 1] = (value >>> 8) & 0xff;
|
||||
target[offset + 2] = (value >>> 16) & 0xff;
|
||||
target[offset + 3] = (value >>> 24) & 0xff;
|
||||
}
|
||||
|
||||
function crc32(bytes: Uint8Array) {
|
||||
let crc = 0xffffffff;
|
||||
for (const byte of bytes) {
|
||||
crc ^= byte;
|
||||
for (let bit = 0; bit < 8; bit += 1) {
|
||||
crc = (crc & 1) === 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
|
||||
}
|
||||
}
|
||||
return (crc ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function collectTextFiles(root: string, current: string, files: Record<string, string>) {
|
||||
for (const entry of readdirSync(current, { withFileTypes: true })) {
|
||||
const absolutePath = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
collectTextFiles(root, absolutePath, files);
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) continue;
|
||||
const relativePath = path.relative(root, absolutePath).replace(/\\/g, "/");
|
||||
files[relativePath] = readFileSync(absolutePath, "utf8");
|
||||
}
|
||||
}
|
||||
|
||||
function createStoredZipArchive(files: Record<string, string>, rootPath: string) {
|
||||
const encoder = new TextEncoder();
|
||||
const localChunks: Uint8Array[] = [];
|
||||
const centralChunks: Uint8Array[] = [];
|
||||
let localOffset = 0;
|
||||
let entryCount = 0;
|
||||
|
||||
for (const [relativePath, content] of Object.entries(files).sort(([left], [right]) => left.localeCompare(right))) {
|
||||
const fileName = encoder.encode(`${rootPath}/${relativePath}`);
|
||||
const body = encoder.encode(content);
|
||||
const checksum = crc32(body);
|
||||
|
||||
const localHeader = new Uint8Array(30 + fileName.length);
|
||||
writeUint32(localHeader, 0, 0x04034b50);
|
||||
writeUint16(localHeader, 4, 20);
|
||||
writeUint16(localHeader, 6, 0x0800);
|
||||
writeUint16(localHeader, 8, 0);
|
||||
writeUint32(localHeader, 14, checksum);
|
||||
writeUint32(localHeader, 18, body.length);
|
||||
writeUint32(localHeader, 22, body.length);
|
||||
writeUint16(localHeader, 26, fileName.length);
|
||||
localHeader.set(fileName, 30);
|
||||
|
||||
const centralHeader = new Uint8Array(46 + fileName.length);
|
||||
writeUint32(centralHeader, 0, 0x02014b50);
|
||||
writeUint16(centralHeader, 4, 20);
|
||||
writeUint16(centralHeader, 6, 20);
|
||||
writeUint16(centralHeader, 8, 0x0800);
|
||||
writeUint16(centralHeader, 10, 0);
|
||||
writeUint32(centralHeader, 16, checksum);
|
||||
writeUint32(centralHeader, 20, body.length);
|
||||
writeUint32(centralHeader, 24, body.length);
|
||||
writeUint16(centralHeader, 28, fileName.length);
|
||||
writeUint32(centralHeader, 42, localOffset);
|
||||
centralHeader.set(fileName, 46);
|
||||
|
||||
localChunks.push(localHeader, body);
|
||||
centralChunks.push(centralHeader);
|
||||
localOffset += localHeader.length + body.length;
|
||||
entryCount += 1;
|
||||
}
|
||||
|
||||
const centralDirectoryLength = centralChunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const archive = new Uint8Array(
|
||||
localChunks.reduce((sum, chunk) => sum + chunk.length, 0) + centralDirectoryLength + 22,
|
||||
);
|
||||
let offset = 0;
|
||||
for (const chunk of localChunks) {
|
||||
archive.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
const centralDirectoryOffset = offset;
|
||||
for (const chunk of centralChunks) {
|
||||
archive.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
writeUint32(archive, offset, 0x06054b50);
|
||||
writeUint16(archive, offset + 8, entryCount);
|
||||
writeUint16(archive, offset + 10, entryCount);
|
||||
writeUint32(archive, offset + 12, centralDirectoryLength);
|
||||
writeUint32(archive, offset + 16, centralDirectoryOffset);
|
||||
|
||||
return archive;
|
||||
}
|
||||
|
||||
async function stopServerProcess(child: ServerProcess | null) {
|
||||
if (!child || child.exitCode !== null) return;
|
||||
child.kill("SIGTERM");
|
||||
|
|
@ -345,6 +446,8 @@ describe("paperclipai company import/export e2e", () => {
|
|||
},
|
||||
);
|
||||
|
||||
const largeIssueDescription = `Round-trip the company package through the CLI.\n\n${"portable-data ".repeat(12_000)}`;
|
||||
|
||||
const sourceIssue = await api<{ id: string; title: string; identifier: string }>(
|
||||
apiBase,
|
||||
`/api/companies/${sourceCompany.id}/issues`,
|
||||
|
|
@ -353,7 +456,7 @@ describe("paperclipai company import/export e2e", () => {
|
|||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
title: "Validate company import/export",
|
||||
description: "Round-trip the company package through the CLI.",
|
||||
description: largeIssueDescription,
|
||||
status: "todo",
|
||||
projectId: sourceProject.id,
|
||||
assigneeAgentId: sourceAgent.id,
|
||||
|
|
@ -397,6 +500,7 @@ describe("paperclipai company import/export e2e", () => {
|
|||
`Imported ${sourceCompany.name}`,
|
||||
"--include",
|
||||
"company,agents,projects,issues",
|
||||
"--yes",
|
||||
],
|
||||
{ apiBase, configPath },
|
||||
);
|
||||
|
|
@ -470,6 +574,7 @@ describe("paperclipai company import/export e2e", () => {
|
|||
"company,agents,projects,issues",
|
||||
"--collision",
|
||||
"rename",
|
||||
"--yes",
|
||||
],
|
||||
{ apiBase, configPath },
|
||||
);
|
||||
|
|
@ -494,5 +599,32 @@ describe("paperclipai company import/export e2e", () => {
|
|||
expect(new Set(twiceImportedAgents.map((agent) => agent.name)).size).toBe(2);
|
||||
expect(twiceImportedProjects).toHaveLength(2);
|
||||
expect(twiceImportedIssues).toHaveLength(2);
|
||||
|
||||
const zipPath = path.join(tempRoot, "exported-company.zip");
|
||||
const portableFiles: Record<string, string> = {};
|
||||
collectTextFiles(exportDir, exportDir, portableFiles);
|
||||
writeFileSync(zipPath, createStoredZipArchive(portableFiles, "paperclip-demo"));
|
||||
|
||||
const importedFromZip = await runCliJson<{
|
||||
company: { id: string; name: string; action: string };
|
||||
agents: Array<{ id: string | null; action: string; name: string }>;
|
||||
}>(
|
||||
[
|
||||
"company",
|
||||
"import",
|
||||
zipPath,
|
||||
"--target",
|
||||
"new",
|
||||
"--new-company-name",
|
||||
`Zip Imported ${sourceCompany.name}`,
|
||||
"--include",
|
||||
"company,agents,projects,issues",
|
||||
"--yes",
|
||||
],
|
||||
{ apiBase, configPath },
|
||||
);
|
||||
|
||||
expect(importedFromZip.company.action).toBe("created");
|
||||
expect(importedFromZip.agents.some((agent) => agent.action === "created")).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
|
|
|
|||
133
cli/src/__tests__/company-import-zip.test.ts
Normal file
133
cli/src/__tests__/company-import-zip.test.ts
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { resolveInlineSourceFromPath } from "../commands/client/company.js";
|
||||
|
||||
function writeUint16(target: Uint8Array, offset: number, value: number) {
|
||||
target[offset] = value & 0xff;
|
||||
target[offset + 1] = (value >>> 8) & 0xff;
|
||||
}
|
||||
|
||||
function writeUint32(target: Uint8Array, offset: number, value: number) {
|
||||
target[offset] = value & 0xff;
|
||||
target[offset + 1] = (value >>> 8) & 0xff;
|
||||
target[offset + 2] = (value >>> 16) & 0xff;
|
||||
target[offset + 3] = (value >>> 24) & 0xff;
|
||||
}
|
||||
|
||||
function crc32(bytes: Uint8Array) {
|
||||
let crc = 0xffffffff;
|
||||
for (const byte of bytes) {
|
||||
crc ^= byte;
|
||||
for (let bit = 0; bit < 8; bit += 1) {
|
||||
crc = (crc & 1) === 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
|
||||
}
|
||||
}
|
||||
return (crc ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function createStoredZipArchive(files: Record<string, string>, rootPath: string) {
|
||||
const encoder = new TextEncoder();
|
||||
const localChunks: Uint8Array[] = [];
|
||||
const centralChunks: Uint8Array[] = [];
|
||||
let localOffset = 0;
|
||||
let entryCount = 0;
|
||||
|
||||
for (const [relativePath, content] of Object.entries(files).sort(([left], [right]) => left.localeCompare(right))) {
|
||||
const fileName = encoder.encode(`${rootPath}/${relativePath}`);
|
||||
const body = encoder.encode(content);
|
||||
const checksum = crc32(body);
|
||||
|
||||
const localHeader = new Uint8Array(30 + fileName.length);
|
||||
writeUint32(localHeader, 0, 0x04034b50);
|
||||
writeUint16(localHeader, 4, 20);
|
||||
writeUint16(localHeader, 6, 0x0800);
|
||||
writeUint16(localHeader, 8, 0);
|
||||
writeUint32(localHeader, 14, checksum);
|
||||
writeUint32(localHeader, 18, body.length);
|
||||
writeUint32(localHeader, 22, body.length);
|
||||
writeUint16(localHeader, 26, fileName.length);
|
||||
localHeader.set(fileName, 30);
|
||||
|
||||
const centralHeader = new Uint8Array(46 + fileName.length);
|
||||
writeUint32(centralHeader, 0, 0x02014b50);
|
||||
writeUint16(centralHeader, 4, 20);
|
||||
writeUint16(centralHeader, 6, 20);
|
||||
writeUint16(centralHeader, 8, 0x0800);
|
||||
writeUint16(centralHeader, 10, 0);
|
||||
writeUint32(centralHeader, 16, checksum);
|
||||
writeUint32(centralHeader, 20, body.length);
|
||||
writeUint32(centralHeader, 24, body.length);
|
||||
writeUint16(centralHeader, 28, fileName.length);
|
||||
writeUint32(centralHeader, 42, localOffset);
|
||||
centralHeader.set(fileName, 46);
|
||||
|
||||
localChunks.push(localHeader, body);
|
||||
centralChunks.push(centralHeader);
|
||||
localOffset += localHeader.length + body.length;
|
||||
entryCount += 1;
|
||||
}
|
||||
|
||||
const centralDirectoryLength = centralChunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const archive = new Uint8Array(
|
||||
localChunks.reduce((sum, chunk) => sum + chunk.length, 0) + centralDirectoryLength + 22,
|
||||
);
|
||||
|
||||
let offset = 0;
|
||||
for (const chunk of localChunks) {
|
||||
archive.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
const centralDirectoryOffset = offset;
|
||||
for (const chunk of centralChunks) {
|
||||
archive.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
writeUint32(archive, offset, 0x06054b50);
|
||||
writeUint16(archive, offset + 8, entryCount);
|
||||
writeUint16(archive, offset + 10, entryCount);
|
||||
writeUint32(archive, offset + 12, centralDirectoryLength);
|
||||
writeUint32(archive, offset + 16, centralDirectoryOffset);
|
||||
|
||||
return archive;
|
||||
}
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("resolveInlineSourceFromPath", () => {
|
||||
it("imports portable files from a zip archive instead of scanning the parent directory", async () => {
|
||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-company-import-zip-"));
|
||||
tempDirs.push(tempDir);
|
||||
|
||||
const archivePath = path.join(tempDir, "paperclip-demo.zip");
|
||||
const archive = createStoredZipArchive(
|
||||
{
|
||||
"COMPANY.md": "# Company\n",
|
||||
".paperclip.yaml": "schema: paperclip/v1\n",
|
||||
"agents/ceo/AGENT.md": "# CEO\n",
|
||||
"notes/todo.txt": "ignore me\n",
|
||||
},
|
||||
"paperclip-demo",
|
||||
);
|
||||
await writeFile(archivePath, archive);
|
||||
|
||||
const resolved = await resolveInlineSourceFromPath(archivePath);
|
||||
|
||||
expect(resolved).toEqual({
|
||||
rootPath: "paperclip-demo",
|
||||
files: {
|
||||
"COMPANY.md": "# Company\n",
|
||||
".paperclip.yaml": "schema: paperclip/v1\n",
|
||||
"agents/ceo/AGENT.md": "# CEO\n",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue