mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-19 12:10:37 +09:00
[codex] Add skills CLI and catalog management (#6782)
## Thinking Path > - Paperclip orchestrates AI agents for zero-human companies through company-scoped control-plane workflows. > - Agents need reusable, inspectable skills that can be installed, reset, audited, exported, and assigned without bespoke local setup. > - The existing skill truth model needed cleanup so bundled skills, optional catalog skills, runtime skills, and adapter-provided skills have clear provenance. > - Operators also need a practical CLI and board UI for discovering and managing company skills. > - This pull request adds the skills CLI, packaged skills catalog, company skills APIs, and catalog-aware board UI. > - The benefit is a more reusable Paperclip company setup where skills are portable, auditable, and easier for operators and agents to manage. ## What Changed - Added `paperclipai skills` CLI commands and coverage for catalog listing, installing, resetting, and inspecting company skills. - Added a packaged `@paperclipai/skills-catalog` workspace with bundled and optional skill content plus validation/build tests. - Added shared company-skill types and validators used across CLI, server, and UI contracts. - Added server catalog APIs/services for company skill catalog operations, reset semantics, audit behavior, and portability provenance. - Updated adapter skill handling so runtime/catalog provenance remains explicit across local adapters. - Added board UI support for browsing and managing catalog-backed company skills. - Updated docs for the skills CLI/catalog flow and the company skills Paperclip skill reference. - Rebased the branch onto current `paperclipai/paperclip:master`; no `pnpm-lock.yaml`, `.github/workflows`, or migration files are included in the final PR diff. ## Verification - Passed: `pnpm run preflight:workspace-links && pnpm exec vitest run cli/src/__tests__/skills.test.ts packages/skills-catalog/src/catalog-builder.test.ts packages/skills-catalog/src/shipped-catalog.test.ts packages/shared/src/validators/company-skill.test.ts packages/adapter-utils/src/server-utils.test.ts packages/plugins/create-paperclip-plugin/src/entrypoints.test.ts server/src/__tests__/company-skills-catalog-service.test.ts server/src/__tests__/company-skills-routes.test.ts server/src/__tests__/company-portability.test.ts`. - Passed: `pnpm exec vitest run server/src/__tests__/workspace-runtime.test.ts -t "default branch|origin/master|symbolic-ref"`. - Attempted: full `server/src/__tests__/workspace-runtime.test.ts`. Four provisioning tests failed while seeding an isolated worktree database from the local Paperclip instance because the local plugin schema dump contains a duplicate-column foreign key (`plugin_content_machine_18a7bc327b.content_case_signals`). The default-branch tests touched by the rebase conflict passed in the focused run above. - Checked final diff: no `pnpm-lock.yaml`, no `.github/workflows`, and no migration-file changes relative to `master`. ## Risks - Medium: this is a broad skills/catalog change touching CLI, server APIs, shared contracts, adapter skill sync, and UI. - Catalog validation and reset semantics need careful reviewer attention because they affect reusable company setup and portability. - No database migrations are included in this PR, so there is no migration ordering/idempotency risk in the final diff. - No lockfile is included by design; dependency resolution will be handled by the repository lockfile workflow. ## Model Used - OpenAI Codex coding agent based on GPT-5, running in Paperclip via the `codex_local` adapter with shell, git, GitHub CLI, and code-editing tool access. Exact hosted model build/context-window metadata is not exposed in this runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run targeted tests locally and documented the local workspace-runtime seed failure above - [x] I have added or updated tests where applicable - [x] If this change affects the UI, screenshots were intentionally omitted per PAP-10124 instructions; UI behavior is covered by tests and reviewer inspection - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
parent
8da50dbcf8
commit
9eac727cf1
77 changed files with 9704 additions and 530 deletions
165
packages/skills-catalog/src/catalog-builder.test.ts
Normal file
165
packages/skills-catalog/src/catalog-builder.test.ts
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildCatalogManifest,
|
||||
formatCatalogManifest,
|
||||
validateCatalog,
|
||||
} from "./catalog-builder.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
describe("skills catalog manifest", () => {
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
it("builds stable manifest entries from catalog skill directories", async () => {
|
||||
const packageDir = await createCatalogPackage();
|
||||
await writeSkill(packageDir, "bundled", "software-development", "github-pr-workflow", {
|
||||
frontmatter: [
|
||||
"name: GitHub PR Workflow",
|
||||
"description: Prepare pull requests and verification notes.",
|
||||
"key: paperclipai/bundled/software-development/github-pr-workflow",
|
||||
"recommendedForRoles:",
|
||||
" - engineer",
|
||||
"tags:",
|
||||
" - github",
|
||||
" - pull-requests",
|
||||
],
|
||||
files: {
|
||||
"references/checklist.md": "# Checklist\n",
|
||||
},
|
||||
});
|
||||
|
||||
const result = await buildCatalogManifest({
|
||||
packageDir,
|
||||
generatedAt: "2026-05-26T00:00:00.000Z",
|
||||
});
|
||||
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(result.manifest.skills).toHaveLength(1);
|
||||
expect(result.manifest.skills[0]).toMatchObject({
|
||||
id: "paperclipai:bundled:software-development:github-pr-workflow",
|
||||
key: "paperclipai/bundled/software-development/github-pr-workflow",
|
||||
kind: "bundled",
|
||||
category: "software-development",
|
||||
slug: "github-pr-workflow",
|
||||
name: "GitHub PR Workflow",
|
||||
trustLevel: "markdown_only",
|
||||
compatibility: "compatible",
|
||||
recommendedForRoles: ["engineer"],
|
||||
tags: ["github", "pull-requests"],
|
||||
});
|
||||
expect(result.manifest.skills[0]!.files.map((file) => file.path)).toEqual([
|
||||
"SKILL.md",
|
||||
"references/checklist.md",
|
||||
]);
|
||||
expect(result.manifest.skills[0]!.contentHash).toMatch(/^sha256:[a-f0-9]{64}$/);
|
||||
});
|
||||
|
||||
it("reports frontmatter, directory, uniqueness, and inventory errors together", async () => {
|
||||
const packageDir = await createCatalogPackage();
|
||||
await writeSkill(packageDir, "bundled", "Bad_Category", "duplicate", {
|
||||
frontmatter: [
|
||||
"name: Duplicate",
|
||||
"key: paperclipai/bundled/software-development/other",
|
||||
"recommendedForRoles: engineer",
|
||||
],
|
||||
});
|
||||
await writeSkill(packageDir, "optional", "software-development", "duplicate", {
|
||||
frontmatter: [
|
||||
"name: Duplicate Optional",
|
||||
"description: Optional duplicate slug.",
|
||||
],
|
||||
});
|
||||
await fs.mkdir(path.join(packageDir, "catalog", "bundled", "software-development", "missing-skill"), {
|
||||
recursive: true,
|
||||
});
|
||||
await fs.mkdir(path.join(packageDir, "catalog", "misc"), { recursive: true });
|
||||
await fs.writeFile(path.join(packageDir, "catalog", "misc", "SKILL.md"), "# Misplaced\n", "utf8");
|
||||
|
||||
const result = await buildCatalogManifest({
|
||||
packageDir,
|
||||
generatedAt: "2026-05-26T00:00:00.000Z",
|
||||
});
|
||||
|
||||
expect(result.errors).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining("catalog/misc/SKILL.md is not under catalog/<bundled|optional>/<category>/<slug>/SKILL.md"),
|
||||
expect.stringContaining("catalog/bundled/software-development/missing-skill is missing SKILL.md"),
|
||||
expect.stringContaining("has invalid category"),
|
||||
expect.stringContaining("frontmatter must include description"),
|
||||
expect.stringContaining("key must be paperclipai/bundled/Bad_Category/duplicate"),
|
||||
expect.stringContaining("field recommendedForRoles must be an array of strings"),
|
||||
expect.stringContaining("Duplicate catalog slug \"duplicate\""),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("detects stale generated manifests", async () => {
|
||||
const packageDir = await createCatalogPackage();
|
||||
await writeSkill(packageDir, "bundled", "software-development", "review", {
|
||||
frontmatter: [
|
||||
"name: Review",
|
||||
"description: Review implementation work.",
|
||||
],
|
||||
});
|
||||
await fs.mkdir(path.join(packageDir, "generated"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(packageDir, "generated", "catalog.json"),
|
||||
formatCatalogManifest({
|
||||
schemaVersion: 1,
|
||||
packageName: "@paperclipai/skills-catalog",
|
||||
packageVersion: "0.3.1",
|
||||
generatedAt: "2026-05-26T00:00:00.000Z",
|
||||
skills: [],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await validateCatalog(packageDir);
|
||||
|
||||
expect(result.errors).toContain(
|
||||
"generated/catalog.json is stale. Run pnpm --filter @paperclipai/skills-catalog build:manifest.",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
async function createCatalogPackage() {
|
||||
const packageDir = await fs.mkdtemp(path.join(os.tmpdir(), "skills-catalog-"));
|
||||
tempDirs.push(packageDir);
|
||||
await fs.mkdir(path.join(packageDir, "catalog", "bundled"), { recursive: true });
|
||||
await fs.mkdir(path.join(packageDir, "catalog", "optional"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(packageDir, "package.json"),
|
||||
JSON.stringify({ version: "0.3.1" }),
|
||||
"utf8",
|
||||
);
|
||||
return packageDir;
|
||||
}
|
||||
|
||||
async function writeSkill(
|
||||
packageDir: string,
|
||||
kind: "bundled" | "optional",
|
||||
category: string,
|
||||
slug: string,
|
||||
options: {
|
||||
frontmatter: string[];
|
||||
files?: Record<string, string>;
|
||||
},
|
||||
) {
|
||||
const skillDir = path.join(packageDir, "catalog", kind, category, slug);
|
||||
await fs.mkdir(skillDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(skillDir, "SKILL.md"),
|
||||
`---\n${options.frontmatter.join("\n")}\n---\n\nUse this skill.\n`,
|
||||
"utf8",
|
||||
);
|
||||
for (const [relativePath, content] of Object.entries(options.files ?? {})) {
|
||||
const filePath = path.join(skillDir, relativePath);
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, content, "utf8");
|
||||
}
|
||||
}
|
||||
443
packages/skills-catalog/src/catalog-builder.ts
Normal file
443
packages/skills-catalog/src/catalog-builder.ts
Normal file
|
|
@ -0,0 +1,443 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
asBoolean,
|
||||
asString,
|
||||
asStringArray,
|
||||
parseFrontmatterMarkdown,
|
||||
} from "./frontmatter.js";
|
||||
import type {
|
||||
CatalogManifest,
|
||||
CatalogSkill,
|
||||
CatalogSkillFile,
|
||||
CatalogSkillFileKind,
|
||||
CatalogSkillKind,
|
||||
CatalogTrustLevel,
|
||||
} from "./types.js";
|
||||
|
||||
const CATALOG_PACKAGE_NAME = "@paperclipai/skills-catalog";
|
||||
const CATALOG_SCHEMA_VERSION = 1;
|
||||
const SKILL_ENTRYPOINT = "SKILL.md";
|
||||
const MAX_CATALOG_FILE_BYTES = 1024 * 1024;
|
||||
const SLUG_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
||||
const CATALOG_KINDS = new Set<CatalogSkillKind>(["bundled", "optional"]);
|
||||
|
||||
interface SkillCandidate {
|
||||
kind: CatalogSkillKind;
|
||||
category: string;
|
||||
slug: string;
|
||||
absolutePath: string;
|
||||
}
|
||||
|
||||
interface BuildCatalogManifestOptions {
|
||||
packageDir: string;
|
||||
generatedAt?: string;
|
||||
}
|
||||
|
||||
interface BuildCatalogManifestResult {
|
||||
manifest: CatalogManifest;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
export function formatCatalogManifest(manifest: CatalogManifest): string {
|
||||
return `${JSON.stringify(manifest, null, 2)}\n`;
|
||||
}
|
||||
|
||||
export async function buildExpectedCatalogManifest(
|
||||
packageDir: string,
|
||||
): Promise<BuildCatalogManifestResult> {
|
||||
const existing = await readExistingManifest(packageDir);
|
||||
const firstPass = await buildCatalogManifest({
|
||||
packageDir,
|
||||
generatedAt: existing?.generatedAt ?? new Date().toISOString(),
|
||||
});
|
||||
|
||||
if (existing && sameManifestExceptGeneratedAt(existing, firstPass.manifest)) {
|
||||
return firstPass;
|
||||
}
|
||||
|
||||
return buildCatalogManifest({
|
||||
packageDir,
|
||||
generatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
export async function buildCatalogManifest(
|
||||
options: BuildCatalogManifestOptions,
|
||||
): Promise<BuildCatalogManifestResult> {
|
||||
const packageDir = path.resolve(options.packageDir);
|
||||
const packageJson = await readPackageJson(packageDir);
|
||||
const errors: string[] = [];
|
||||
const candidates = await discoverSkillCandidates(packageDir, errors);
|
||||
const skills: CatalogSkill[] = [];
|
||||
|
||||
collectCandidateUniquenessErrors(candidates, errors);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const skill = await buildCatalogSkill(packageDir, candidate, errors);
|
||||
if (skill) skills.push(skill);
|
||||
}
|
||||
|
||||
skills.sort((a, b) => a.id.localeCompare(b.id));
|
||||
collectUniquenessErrors(skills, errors);
|
||||
|
||||
return {
|
||||
manifest: {
|
||||
schemaVersion: CATALOG_SCHEMA_VERSION,
|
||||
packageName: CATALOG_PACKAGE_NAME,
|
||||
packageVersion: packageJson.version,
|
||||
generatedAt: options.generatedAt ?? new Date().toISOString(),
|
||||
skills,
|
||||
},
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
export async function validateCatalog(packageDir: string): Promise<BuildCatalogManifestResult> {
|
||||
const expected = await buildExpectedCatalogManifest(packageDir);
|
||||
const generatedPath = path.join(packageDir, "generated", "catalog.json");
|
||||
const errors = [...expected.errors];
|
||||
|
||||
let generatedText: string | null = null;
|
||||
try {
|
||||
generatedText = await fs.readFile(generatedPath, "utf8");
|
||||
JSON.parse(generatedText);
|
||||
} catch (error) {
|
||||
errors.push(`generated/catalog.json is missing or invalid: ${errorMessage(error)}`);
|
||||
}
|
||||
|
||||
if (generatedText !== null) {
|
||||
const expectedText = formatCatalogManifest(expected.manifest);
|
||||
if (generatedText !== expectedText) {
|
||||
errors.push("generated/catalog.json is stale. Run pnpm --filter @paperclipai/skills-catalog build:manifest.");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
manifest: expected.manifest,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeCatalogManifest(packageDir: string) {
|
||||
const result = await buildExpectedCatalogManifest(packageDir);
|
||||
if (result.errors.length > 0) return result;
|
||||
|
||||
const generatedDir = path.join(packageDir, "generated");
|
||||
await fs.mkdir(generatedDir, { recursive: true });
|
||||
await fs.writeFile(path.join(generatedDir, "catalog.json"), formatCatalogManifest(result.manifest), "utf8");
|
||||
return result;
|
||||
}
|
||||
|
||||
async function readPackageJson(packageDir: string) {
|
||||
const packageJsonPath = path.join(packageDir, "package.json");
|
||||
const packageJson = JSON.parse(await fs.readFile(packageJsonPath, "utf8")) as { version?: unknown };
|
||||
const version = asString(packageJson.version);
|
||||
if (!version) throw new Error(`${packageJsonPath} must declare a package version.`);
|
||||
return { version };
|
||||
}
|
||||
|
||||
async function readExistingManifest(packageDir: string): Promise<CatalogManifest | null> {
|
||||
try {
|
||||
return JSON.parse(await fs.readFile(path.join(packageDir, "generated", "catalog.json"), "utf8")) as CatalogManifest;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function discoverSkillCandidates(packageDir: string, errors: string[]) {
|
||||
const catalogDir = path.join(packageDir, "catalog");
|
||||
const candidates: SkillCandidate[] = [];
|
||||
|
||||
if (!existsSync(catalogDir)) {
|
||||
errors.push("catalog directory is missing.");
|
||||
return candidates;
|
||||
}
|
||||
|
||||
await collectMisplacedSkillFiles(catalogDir, errors);
|
||||
|
||||
for (const kind of ["bundled", "optional"] as const) {
|
||||
const kindDir = path.join(catalogDir, kind);
|
||||
if (!existsSync(kindDir)) continue;
|
||||
|
||||
for (const categoryEntry of await sortedDirEntries(kindDir)) {
|
||||
if (!categoryEntry.isDirectory()) continue;
|
||||
const category = categoryEntry.name;
|
||||
const categoryDir = path.join(kindDir, category);
|
||||
|
||||
for (const slugEntry of await sortedDirEntries(categoryDir)) {
|
||||
if (!slugEntry.isDirectory()) continue;
|
||||
const slug = slugEntry.name;
|
||||
const skillDir = path.join(categoryDir, slug);
|
||||
if (!existsSync(path.join(skillDir, SKILL_ENTRYPOINT))) {
|
||||
errors.push(`${relativePackagePath(packageDir, skillDir)} is missing SKILL.md.`);
|
||||
continue;
|
||||
}
|
||||
candidates.push({ kind, category, slug, absolutePath: skillDir });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
async function collectMisplacedSkillFiles(catalogDir: string, errors: string[]) {
|
||||
async function visit(dir: string) {
|
||||
for (const entry of await sortedDirEntries(dir)) {
|
||||
const absolutePath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await visit(absolutePath);
|
||||
continue;
|
||||
}
|
||||
if (entry.name !== SKILL_ENTRYPOINT) continue;
|
||||
|
||||
const relativePath = toPosixPath(path.relative(catalogDir, absolutePath));
|
||||
const parts = relativePath.split("/");
|
||||
const kind = parts[0];
|
||||
if (parts.length !== 4 || !CATALOG_KINDS.has(kind as CatalogSkillKind)) {
|
||||
errors.push(`catalog/${relativePath} is not under catalog/<bundled|optional>/<category>/<slug>/SKILL.md.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await visit(catalogDir);
|
||||
}
|
||||
|
||||
async function buildCatalogSkill(
|
||||
packageDir: string,
|
||||
candidate: SkillCandidate,
|
||||
errors: string[],
|
||||
): Promise<CatalogSkill | null> {
|
||||
const prefix = relativePackagePath(packageDir, candidate.absolutePath);
|
||||
validateSlug("category", candidate.category, prefix, errors);
|
||||
validateSlug("slug", candidate.slug, prefix, errors);
|
||||
|
||||
const id = `paperclipai:${candidate.kind}:${candidate.category}:${candidate.slug}`;
|
||||
const key = `paperclipai/${candidate.kind}/${candidate.category}/${candidate.slug}`;
|
||||
const skillMarkdownPath = path.join(candidate.absolutePath, SKILL_ENTRYPOINT);
|
||||
const parsed = parseFrontmatterMarkdown(await fs.readFile(skillMarkdownPath, "utf8"));
|
||||
|
||||
if (!parsed.hasFrontmatter) {
|
||||
errors.push(`${prefix}/SKILL.md must start with YAML frontmatter.`);
|
||||
}
|
||||
|
||||
const name = asString(parsed.frontmatter.name);
|
||||
if (!name) errors.push(`${prefix}/SKILL.md frontmatter must include name.`);
|
||||
|
||||
const description = asString(parsed.frontmatter.description);
|
||||
if (!description) errors.push(`${prefix}/SKILL.md frontmatter must include description.`);
|
||||
|
||||
const explicitKey = asString(parsed.frontmatter.key);
|
||||
if (explicitKey && explicitKey !== key) {
|
||||
errors.push(`${prefix}/SKILL.md key must be ${key}.`);
|
||||
}
|
||||
|
||||
const explicitSlug = asString(parsed.frontmatter.slug);
|
||||
if (explicitSlug && explicitSlug !== candidate.slug) {
|
||||
errors.push(`${prefix}/SKILL.md slug must be ${candidate.slug}.`);
|
||||
}
|
||||
|
||||
const defaultInstall = asBoolean(parsed.frontmatter.defaultInstall) ?? false;
|
||||
const recommendedForRoles = readStringArrayField(parsed.frontmatter.recommendedForRoles, "recommendedForRoles", prefix, errors);
|
||||
const requires = readStringArrayField(parsed.frontmatter.requires, "requires", prefix, errors);
|
||||
const tags = readStringArrayField(parsed.frontmatter.tags, "tags", prefix, errors);
|
||||
const files = await collectSkillFiles(packageDir, candidate.absolutePath, prefix, errors);
|
||||
|
||||
if (!name || !description) return null;
|
||||
|
||||
return {
|
||||
id,
|
||||
key,
|
||||
kind: candidate.kind,
|
||||
category: candidate.category,
|
||||
slug: candidate.slug,
|
||||
name,
|
||||
description,
|
||||
path: toPosixPath(path.relative(packageDir, candidate.absolutePath)),
|
||||
entrypoint: SKILL_ENTRYPOINT,
|
||||
trustLevel: deriveTrustLevel(files),
|
||||
compatibility: "compatible",
|
||||
defaultInstall,
|
||||
recommendedForRoles,
|
||||
requires,
|
||||
tags,
|
||||
files,
|
||||
contentHash: buildContentHash(files),
|
||||
};
|
||||
}
|
||||
|
||||
async function collectSkillFiles(
|
||||
packageDir: string,
|
||||
skillDir: string,
|
||||
prefix: string,
|
||||
errors: string[],
|
||||
): Promise<CatalogSkillFile[]> {
|
||||
const files: CatalogSkillFile[] = [];
|
||||
const skillRoot = await fs.realpath(skillDir);
|
||||
|
||||
async function visit(dir: string) {
|
||||
for (const entry of await sortedDirEntries(dir)) {
|
||||
const absolutePath = path.join(dir, entry.name);
|
||||
const lstat = await fs.lstat(absolutePath);
|
||||
let stat = lstat;
|
||||
let realPath = absolutePath;
|
||||
|
||||
if (lstat.isSymbolicLink()) {
|
||||
try {
|
||||
realPath = await fs.realpath(absolutePath);
|
||||
stat = await fs.stat(absolutePath);
|
||||
} catch {
|
||||
errors.push(`${relativePackagePath(packageDir, absolutePath)} is a broken symlink.`);
|
||||
continue;
|
||||
}
|
||||
if (!isPathInside(skillRoot, realPath)) {
|
||||
errors.push(`${relativePackagePath(packageDir, absolutePath)} points outside its skill directory.`);
|
||||
continue;
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
errors.push(`${relativePackagePath(packageDir, absolutePath)} is a directory symlink; copy files into the skill directory instead.`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (stat.isDirectory()) {
|
||||
await visit(absolutePath);
|
||||
continue;
|
||||
}
|
||||
if (!stat.isFile()) continue;
|
||||
|
||||
const relativePath = toPosixPath(path.relative(skillDir, absolutePath));
|
||||
if (path.isAbsolute(relativePath) || relativePath.split("/").includes("..")) {
|
||||
errors.push(`${prefix}/${relativePath} has an invalid inventory path.`);
|
||||
continue;
|
||||
}
|
||||
if (stat.size > MAX_CATALOG_FILE_BYTES) {
|
||||
errors.push(`${prefix}/${relativePath} exceeds ${MAX_CATALOG_FILE_BYTES} bytes.`);
|
||||
}
|
||||
|
||||
const contents = await fs.readFile(absolutePath);
|
||||
files.push({
|
||||
path: relativePath,
|
||||
kind: classifyCatalogFile(relativePath),
|
||||
sizeBytes: stat.size,
|
||||
sha256: sha256(contents),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await visit(skillDir);
|
||||
files.sort((a, b) => {
|
||||
if (a.path === SKILL_ENTRYPOINT) return -1;
|
||||
if (b.path === SKILL_ENTRYPOINT) return 1;
|
||||
return a.path.localeCompare(b.path);
|
||||
});
|
||||
|
||||
if (!files.some((file) => file.path === SKILL_ENTRYPOINT && file.kind === "skill")) {
|
||||
errors.push(`${prefix} inventory does not contain SKILL.md.`);
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
function readStringArrayField(
|
||||
value: unknown,
|
||||
field: string,
|
||||
prefix: string,
|
||||
errors: string[],
|
||||
) {
|
||||
const parsed = asStringArray(value);
|
||||
if (!parsed) {
|
||||
errors.push(`${prefix}/SKILL.md frontmatter field ${field} must be an array of strings.`);
|
||||
return [];
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function classifyCatalogFile(relativePath: string): CatalogSkillFileKind {
|
||||
if (relativePath === SKILL_ENTRYPOINT) return "skill";
|
||||
if (relativePath.startsWith("references/")) return "reference";
|
||||
if (relativePath.startsWith("scripts/")) return "script";
|
||||
if (relativePath.startsWith("assets/")) return "asset";
|
||||
if (relativePath.endsWith(".md") || relativePath.endsWith(".mdx")) return "markdown";
|
||||
return "other";
|
||||
}
|
||||
|
||||
function deriveTrustLevel(files: CatalogSkillFile[]): CatalogTrustLevel {
|
||||
if (files.some((file) => file.kind === "script")) return "scripts_executables";
|
||||
if (files.some((file) => file.kind === "asset" || file.kind === "other")) return "assets";
|
||||
return "markdown_only";
|
||||
}
|
||||
|
||||
function buildContentHash(files: CatalogSkillFile[]) {
|
||||
const hashInput = files.map((file) => ({
|
||||
path: file.path,
|
||||
sha256: file.sha256,
|
||||
}));
|
||||
return `sha256:${sha256(Buffer.from(JSON.stringify(hashInput)))}`;
|
||||
}
|
||||
|
||||
function collectUniquenessErrors(skills: CatalogSkill[], errors: string[]) {
|
||||
collectDuplicateErrors(skills, "id", errors);
|
||||
collectDuplicateErrors(skills, "key", errors);
|
||||
collectDuplicateErrors(skills, "slug", errors);
|
||||
}
|
||||
|
||||
function collectCandidateUniquenessErrors(candidates: SkillCandidate[], errors: string[]) {
|
||||
const projected = candidates.map((candidate) => ({
|
||||
id: `paperclipai:${candidate.kind}:${candidate.category}:${candidate.slug}`,
|
||||
key: `paperclipai/${candidate.kind}/${candidate.category}/${candidate.slug}`,
|
||||
slug: candidate.slug,
|
||||
path: toPosixPath(path.join("catalog", candidate.kind, candidate.category, candidate.slug)),
|
||||
})) as CatalogSkill[];
|
||||
collectUniquenessErrors(projected, errors);
|
||||
}
|
||||
|
||||
function collectDuplicateErrors(fieldSkills: CatalogSkill[], field: "id" | "key" | "slug", errors: string[]) {
|
||||
const seen = new Map<string, string>();
|
||||
for (const skill of fieldSkills) {
|
||||
const value = skill[field];
|
||||
const first = seen.get(value);
|
||||
if (first) {
|
||||
errors.push(`Duplicate catalog ${field} "${value}" in ${first} and ${skill.path}.`);
|
||||
continue;
|
||||
}
|
||||
seen.set(value, skill.path);
|
||||
}
|
||||
}
|
||||
|
||||
function validateSlug(label: string, value: string, prefix: string, errors: string[]) {
|
||||
if (!SLUG_PATTERN.test(value)) {
|
||||
errors.push(`${prefix} has invalid ${label} "${value}"; use lowercase URL slugs.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function sortedDirEntries(dir: string) {
|
||||
return (await fs.readdir(dir, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
function sameManifestExceptGeneratedAt(a: CatalogManifest, b: CatalogManifest) {
|
||||
return JSON.stringify({ ...a, generatedAt: "" }) === JSON.stringify({ ...b, generatedAt: "" });
|
||||
}
|
||||
|
||||
function sha256(contents: Buffer) {
|
||||
return createHash("sha256").update(contents).digest("hex");
|
||||
}
|
||||
|
||||
function relativePackagePath(packageDir: string, absolutePath: string) {
|
||||
return toPosixPath(path.relative(packageDir, absolutePath));
|
||||
}
|
||||
|
||||
function toPosixPath(input: string) {
|
||||
return input.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function isPathInside(parent: string, child: string) {
|
||||
const relativePath = path.relative(parent, child);
|
||||
return relativePath === "" || (!relativePath.startsWith("..") && !path.isAbsolute(relativePath));
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
154
packages/skills-catalog/src/frontmatter.ts
Normal file
154
packages/skills-catalog/src/frontmatter.ts
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
export interface MarkdownDoc {
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
hasFrontmatter: boolean;
|
||||
}
|
||||
|
||||
export function isPlainRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function asString(value: unknown): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
export function asBoolean(value: unknown): boolean | null {
|
||||
return typeof value === "boolean" ? value : null;
|
||||
}
|
||||
|
||||
export function asStringArray(value: unknown): string[] | null {
|
||||
if (value === undefined) return [];
|
||||
if (!Array.isArray(value)) return null;
|
||||
|
||||
const out: string[] = [];
|
||||
for (const item of value) {
|
||||
const text = asString(item);
|
||||
if (!text) return null;
|
||||
out.push(text);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function parseFrontmatterMarkdown(raw: string): MarkdownDoc {
|
||||
const normalized = raw.replace(/\r\n/g, "\n");
|
||||
if (!normalized.startsWith("---\n")) {
|
||||
return { frontmatter: {}, body: normalized.trim(), hasFrontmatter: false };
|
||||
}
|
||||
|
||||
const closing = normalized.indexOf("\n---\n", 4);
|
||||
if (closing < 0) {
|
||||
return { frontmatter: {}, body: normalized.trim(), hasFrontmatter: false };
|
||||
}
|
||||
|
||||
const frontmatterRaw = normalized.slice(4, closing).trim();
|
||||
const body = normalized.slice(closing + 5).trim();
|
||||
return {
|
||||
frontmatter: parseYamlFrontmatter(frontmatterRaw),
|
||||
body,
|
||||
hasFrontmatter: true,
|
||||
};
|
||||
}
|
||||
|
||||
function parseYamlFrontmatter(raw: string): Record<string, unknown> {
|
||||
const prepared = prepareYamlLines(raw);
|
||||
if (prepared.length === 0) return {};
|
||||
const parsed = parseYamlBlock(prepared, 0, prepared[0]!.indent);
|
||||
return isPlainRecord(parsed.value) ? parsed.value : {};
|
||||
}
|
||||
|
||||
function prepareYamlLines(raw: string) {
|
||||
return raw
|
||||
.split("\n")
|
||||
.map((line) => ({
|
||||
indent: line.match(/^ */)?.[0].length ?? 0,
|
||||
content: line.trim(),
|
||||
}))
|
||||
.filter((line) => line.content.length > 0 && !line.content.startsWith("#"));
|
||||
}
|
||||
|
||||
function parseYamlBlock(
|
||||
lines: Array<{ indent: number; content: string }>,
|
||||
startIndex: number,
|
||||
indentLevel: number,
|
||||
): { value: unknown; nextIndex: number } {
|
||||
let index = startIndex;
|
||||
if (index >= lines.length || lines[index]!.indent < indentLevel) {
|
||||
return { value: {}, nextIndex: index };
|
||||
}
|
||||
|
||||
const isArray = lines[index]!.indent === indentLevel && lines[index]!.content.startsWith("-");
|
||||
if (isArray) {
|
||||
const values: unknown[] = [];
|
||||
while (index < lines.length) {
|
||||
const line = lines[index]!;
|
||||
if (line.indent < indentLevel) break;
|
||||
if (line.indent !== indentLevel || !line.content.startsWith("-")) break;
|
||||
|
||||
const remainder = line.content.slice(1).trim();
|
||||
index += 1;
|
||||
if (!remainder) {
|
||||
const nested = parseYamlBlock(lines, index, indentLevel + 2);
|
||||
values.push(nested.value);
|
||||
index = nested.nextIndex;
|
||||
continue;
|
||||
}
|
||||
|
||||
values.push(parseYamlScalar(remainder));
|
||||
}
|
||||
return { value: values, nextIndex: index };
|
||||
}
|
||||
|
||||
const record: Record<string, unknown> = {};
|
||||
while (index < lines.length) {
|
||||
const line = lines[index]!;
|
||||
if (line.indent < indentLevel) break;
|
||||
if (line.indent !== indentLevel) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const separatorIndex = line.content.indexOf(":");
|
||||
if (separatorIndex <= 0) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = line.content.slice(0, separatorIndex).trim();
|
||||
const remainder = line.content.slice(separatorIndex + 1).trim();
|
||||
index += 1;
|
||||
if (!remainder) {
|
||||
const nested = parseYamlBlock(lines, index, indentLevel + 2);
|
||||
record[key] = nested.value;
|
||||
index = nested.nextIndex;
|
||||
continue;
|
||||
}
|
||||
record[key] = parseYamlScalar(remainder);
|
||||
}
|
||||
|
||||
return { value: record, nextIndex: index };
|
||||
}
|
||||
|
||||
function parseYamlScalar(rawValue: string): unknown {
|
||||
const trimmed = rawValue.trim();
|
||||
if (trimmed === "") return "";
|
||||
if (trimmed === "null" || trimmed === "~") return null;
|
||||
if (trimmed === "true") return true;
|
||||
if (trimmed === "false") return false;
|
||||
if (trimmed === "[]") return [];
|
||||
if (trimmed === "{}") return {};
|
||||
if (/^-?\d+(\.\d+)?$/.test(trimmed)) return Number(trimmed);
|
||||
if (
|
||||
trimmed.startsWith("\"") ||
|
||||
trimmed.startsWith("[") ||
|
||||
trimmed.startsWith("{")
|
||||
) {
|
||||
try {
|
||||
return JSON.parse(trimmed);
|
||||
} catch {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
37
packages/skills-catalog/src/index.ts
Normal file
37
packages/skills-catalog/src/index.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import catalogManifestJson from "../generated/catalog.json" with { type: "json" };
|
||||
import type { CatalogManifest, CatalogSkill } from "./types.js";
|
||||
|
||||
export type {
|
||||
CatalogCompatibility,
|
||||
CatalogManifest,
|
||||
CatalogSkill,
|
||||
CatalogSkillFile,
|
||||
CatalogSkillFileKind,
|
||||
CatalogSkillKind,
|
||||
CatalogTrustLevel,
|
||||
CatalogValidationResult,
|
||||
} from "./types.js";
|
||||
|
||||
export const catalogManifest = catalogManifestJson as CatalogManifest;
|
||||
|
||||
export const catalogSkills: CatalogSkill[] = catalogManifest.skills;
|
||||
|
||||
const skillsById = new Map(catalogSkills.map((skill) => [skill.id, skill]));
|
||||
const skillsByKey = new Map(catalogSkills.map((skill) => [skill.key, skill]));
|
||||
|
||||
export function getCatalogSkill(id: string): CatalogSkill | null {
|
||||
return skillsById.get(id) ?? null;
|
||||
}
|
||||
|
||||
export function resolveCatalogSkillRef(ref: string): CatalogSkill | null {
|
||||
const normalized = ref.trim();
|
||||
if (normalized.length === 0) return null;
|
||||
|
||||
const exactMatch = skillsById.get(normalized) ?? skillsByKey.get(normalized);
|
||||
if (exactMatch) return exactMatch;
|
||||
|
||||
const slugMatches = catalogSkills.filter((skill) => skill.slug === normalized);
|
||||
if (slugMatches.length === 1) return slugMatches[0]!;
|
||||
|
||||
return null;
|
||||
}
|
||||
90
packages/skills-catalog/src/shipped-catalog.test.ts
Normal file
90
packages/skills-catalog/src/shipped-catalog.test.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { catalogManifest, catalogSkills, resolveCatalogSkillRef } from "./index.js";
|
||||
import type { CatalogSkill } from "./types.js";
|
||||
|
||||
const EXPECTED_BUNDLED_KEYS = [
|
||||
"paperclipai/bundled/docs/doc-maintenance",
|
||||
"paperclipai/bundled/paperclip-operations/issue-triage",
|
||||
"paperclipai/bundled/paperclip-operations/task-planning",
|
||||
"paperclipai/bundled/quality/qa-acceptance",
|
||||
"paperclipai/bundled/software-development/github-pr-workflow",
|
||||
];
|
||||
|
||||
const EXPECTED_OPTIONAL_KEYS = [
|
||||
"paperclipai/optional/browser/agent-browser",
|
||||
"paperclipai/optional/content/release-announcement",
|
||||
"paperclipai/optional/product/design-critique",
|
||||
];
|
||||
|
||||
describe("shipped skills catalog", () => {
|
||||
it("ships the expected bundled and optional skill set", () => {
|
||||
const bundledKeys = catalogSkills
|
||||
.filter((skill) => skill.kind === "bundled")
|
||||
.map((skill) => skill.key)
|
||||
.sort();
|
||||
const optionalKeys = catalogSkills
|
||||
.filter((skill) => skill.kind === "optional")
|
||||
.map((skill) => skill.key)
|
||||
.sort();
|
||||
|
||||
expect(bundledKeys).toEqual(EXPECTED_BUNDLED_KEYS);
|
||||
expect(optionalKeys).toEqual(EXPECTED_OPTIONAL_KEYS);
|
||||
});
|
||||
|
||||
it("keeps every shipped skill markdown-only until a script-bearing skill clears security review", () => {
|
||||
const scriptBearing = catalogSkills.filter((skill) => skill.trustLevel !== "markdown_only");
|
||||
expect(scriptBearing, formatViolations("script-bearing skills require security review", scriptBearing)).toEqual([]);
|
||||
});
|
||||
|
||||
it("populates browse/search-relevant fields for every shipped skill", () => {
|
||||
const issues: string[] = [];
|
||||
for (const skill of catalogSkills) {
|
||||
if (skill.compatibility !== "compatible") {
|
||||
issues.push(`${skill.key} compatibility=${skill.compatibility}`);
|
||||
}
|
||||
if (!skill.description || skill.description.length < 40) {
|
||||
issues.push(`${skill.key} description must be at least 40 characters for catalog browse/search`);
|
||||
}
|
||||
if (skill.recommendedForRoles.length === 0) {
|
||||
issues.push(`${skill.key} must list recommendedForRoles`);
|
||||
}
|
||||
if (skill.tags.length === 0) {
|
||||
issues.push(`${skill.key} must list tags`);
|
||||
}
|
||||
}
|
||||
expect(issues).toEqual([]);
|
||||
});
|
||||
|
||||
it("uses canonical paperclipai keys derived from kind/category/slug", () => {
|
||||
const violations: string[] = [];
|
||||
for (const skill of catalogSkills) {
|
||||
const expectedKey = `paperclipai/${skill.kind}/${skill.category}/${skill.slug}`;
|
||||
const expectedId = `paperclipai:${skill.kind}:${skill.category}:${skill.slug}`;
|
||||
if (skill.key !== expectedKey) violations.push(`${skill.key} should be ${expectedKey}`);
|
||||
if (skill.id !== expectedId) violations.push(`${skill.id} should be ${expectedId}`);
|
||||
}
|
||||
expect(violations).toEqual([]);
|
||||
});
|
||||
|
||||
it("exposes a stable manifest header for downstream consumers", () => {
|
||||
expect(catalogManifest.schemaVersion).toBe(1);
|
||||
expect(catalogManifest.packageName).toBe("@paperclipai/skills-catalog");
|
||||
expect(catalogSkills.length).toBe(EXPECTED_BUNDLED_KEYS.length + EXPECTED_OPTIONAL_KEYS.length);
|
||||
});
|
||||
|
||||
it("resolves shipped skills by id, key, and unique slug", () => {
|
||||
const sample = catalogSkills.find((skill) => skill.key === "paperclipai/bundled/software-development/github-pr-workflow");
|
||||
expect(sample, "expected github-pr-workflow to ship in the bundled catalog").toBeDefined();
|
||||
if (!sample) return;
|
||||
|
||||
expect(resolveCatalogSkillRef(sample.id)).toMatchObject({ key: sample.key });
|
||||
expect(resolveCatalogSkillRef(sample.key)).toMatchObject({ key: sample.key });
|
||||
expect(resolveCatalogSkillRef(sample.slug)).toMatchObject({ key: sample.key });
|
||||
});
|
||||
});
|
||||
|
||||
function formatViolations(label: string, skills: CatalogSkill[]) {
|
||||
if (skills.length === 0) return label;
|
||||
const detail = skills.map((skill) => `${skill.key} (${skill.trustLevel})`).join(", ");
|
||||
return `${label}: ${detail}`;
|
||||
}
|
||||
48
packages/skills-catalog/src/types.ts
Normal file
48
packages/skills-catalog/src/types.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
export type CatalogSkillKind = "bundled" | "optional";
|
||||
|
||||
export type CatalogTrustLevel = "markdown_only" | "assets" | "scripts_executables";
|
||||
|
||||
export type CatalogCompatibility = "compatible" | "unknown" | "invalid";
|
||||
|
||||
export type CatalogSkillFileKind = "skill" | "markdown" | "reference" | "script" | "asset" | "other";
|
||||
|
||||
export interface CatalogSkillFile {
|
||||
path: string;
|
||||
kind: CatalogSkillFileKind;
|
||||
sizeBytes: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export interface CatalogSkill {
|
||||
id: string;
|
||||
key: string;
|
||||
kind: CatalogSkillKind;
|
||||
category: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
description: string;
|
||||
path: string;
|
||||
entrypoint: "SKILL.md";
|
||||
trustLevel: CatalogTrustLevel;
|
||||
compatibility: CatalogCompatibility;
|
||||
defaultInstall: boolean;
|
||||
recommendedForRoles: string[];
|
||||
requires: string[];
|
||||
tags: string[];
|
||||
files: CatalogSkillFile[];
|
||||
contentHash: string;
|
||||
}
|
||||
|
||||
export interface CatalogManifest {
|
||||
schemaVersion: 1;
|
||||
packageName: "@paperclipai/skills-catalog";
|
||||
packageVersion: string;
|
||||
generatedAt: string;
|
||||
skills: CatalogSkill[];
|
||||
}
|
||||
|
||||
export interface CatalogValidationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
manifest: CatalogManifest;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue