ake/paperclip
1
0
Fork 0
mirror of https://github.com/alkimake/paperclip.git synced 2026-06-15 02:20:38 +09:00
Code Issues Projects Releases Packages Wiki Activity

feat: support GitHub Enterprise URLs for skill and company imports

Browse source
This commit is contained in:
statxc 2026-04-01 20:42:48 +00:00
parent 6c2c63e0f1
commit 9e1ee925cd
6 changed files with 116 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

2
server/src/__tests__/company-portability.test.ts
View file

@ -375,6 +375,7 @@ describe("company portability", () => {
expect(
parseGitHubSourceUrl("https://github.com/paperclipai/companies?ref=feature%2Fdemo&path=gstack"),
).toEqual({
hostname: "github.com",
owner: "paperclipai",
repo: "companies",
ref: "feature/demo",
@ -389,6 +390,7 @@ describe("company portability", () => {
"https://github.com/paperclipai/companies?ref=abc123&companyPath=gstack%2FCOMPANY.md",
),
).toEqual({
hostname: "github.com",
owner: "paperclipai",
repo: "companies",
ref: "abc123",

57
server/src/services/company-portability.ts
View file

@ -2100,8 +2100,16 @@ function parseFrontmatterMarkdown(raw: string): MarkdownDoc {
};
}
async function ghFetch(url: string, init?: RequestInit): Promise<Response> {
try {
return await fetch(url, init);
} catch {
throw unprocessable(`Could not connect to ${new URL(url).hostname} — ensure the URL points to a GitHub or GitHub Enterprise instance`);
}
}
async function fetchText(url: string) {
const response = await fetch(url);
const response = await ghFetch(url);
if (!response.ok) {
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
}
@ -2109,7 +2117,7 @@ async function fetchText(url: string) {
}
async function fetchOptionalText(url: string) {
const response = await fetch(url);
const response = await ghFetch(url);
if (response.status === 404) return null;
if (!response.ok) {
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
@ -2118,7 +2126,7 @@ async function fetchOptionalText(url: string) {
}
async function fetchBinary(url: string) {
const response = await fetch(url);
const response = await ghFetch(url);
if (!response.ok) {
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
}
@ -2126,7 +2134,7 @@ async function fetchBinary(url: string) {
}
async function fetchJson<T>(url: string): Promise<T> {
const response = await fetch(url, {
const response = await ghFetch(url, {
headers: {
accept: "application/vnd.github+json",
},
@ -2411,14 +2419,16 @@ function buildManifestFromPackageFiles(
const repoPath = asString(primarySource?.path);
const commit = asString(primarySource?.commit);
const trackingRef = asString(primarySource?.trackingRef);
const sourceHostname = asString(primarySource?.hostname) || "github.com";
const [owner, repoName] = (repo ?? "").split("/");
sourceType = "github";
sourceLocator = asString(primarySource?.url)
?? (repo ? `https://github.com/${repo}${repoPath ? `/tree/${trackingRef ?? commit ?? "main"}/${repoPath}` : ""}` : null);
?? (repo ? `https://${sourceHostname}/${repo}${repoPath ? `/tree/${trackingRef ?? commit ?? "main"}/${repoPath}` : ""}` : null);
sourceRef = commit;
normalizedMetadata = owner && repoName
? {
sourceKind: "github",
...(sourceHostname !== "github.com" ? { hostname: sourceHostname } : {}),
owner,
repo: repoName,
ref: commit,
@ -2564,9 +2574,7 @@ function normalizeGitHubSourcePath(value: string | null | undefined) {
export function parseGitHubSourceUrl(rawUrl: string) {
const url = new URL(rawUrl);
if (url.hostname !== "github.com") {
throw unprocessable("GitHub source must use github.com URL");
}
const hostname = url.hostname;
const parts = url.pathname.split("/").filter(Boolean);
if (parts.length < 2) {
throw unprocessable("Invalid GitHub URL");
@ -2584,6 +2592,7 @@ export function parseGitHubSourceUrl(rawUrl: string) {
if (basePath === ".") basePath = "";
}
return {
hostname,
owner,
repo,
ref: queryRef || "main",
@ -2607,12 +2616,23 @@ export function parseGitHubSourceUrl(rawUrl: string) {
basePath = path.posix.dirname(blobPath);
if (basePath === ".") basePath = "";
}
return { owner, repo, ref, basePath, companyPath };
return { hostname, owner, repo, ref, basePath, companyPath };
}
function resolveRawGitHubUrl(owner: string, repo: string, ref: string, filePath: string) {
const normalizedFilePath = filePath.replace(/^\/+/, "");
return `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${normalizedFilePath}`;
function isGitHubDotCom(hostname: string) {
const h = hostname.toLowerCase();
return h === "github.com" || h === "www.github.com";
}
function resolveRawGitHubUrl(hostname: string, owner: string, repo: string, ref: string, filePath: string) {
const p = filePath.replace(/^\/+/, "");
return isGitHubDotCom(hostname)
? `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${p}`
: `https://${hostname}/raw/${owner}/${repo}/${ref}/${p}`;
}
function gitHubApiBase(hostname: string) {
return isGitHubDotCom(hostname) ? "https://api.github.com" : `https://${hostname}/api/v3`;
}
export function companyPortabilityService(db: Db, storage?: StorageService) {
@ -2641,14 +2661,14 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
let companyMarkdown: string | null = null;
try {
companyMarkdown = await fetchOptionalText(
resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, companyRelativePath),
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath),
);
} catch (err) {
if (ref === "main") {
ref = "master";
warnings.push("GitHub ref main not found; falling back to master.");
companyMarkdown = await fetchOptionalText(
resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, companyRelativePath),
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath),
);
} else {
throw err;
@ -2664,8 +2684,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
const files: Record<string, CompanyPortabilityFileEntry> = {
[companyPath]: companyMarkdown,
};
const apiBase = gitHubApiBase(parsed.hostname);
const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>(
`https://api.github.com/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`,
`${apiBase}/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`,
).catch(() => ({ tree: [] }));
const basePrefix = parsed.basePath ? `${parsed.basePath.replace(/^\/+|\/+$/g, "")}/` : "";
const candidatePaths = (tree.tree ?? [])
@ -2686,7 +2707,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
const relativePath = basePrefix ? repoPath.slice(basePrefix.length) : repoPath;
if (files[relativePath] !== undefined) continue;
files[normalizePortablePath(relativePath)] = await fetchText(
resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoPath),
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath),
);
}
const companyDoc = parseFrontmatterMarkdown(companyMarkdown);
@ -2697,7 +2718,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
if (files[relativePath] !== undefined) continue;
if (!(repoPath.endsWith(".md") || repoPath.endsWith(".yaml") || repoPath.endsWith(".yml"))) continue;
files[relativePath] = await fetchText(
resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoPath),
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath),
);
}
@ -2707,7 +2728,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
const repoPath = [parsed.basePath, companyLogoPath].filter(Boolean).join("/");
try {
const binary = await fetchBinary(
resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoPath),
resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath),
);
resolved.files[companyLogoPath] = bufferToPortableBinaryFile(binary, inferContentTypeFromPath(companyLogoPath));
} catch (err) {

73
server/src/services/company-skills.ts
View file

@ -73,6 +73,7 @@ type ParsedSkillImportSource = {
type SkillSourceMeta = {
skillKey?: string;
sourceKind?: string;
hostname?: string;
owner?: string;
repo?: string;
ref?: string;
@ -469,8 +470,16 @@ function parseFrontmatterMarkdown(raw: string): { frontmatter: Record<string, un
};
}
async function ghFetch(url: string, init?: RequestInit): Promise<Response> {
try {
return await fetch(url, init);
} catch {
throw unprocessable(`Could not connect to ${new URL(url).hostname} — ensure the URL points to a GitHub or GitHub Enterprise instance`);
}
}
async function fetchText(url: string) {
const response = await fetch(url);
const response = await ghFetch(url);
if (!response.ok) {
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
}
@ -478,7 +487,7 @@ async function fetchText(url: string) {
}
async function fetchJson<T>(url: string): Promise<T> {
const response = await fetch(url, {
const response = await ghFetch(url, {
headers: {
accept: "application/vnd.github+json",
},
@ -489,16 +498,25 @@ async function fetchJson<T>(url: string): Promise<T> {
return response.json() as Promise<T>;
}
async function resolveGitHubDefaultBranch(owner: string, repo: string) {
function isGitHubDotCom(hostname: string) {
const h = hostname.toLowerCase();
return h === "github.com" || h === "www.github.com";
}
function gitHubApiBase(hostname: string) {
return isGitHubDotCom(hostname) ? "https://api.github.com" : `https://${hostname}/api/v3`;
}
async function resolveGitHubDefaultBranch(owner: string, repo: string, apiBase: string) {
const response = await fetchJson<{ default_branch?: string }>(
`https://api.github.com/repos/${owner}/${repo}`,
`${apiBase}/repos/${owner}/${repo}`,
);
return asString(response.default_branch) ?? "main";
}
async function resolveGitHubCommitSha(owner: string, repo: string, ref: string) {
async function resolveGitHubCommitSha(owner: string, repo: string, ref: string, apiBase: string) {
const response = await fetchJson<{ sha?: string }>(
`https://api.github.com/repos/${owner}/${repo}/commits/${encodeURIComponent(ref)}`,
`${apiBase}/repos/${owner}/${repo}/commits/${encodeURIComponent(ref)}`,
);
const sha = asString(response.sha);
if (!sha) {
@ -509,9 +527,6 @@ async function resolveGitHubCommitSha(owner: string, repo: string, ref: string)
function parseGitHubSourceUrl(rawUrl: string) {
const url = new URL(rawUrl);
if (url.hostname !== "github.com") {
throw unprocessable("GitHub source must use github.com URL");
}
const parts = url.pathname.split("/").filter(Boolean);
if (parts.length < 2) {
throw unprocessable("Invalid GitHub URL");
@ -532,10 +547,11 @@ function parseGitHubSourceUrl(rawUrl: string) {
basePath = filePath ? path.posix.dirname(filePath) : "";
explicitRef = true;
}
return { owner, repo, ref, basePath, filePath, explicitRef };
return { hostname: url.hostname, owner, repo, ref, basePath, filePath, explicitRef };
}
async function resolveGitHubPinnedRef(parsed: ReturnType<typeof parseGitHubSourceUrl>) {
const apiBase = gitHubApiBase(parsed.hostname);
if (/^[0-9a-f]{40}$/i.test(parsed.ref.trim())) {
return {
pinnedRef: parsed.ref,
@ -545,13 +561,16 @@ async function resolveGitHubPinnedRef(parsed: ReturnType<typeof parseGitHubSourc
const trackingRef = parsed.explicitRef
? parsed.ref
: await resolveGitHubDefaultBranch(parsed.owner, parsed.repo);
const pinnedRef = await resolveGitHubCommitSha(parsed.owner, parsed.repo, trackingRef);
: await resolveGitHubDefaultBranch(parsed.owner, parsed.repo, apiBase);
const pinnedRef = await resolveGitHubCommitSha(parsed.owner, parsed.repo, trackingRef, apiBase);
return { pinnedRef, trackingRef };
}
function resolveRawGitHubUrl(owner: string, repo: string, ref: string, filePath: string) {
return `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${filePath.replace(/^\/+/, "")}`;
function resolveRawGitHubUrl(hostname: string, owner: string, repo: string, ref: string, filePath: string) {
const p = filePath.replace(/^\/+/, "");
return isGitHubDotCom(hostname)
? `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${p}`
: `https://${hostname}/raw/${owner}/${repo}/${ref}/${p}`;
}
function extractCommandTokens(raw: string) {
@ -675,9 +694,10 @@ function deriveImportedSkillSource(
const repoPath = asString(sourceEntry?.path);
const commit = asString(sourceEntry?.commit);
const trackingRef = asString(sourceEntry?.trackingRef);
const sourceHostname = asString(sourceEntry?.hostname) || "github.com";
const url = asString(sourceEntry?.url)
?? (repo
? `https://github.com/${repo}${repoPath ? `/tree/${trackingRef ?? commit ?? "main"}/${repoPath}` : ""}`
? `https://${sourceHostname}/${repo}${repoPath ? `/tree/${trackingRef ?? commit ?? "main"}/${repoPath}` : ""}`
: null);
const [owner, repoName] = (repo ?? "").split("/");
if (repo && owner && repoName) {
@ -688,6 +708,7 @@ function deriveImportedSkillSource(
metadata: {
...(canonicalKey ? { skillKey: canonicalKey } : {}),
sourceKind: "github",
...(sourceHostname !== "github.com" ? { hostname: sourceHostname } : {}),
owner,
repo: repoName,
ref: commit,
@ -981,12 +1002,18 @@ async function readUrlSkillImports(
): Promise<{ skills: ImportedSkill[]; warnings: string[] }> {
const url = sourceUrl.trim();
const warnings: string[] = [];
if (url.includes("github.com/")) {
const isGitHubRepoUrl = (() => { try {
const parsed = new URL(url);
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return false;
return parsed.pathname.split("/").filter(Boolean).length >= 2 && !parsed.pathname.endsWith(".md");
} catch { return false; } })();
if (isGitHubRepoUrl) {
const parsed = parseGitHubSourceUrl(url);
const apiBase = gitHubApiBase(parsed.hostname);
const { pinnedRef, trackingRef } = await resolveGitHubPinnedRef(parsed);
let ref = pinnedRef;
const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>(
`https://api.github.com/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`,
`${apiBase}/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`,
).catch(() => {
throw unprocessable(`Failed to read GitHub tree for ${url}`);
});
@ -1013,7 +1040,7 @@ async function readUrlSkillImports(
const skills: ImportedSkill[] = [];
for (const relativeSkillPath of skillPaths) {
const repoSkillPath = basePrefix ? `${basePrefix}${relativeSkillPath}` : relativeSkillPath;
const markdown = await fetchText(resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoSkillPath));
const markdown = await fetchText(resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoSkillPath));
const parsedMarkdown = parseFrontmatterMarkdown(markdown);
const skillDir = path.posix.dirname(relativeSkillPath);
const slug = deriveImportedSkillSlug(parsedMarkdown.frontmatter, path.posix.basename(skillDir));
@ -1027,9 +1054,10 @@ async function readUrlSkillImports(
const metadata = {
...(skillKey ? { skillKey } : {}),
sourceKind: "github",
...(parsed.hostname !== "github.com" ? { hostname: parsed.hostname } : {}),
owner: parsed.owner,
repo: parsed.repo,
ref: ref,
ref,
trackingRef,
repoSkillDir: normalizeGitHubSkillDirectory(
basePrefix ? `${basePrefix}${skillDir}` : skillDir,
@ -1641,7 +1669,9 @@ export function companySkillService(db: Db) {
};
}
const latestRef = await resolveGitHubCommitSha(owner, repo, trackingRef);
const hostname = asString(metadata.hostname) || "github.com";
const apiBase = gitHubApiBase(hostname);
const latestRef = await resolveGitHubCommitSha(owner, repo, trackingRef, apiBase);
return {
supported: true,
reason: null,
@ -1679,13 +1709,14 @@ export function companySkillService(db: Db) {
const metadata = getSkillMeta(skill);
const owner = asString(metadata.owner);
const repo = asString(metadata.repo);
const hostname = asString(metadata.hostname) || "github.com";
const ref = skill.sourceRef ?? asString(metadata.ref) ?? "main";
const repoSkillDir = normalizeGitHubSkillDirectory(asString(metadata.repoSkillDir), skill.slug);
if (!owner || !repo) {
throw unprocessable("Skill source metadata is incomplete.");
}
const repoPath = normalizePortablePath(path.posix.join(repoSkillDir, normalizedPath));
content = await fetchText(resolveRawGitHubUrl(owner, repo, ref, repoPath));
content = await fetchText(resolveRawGitHubUrl(hostname, owner, repo, ref, repoPath));
} else if (skill.sourceType === "url") {
if (normalizedPath !== "SKILL.md") {
throw notFound("This skill source only exposes SKILL.md");