364 lines
13 KiB
TypeScript
364 lines
13 KiB
TypeScript
/**
|
|
* Git URL install handler for skills.
|
|
* POST /api/skills/install-from-url
|
|
*
|
|
* Two modes:
|
|
* - Preview: POST { url } → returns detected skills + scan findings
|
|
* - Install: POST { url, selectedSkills: [...] } → installs selected skills
|
|
*/
|
|
|
|
import type { Request, Response } from 'express';
|
|
import {
|
|
existsSync, mkdirSync, mkdtempSync, rmSync, cpSync,
|
|
readdirSync, lstatSync, readFileSync, writeFileSync,
|
|
} from 'fs';
|
|
import { join, relative } from 'path';
|
|
import { execFileSync } from 'child_process';
|
|
import { tmpdir } from 'os';
|
|
import matter from 'gray-matter';
|
|
import type { SkillCatalog } from '../engine/skills.js';
|
|
import { VALID_SKILL_NAME } from '../engine/skills.js';
|
|
import { scanSkillContent, scanSkillDirectory, maxSeverity, type ScanFinding } from '../engine/skills-scanner.js';
|
|
import { logger } from '../logger.js';
|
|
|
|
// ── Types ───────────────────────────────────────────────────────────────────
|
|
|
|
export interface GitInstallDeps {
|
|
skillCatalog: SkillCatalog;
|
|
auditLog?: (jobId: string | null, action: string, actor: string, detail: object) => Promise<void>;
|
|
}
|
|
|
|
export interface DetectedSkill {
|
|
name: string;
|
|
description: string;
|
|
relativePath: string;
|
|
fullPath: string;
|
|
isDir: boolean;
|
|
findings: ScanFinding[];
|
|
maxSeverity: 'high' | 'medium' | 'none';
|
|
}
|
|
|
|
// ── Constants ───────────────────────────────────────────────────────────────
|
|
|
|
const MAX_REPO_SIZE_BYTES = 50 * 1024 * 1024; // 50 MB
|
|
const GIT_CLONE_TIMEOUT_MS = 30_000;
|
|
const SKIP_DIRS = new Set(['.git', 'node_modules', '.github', '.vscode']);
|
|
|
|
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Walk a cloned directory looking for skills:
|
|
* - Directories containing SKILL.md
|
|
* - Standalone .md files with frontmatter `name`
|
|
*/
|
|
export function detectSkillsInDir(rootDir: string): DetectedSkill[] {
|
|
const results: DetectedSkill[] = [];
|
|
|
|
function walk(dir: string): void {
|
|
let entries: string[];
|
|
try {
|
|
entries = readdirSync(dir);
|
|
} catch {
|
|
return;
|
|
}
|
|
|
|
for (const entry of entries) {
|
|
const fullPath = join(dir, entry);
|
|
|
|
let stat;
|
|
try {
|
|
stat = lstatSync(fullPath);
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
// Skip symlinks entirely (security)
|
|
if (stat.isSymbolicLink()) continue;
|
|
|
|
if (stat.isDirectory()) {
|
|
if (SKIP_DIRS.has(entry)) continue;
|
|
|
|
// Check if this directory is a skill (has SKILL.md)
|
|
const skillMdPath = join(fullPath, 'SKILL.md');
|
|
if (existsSync(skillMdPath)) {
|
|
try {
|
|
const skillStat = lstatSync(skillMdPath);
|
|
if (skillStat.isSymbolicLink()) continue; // Skip symlinked SKILL.md
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
const raw = readFileSync(skillMdPath, 'utf-8');
|
|
const { data } = matter(raw);
|
|
if (data && typeof data.name === 'string' && data.name && VALID_SKILL_NAME.test(data.name)) {
|
|
const findings = scanSkillDirectory(fullPath);
|
|
results.push({
|
|
name: data.name,
|
|
description: typeof data.description === 'string' ? data.description : '',
|
|
relativePath: relative(rootDir, fullPath),
|
|
fullPath,
|
|
isDir: true,
|
|
findings,
|
|
maxSeverity: maxSeverity(findings),
|
|
});
|
|
}
|
|
} catch {
|
|
// Unreadable SKILL.md — skip
|
|
}
|
|
// Don't recurse into skill directories (they're self-contained)
|
|
continue;
|
|
}
|
|
|
|
// Not a skill dir — recurse
|
|
walk(fullPath);
|
|
continue;
|
|
}
|
|
|
|
// Standalone .md file
|
|
if (stat.isFile() && entry.endsWith('.md')) {
|
|
try {
|
|
const raw = readFileSync(fullPath, 'utf-8');
|
|
const { data } = matter(raw);
|
|
if (data && typeof data.name === 'string' && data.name && VALID_SKILL_NAME.test(data.name)) {
|
|
const findings = scanSkillContent(raw);
|
|
results.push({
|
|
name: data.name,
|
|
description: typeof data.description === 'string' ? data.description : '',
|
|
relativePath: relative(rootDir, fullPath),
|
|
fullPath,
|
|
isDir: false,
|
|
findings,
|
|
maxSeverity: maxSeverity(findings),
|
|
});
|
|
}
|
|
} catch {
|
|
// Unreadable .md — skip
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
walk(rootDir);
|
|
return results;
|
|
}
|
|
|
|
/**
|
|
* Validate that a URL is safe for git clone (SSRF defense).
|
|
* Only HTTPS URLs are allowed.
|
|
*/
|
|
export function validateUrl(url: string): string | null {
|
|
if (!url || typeof url !== 'string') return 'url is required';
|
|
const trimmed = url.trim();
|
|
if (!trimmed.startsWith('https://')) {
|
|
return 'Only https:// URLs are allowed (http://, file://, ssh://, git:// and local paths are rejected for security)';
|
|
}
|
|
// Reject control characters (newlines, NUL, etc.), shell metacharacters,
|
|
// quotes and whitespace. The clone now runs via execFile (no shell), so this
|
|
// is defense-in-depth, but it also prevents a newline+quote breakout if the
|
|
// URL is ever reused in a shell context.
|
|
if (/[\u0000-\u001f\u007f;&|`$"'\\<>(){}\s]/.test(trimmed)) {
|
|
return 'URL contains disallowed characters';
|
|
}
|
|
// Must parse as a real https URL.
|
|
let parsed: URL;
|
|
try {
|
|
parsed = new URL(trimmed);
|
|
} catch {
|
|
return 'URL is not a valid URL';
|
|
}
|
|
if (parsed.protocol !== 'https:') {
|
|
return 'Only https:// URLs are allowed';
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// ── Handler ─────────────────────────────────────────────────────────────────
|
|
|
|
export function handleInstallFromUrl(deps: GitInstallDeps): (req: Request, res: Response) => Promise<void> {
|
|
return async (req: Request, res: Response): Promise<void> => {
|
|
const { url, scope: rawScope, selectedSkills } = req.body ?? {};
|
|
|
|
// 1. Validate URL
|
|
const urlError = validateUrl(url);
|
|
if (urlError) {
|
|
res.status(400).json({ error: urlError });
|
|
return;
|
|
}
|
|
|
|
// 2. Validate scope
|
|
const scope: 'system' | 'user' = rawScope === 'system' ? 'system' : 'user';
|
|
|
|
// 3. System scope requires admin
|
|
const user = req.user as Express.User | undefined;
|
|
if (scope === 'system' && (!user || user.role !== 'admin')) {
|
|
res.status(403).json({ error: 'System-scope install requires admin role' });
|
|
return;
|
|
}
|
|
|
|
const userId = user?.id ?? 'anonymous';
|
|
|
|
// 4. Clone to temp directory
|
|
const tmpBase = mkdtempSync(join(tmpdir(), 'skill-git-'));
|
|
const cloneDir = join(tmpBase, 'repo');
|
|
|
|
try {
|
|
try {
|
|
// execFile (no shell): url and cloneDir are passed as literal argv
|
|
// entries, so shell metacharacters in `url` cannot inject commands.
|
|
// `--` terminates option parsing so a `url` starting with `-` cannot
|
|
// be treated as a git flag (defense-in-depth; validateUrl already
|
|
// requires an https:// prefix).
|
|
execFileSync(
|
|
'git',
|
|
['clone', '--depth', '1', '--no-recurse-submodules', '--no-checkout', '--', url, cloneDir],
|
|
{ timeout: GIT_CLONE_TIMEOUT_MS, stdio: 'pipe' },
|
|
);
|
|
execFileSync(
|
|
'git',
|
|
['-C', cloneDir, 'checkout', 'HEAD', '--', '.'],
|
|
{ timeout: GIT_CLONE_TIMEOUT_MS, stdio: 'pipe' },
|
|
);
|
|
} catch (cloneErr: unknown) {
|
|
const isTimeout = cloneErr instanceof Error && 'killed' in cloneErr && (cloneErr as any).killed;
|
|
if (isTimeout) {
|
|
res.status(408).json({ error: 'Git clone timed out (30s limit)' });
|
|
return;
|
|
}
|
|
const msg = cloneErr instanceof Error ? cloneErr.message : String(cloneErr);
|
|
res.status(400).json({ error: `Git clone failed: ${msg.slice(0, 300)}` });
|
|
return;
|
|
}
|
|
|
|
// 5. Verify clone exists
|
|
if (!existsSync(cloneDir)) {
|
|
res.status(400).json({ error: 'Git clone produced no output directory' });
|
|
return;
|
|
}
|
|
|
|
// 6. Size check
|
|
try {
|
|
const duOutput = execFileSync('du', ['-sb', cloneDir], { encoding: 'utf-8', timeout: 10_000 });
|
|
const sizeBytes = parseInt(duOutput.split('\t')[0], 10);
|
|
if (sizeBytes > MAX_REPO_SIZE_BYTES) {
|
|
res.status(400).json({
|
|
error: `Repository too large: ${Math.round(sizeBytes / 1024 / 1024)}MB exceeds 50MB limit`,
|
|
});
|
|
return;
|
|
}
|
|
} catch {
|
|
// du failed — continue (non-critical)
|
|
logger.warn('[skills-git-install] du -sb failed, skipping size check');
|
|
}
|
|
|
|
// 7. Detect skills
|
|
const detected = detectSkillsInDir(cloneDir);
|
|
|
|
if (detected.length === 0) {
|
|
res.status(400).json({
|
|
error: 'No skills found in repository. Skills must be directories with SKILL.md or standalone .md files with frontmatter "name".',
|
|
});
|
|
return;
|
|
}
|
|
|
|
// 8. Preview mode — explicitly requested via preview flag
|
|
const previewMode = req.body?.preview === true;
|
|
if (previewMode) {
|
|
const preview = detected.map(s => ({
|
|
name: s.name,
|
|
description: s.description,
|
|
relativePath: s.relativePath,
|
|
isDir: s.isDir,
|
|
findings: s.findings,
|
|
maxSeverity: s.maxSeverity,
|
|
}));
|
|
res.json({ preview, totalDetected: detected.length });
|
|
return;
|
|
}
|
|
|
|
// 9. Install mode — selectedSkills or all detected
|
|
const selectedSet = Array.isArray(selectedSkills) && selectedSkills.length > 0
|
|
? new Set(selectedSkills.filter((s: unknown) => typeof s === 'string'))
|
|
: new Set(detected.map(s => s.name));
|
|
if (selectedSet.size === 0) {
|
|
res.status(400).json({ error: 'No skills detected in repository' });
|
|
return;
|
|
}
|
|
|
|
// Resolve target directory
|
|
const targetDir = scope === 'system'
|
|
? deps.skillCatalog.getSystemDir()
|
|
: deps.skillCatalog.getUserSkillDir(userId);
|
|
|
|
if (!existsSync(targetDir)) {
|
|
mkdirSync(targetDir, { recursive: true });
|
|
}
|
|
|
|
const installed: string[] = [];
|
|
const errors: string[] = [];
|
|
|
|
for (const skill of detected) {
|
|
if (!selectedSet.has(skill.name)) continue;
|
|
|
|
try {
|
|
if (skill.isDir) {
|
|
// Copy entire skill directory
|
|
const destDir = join(targetDir, skill.name);
|
|
cpSync(skill.fullPath, destDir, { recursive: true });
|
|
} else {
|
|
// Single .md file → create as directory format ({name}/SKILL.md)
|
|
const content = readFileSync(skill.fullPath, 'utf-8');
|
|
const destDir = join(targetDir, skill.name);
|
|
mkdirSync(destDir, { recursive: true });
|
|
writeFileSync(join(destDir, 'SKILL.md'), content, 'utf-8');
|
|
}
|
|
installed.push(skill.name);
|
|
} catch (err: unknown) {
|
|
const msg = err instanceof Error ? err.message : String(err);
|
|
errors.push(`${skill.name}: ${msg.slice(0, 200)}`);
|
|
logger.warn(`[skills-git-install] failed to install skill=${skill.name} err=${msg}`);
|
|
}
|
|
}
|
|
|
|
// Check for requested skills that weren't found in the repo
|
|
for (const name of selectedSet) {
|
|
if (!installed.includes(name) && !errors.some(e => e.startsWith(`${name}:`))) {
|
|
errors.push(`${name}: not found in repository`);
|
|
}
|
|
}
|
|
|
|
// 10. Invalidate cache
|
|
if (scope === 'system') {
|
|
deps.skillCatalog.refreshSystem();
|
|
} else {
|
|
deps.skillCatalog.invalidate(userId);
|
|
}
|
|
|
|
// 11. Audit log
|
|
if (deps.auditLog && installed.length > 0) {
|
|
deps.auditLog(null, 'skill_install_from_url', userId, {
|
|
url,
|
|
scope,
|
|
installed,
|
|
errors: errors.length > 0 ? errors : undefined,
|
|
}).catch(err => {
|
|
logger.warn(`[skills-git-install] audit log failed err=${err}`);
|
|
});
|
|
}
|
|
|
|
logger.info(`[skills-git-install] installed=${installed.length} errors=${errors.length} scope=${scope} user=${userId}`);
|
|
|
|
// 12. Return result
|
|
res.json({ installed, errors: errors.length > 0 ? errors : undefined });
|
|
|
|
} finally {
|
|
// Always clean up temp directory
|
|
try {
|
|
rmSync(tmpBase, { recursive: true, force: true });
|
|
} catch (cleanupErr) {
|
|
logger.warn(`[skills-git-install] tmpdir cleanup failed: ${cleanupErr}`);
|
|
}
|
|
}
|
|
};
|
|
}
|