diff --git a/AGENTS.md b/AGENTS.md index 9f5af3b30..e53b620c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,3 +9,4 @@ Open source framework for structured, agent-assisted software delivery. `quality` mirrors the checks in `.github/workflows/quality.yaml`. - Skill validation rules are in `tools/skill-validator.md`. +- Deterministic skill checks run via `npm run validate:skills` (included in `quality`). diff --git a/package.json b/package.json index e76f9d0dc..1eb1df26e 100644 --- a/package.json +++ b/package.json @@ -39,12 +39,13 @@ "lint:fix": "eslint . --ext .js,.cjs,.mjs,.yaml --fix", "lint:md": "markdownlint-cli2 \"**/*.md\"", "prepare": "command -v husky >/dev/null 2>&1 && husky || exit 0", - "quality": "npm run format:check && npm run lint && npm run lint:md && npm run docs:build && npm run test:install && npm run validate:refs", + "quality": "npm run format:check && npm run lint && npm run lint:md && npm run docs:build && npm run test:install && npm run validate:refs && npm run validate:skills", "rebundle": "node tools/cli/bundlers/bundle-web.js rebundle", "test": "npm run test:refs && npm run test:install && npm run lint && npm run lint:md && npm run format:check", "test:install": "node test/test-installation-components.js", "test:refs": "node test/test-file-refs-csv.js", - "validate:refs": "node tools/validate-file-refs.js --strict" + "validate:refs": "node tools/validate-file-refs.js --strict", + "validate:skills": "node tools/validate-skills.js --strict" }, "lint-staged": { "*.{js,cjs,mjs}": [ diff --git a/tools/skill-validator.md b/tools/skill-validator.md index 4ed4b3eda..543c8370a 100644 --- a/tools/skill-validator.md +++ b/tools/skill-validator.md @@ -2,14 +2,27 @@ An LLM-readable validation prompt for skills following the Agent Skills open standard. +## First Pass — Deterministic Checks + +Before running inference-based validation, run the deterministic validator: + +```bash +node tools/validate-skills.js --json path/to/skill-dir +``` + +This checks 13 rules deterministically: SKILL-01, SKILL-02, SKILL-03, SKILL-04, SKILL-05, SKILL-06, WF-01, WF-02, PATH-02, STEP-01, STEP-06, STEP-07, SEQ-02. + +Review its JSON output. For any rule that produced **zero findings** in the first pass, **skip it** during inference-based validation below — it has already been verified. If a rule produced any findings, the inference validator should still review that rule (some rules like SKILL-04 and SKILL-06 have sub-checks that benefit from judgment). Focus your inference effort on the remaining rules that require judgment (PATH-01, PATH-03, PATH-04, PATH-05, WF-03, STEP-02, STEP-03, STEP-04, STEP-05, SEQ-01, REF-01, REF-02, REF-03). + ## How to Use 1. You are given a **skill directory path** to validate. -2. Read every file in the skill directory recursively. -3. Apply every rule in the catalog below to every applicable file. -4. Produce a findings report using the report template at the end. +2. Run the deterministic first pass (see above) and note which rules passed. +3. Read every file in the skill directory recursively. +4. Apply every rule in the catalog below to every applicable file, **skipping rules that passed the deterministic first pass**. +5. Produce a findings report using the report template at the end, including any deterministic findings from the first pass. -If no findings are generated, the skill passes validation. +If no findings are generated (from either pass), the skill passes validation. --- @@ -103,6 +116,7 @@ If no findings are generated, the skill passes validation. - A legitimate external path expression (must not violate PATH-05 — no paths into another skill's directory) It must NOT be a path to a file within the skill directory (see PATH-04), nor a path into another skill's directory (see PATH-05). + - **Detection:** For each frontmatter variable, check if its value resolves to a file inside the skill (e.g., starts with `./`, `{installed_path}`, or is a bare relative path to a sibling file). If so, it is an intra-skill path variable. Also check if the value is a path into another skill's directory — if so, it violates PATH-05 and is not a legitimate external path. - **Fix:** Remove the variable. Use a hardcoded relative path inline where the file is referenced. @@ -294,11 +308,11 @@ When reporting findings, use this format: ## Summary | Severity | Count | -|----------|-------| -| CRITICAL | N | -| HIGH | N | -| MEDIUM | N | -| LOW | N | +| -------- | ----- | +| CRITICAL | N | +| HIGH | N | +| MEDIUM | N | +| LOW | N | ## Findings @@ -329,28 +343,34 @@ Quick-reference for the Agent Skills open standard. For the full standard, see: [Agent Skills specification](https://agentskills.io/specification) ### Structure + - Every skill is a directory with `SKILL.md` as the required entrypoint - YAML frontmatter between `---` markers provides metadata; markdown body provides instructions - Supporting files (scripts, templates, references) live alongside SKILL.md ### Path resolution + - Relative file references resolve from the directory of the file that contains the reference, not from the skill root - Example: from `branch-a/deep/next.md`, `./deeper/final.md` resolves to `branch-a/deep/deeper/final.md` - Example: from `branch-a/deep/next.md`, `./branch-b/alt/leaf.md` incorrectly resolves to `branch-a/deep/branch-b/alt/leaf.md` ### Frontmatter fields (standard) + - `name`: lowercase letters, numbers, hyphens only; max 64 chars; no "anthropic" or "claude" - `description`: required, max 1024 chars; should state what the skill does AND when to use it ### Progressive disclosure — three loading levels + - **L1 Metadata** (~100 tokens): `name` + `description` loaded at startup into system prompt - **L2 Instructions** (<5k tokens): SKILL.md body loaded only when skill is triggered - **L3 Resources** (unlimited): additional files + scripts loaded/executed on demand; script output enters context, script code does not ### Key design principle + - Skills are filesystem-based directories, not API payloads — Claude reads them via bash/file tools - Keep SKILL.md focused; offload detailed reference to separate files ### Practical tips + - Keep SKILL.md under 500 lines - `description` drives auto-discovery — use keywords users would naturally say diff --git a/tools/validate-skills.js b/tools/validate-skills.js new file mode 100644 index 000000000..ea13b80e3 --- /dev/null +++ b/tools/validate-skills.js @@ -0,0 +1,705 @@ +/** + * Deterministic Skill Validator + * + * Validates 13 deterministic rules across all skill directories. + * Acts as a fast first-pass complement to the inference-based skill validator. + * + * What it checks: + * - SKILL-01: SKILL.md exists + * - SKILL-02: SKILL.md frontmatter has name + * - SKILL-03: SKILL.md frontmatter has description + * - SKILL-04: name format (lowercase, hyphens, no forbidden substrings) + * - SKILL-05: name matches directory basename + * - SKILL-06: description quality (length, "Use when"/"Use if") + * - WF-01: workflow.md frontmatter has no name + * - WF-02: workflow.md frontmatter has no description + * - PATH-02: no installed_path variable + * - STEP-01: step filename format + * - STEP-06: step frontmatter has no name/description + * - STEP-07: step count 2-10 + * - SEQ-02: no time estimates + * + * Usage: + * node tools/validate-skills.js # All skills, human-readable + * node tools/validate-skills.js path/to/skill-dir # Single skill + * node tools/validate-skills.js --strict # Exit 1 on HIGH+ findings + * node tools/validate-skills.js --json # JSON output + */ + +const fs = require('node:fs'); +const path = require('node:path'); + +const PROJECT_ROOT = path.resolve(__dirname, '..'); +const SRC_DIR = path.join(PROJECT_ROOT, 'src'); + +// --- CLI Parsing --- + +const args = process.argv.slice(2); +const STRICT = args.includes('--strict'); +const JSON_OUTPUT = args.includes('--json'); +const positionalArgs = args.filter((a) => !a.startsWith('--')); + +// --- Constants --- + +const SKILL_LOCATIONS = [path.join(SRC_DIR, 'core', 'skills'), path.join(SRC_DIR, 'core', 'tasks'), path.join(SRC_DIR, 'bmm', 'workflows')]; + +// Agent skills live separately +const AGENT_LOCATION = path.join(SRC_DIR, 'bmm', 'agents'); + +const NAME_REGEX = /^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$/; +const STEP_FILENAME_REGEX = /^step-\d{2}[a-z]?-[a-z0-9-]+\.md$/; +const FORBIDDEN_NAME_SUBSTRINGS = ['anthropic', 'claude']; +const TIME_ESTIMATE_PATTERNS = [/takes?\s+\d+\s*min/i, /~\s*\d+\s*min/i, /estimated\s+time/i, /\bETA\b/]; + +const SEVERITY_ORDER = { CRITICAL: 0, HIGH: 1, MEDIUM: 2, LOW: 3 }; + +// --- Output Escaping --- + +function escapeAnnotation(str) { + return str.replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A'); +} + +function escapeTableCell(str) { + return String(str).replaceAll('|', String.raw`\|`); +} + +// --- Frontmatter Parsing --- + +/** + * Parse YAML frontmatter from a markdown file. + * Returns an object with key-value pairs, or null if no frontmatter. + */ +function parseFrontmatter(content) { + const trimmed = content.trimStart(); + if (!trimmed.startsWith('---')) return null; + + const endIndex = trimmed.indexOf('\n---', 3); + if (endIndex === -1) return null; + + const fmBlock = trimmed.slice(3, endIndex).trim(); + if (fmBlock === '') return {}; + + const result = {}; + for (const line of fmBlock.split('\n')) { + const colonIndex = line.indexOf(':'); + if (colonIndex === -1) continue; + // Skip indented lines (nested YAML values) + if (line[0] === ' ' || line[0] === '\t') continue; + const key = line.slice(0, colonIndex).trim(); + let value = line.slice(colonIndex + 1).trim(); + // Strip surrounding quotes (single or double) + if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith('"') && value.endsWith('"'))) { + value = value.slice(1, -1); + } + result[key] = value; + } + + return result; +} + +/** + * Parse YAML frontmatter, handling multiline values (description often spans lines). + * Returns an object with key-value pairs, or null if no frontmatter. + */ +function parseFrontmatterMultiline(content) { + const trimmed = content.trimStart(); + if (!trimmed.startsWith('---')) return null; + + const endIndex = trimmed.indexOf('\n---', 3); + if (endIndex === -1) return null; + + const fmBlock = trimmed.slice(3, endIndex).trim(); + if (fmBlock === '') return {}; + + const result = {}; + let currentKey = null; + let currentValue = ''; + + for (const line of fmBlock.split('\n')) { + const colonIndex = line.indexOf(':'); + // New key-value pair: must start at column 0 (no leading whitespace) and have a colon + if (colonIndex > 0 && line[0] !== ' ' && line[0] !== '\t') { + // Save previous key + if (currentKey !== null) { + result[currentKey] = stripQuotes(currentValue.trim()); + } + currentKey = line.slice(0, colonIndex).trim(); + currentValue = line.slice(colonIndex + 1); + } else if (currentKey !== null) { + // Continuation of multiline value + currentValue += '\n' + line; + } + } + + // Save last key + if (currentKey !== null) { + result[currentKey] = stripQuotes(currentValue.trim()); + } + + return result; +} + +function stripQuotes(value) { + if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith('"') && value.endsWith('"'))) { + return value.slice(1, -1); + } + return value; +} + +// --- Safe File Reading --- + +/** + * Read a file safely, returning null on error. + * Pushes a warning finding if the file cannot be read. + */ +function safeReadFile(filePath, findings, relFile) { + try { + return fs.readFileSync(filePath, 'utf-8'); + } catch (error) { + findings.push({ + rule: 'READ-ERR', + title: 'File Read Error', + severity: 'MEDIUM', + file: relFile || path.basename(filePath), + detail: `Cannot read file: ${error.message}`, + fix: 'Check file permissions and ensure the file exists.', + }); + return null; + } +} + +// --- Code Block Stripping --- + +function stripCodeBlocks(content) { + return content.replaceAll(/```[\s\S]*?```/g, (m) => m.replaceAll(/[^\n]/g, '')); +} + +// --- Skill Discovery --- + +function discoverSkillDirs(rootDirs) { + const skillDirs = []; + + function walk(dir) { + if (!fs.existsSync(dir)) return; + const entries = fs.readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (entry.name === 'node_modules' || entry.name === '.git') continue; + + const fullPath = path.join(dir, entry.name); + const skillMd = path.join(fullPath, 'SKILL.md'); + + if (fs.existsSync(skillMd)) { + skillDirs.push(fullPath); + } + + // Keep walking into subdirectories to find nested skills + walk(fullPath); + } + } + + for (const rootDir of rootDirs) { + walk(rootDir); + } + + return skillDirs.sort(); +} + +// --- File Collection --- + +function collectSkillFiles(skillDir) { + const files = []; + + function walk(dir) { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name === 'node_modules' || entry.name === '.git') continue; + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(fullPath); + } else if (entry.isFile()) { + files.push(fullPath); + } + } + } + + walk(skillDir); + return files; +} + +// --- Rule Checks --- + +function validateSkill(skillDir) { + const findings = []; + const dirName = path.basename(skillDir); + const skillMdPath = path.join(skillDir, 'SKILL.md'); + const workflowMdPath = path.join(skillDir, 'workflow.md'); + const stepsDir = path.join(skillDir, 'steps'); + + // Collect all files in the skill for PATH-02 and SEQ-02 + const allFiles = collectSkillFiles(skillDir); + + // --- SKILL-01: SKILL.md must exist --- + if (!fs.existsSync(skillMdPath)) { + findings.push({ + rule: 'SKILL-01', + title: 'SKILL.md Must Exist', + severity: 'CRITICAL', + file: 'SKILL.md', + detail: 'SKILL.md not found in skill directory.', + fix: 'Create SKILL.md as the skill entrypoint.', + }); + // Cannot check SKILL-02 through SKILL-06 without SKILL.md + return findings; + } + + const skillContent = safeReadFile(skillMdPath, findings, 'SKILL.md'); + if (skillContent === null) return findings; + const skillFm = parseFrontmatterMultiline(skillContent); + + // --- SKILL-02: frontmatter has name --- + if (!skillFm || !('name' in skillFm)) { + findings.push({ + rule: 'SKILL-02', + title: 'SKILL.md Must Have name in Frontmatter', + severity: 'CRITICAL', + file: 'SKILL.md', + detail: 'Frontmatter is missing the `name` field.', + fix: 'Add `name: ` to the frontmatter.', + }); + } else if (skillFm.name === '') { + findings.push({ + rule: 'SKILL-02', + title: 'SKILL.md Must Have name in Frontmatter', + severity: 'CRITICAL', + file: 'SKILL.md', + detail: 'Frontmatter `name` field is empty.', + fix: 'Set `name` to the skill directory name (kebab-case).', + }); + } + + // --- SKILL-03: frontmatter has description --- + if (!skillFm || !('description' in skillFm)) { + findings.push({ + rule: 'SKILL-03', + title: 'SKILL.md Must Have description in Frontmatter', + severity: 'CRITICAL', + file: 'SKILL.md', + detail: 'Frontmatter is missing the `description` field.', + fix: 'Add `description: ` to the frontmatter.', + }); + } else if (skillFm.description === '') { + findings.push({ + rule: 'SKILL-03', + title: 'SKILL.md Must Have description in Frontmatter', + severity: 'CRITICAL', + file: 'SKILL.md', + detail: 'Frontmatter `description` field is empty.', + fix: 'Add a description stating what the skill does and when to use it.', + }); + } + + const name = skillFm && skillFm.name; + const description = skillFm && skillFm.description; + + // --- SKILL-04: name format --- + if (name) { + if (!NAME_REGEX.test(name)) { + findings.push({ + rule: 'SKILL-04', + title: 'name Format', + severity: 'HIGH', + file: 'SKILL.md', + detail: `name "${name}" does not match pattern: ${NAME_REGEX}`, + fix: 'Rename to comply with lowercase letters, numbers, and hyphens only (max 64 chars).', + }); + } + + for (const forbidden of FORBIDDEN_NAME_SUBSTRINGS) { + if (name.toLowerCase().includes(forbidden)) { + findings.push({ + rule: 'SKILL-04', + title: 'name Format', + severity: 'HIGH', + file: 'SKILL.md', + detail: `name "${name}" contains forbidden substring "${forbidden}".`, + fix: `Remove "${forbidden}" from the name.`, + }); + } + } + } + + // --- SKILL-05: name matches directory --- + if (name && name !== dirName) { + findings.push({ + rule: 'SKILL-05', + title: 'name Must Match Directory Name', + severity: 'HIGH', + file: 'SKILL.md', + detail: `name "${name}" does not match directory name "${dirName}".`, + fix: `Change name to "${dirName}" or rename the directory.`, + }); + } + + // --- SKILL-06: description quality --- + if (description) { + if (description.length > 1024) { + findings.push({ + rule: 'SKILL-06', + title: 'description Quality', + severity: 'MEDIUM', + file: 'SKILL.md', + detail: `description is ${description.length} characters (max 1024).`, + fix: 'Shorten the description to 1024 characters or less.', + }); + } + + if (!/use\s+when\b/i.test(description) && !/use\s+if\b/i.test(description)) { + findings.push({ + rule: 'SKILL-06', + title: 'description Quality', + severity: 'MEDIUM', + file: 'SKILL.md', + detail: 'description does not contain "Use when" or "Use if" trigger phrase.', + fix: 'Append a "Use when..." clause to explain when to invoke this skill.', + }); + } + } + + // --- WF-01 / WF-02: workflow.md must NOT have name/description --- + if (fs.existsSync(workflowMdPath)) { + const wfContent = safeReadFile(workflowMdPath, findings, 'workflow.md'); + const wfFm = wfContent ? parseFrontmatter(wfContent) : null; + + if (wfFm && 'name' in wfFm) { + findings.push({ + rule: 'WF-01', + title: 'workflow.md Must NOT Have name in Frontmatter', + severity: 'HIGH', + file: 'workflow.md', + detail: 'workflow.md frontmatter contains `name` — this belongs only in SKILL.md.', + fix: 'Remove the `name:` line from workflow.md frontmatter.', + }); + } + + if (wfFm && 'description' in wfFm) { + findings.push({ + rule: 'WF-02', + title: 'workflow.md Must NOT Have description in Frontmatter', + severity: 'HIGH', + file: 'workflow.md', + detail: 'workflow.md frontmatter contains `description` — this belongs only in SKILL.md.', + fix: 'Remove the `description:` line from workflow.md frontmatter.', + }); + } + } + + // --- PATH-02: no installed_path --- + for (const filePath of allFiles) { + // Only check markdown and yaml files + const ext = path.extname(filePath); + if (!['.md', '.yaml', '.yml'].includes(ext)) continue; + + const relFile = path.relative(skillDir, filePath); + const content = safeReadFile(filePath, findings, relFile); + if (content === null) continue; + + // Check frontmatter for installed_path key + const fm = parseFrontmatter(content); + if (fm && 'installed_path' in fm) { + findings.push({ + rule: 'PATH-02', + title: 'No installed_path Variable', + severity: 'HIGH', + file: relFile, + detail: 'Frontmatter contains `installed_path:` key.', + fix: 'Remove `installed_path` from frontmatter. Use relative paths instead.', + }); + } + + // Check content for {installed_path} + const stripped = stripCodeBlocks(content); + const lines = stripped.split('\n'); + for (const [i, line] of lines.entries()) { + if (line.includes('{installed_path}')) { + findings.push({ + rule: 'PATH-02', + title: 'No installed_path Variable', + severity: 'HIGH', + file: relFile, + line: i + 1, + detail: '`{installed_path}` reference found in content.', + fix: 'Replace `{installed_path}/path` with a relative path (`./path` or `../path`).', + }); + } + } + } + + // --- STEP-01: step filename format --- + // --- STEP-06: step frontmatter no name/description --- + // --- STEP-07: step count --- + // Only check the literal steps/ directory (variant directories like steps-c, steps-v + // use different naming conventions and are excluded per the rule specification) + if (fs.existsSync(stepsDir) && fs.statSync(stepsDir).isDirectory()) { + const stepDirName = 'steps'; + const stepFiles = fs.readdirSync(stepsDir).filter((f) => f.endsWith('.md')); + + // STEP-01: filename format + for (const stepFile of stepFiles) { + if (!STEP_FILENAME_REGEX.test(stepFile)) { + findings.push({ + rule: 'STEP-01', + title: 'Step File Naming', + severity: 'MEDIUM', + file: path.join(stepDirName, stepFile), + detail: `Filename "${stepFile}" does not match pattern: ${STEP_FILENAME_REGEX}`, + fix: 'Rename to step-NN-description.md (NN = zero-padded number, optional letter suffix).', + }); + } + } + + // STEP-06: step frontmatter has no name/description + for (const stepFile of stepFiles) { + const stepPath = path.join(stepsDir, stepFile); + const stepContent = safeReadFile(stepPath, findings, path.join(stepDirName, stepFile)); + if (stepContent === null) continue; + const stepFm = parseFrontmatter(stepContent); + + if (stepFm) { + if ('name' in stepFm) { + findings.push({ + rule: 'STEP-06', + title: 'Step File Frontmatter: No name or description', + severity: 'MEDIUM', + file: path.join(stepDirName, stepFile), + detail: 'Step file frontmatter contains `name:` — this is metadata noise.', + fix: 'Remove `name:` from step file frontmatter.', + }); + } + if ('description' in stepFm) { + findings.push({ + rule: 'STEP-06', + title: 'Step File Frontmatter: No name or description', + severity: 'MEDIUM', + file: path.join(stepDirName, stepFile), + detail: 'Step file frontmatter contains `description:` — this is metadata noise.', + fix: 'Remove `description:` from step file frontmatter.', + }); + } + } + } + + // STEP-07: step count 2-10 + const stepCount = stepFiles.filter((f) => f.startsWith('step-')).length; + if (stepCount > 0 && (stepCount < 2 || stepCount > 10)) { + const detail = + stepCount < 2 + ? `Only ${stepCount} step file found — consider inlining into workflow.md.` + : `${stepCount} step files found — more than 10 risks LLM context degradation.`; + findings.push({ + rule: 'STEP-07', + title: 'Step Count', + severity: 'LOW', + file: stepDirName + '/', + detail, + fix: stepCount > 10 ? 'Consider consolidating steps.' : 'Consider expanding or inlining.', + }); + } + } + + // --- SEQ-02: no time estimates --- + for (const filePath of allFiles) { + const ext = path.extname(filePath); + if (!['.md', '.yaml', '.yml'].includes(ext)) continue; + + const relFile = path.relative(skillDir, filePath); + const content = safeReadFile(filePath, findings, relFile); + if (content === null) continue; + const stripped = stripCodeBlocks(content); + const lines = stripped.split('\n'); + + for (const [i, line] of lines.entries()) { + for (const pattern of TIME_ESTIMATE_PATTERNS) { + if (pattern.test(line)) { + findings.push({ + rule: 'SEQ-02', + title: 'No Time Estimates', + severity: 'LOW', + file: relFile, + line: i + 1, + detail: `Time estimate pattern found: "${line.trim()}"`, + fix: 'Remove time estimates — AI execution speed varies too much.', + }); + break; // Only report once per line + } + } + } + } + + return findings; +} + +// --- Output Formatting --- + +function formatHumanReadable(results) { + const output = []; + let totalFindings = 0; + const severityCounts = { CRITICAL: 0, HIGH: 0, MEDIUM: 0, LOW: 0 }; + + output.push( + `\nValidating skills in: ${SRC_DIR}`, + `Mode: ${STRICT ? 'STRICT (exit 1 on HIGH+)' : 'WARNING (exit 0)'}${JSON_OUTPUT ? ' + JSON' : ''}\n`, + ); + + let totalSkills = 0; + let skillsWithFindings = 0; + + for (const { skillDir, findings } of results) { + totalSkills++; + const relDir = path.relative(PROJECT_ROOT, skillDir); + + if (findings.length > 0) { + skillsWithFindings++; + output.push(`\n${relDir}`); + + for (const f of findings) { + totalFindings++; + severityCounts[f.severity]++; + const location = f.line ? ` (line ${f.line})` : ''; + output.push(` [${f.severity}] ${f.rule} — ${f.title}`, ` File: ${f.file}${location}`, ` ${f.detail}`); + + if (process.env.GITHUB_ACTIONS) { + const absFile = path.join(skillDir, f.file); + const ghFile = path.relative(PROJECT_ROOT, absFile); + const line = f.line || 1; + const level = f.severity === 'LOW' ? 'notice' : 'warning'; + console.log(`::${level} file=${ghFile},line=${line}::${escapeAnnotation(`${f.rule}: ${f.detail}`)}`); + } + } + } + } + + // Summary + output.push( + `\n${'─'.repeat(60)}`, + `\nSummary:`, + ` Skills scanned: ${totalSkills}`, + ` Skills with findings: ${skillsWithFindings}`, + ` Total findings: ${totalFindings}`, + ); + + if (totalFindings > 0) { + output.push('', ` | Severity | Count |`, ` |----------|-------|`); + for (const sev of ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']) { + if (severityCounts[sev] > 0) { + output.push(` | ${sev.padEnd(8)} | ${String(severityCounts[sev]).padStart(5)} |`); + } + } + } + + const hasHighPlus = severityCounts.CRITICAL > 0 || severityCounts.HIGH > 0; + + if (totalFindings === 0) { + output.push(`\n All skills passed validation!`); + } else if (STRICT && hasHighPlus) { + output.push(`\n [STRICT MODE] HIGH+ findings found — exiting with failure.`); + } else if (STRICT) { + output.push(`\n [STRICT MODE] Only MEDIUM/LOW findings — pass.`); + } else { + output.push(`\n Run with --strict to treat HIGH+ findings as errors.`); + } + + output.push(''); + + // Write GitHub Actions step summary + if (process.env.GITHUB_STEP_SUMMARY) { + let summary = '## Skill Validation\n\n'; + if (totalFindings > 0) { + summary += '| Skill | Rule | Severity | File | Detail |\n'; + summary += '|-------|------|----------|------|--------|\n'; + for (const { skillDir, findings } of results) { + const relDir = path.relative(PROJECT_ROOT, skillDir); + for (const f of findings) { + summary += `| ${escapeTableCell(relDir)} | ${f.rule} | ${f.severity} | ${escapeTableCell(f.file)} | ${escapeTableCell(f.detail)} |\n`; + } + } + summary += '\n'; + } + summary += `**${totalSkills} skills scanned, ${totalFindings} findings**\n`; + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + } + + return { output: output.join('\n'), hasHighPlus }; +} + +function formatJson(results) { + const allFindings = []; + for (const { skillDir, findings } of results) { + const relDir = path.relative(PROJECT_ROOT, skillDir); + for (const f of findings) { + allFindings.push({ + skill: relDir, + rule: f.rule, + title: f.title, + severity: f.severity, + file: f.file, + line: f.line || null, + detail: f.detail, + fix: f.fix, + }); + } + } + + // Sort by severity + allFindings.sort((a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]); + + const hasHighPlus = allFindings.some((f) => f.severity === 'CRITICAL' || f.severity === 'HIGH'); + + return { output: JSON.stringify(allFindings, null, 2), hasHighPlus }; +} + +// --- Main --- + +if (require.main === module) { + // Determine which skills to validate + let skillDirs; + + if (positionalArgs.length > 0) { + // Single skill directory specified + const target = path.resolve(positionalArgs[0]); + if (!fs.existsSync(target) || !fs.statSync(target).isDirectory()) { + console.error(`Error: "${positionalArgs[0]}" is not a valid directory.`); + process.exit(2); + } + skillDirs = [target]; + } else { + // Discover all skills + const allLocations = [...SKILL_LOCATIONS, AGENT_LOCATION]; + skillDirs = discoverSkillDirs(allLocations); + } + + if (skillDirs.length === 0) { + console.error('No skill directories found.'); + process.exit(2); + } + + // Validate each skill + const results = []; + for (const skillDir of skillDirs) { + const findings = validateSkill(skillDir); + results.push({ skillDir, findings }); + } + + // Format output + const { output, hasHighPlus } = JSON_OUTPUT ? formatJson(results) : formatHumanReadable(results); + console.log(output); + + // Exit code + if (STRICT && hasHighPlus) { + process.exit(1); + } +} + +// --- Exports (for testing) --- +module.exports = { parseFrontmatter, parseFrontmatterMultiline, validateSkill, discoverSkillDirs };