feat(tools): add deterministic skill validator for CI

Add tools/validate-skills.js — a Node CLI that checks 13 deterministic
rules (SKILL-01–06, WF-01–02, PATH-02, STEP-01/06/07, SEQ-02) across
all skill directories. Runs in under a second, exits non-zero on HIGH+
findings in strict mode, and outputs JSON for the inference validator.

- Add validate:skills npm script to quality chain
- Update skill-validator.md with first-pass integration instructions
- Update AGENTS.md push gate documentation

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Alex Verkhovsky 2026-03-17 23:49:01 -06:00
parent 21c2a48ab2
commit 5a1f356e2c
4 changed files with 738 additions and 11 deletions

View File

@ -9,3 +9,4 @@ Open source framework for structured, agent-assisted software delivery.
`quality` mirrors the checks in `.github/workflows/quality.yaml`. `quality` mirrors the checks in `.github/workflows/quality.yaml`.
- Skill validation rules are in `tools/skill-validator.md`. - Skill validation rules are in `tools/skill-validator.md`.
- Deterministic skill checks run via `npm run validate:skills` (included in `quality`).

View File

@ -39,12 +39,13 @@
"lint:fix": "eslint . --ext .js,.cjs,.mjs,.yaml --fix", "lint:fix": "eslint . --ext .js,.cjs,.mjs,.yaml --fix",
"lint:md": "markdownlint-cli2 \"**/*.md\"", "lint:md": "markdownlint-cli2 \"**/*.md\"",
"prepare": "command -v husky >/dev/null 2>&1 && husky || exit 0", "prepare": "command -v husky >/dev/null 2>&1 && husky || exit 0",
"quality": "npm run format:check && npm run lint && npm run lint:md && npm run docs:build && npm run test:install && npm run validate:refs", "quality": "npm run format:check && npm run lint && npm run lint:md && npm run docs:build && npm run test:install && npm run validate:refs && npm run validate:skills",
"rebundle": "node tools/cli/bundlers/bundle-web.js rebundle", "rebundle": "node tools/cli/bundlers/bundle-web.js rebundle",
"test": "npm run test:refs && npm run test:install && npm run lint && npm run lint:md && npm run format:check", "test": "npm run test:refs && npm run test:install && npm run lint && npm run lint:md && npm run format:check",
"test:install": "node test/test-installation-components.js", "test:install": "node test/test-installation-components.js",
"test:refs": "node test/test-file-refs-csv.js", "test:refs": "node test/test-file-refs-csv.js",
"validate:refs": "node tools/validate-file-refs.js --strict" "validate:refs": "node tools/validate-file-refs.js --strict",
"validate:skills": "node tools/validate-skills.js --strict"
}, },
"lint-staged": { "lint-staged": {
"*.{js,cjs,mjs}": [ "*.{js,cjs,mjs}": [

View File

@ -2,14 +2,27 @@
An LLM-readable validation prompt for skills following the Agent Skills open standard. An LLM-readable validation prompt for skills following the Agent Skills open standard.
## First Pass — Deterministic Checks
Before running inference-based validation, run the deterministic validator:
```bash
node tools/validate-skills.js --json path/to/skill-dir
```
This checks 13 rules deterministically: SKILL-01, SKILL-02, SKILL-03, SKILL-04, SKILL-05, SKILL-06, WF-01, WF-02, PATH-02, STEP-01, STEP-06, STEP-07, SEQ-02.
Review its JSON output. For any rule that produced **zero findings** in the first pass, **skip it** during inference-based validation below — it has already been verified. If a rule produced any findings, the inference validator should still review that rule (some rules like SKILL-04 and SKILL-06 have sub-checks that benefit from judgment). Focus your inference effort on the remaining rules that require judgment (PATH-01, PATH-03, PATH-04, PATH-05, WF-03, STEP-02, STEP-03, STEP-04, STEP-05, SEQ-01, REF-01, REF-02, REF-03).
## How to Use ## How to Use
1. You are given a **skill directory path** to validate. 1. You are given a **skill directory path** to validate.
2. Read every file in the skill directory recursively. 2. Run the deterministic first pass (see above) and note which rules passed.
3. Apply every rule in the catalog below to every applicable file. 3. Read every file in the skill directory recursively.
4. Produce a findings report using the report template at the end. 4. Apply every rule in the catalog below to every applicable file, **skipping rules that passed the deterministic first pass**.
5. Produce a findings report using the report template at the end, including any deterministic findings from the first pass.
If no findings are generated, the skill passes validation. If no findings are generated (from either pass), the skill passes validation.
--- ---
@ -103,6 +116,7 @@ If no findings are generated, the skill passes validation.
- A legitimate external path expression (must not violate PATH-05 — no paths into another skill's directory) - A legitimate external path expression (must not violate PATH-05 — no paths into another skill's directory)
It must NOT be a path to a file within the skill directory (see PATH-04), nor a path into another skill's directory (see PATH-05). It must NOT be a path to a file within the skill directory (see PATH-04), nor a path into another skill's directory (see PATH-05).
- **Detection:** For each frontmatter variable, check if its value resolves to a file inside the skill (e.g., starts with `./`, `{installed_path}`, or is a bare relative path to a sibling file). If so, it is an intra-skill path variable. Also check if the value is a path into another skill's directory — if so, it violates PATH-05 and is not a legitimate external path. - **Detection:** For each frontmatter variable, check if its value resolves to a file inside the skill (e.g., starts with `./`, `{installed_path}`, or is a bare relative path to a sibling file). If so, it is an intra-skill path variable. Also check if the value is a path into another skill's directory — if so, it violates PATH-05 and is not a legitimate external path.
- **Fix:** Remove the variable. Use a hardcoded relative path inline where the file is referenced. - **Fix:** Remove the variable. Use a hardcoded relative path inline where the file is referenced.
@ -294,7 +308,7 @@ When reporting findings, use this format:
## Summary ## Summary
| Severity | Count | | Severity | Count |
|----------|-------| | -------- | ----- |
| CRITICAL | N | | CRITICAL | N |
| HIGH | N | | HIGH | N |
| MEDIUM | N | | MEDIUM | N |
@ -329,28 +343,34 @@ Quick-reference for the Agent Skills open standard.
For the full standard, see: [Agent Skills specification](https://agentskills.io/specification) For the full standard, see: [Agent Skills specification](https://agentskills.io/specification)
### Structure ### Structure
- Every skill is a directory with `SKILL.md` as the required entrypoint - Every skill is a directory with `SKILL.md` as the required entrypoint
- YAML frontmatter between `---` markers provides metadata; markdown body provides instructions - YAML frontmatter between `---` markers provides metadata; markdown body provides instructions
- Supporting files (scripts, templates, references) live alongside SKILL.md - Supporting files (scripts, templates, references) live alongside SKILL.md
### Path resolution ### Path resolution
- Relative file references resolve from the directory of the file that contains the reference, not from the skill root - Relative file references resolve from the directory of the file that contains the reference, not from the skill root
- Example: from `branch-a/deep/next.md`, `./deeper/final.md` resolves to `branch-a/deep/deeper/final.md` - Example: from `branch-a/deep/next.md`, `./deeper/final.md` resolves to `branch-a/deep/deeper/final.md`
- Example: from `branch-a/deep/next.md`, `./branch-b/alt/leaf.md` incorrectly resolves to `branch-a/deep/branch-b/alt/leaf.md` - Example: from `branch-a/deep/next.md`, `./branch-b/alt/leaf.md` incorrectly resolves to `branch-a/deep/branch-b/alt/leaf.md`
### Frontmatter fields (standard) ### Frontmatter fields (standard)
- `name`: lowercase letters, numbers, hyphens only; max 64 chars; no "anthropic" or "claude" - `name`: lowercase letters, numbers, hyphens only; max 64 chars; no "anthropic" or "claude"
- `description`: required, max 1024 chars; should state what the skill does AND when to use it - `description`: required, max 1024 chars; should state what the skill does AND when to use it
### Progressive disclosure — three loading levels ### Progressive disclosure — three loading levels
- **L1 Metadata** (~100 tokens): `name` + `description` loaded at startup into system prompt - **L1 Metadata** (~100 tokens): `name` + `description` loaded at startup into system prompt
- **L2 Instructions** (<5k tokens): SKILL.md body loaded only when skill is triggered - **L2 Instructions** (<5k tokens): SKILL.md body loaded only when skill is triggered
- **L3 Resources** (unlimited): additional files + scripts loaded/executed on demand; script output enters context, script code does not - **L3 Resources** (unlimited): additional files + scripts loaded/executed on demand; script output enters context, script code does not
### Key design principle ### Key design principle
- Skills are filesystem-based directories, not API payloads — Claude reads them via bash/file tools - Skills are filesystem-based directories, not API payloads — Claude reads them via bash/file tools
- Keep SKILL.md focused; offload detailed reference to separate files - Keep SKILL.md focused; offload detailed reference to separate files
### Practical tips ### Practical tips
- Keep SKILL.md under 500 lines - Keep SKILL.md under 500 lines
- `description` drives auto-discovery — use keywords users would naturally say - `description` drives auto-discovery — use keywords users would naturally say

705
tools/validate-skills.js Normal file
View File

@ -0,0 +1,705 @@
/**
* Deterministic Skill Validator
*
* Validates 13 deterministic rules across all skill directories.
* Acts as a fast first-pass complement to the inference-based skill validator.
*
* What it checks:
* - SKILL-01: SKILL.md exists
* - SKILL-02: SKILL.md frontmatter has name
* - SKILL-03: SKILL.md frontmatter has description
* - SKILL-04: name format (lowercase, hyphens, no forbidden substrings)
* - SKILL-05: name matches directory basename
* - SKILL-06: description quality (length, "Use when"/"Use if")
* - WF-01: workflow.md frontmatter has no name
* - WF-02: workflow.md frontmatter has no description
* - PATH-02: no installed_path variable
* - STEP-01: step filename format
* - STEP-06: step frontmatter has no name/description
* - STEP-07: step count 2-10
* - SEQ-02: no time estimates
*
* Usage:
* node tools/validate-skills.js # All skills, human-readable
* node tools/validate-skills.js path/to/skill-dir # Single skill
* node tools/validate-skills.js --strict # Exit 1 on HIGH+ findings
* node tools/validate-skills.js --json # JSON output
*/
const fs = require('node:fs');
const path = require('node:path');
const PROJECT_ROOT = path.resolve(__dirname, '..');
const SRC_DIR = path.join(PROJECT_ROOT, 'src');
// --- CLI Parsing ---
const args = process.argv.slice(2);
const STRICT = args.includes('--strict');
const JSON_OUTPUT = args.includes('--json');
const positionalArgs = args.filter((a) => !a.startsWith('--'));
// --- Constants ---
const SKILL_LOCATIONS = [path.join(SRC_DIR, 'core', 'skills'), path.join(SRC_DIR, 'core', 'tasks'), path.join(SRC_DIR, 'bmm', 'workflows')];
// Agent skills live separately
const AGENT_LOCATION = path.join(SRC_DIR, 'bmm', 'agents');
const NAME_REGEX = /^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$/;
const STEP_FILENAME_REGEX = /^step-\d{2}[a-z]?-[a-z0-9-]+\.md$/;
const FORBIDDEN_NAME_SUBSTRINGS = ['anthropic', 'claude'];
const TIME_ESTIMATE_PATTERNS = [/takes?\s+\d+\s*min/i, /~\s*\d+\s*min/i, /estimated\s+time/i, /\bETA\b/];
const SEVERITY_ORDER = { CRITICAL: 0, HIGH: 1, MEDIUM: 2, LOW: 3 };
// --- Output Escaping ---
function escapeAnnotation(str) {
return str.replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A');
}
function escapeTableCell(str) {
return String(str).replaceAll('|', String.raw`\|`);
}
// --- Frontmatter Parsing ---
/**
* Parse YAML frontmatter from a markdown file.
* Returns an object with key-value pairs, or null if no frontmatter.
*/
function parseFrontmatter(content) {
const trimmed = content.trimStart();
if (!trimmed.startsWith('---')) return null;
const endIndex = trimmed.indexOf('\n---', 3);
if (endIndex === -1) return null;
const fmBlock = trimmed.slice(3, endIndex).trim();
if (fmBlock === '') return {};
const result = {};
for (const line of fmBlock.split('\n')) {
const colonIndex = line.indexOf(':');
if (colonIndex === -1) continue;
// Skip indented lines (nested YAML values)
if (line[0] === ' ' || line[0] === '\t') continue;
const key = line.slice(0, colonIndex).trim();
let value = line.slice(colonIndex + 1).trim();
// Strip surrounding quotes (single or double)
if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith('"') && value.endsWith('"'))) {
value = value.slice(1, -1);
}
result[key] = value;
}
return result;
}
/**
* Parse YAML frontmatter, handling multiline values (description often spans lines).
* Returns an object with key-value pairs, or null if no frontmatter.
*/
function parseFrontmatterMultiline(content) {
const trimmed = content.trimStart();
if (!trimmed.startsWith('---')) return null;
const endIndex = trimmed.indexOf('\n---', 3);
if (endIndex === -1) return null;
const fmBlock = trimmed.slice(3, endIndex).trim();
if (fmBlock === '') return {};
const result = {};
let currentKey = null;
let currentValue = '';
for (const line of fmBlock.split('\n')) {
const colonIndex = line.indexOf(':');
// New key-value pair: must start at column 0 (no leading whitespace) and have a colon
if (colonIndex > 0 && line[0] !== ' ' && line[0] !== '\t') {
// Save previous key
if (currentKey !== null) {
result[currentKey] = stripQuotes(currentValue.trim());
}
currentKey = line.slice(0, colonIndex).trim();
currentValue = line.slice(colonIndex + 1);
} else if (currentKey !== null) {
// Continuation of multiline value
currentValue += '\n' + line;
}
}
// Save last key
if (currentKey !== null) {
result[currentKey] = stripQuotes(currentValue.trim());
}
return result;
}
function stripQuotes(value) {
if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith('"') && value.endsWith('"'))) {
return value.slice(1, -1);
}
return value;
}
// --- Safe File Reading ---
/**
* Read a file safely, returning null on error.
* Pushes a warning finding if the file cannot be read.
*/
function safeReadFile(filePath, findings, relFile) {
try {
return fs.readFileSync(filePath, 'utf-8');
} catch (error) {
findings.push({
rule: 'READ-ERR',
title: 'File Read Error',
severity: 'MEDIUM',
file: relFile || path.basename(filePath),
detail: `Cannot read file: ${error.message}`,
fix: 'Check file permissions and ensure the file exists.',
});
return null;
}
}
// --- Code Block Stripping ---
function stripCodeBlocks(content) {
return content.replaceAll(/```[\s\S]*?```/g, (m) => m.replaceAll(/[^\n]/g, ''));
}
// --- Skill Discovery ---
function discoverSkillDirs(rootDirs) {
const skillDirs = [];
function walk(dir) {
if (!fs.existsSync(dir)) return;
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
if (entry.name === 'node_modules' || entry.name === '.git') continue;
const fullPath = path.join(dir, entry.name);
const skillMd = path.join(fullPath, 'SKILL.md');
if (fs.existsSync(skillMd)) {
skillDirs.push(fullPath);
}
// Keep walking into subdirectories to find nested skills
walk(fullPath);
}
}
for (const rootDir of rootDirs) {
walk(rootDir);
}
return skillDirs.sort();
}
// --- File Collection ---
function collectSkillFiles(skillDir) {
const files = [];
function walk(dir) {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
if (entry.name === 'node_modules' || entry.name === '.git') continue;
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
walk(fullPath);
} else if (entry.isFile()) {
files.push(fullPath);
}
}
}
walk(skillDir);
return files;
}
// --- Rule Checks ---
function validateSkill(skillDir) {
const findings = [];
const dirName = path.basename(skillDir);
const skillMdPath = path.join(skillDir, 'SKILL.md');
const workflowMdPath = path.join(skillDir, 'workflow.md');
const stepsDir = path.join(skillDir, 'steps');
// Collect all files in the skill for PATH-02 and SEQ-02
const allFiles = collectSkillFiles(skillDir);
// --- SKILL-01: SKILL.md must exist ---
if (!fs.existsSync(skillMdPath)) {
findings.push({
rule: 'SKILL-01',
title: 'SKILL.md Must Exist',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'SKILL.md not found in skill directory.',
fix: 'Create SKILL.md as the skill entrypoint.',
});
// Cannot check SKILL-02 through SKILL-06 without SKILL.md
return findings;
}
const skillContent = safeReadFile(skillMdPath, findings, 'SKILL.md');
if (skillContent === null) return findings;
const skillFm = parseFrontmatterMultiline(skillContent);
// --- SKILL-02: frontmatter has name ---
if (!skillFm || !('name' in skillFm)) {
findings.push({
rule: 'SKILL-02',
title: 'SKILL.md Must Have name in Frontmatter',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'Frontmatter is missing the `name` field.',
fix: 'Add `name: <skill-name>` to the frontmatter.',
});
} else if (skillFm.name === '') {
findings.push({
rule: 'SKILL-02',
title: 'SKILL.md Must Have name in Frontmatter',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'Frontmatter `name` field is empty.',
fix: 'Set `name` to the skill directory name (kebab-case).',
});
}
// --- SKILL-03: frontmatter has description ---
if (!skillFm || !('description' in skillFm)) {
findings.push({
rule: 'SKILL-03',
title: 'SKILL.md Must Have description in Frontmatter',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'Frontmatter is missing the `description` field.',
fix: 'Add `description: <what it does and when to use it>` to the frontmatter.',
});
} else if (skillFm.description === '') {
findings.push({
rule: 'SKILL-03',
title: 'SKILL.md Must Have description in Frontmatter',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'Frontmatter `description` field is empty.',
fix: 'Add a description stating what the skill does and when to use it.',
});
}
const name = skillFm && skillFm.name;
const description = skillFm && skillFm.description;
// --- SKILL-04: name format ---
if (name) {
if (!NAME_REGEX.test(name)) {
findings.push({
rule: 'SKILL-04',
title: 'name Format',
severity: 'HIGH',
file: 'SKILL.md',
detail: `name "${name}" does not match pattern: ${NAME_REGEX}`,
fix: 'Rename to comply with lowercase letters, numbers, and hyphens only (max 64 chars).',
});
}
for (const forbidden of FORBIDDEN_NAME_SUBSTRINGS) {
if (name.toLowerCase().includes(forbidden)) {
findings.push({
rule: 'SKILL-04',
title: 'name Format',
severity: 'HIGH',
file: 'SKILL.md',
detail: `name "${name}" contains forbidden substring "${forbidden}".`,
fix: `Remove "${forbidden}" from the name.`,
});
}
}
}
// --- SKILL-05: name matches directory ---
if (name && name !== dirName) {
findings.push({
rule: 'SKILL-05',
title: 'name Must Match Directory Name',
severity: 'HIGH',
file: 'SKILL.md',
detail: `name "${name}" does not match directory name "${dirName}".`,
fix: `Change name to "${dirName}" or rename the directory.`,
});
}
// --- SKILL-06: description quality ---
if (description) {
if (description.length > 1024) {
findings.push({
rule: 'SKILL-06',
title: 'description Quality',
severity: 'MEDIUM',
file: 'SKILL.md',
detail: `description is ${description.length} characters (max 1024).`,
fix: 'Shorten the description to 1024 characters or less.',
});
}
if (!/use\s+when\b/i.test(description) && !/use\s+if\b/i.test(description)) {
findings.push({
rule: 'SKILL-06',
title: 'description Quality',
severity: 'MEDIUM',
file: 'SKILL.md',
detail: 'description does not contain "Use when" or "Use if" trigger phrase.',
fix: 'Append a "Use when..." clause to explain when to invoke this skill.',
});
}
}
// --- WF-01 / WF-02: workflow.md must NOT have name/description ---
if (fs.existsSync(workflowMdPath)) {
const wfContent = safeReadFile(workflowMdPath, findings, 'workflow.md');
const wfFm = wfContent ? parseFrontmatter(wfContent) : null;
if (wfFm && 'name' in wfFm) {
findings.push({
rule: 'WF-01',
title: 'workflow.md Must NOT Have name in Frontmatter',
severity: 'HIGH',
file: 'workflow.md',
detail: 'workflow.md frontmatter contains `name` — this belongs only in SKILL.md.',
fix: 'Remove the `name:` line from workflow.md frontmatter.',
});
}
if (wfFm && 'description' in wfFm) {
findings.push({
rule: 'WF-02',
title: 'workflow.md Must NOT Have description in Frontmatter',
severity: 'HIGH',
file: 'workflow.md',
detail: 'workflow.md frontmatter contains `description` — this belongs only in SKILL.md.',
fix: 'Remove the `description:` line from workflow.md frontmatter.',
});
}
}
// --- PATH-02: no installed_path ---
for (const filePath of allFiles) {
// Only check markdown and yaml files
const ext = path.extname(filePath);
if (!['.md', '.yaml', '.yml'].includes(ext)) continue;
const relFile = path.relative(skillDir, filePath);
const content = safeReadFile(filePath, findings, relFile);
if (content === null) continue;
// Check frontmatter for installed_path key
const fm = parseFrontmatter(content);
if (fm && 'installed_path' in fm) {
findings.push({
rule: 'PATH-02',
title: 'No installed_path Variable',
severity: 'HIGH',
file: relFile,
detail: 'Frontmatter contains `installed_path:` key.',
fix: 'Remove `installed_path` from frontmatter. Use relative paths instead.',
});
}
// Check content for {installed_path}
const stripped = stripCodeBlocks(content);
const lines = stripped.split('\n');
for (const [i, line] of lines.entries()) {
if (line.includes('{installed_path}')) {
findings.push({
rule: 'PATH-02',
title: 'No installed_path Variable',
severity: 'HIGH',
file: relFile,
line: i + 1,
detail: '`{installed_path}` reference found in content.',
fix: 'Replace `{installed_path}/path` with a relative path (`./path` or `../path`).',
});
}
}
}
// --- STEP-01: step filename format ---
// --- STEP-06: step frontmatter no name/description ---
// --- STEP-07: step count ---
// Only check the literal steps/ directory (variant directories like steps-c, steps-v
// use different naming conventions and are excluded per the rule specification)
if (fs.existsSync(stepsDir) && fs.statSync(stepsDir).isDirectory()) {
const stepDirName = 'steps';
const stepFiles = fs.readdirSync(stepsDir).filter((f) => f.endsWith('.md'));
// STEP-01: filename format
for (const stepFile of stepFiles) {
if (!STEP_FILENAME_REGEX.test(stepFile)) {
findings.push({
rule: 'STEP-01',
title: 'Step File Naming',
severity: 'MEDIUM',
file: path.join(stepDirName, stepFile),
detail: `Filename "${stepFile}" does not match pattern: ${STEP_FILENAME_REGEX}`,
fix: 'Rename to step-NN-description.md (NN = zero-padded number, optional letter suffix).',
});
}
}
// STEP-06: step frontmatter has no name/description
for (const stepFile of stepFiles) {
const stepPath = path.join(stepsDir, stepFile);
const stepContent = safeReadFile(stepPath, findings, path.join(stepDirName, stepFile));
if (stepContent === null) continue;
const stepFm = parseFrontmatter(stepContent);
if (stepFm) {
if ('name' in stepFm) {
findings.push({
rule: 'STEP-06',
title: 'Step File Frontmatter: No name or description',
severity: 'MEDIUM',
file: path.join(stepDirName, stepFile),
detail: 'Step file frontmatter contains `name:` — this is metadata noise.',
fix: 'Remove `name:` from step file frontmatter.',
});
}
if ('description' in stepFm) {
findings.push({
rule: 'STEP-06',
title: 'Step File Frontmatter: No name or description',
severity: 'MEDIUM',
file: path.join(stepDirName, stepFile),
detail: 'Step file frontmatter contains `description:` — this is metadata noise.',
fix: 'Remove `description:` from step file frontmatter.',
});
}
}
}
// STEP-07: step count 2-10
const stepCount = stepFiles.filter((f) => f.startsWith('step-')).length;
if (stepCount > 0 && (stepCount < 2 || stepCount > 10)) {
const detail =
stepCount < 2
? `Only ${stepCount} step file found — consider inlining into workflow.md.`
: `${stepCount} step files found — more than 10 risks LLM context degradation.`;
findings.push({
rule: 'STEP-07',
title: 'Step Count',
severity: 'LOW',
file: stepDirName + '/',
detail,
fix: stepCount > 10 ? 'Consider consolidating steps.' : 'Consider expanding or inlining.',
});
}
}
// --- SEQ-02: no time estimates ---
for (const filePath of allFiles) {
const ext = path.extname(filePath);
if (!['.md', '.yaml', '.yml'].includes(ext)) continue;
const relFile = path.relative(skillDir, filePath);
const content = safeReadFile(filePath, findings, relFile);
if (content === null) continue;
const stripped = stripCodeBlocks(content);
const lines = stripped.split('\n');
for (const [i, line] of lines.entries()) {
for (const pattern of TIME_ESTIMATE_PATTERNS) {
if (pattern.test(line)) {
findings.push({
rule: 'SEQ-02',
title: 'No Time Estimates',
severity: 'LOW',
file: relFile,
line: i + 1,
detail: `Time estimate pattern found: "${line.trim()}"`,
fix: 'Remove time estimates — AI execution speed varies too much.',
});
break; // Only report once per line
}
}
}
}
return findings;
}
// --- Output Formatting ---
function formatHumanReadable(results) {
const output = [];
let totalFindings = 0;
const severityCounts = { CRITICAL: 0, HIGH: 0, MEDIUM: 0, LOW: 0 };
output.push(
`\nValidating skills in: ${SRC_DIR}`,
`Mode: ${STRICT ? 'STRICT (exit 1 on HIGH+)' : 'WARNING (exit 0)'}${JSON_OUTPUT ? ' + JSON' : ''}\n`,
);
let totalSkills = 0;
let skillsWithFindings = 0;
for (const { skillDir, findings } of results) {
totalSkills++;
const relDir = path.relative(PROJECT_ROOT, skillDir);
if (findings.length > 0) {
skillsWithFindings++;
output.push(`\n${relDir}`);
for (const f of findings) {
totalFindings++;
severityCounts[f.severity]++;
const location = f.line ? ` (line ${f.line})` : '';
output.push(` [${f.severity}] ${f.rule}${f.title}`, ` File: ${f.file}${location}`, ` ${f.detail}`);
if (process.env.GITHUB_ACTIONS) {
const absFile = path.join(skillDir, f.file);
const ghFile = path.relative(PROJECT_ROOT, absFile);
const line = f.line || 1;
const level = f.severity === 'LOW' ? 'notice' : 'warning';
console.log(`::${level} file=${ghFile},line=${line}::${escapeAnnotation(`${f.rule}: ${f.detail}`)}`);
}
}
}
}
// Summary
output.push(
`\n${'─'.repeat(60)}`,
`\nSummary:`,
` Skills scanned: ${totalSkills}`,
` Skills with findings: ${skillsWithFindings}`,
` Total findings: ${totalFindings}`,
);
if (totalFindings > 0) {
output.push('', ` | Severity | Count |`, ` |----------|-------|`);
for (const sev of ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']) {
if (severityCounts[sev] > 0) {
output.push(` | ${sev.padEnd(8)} | ${String(severityCounts[sev]).padStart(5)} |`);
}
}
}
const hasHighPlus = severityCounts.CRITICAL > 0 || severityCounts.HIGH > 0;
if (totalFindings === 0) {
output.push(`\n All skills passed validation!`);
} else if (STRICT && hasHighPlus) {
output.push(`\n [STRICT MODE] HIGH+ findings found — exiting with failure.`);
} else if (STRICT) {
output.push(`\n [STRICT MODE] Only MEDIUM/LOW findings — pass.`);
} else {
output.push(`\n Run with --strict to treat HIGH+ findings as errors.`);
}
output.push('');
// Write GitHub Actions step summary
if (process.env.GITHUB_STEP_SUMMARY) {
let summary = '## Skill Validation\n\n';
if (totalFindings > 0) {
summary += '| Skill | Rule | Severity | File | Detail |\n';
summary += '|-------|------|----------|------|--------|\n';
for (const { skillDir, findings } of results) {
const relDir = path.relative(PROJECT_ROOT, skillDir);
for (const f of findings) {
summary += `| ${escapeTableCell(relDir)} | ${f.rule} | ${f.severity} | ${escapeTableCell(f.file)} | ${escapeTableCell(f.detail)} |\n`;
}
}
summary += '\n';
}
summary += `**${totalSkills} skills scanned, ${totalFindings} findings**\n`;
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary);
}
return { output: output.join('\n'), hasHighPlus };
}
function formatJson(results) {
const allFindings = [];
for (const { skillDir, findings } of results) {
const relDir = path.relative(PROJECT_ROOT, skillDir);
for (const f of findings) {
allFindings.push({
skill: relDir,
rule: f.rule,
title: f.title,
severity: f.severity,
file: f.file,
line: f.line || null,
detail: f.detail,
fix: f.fix,
});
}
}
// Sort by severity
allFindings.sort((a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]);
const hasHighPlus = allFindings.some((f) => f.severity === 'CRITICAL' || f.severity === 'HIGH');
return { output: JSON.stringify(allFindings, null, 2), hasHighPlus };
}
// --- Main ---
if (require.main === module) {
// Determine which skills to validate
let skillDirs;
if (positionalArgs.length > 0) {
// Single skill directory specified
const target = path.resolve(positionalArgs[0]);
if (!fs.existsSync(target) || !fs.statSync(target).isDirectory()) {
console.error(`Error: "${positionalArgs[0]}" is not a valid directory.`);
process.exit(2);
}
skillDirs = [target];
} else {
// Discover all skills
const allLocations = [...SKILL_LOCATIONS, AGENT_LOCATION];
skillDirs = discoverSkillDirs(allLocations);
}
if (skillDirs.length === 0) {
console.error('No skill directories found.');
process.exit(2);
}
// Validate each skill
const results = [];
for (const skillDir of skillDirs) {
const findings = validateSkill(skillDir);
results.push({ skillDir, findings });
}
// Format output
const { output, hasHighPlus } = JSON_OUTPUT ? formatJson(results) : formatHumanReadable(results);
console.log(output);
// Exit code
if (STRICT && hasHighPlus) {
process.exit(1);
}
}
// --- Exports (for testing) ---
module.exports = { parseFrontmatter, parseFrontmatterMultiline, validateSkill, discoverSkillDirs };