Merge branch 'main' into fix-elicitation-trigger

This commit is contained in:
Alex Verkhovsky 2026-03-18 17:20:52 -06:00 committed by GitHub
commit 9973b3c35a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 795 additions and 24 deletions

View File

@ -108,3 +108,6 @@ jobs:
- name: Validate file references - name: Validate file references
run: npm run validate:refs run: npm run validate:refs
- name: Validate skills
run: npm run validate:skills

View File

@ -9,3 +9,4 @@ Open source framework for structured, agent-assisted software delivery.
`quality` mirrors the checks in `.github/workflows/quality.yaml`. `quality` mirrors the checks in `.github/workflows/quality.yaml`.
- Skill validation rules are in `tools/skill-validator.md`. - Skill validation rules are in `tools/skill-validator.md`.
- Deterministic skill checks run via `npm run validate:skills` (included in `quality`).

View File

@ -39,12 +39,13 @@
"lint:fix": "eslint . --ext .js,.cjs,.mjs,.yaml --fix", "lint:fix": "eslint . --ext .js,.cjs,.mjs,.yaml --fix",
"lint:md": "markdownlint-cli2 \"**/*.md\"", "lint:md": "markdownlint-cli2 \"**/*.md\"",
"prepare": "command -v husky >/dev/null 2>&1 && husky || exit 0", "prepare": "command -v husky >/dev/null 2>&1 && husky || exit 0",
"quality": "npm run format:check && npm run lint && npm run lint:md && npm run docs:build && npm run test:install && npm run validate:refs", "quality": "npm run format:check && npm run lint && npm run lint:md && npm run docs:build && npm run test:install && npm run validate:refs && npm run validate:skills",
"rebundle": "node tools/cli/bundlers/bundle-web.js rebundle", "rebundle": "node tools/cli/bundlers/bundle-web.js rebundle",
"test": "npm run test:refs && npm run test:install && npm run lint && npm run lint:md && npm run format:check", "test": "npm run test:refs && npm run test:install && npm run lint && npm run lint:md && npm run format:check",
"test:install": "node test/test-installation-components.js", "test:install": "node test/test-installation-components.js",
"test:refs": "node test/test-file-refs-csv.js", "test:refs": "node test/test-file-refs-csv.js",
"validate:refs": "node tools/validate-file-refs.js --strict" "validate:refs": "node tools/validate-file-refs.js --strict",
"validate:skills": "node tools/validate-skills.js --strict"
}, },
"lint-staged": { "lint-staged": {
"*.{js,cjs,mjs}": [ "*.{js,cjs,mjs}": [

View File

@ -2,14 +2,27 @@
An LLM-readable validation prompt for skills following the Agent Skills open standard. An LLM-readable validation prompt for skills following the Agent Skills open standard.
## First Pass — Deterministic Checks
Before running inference-based validation, run the deterministic validator:
```bash
node tools/validate-skills.js --json path/to/skill-dir
```
This checks 14 rules deterministically: SKILL-01, SKILL-02, SKILL-03, SKILL-04, SKILL-05, SKILL-06, SKILL-07, WF-01, WF-02, PATH-02, STEP-01, STEP-06, STEP-07, SEQ-02.
Review its JSON output. For any rule that produced **zero findings** in the first pass, **skip it** during inference-based validation below — it has already been verified. If a rule produced any findings, the inference validator should still review that rule (some rules like SKILL-04 and SKILL-06 have sub-checks that benefit from judgment). Focus your inference effort on the remaining rules that require judgment (PATH-01, PATH-03, PATH-04, PATH-05, WF-03, STEP-02, STEP-03, STEP-04, STEP-05, SEQ-01, REF-01, REF-02, REF-03).
## How to Use ## How to Use
1. You are given a **skill directory path** to validate. 1. You are given a **skill directory path** to validate.
2. Read every file in the skill directory recursively. 2. Run the deterministic first pass (see above) and note which rules passed.
3. Apply every rule in the catalog below to every applicable file. 3. Read every file in the skill directory recursively.
4. Produce a findings report using the report template at the end. 4. Apply every rule in the catalog below to every applicable file, **skipping rules that passed the deterministic first pass**.
5. Produce a findings report using the report template at the end, including any deterministic findings from the first pass.
If no findings are generated, the skill passes validation. If no findings are generated (from either pass), the skill passes validation.
--- ---
@ -55,9 +68,9 @@ If no findings are generated, the skill passes validation.
- **Severity:** HIGH - **Severity:** HIGH
- **Applies to:** `SKILL.md` - **Applies to:** `SKILL.md`
- **Rule:** The `name` value must use only lowercase letters, numbers, and hyphens. Max 64 characters. Must not contain "anthropic" or "claude". - **Rule:** The `name` value must start with `bmad-`, use only lowercase letters, numbers, and single hyphens between segments.
- **Detection:** Regex test: `^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$`. String search for forbidden substrings. - **Detection:** Regex test: `^bmad-[a-z0-9]+(-[a-z0-9]+)*$`.
- **Fix:** Rename to comply with the format. - **Fix:** Rename to comply with the format (e.g., `bmad-my-skill`).
### SKILL-05 — `name` Must Match Directory Name ### SKILL-05 — `name` Must Match Directory Name
@ -75,23 +88,33 @@ If no findings are generated, the skill passes validation.
- **Detection:** Check length. Look for trigger phrases like "Use when" or "Use if" — their absence suggests the description only says _what_ but not _when_. - **Detection:** Check length. Look for trigger phrases like "Use when" or "Use if" — their absence suggests the description only says _what_ but not _when_.
- **Fix:** Append a "Use when..." clause to the description. - **Fix:** Append a "Use when..." clause to the description.
### SKILL-07 — SKILL.md Must Have Body Content
- **Severity:** HIGH
- **Applies to:** `SKILL.md`
- **Rule:** SKILL.md must have non-empty markdown body content after the frontmatter. The body provides L2 instructions — a SKILL.md with only frontmatter is incomplete.
- **Detection:** Extract content after the closing `---` frontmatter delimiter and check it is non-empty after trimming whitespace.
- **Fix:** Add markdown body with skill instructions after the closing `---`.
--- ---
### WF-01 — workflow.md Must NOT Have `name` in Frontmatter ### WF-01 — Only SKILL.md May Have `name` in Frontmatter
- **Severity:** HIGH - **Severity:** HIGH
- **Applies to:** `workflow.md` (if it exists) - **Applies to:** all `.md` files except `SKILL.md`
- **Rule:** The `name` field belongs only in `SKILL.md`. If `workflow.md` has YAML frontmatter, it must not contain `name:`. - **Rule:** The `name` field belongs only in `SKILL.md`. No other markdown file in the skill directory may have `name:` in its frontmatter.
- **Detection:** Parse frontmatter and check for `name:` key. - **Detection:** Parse frontmatter of every non-SKILL.md markdown file and check for `name:` key.
- **Fix:** Remove the `name:` line from workflow.md frontmatter. - **Fix:** Remove the `name:` line from the file's frontmatter.
- **Exception:** `bmad-agent-tech-writer` — has sub-skill files with intentional `name` fields (to be revisited).
### WF-02 — workflow.md Must NOT Have `description` in Frontmatter ### WF-02 — Only SKILL.md May Have `description` in Frontmatter
- **Severity:** HIGH - **Severity:** HIGH
- **Applies to:** `workflow.md` (if it exists) - **Applies to:** all `.md` files except `SKILL.md`
- **Rule:** The `description` field belongs only in `SKILL.md`. If `workflow.md` has YAML frontmatter, it must not contain `description:`. - **Rule:** The `description` field belongs only in `SKILL.md`. No other markdown file in the skill directory may have `description:` in its frontmatter.
- **Detection:** Parse frontmatter and check for `description:` key. - **Detection:** Parse frontmatter of every non-SKILL.md markdown file and check for `description:` key.
- **Fix:** Remove the `description:` line from workflow.md frontmatter. - **Fix:** Remove the `description:` line from the file's frontmatter.
- **Exception:** `bmad-agent-tech-writer` — has sub-skill files with intentional `description` fields (to be revisited).
### WF-03 — workflow.md Frontmatter Variables Must Be Config or Runtime Only ### WF-03 — workflow.md Frontmatter Variables Must Be Config or Runtime Only
@ -103,6 +126,7 @@ If no findings are generated, the skill passes validation.
- A legitimate external path expression (must not violate PATH-05 — no paths into another skill's directory) - A legitimate external path expression (must not violate PATH-05 — no paths into another skill's directory)
It must NOT be a path to a file within the skill directory (see PATH-04), nor a path into another skill's directory (see PATH-05). It must NOT be a path to a file within the skill directory (see PATH-04), nor a path into another skill's directory (see PATH-05).
- **Detection:** For each frontmatter variable, check if its value resolves to a file inside the skill (e.g., starts with `./`, `{installed_path}`, or is a bare relative path to a sibling file). If so, it is an intra-skill path variable. Also check if the value is a path into another skill's directory — if so, it violates PATH-05 and is not a legitimate external path. - **Detection:** For each frontmatter variable, check if its value resolves to a file inside the skill (e.g., starts with `./`, `{installed_path}`, or is a bare relative path to a sibling file). If so, it is an intra-skill path variable. Also check if the value is a path into another skill's directory — if so, it violates PATH-05 and is not a legitimate external path.
- **Fix:** Remove the variable. Use a hardcoded relative path inline where the file is referenced. - **Fix:** Remove the variable. Use a hardcoded relative path inline where the file is referenced.
@ -294,11 +318,11 @@ When reporting findings, use this format:
## Summary ## Summary
| Severity | Count | | Severity | Count |
|----------|-------| | -------- | ----- |
| CRITICAL | N | | CRITICAL | N |
| HIGH | N | | HIGH | N |
| MEDIUM | N | | MEDIUM | N |
| LOW | N | | LOW | N |
## Findings ## Findings
@ -329,28 +353,34 @@ Quick-reference for the Agent Skills open standard.
For the full standard, see: [Agent Skills specification](https://agentskills.io/specification) For the full standard, see: [Agent Skills specification](https://agentskills.io/specification)
### Structure ### Structure
- Every skill is a directory with `SKILL.md` as the required entrypoint - Every skill is a directory with `SKILL.md` as the required entrypoint
- YAML frontmatter between `---` markers provides metadata; markdown body provides instructions - YAML frontmatter between `---` markers provides metadata; markdown body provides instructions
- Supporting files (scripts, templates, references) live alongside SKILL.md - Supporting files (scripts, templates, references) live alongside SKILL.md
### Path resolution ### Path resolution
- Relative file references resolve from the directory of the file that contains the reference, not from the skill root - Relative file references resolve from the directory of the file that contains the reference, not from the skill root
- Example: from `branch-a/deep/next.md`, `./deeper/final.md` resolves to `branch-a/deep/deeper/final.md` - Example: from `branch-a/deep/next.md`, `./deeper/final.md` resolves to `branch-a/deep/deeper/final.md`
- Example: from `branch-a/deep/next.md`, `./branch-b/alt/leaf.md` incorrectly resolves to `branch-a/deep/branch-b/alt/leaf.md` - Example: from `branch-a/deep/next.md`, `./branch-b/alt/leaf.md` incorrectly resolves to `branch-a/deep/branch-b/alt/leaf.md`
### Frontmatter fields (standard) ### Frontmatter fields (standard)
- `name`: lowercase letters, numbers, hyphens only; max 64 chars; no "anthropic" or "claude" - `name`: lowercase letters, numbers, hyphens only; max 64 chars; no "anthropic" or "claude"
- `description`: required, max 1024 chars; should state what the skill does AND when to use it - `description`: required, max 1024 chars; should state what the skill does AND when to use it
### Progressive disclosure — three loading levels ### Progressive disclosure — three loading levels
- **L1 Metadata** (~100 tokens): `name` + `description` loaded at startup into system prompt - **L1 Metadata** (~100 tokens): `name` + `description` loaded at startup into system prompt
- **L2 Instructions** (<5k tokens): SKILL.md body loaded only when skill is triggered - **L2 Instructions** (<5k tokens): SKILL.md body loaded only when skill is triggered
- **L3 Resources** (unlimited): additional files + scripts loaded/executed on demand; script output enters context, script code does not - **L3 Resources** (unlimited): additional files + scripts loaded/executed on demand; script output enters context, script code does not
### Key design principle ### Key design principle
- Skills are filesystem-based directories, not API payloads — Claude reads them via bash/file tools - Skills are filesystem-based directories, not API payloads — Claude reads them via bash/file tools
- Keep SKILL.md focused; offload detailed reference to separate files - Keep SKILL.md focused; offload detailed reference to separate files
### Practical tips ### Practical tips
- Keep SKILL.md under 500 lines - Keep SKILL.md under 500 lines
- `description` drives auto-discovery — use keywords users would naturally say - `description` drives auto-discovery — use keywords users would naturally say

736
tools/validate-skills.js Normal file
View File

@ -0,0 +1,736 @@
/**
* Deterministic Skill Validator
*
* Validates 14 deterministic rules across all skill directories.
* Acts as a fast first-pass complement to the inference-based skill validator.
*
* What it checks:
* - SKILL-01: SKILL.md exists
* - SKILL-02: SKILL.md frontmatter has name
* - SKILL-03: SKILL.md frontmatter has description
* - SKILL-04: name format (lowercase, hyphens, no forbidden substrings)
* - SKILL-05: name matches directory basename
* - SKILL-06: description quality (length, "Use when"/"Use if")
* - SKILL-07: SKILL.md has body content after frontmatter
* - WF-01: workflow.md frontmatter has no name
* - WF-02: workflow.md frontmatter has no description
* - PATH-02: no installed_path variable
* - STEP-01: step filename format
* - STEP-06: step frontmatter has no name/description
* - STEP-07: step count 2-10
* - SEQ-02: no time estimates
*
* Usage:
* node tools/validate-skills.js # All skills, human-readable
* node tools/validate-skills.js path/to/skill-dir # Single skill
* node tools/validate-skills.js --strict # Exit 1 on HIGH+ findings
* node tools/validate-skills.js --json # JSON output
*/
const fs = require('node:fs');
const path = require('node:path');
const PROJECT_ROOT = path.resolve(__dirname, '..');
const SRC_DIR = path.join(PROJECT_ROOT, 'src');
// --- CLI Parsing ---
const args = process.argv.slice(2);
const STRICT = args.includes('--strict');
const JSON_OUTPUT = args.includes('--json');
const positionalArgs = args.filter((a) => !a.startsWith('--'));
// --- Constants ---
const NAME_REGEX = /^bmad-[a-z0-9]+(-[a-z0-9]+)*$/;
const STEP_FILENAME_REGEX = /^step-\d{2}[a-z]?-[a-z0-9-]+\.md$/;
const TIME_ESTIMATE_PATTERNS = [/takes?\s+\d+\s*min/i, /~\s*\d+\s*min/i, /estimated\s+time/i, /\bETA\b/];
const SEVERITY_ORDER = { CRITICAL: 0, HIGH: 1, MEDIUM: 2, LOW: 3 };
// --- Output Escaping ---
function escapeAnnotation(str) {
return str.replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A');
}
function escapeTableCell(str) {
return String(str).replaceAll('|', String.raw`\|`);
}
// --- Frontmatter Parsing ---
/**
* Parse YAML frontmatter from a markdown file.
* Returns an object with key-value pairs, or null if no frontmatter.
*/
function parseFrontmatter(content) {
const trimmed = content.trimStart();
if (!trimmed.startsWith('---')) return null;
let endIndex = trimmed.indexOf('\n---\n', 3);
if (endIndex === -1) {
// Handle file ending with \n---
if (trimmed.endsWith('\n---')) {
endIndex = trimmed.length - 4;
} else {
return null;
}
}
const fmBlock = trimmed.slice(3, endIndex).trim();
if (fmBlock === '') return {};
const result = {};
for (const line of fmBlock.split('\n')) {
const colonIndex = line.indexOf(':');
if (colonIndex === -1) continue;
// Skip indented lines (nested YAML values)
if (line[0] === ' ' || line[0] === '\t') continue;
const key = line.slice(0, colonIndex).trim();
let value = line.slice(colonIndex + 1).trim();
// Strip surrounding quotes (single or double)
if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith('"') && value.endsWith('"'))) {
value = value.slice(1, -1);
}
result[key] = value;
}
return result;
}
/**
* Parse YAML frontmatter, handling multiline values (description often spans lines).
* Returns an object with key-value pairs, or null if no frontmatter.
*/
function parseFrontmatterMultiline(content) {
const trimmed = content.trimStart();
if (!trimmed.startsWith('---')) return null;
let endIndex = trimmed.indexOf('\n---\n', 3);
if (endIndex === -1) {
// Handle file ending with \n---
if (trimmed.endsWith('\n---')) {
endIndex = trimmed.length - 4;
} else {
return null;
}
}
const fmBlock = trimmed.slice(3, endIndex).trim();
if (fmBlock === '') return {};
const result = {};
let currentKey = null;
let currentValue = '';
for (const line of fmBlock.split('\n')) {
const colonIndex = line.indexOf(':');
// New key-value pair: must start at column 0 (no leading whitespace) and have a colon
if (colonIndex > 0 && line[0] !== ' ' && line[0] !== '\t') {
// Save previous key
if (currentKey !== null) {
result[currentKey] = stripQuotes(currentValue.trim());
}
currentKey = line.slice(0, colonIndex).trim();
currentValue = line.slice(colonIndex + 1);
} else if (currentKey !== null) {
// Skip YAML comment lines
if (line.trimStart().startsWith('#')) continue;
// Continuation of multiline value
currentValue += '\n' + line;
}
}
// Save last key
if (currentKey !== null) {
result[currentKey] = stripQuotes(currentValue.trim());
}
return result;
}
function stripQuotes(value) {
if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith('"') && value.endsWith('"'))) {
return value.slice(1, -1);
}
return value;
}
// --- Safe File Reading ---
/**
* Read a file safely, returning null on error.
* Pushes a warning finding if the file cannot be read.
*/
function safeReadFile(filePath, findings, relFile) {
try {
return fs.readFileSync(filePath, 'utf-8');
} catch (error) {
findings.push({
rule: 'READ-ERR',
title: 'File Read Error',
severity: 'MEDIUM',
file: relFile || path.basename(filePath),
detail: `Cannot read file: ${error.message}`,
fix: 'Check file permissions and ensure the file exists.',
});
return null;
}
}
// --- Code Block Stripping ---
function stripCodeBlocks(content) {
return content.replaceAll(/```[\s\S]*?```/g, (m) => m.replaceAll(/[^\n]/g, ''));
}
// --- Skill Discovery ---
function discoverSkillDirs(rootDirs) {
const skillDirs = [];
function walk(dir) {
if (!fs.existsSync(dir)) return;
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
if (entry.name === 'node_modules' || entry.name === '.git') continue;
const fullPath = path.join(dir, entry.name);
const skillMd = path.join(fullPath, 'SKILL.md');
if (fs.existsSync(skillMd)) {
skillDirs.push(fullPath);
}
// Keep walking into subdirectories to find nested skills
walk(fullPath);
}
}
for (const rootDir of rootDirs) {
walk(rootDir);
}
return skillDirs.sort();
}
// --- File Collection ---
function collectSkillFiles(skillDir) {
const files = [];
function walk(dir) {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
if (entry.name === 'node_modules' || entry.name === '.git') continue;
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
walk(fullPath);
} else if (entry.isFile()) {
files.push(fullPath);
}
}
}
walk(skillDir);
return files;
}
// --- Rule Checks ---
function validateSkill(skillDir) {
const findings = [];
const dirName = path.basename(skillDir);
const skillMdPath = path.join(skillDir, 'SKILL.md');
const workflowMdPath = path.join(skillDir, 'workflow.md');
const stepsDir = path.join(skillDir, 'steps');
// Collect all files in the skill for PATH-02 and SEQ-02
const allFiles = collectSkillFiles(skillDir);
// --- SKILL-01: SKILL.md must exist ---
if (!fs.existsSync(skillMdPath)) {
findings.push({
rule: 'SKILL-01',
title: 'SKILL.md Must Exist',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'SKILL.md not found in skill directory.',
fix: 'Create SKILL.md as the skill entrypoint.',
});
// Cannot check SKILL-02 through SKILL-07 without SKILL.md
return findings;
}
const skillContent = safeReadFile(skillMdPath, findings, 'SKILL.md');
if (skillContent === null) return findings;
const skillFm = parseFrontmatterMultiline(skillContent);
// --- SKILL-02: frontmatter has name ---
if (!skillFm || !('name' in skillFm)) {
findings.push({
rule: 'SKILL-02',
title: 'SKILL.md Must Have name in Frontmatter',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'Frontmatter is missing the `name` field.',
fix: 'Add `name: <skill-name>` to the frontmatter.',
});
} else if (skillFm.name === '') {
findings.push({
rule: 'SKILL-02',
title: 'SKILL.md Must Have name in Frontmatter',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'Frontmatter `name` field is empty.',
fix: 'Set `name` to the skill directory name (kebab-case).',
});
}
// --- SKILL-03: frontmatter has description ---
if (!skillFm || !('description' in skillFm)) {
findings.push({
rule: 'SKILL-03',
title: 'SKILL.md Must Have description in Frontmatter',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'Frontmatter is missing the `description` field.',
fix: 'Add `description: <what it does and when to use it>` to the frontmatter.',
});
} else if (skillFm.description === '') {
findings.push({
rule: 'SKILL-03',
title: 'SKILL.md Must Have description in Frontmatter',
severity: 'CRITICAL',
file: 'SKILL.md',
detail: 'Frontmatter `description` field is empty.',
fix: 'Add a description stating what the skill does and when to use it.',
});
}
const name = skillFm && skillFm.name;
const description = skillFm && skillFm.description;
// --- SKILL-04: name format ---
if (name && !NAME_REGEX.test(name)) {
findings.push({
rule: 'SKILL-04',
title: 'name Format',
severity: 'HIGH',
file: 'SKILL.md',
detail: `name "${name}" does not match pattern: ${NAME_REGEX}`,
fix: 'Rename to comply with lowercase letters, numbers, and hyphens only (max 64 chars).',
});
}
// --- SKILL-05: name matches directory ---
if (name && name !== dirName) {
findings.push({
rule: 'SKILL-05',
title: 'name Must Match Directory Name',
severity: 'HIGH',
file: 'SKILL.md',
detail: `name "${name}" does not match directory name "${dirName}".`,
fix: `Change name to "${dirName}" or rename the directory.`,
});
}
// --- SKILL-06: description quality ---
if (description) {
if (description.length > 1024) {
findings.push({
rule: 'SKILL-06',
title: 'description Quality',
severity: 'MEDIUM',
file: 'SKILL.md',
detail: `description is ${description.length} characters (max 1024).`,
fix: 'Shorten the description to 1024 characters or less.',
});
}
if (!/use\s+when\b/i.test(description) && !/use\s+if\b/i.test(description)) {
findings.push({
rule: 'SKILL-06',
title: 'description Quality',
severity: 'MEDIUM',
file: 'SKILL.md',
detail: 'description does not contain "Use when" or "Use if" trigger phrase.',
fix: 'Append a "Use when..." clause to explain when to invoke this skill.',
});
}
}
// --- SKILL-07: SKILL.md must have body content after frontmatter ---
{
const trimmed = skillContent.trimStart();
let bodyStart = -1;
if (trimmed.startsWith('---')) {
let endIdx = trimmed.indexOf('\n---\n', 3);
if (endIdx !== -1) {
bodyStart = endIdx + 4;
} else if (trimmed.endsWith('\n---')) {
bodyStart = trimmed.length; // no body at all
}
} else {
bodyStart = 0; // no frontmatter, entire file is body
}
const body = bodyStart >= 0 ? trimmed.slice(bodyStart).trim() : '';
if (body === '') {
findings.push({
rule: 'SKILL-07',
title: 'SKILL.md Must Have Body Content',
severity: 'HIGH',
file: 'SKILL.md',
detail: 'SKILL.md has no content after frontmatter. L2 instructions are required.',
fix: 'Add markdown body with skill instructions after the closing ---.',
});
}
}
// --- WF-01 / WF-02: non-SKILL.md files must NOT have name/description ---
// TODO: bmad-agent-tech-writer has sub-skill files with intentional name/description
const WF_SKIP_SKILLS = new Set(['bmad-agent-tech-writer']);
for (const filePath of allFiles) {
if (path.extname(filePath) !== '.md') continue;
if (path.basename(filePath) === 'SKILL.md') continue;
if (WF_SKIP_SKILLS.has(dirName)) continue;
const relFile = path.relative(skillDir, filePath);
const content = safeReadFile(filePath, findings, relFile);
if (content === null) continue;
const fm = parseFrontmatter(content);
if (!fm) continue;
if ('name' in fm) {
findings.push({
rule: 'WF-01',
title: 'Only SKILL.md May Have name in Frontmatter',
severity: 'HIGH',
file: relFile,
detail: `${relFile} frontmatter contains \`name\` — this belongs only in SKILL.md.`,
fix: "Remove the `name:` line from this file's frontmatter.",
});
}
if ('description' in fm) {
findings.push({
rule: 'WF-02',
title: 'Only SKILL.md May Have description in Frontmatter',
severity: 'HIGH',
file: relFile,
detail: `${relFile} frontmatter contains \`description\` — this belongs only in SKILL.md.`,
fix: "Remove the `description:` line from this file's frontmatter.",
});
}
}
// --- PATH-02: no installed_path ---
for (const filePath of allFiles) {
// Only check markdown and yaml files
const ext = path.extname(filePath);
if (!['.md', '.yaml', '.yml'].includes(ext)) continue;
const relFile = path.relative(skillDir, filePath);
const content = safeReadFile(filePath, findings, relFile);
if (content === null) continue;
// Check frontmatter for installed_path key
const fm = parseFrontmatter(content);
if (fm && 'installed_path' in fm) {
findings.push({
rule: 'PATH-02',
title: 'No installed_path Variable',
severity: 'HIGH',
file: relFile,
detail: 'Frontmatter contains `installed_path:` key.',
fix: 'Remove `installed_path` from frontmatter. Use relative paths instead.',
});
}
// Check content for any mention of installed_path (variable ref, prose, bare text)
const stripped = stripCodeBlocks(content);
const lines = stripped.split('\n');
for (const [i, line] of lines.entries()) {
if (/installed_path/i.test(line)) {
findings.push({
rule: 'PATH-02',
title: 'No installed_path Variable',
severity: 'HIGH',
file: relFile,
line: i + 1,
detail: '`installed_path` reference found in content.',
fix: 'Remove all installed_path usage. Use relative paths (`./path` or `../path`) instead.',
});
}
}
}
// --- STEP-01: step filename format ---
// --- STEP-06: step frontmatter no name/description ---
// --- STEP-07: step count ---
// Only check the literal steps/ directory (variant directories like steps-c, steps-v
// use different naming conventions and are excluded per the rule specification)
if (fs.existsSync(stepsDir) && fs.statSync(stepsDir).isDirectory()) {
const stepDirName = 'steps';
const stepFiles = fs.readdirSync(stepsDir).filter((f) => f.endsWith('.md'));
// STEP-01: filename format
for (const stepFile of stepFiles) {
if (!STEP_FILENAME_REGEX.test(stepFile)) {
findings.push({
rule: 'STEP-01',
title: 'Step File Naming',
severity: 'MEDIUM',
file: path.join(stepDirName, stepFile),
detail: `Filename "${stepFile}" does not match pattern: ${STEP_FILENAME_REGEX}`,
fix: 'Rename to step-NN-description.md (NN = zero-padded number, optional letter suffix).',
});
}
}
// STEP-06: step frontmatter has no name/description
for (const stepFile of stepFiles) {
const stepPath = path.join(stepsDir, stepFile);
const stepContent = safeReadFile(stepPath, findings, path.join(stepDirName, stepFile));
if (stepContent === null) continue;
const stepFm = parseFrontmatter(stepContent);
if (stepFm) {
if ('name' in stepFm) {
findings.push({
rule: 'STEP-06',
title: 'Step File Frontmatter: No name or description',
severity: 'MEDIUM',
file: path.join(stepDirName, stepFile),
detail: 'Step file frontmatter contains `name:` — this is metadata noise.',
fix: 'Remove `name:` from step file frontmatter.',
});
}
if ('description' in stepFm) {
findings.push({
rule: 'STEP-06',
title: 'Step File Frontmatter: No name or description',
severity: 'MEDIUM',
file: path.join(stepDirName, stepFile),
detail: 'Step file frontmatter contains `description:` — this is metadata noise.',
fix: 'Remove `description:` from step file frontmatter.',
});
}
}
}
// STEP-07: step count 2-10
const stepCount = stepFiles.filter((f) => f.startsWith('step-')).length;
if (stepCount > 0 && (stepCount < 2 || stepCount > 10)) {
const detail =
stepCount < 2
? `Only ${stepCount} step file found — consider inlining into workflow.md.`
: `${stepCount} step files found — more than 10 risks LLM context degradation.`;
findings.push({
rule: 'STEP-07',
title: 'Step Count',
severity: 'LOW',
file: stepDirName + '/',
detail,
fix: stepCount > 10 ? 'Consider consolidating steps.' : 'Consider expanding or inlining.',
});
}
}
// --- SEQ-02: no time estimates ---
for (const filePath of allFiles) {
const ext = path.extname(filePath);
if (!['.md', '.yaml', '.yml'].includes(ext)) continue;
const relFile = path.relative(skillDir, filePath);
const content = safeReadFile(filePath, findings, relFile);
if (content === null) continue;
const stripped = stripCodeBlocks(content);
const lines = stripped.split('\n');
for (const [i, line] of lines.entries()) {
for (const pattern of TIME_ESTIMATE_PATTERNS) {
if (pattern.test(line)) {
findings.push({
rule: 'SEQ-02',
title: 'No Time Estimates',
severity: 'LOW',
file: relFile,
line: i + 1,
detail: `Time estimate pattern found: "${line.trim()}"`,
fix: 'Remove time estimates — AI execution speed varies too much.',
});
break; // Only report once per line
}
}
}
}
return findings;
}
// --- Output Formatting ---
function formatHumanReadable(results) {
const output = [];
let totalFindings = 0;
const severityCounts = { CRITICAL: 0, HIGH: 0, MEDIUM: 0, LOW: 0 };
output.push(
`\nValidating skills in: ${SRC_DIR}`,
`Mode: ${STRICT ? 'STRICT (exit 1 on HIGH+)' : 'WARNING (exit 0)'}${JSON_OUTPUT ? ' + JSON' : ''}\n`,
);
let totalSkills = 0;
let skillsWithFindings = 0;
for (const { skillDir, findings } of results) {
totalSkills++;
const relDir = path.relative(PROJECT_ROOT, skillDir);
if (findings.length > 0) {
skillsWithFindings++;
output.push(`\n${relDir}`);
for (const f of findings) {
totalFindings++;
severityCounts[f.severity]++;
const location = f.line ? ` (line ${f.line})` : '';
output.push(` [${f.severity}] ${f.rule}${f.title}`, ` File: ${f.file}${location}`, ` ${f.detail}`);
if (process.env.GITHUB_ACTIONS) {
const absFile = path.join(skillDir, f.file);
const ghFile = path.relative(PROJECT_ROOT, absFile);
const line = f.line || 1;
const level = f.severity === 'LOW' ? 'notice' : 'warning';
console.log(`::${level} file=${ghFile},line=${line}::${escapeAnnotation(`${f.rule}: ${f.detail}`)}`);
}
}
}
}
// Summary
output.push(
`\n${'─'.repeat(60)}`,
`\nSummary:`,
` Skills scanned: ${totalSkills}`,
` Skills with findings: ${skillsWithFindings}`,
` Total findings: ${totalFindings}`,
);
if (totalFindings > 0) {
output.push('', ` | Severity | Count |`, ` |----------|-------|`);
for (const sev of ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']) {
if (severityCounts[sev] > 0) {
output.push(` | ${sev.padEnd(8)} | ${String(severityCounts[sev]).padStart(5)} |`);
}
}
}
const hasHighPlus = severityCounts.CRITICAL > 0 || severityCounts.HIGH > 0;
if (totalFindings === 0) {
output.push(`\n All skills passed validation!`);
} else if (STRICT && hasHighPlus) {
output.push(`\n [STRICT MODE] HIGH+ findings found — exiting with failure.`);
} else if (STRICT) {
output.push(`\n [STRICT MODE] Only MEDIUM/LOW findings — pass.`);
} else {
output.push(`\n Run with --strict to treat HIGH+ findings as errors.`);
}
output.push('');
// Write GitHub Actions step summary
if (process.env.GITHUB_STEP_SUMMARY) {
let summary = '## Skill Validation\n\n';
if (totalFindings > 0) {
summary += '| Skill | Rule | Severity | File | Detail |\n';
summary += '|-------|------|----------|------|--------|\n';
for (const { skillDir, findings } of results) {
const relDir = path.relative(PROJECT_ROOT, skillDir);
for (const f of findings) {
summary += `| ${escapeTableCell(relDir)} | ${f.rule} | ${f.severity} | ${escapeTableCell(f.file)} | ${escapeTableCell(f.detail)} |\n`;
}
}
summary += '\n';
}
summary += `**${totalSkills} skills scanned, ${totalFindings} findings**\n`;
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary);
}
return { output: output.join('\n'), hasHighPlus };
}
function formatJson(results) {
const allFindings = [];
for (const { skillDir, findings } of results) {
const relDir = path.relative(PROJECT_ROOT, skillDir);
for (const f of findings) {
allFindings.push({
skill: relDir,
rule: f.rule,
title: f.title,
severity: f.severity,
file: f.file,
line: f.line || null,
detail: f.detail,
fix: f.fix,
});
}
}
// Sort by severity
allFindings.sort((a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]);
const hasHighPlus = allFindings.some((f) => f.severity === 'CRITICAL' || f.severity === 'HIGH');
return { output: JSON.stringify(allFindings, null, 2), hasHighPlus };
}
// --- Main ---
if (require.main === module) {
// Determine which skills to validate
let skillDirs;
if (positionalArgs.length > 0) {
// Single skill directory specified
const target = path.resolve(positionalArgs[0]);
if (!fs.existsSync(target) || !fs.statSync(target).isDirectory()) {
console.error(`Error: "${positionalArgs[0]}" is not a valid directory.`);
process.exit(2);
}
skillDirs = [target];
} else {
// Discover all skills
skillDirs = discoverSkillDirs([SRC_DIR]);
}
if (skillDirs.length === 0) {
console.error('No skill directories found.');
process.exit(2);
}
// Validate each skill
const results = [];
for (const skillDir of skillDirs) {
const findings = validateSkill(skillDir);
results.push({ skillDir, findings });
}
// Format output
const { output, hasHighPlus } = JSON_OUTPUT ? formatJson(results) : formatHumanReadable(results);
console.log(output);
// Exit code
if (STRICT && hasHighPlus) {
process.exit(1);
}
}
// --- Exports (for testing) ---
module.exports = { parseFrontmatter, parseFrontmatterMultiline, validateSkill, discoverSkillDirs };