test: add RED-phase test fixtures and runner for workflow schema validator

27 test fixtures (11 valid + 16 invalid) and a test runner that
exercises the forthcoming Zod schema for workflow.yaml files.
Includes a stub schema that fails all validation, confirming
26 failing / 1 passing (yaml-parse-error) — TDD red state.

Follows the validate-agent-schema pattern (tools/schema/agent.js,
test/test-agent-schema.js).

Ref: MSSCI-12749
This commit is contained in:
Michael Pursifull 2026-02-04 13:11:44 -06:00
parent df176d4206
commit 0b83185a64
No known key found for this signature in database
29 changed files with 674 additions and 0 deletions

View File

@ -0,0 +1,16 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: execution_hints.interactive
# Error expected: boolean
# Error received: string
# Tests: AC19 — execution_hints with non-boolean sub-fields produces error
name: bad-execution-hints-test
description: "A workflow with wrong type in execution_hints"
author: "Test"
standalone: true
web_bundle: false
execution_hints:
interactive: "yes"
autonomous: true
iterative: false

View File

@ -0,0 +1,11 @@
# Expected: FAIL
# Error code: custom
# Error path: instructions
# Error message: Instructions file must end with .md or .xml
# Tests: AC22 — instructions ending in .txt produces error
name: bad-instructions-ext-test
description: "A workflow with invalid instructions file extension"
author: "Test"
standalone: true
web_bundle: false
instructions: "{installed_path}/instructions.txt"

View File

@ -0,0 +1,15 @@
# Expected: FAIL
# Error code: invalid_enum_value
# Error path: input_file_patterns.epics.load_strategy
# Tests: AC17 — invalid load_strategy enum produces error
name: bad-load-strategy-test
description: "A workflow with invalid load_strategy"
author: "Test"
standalone: true
web_bundle: false
input_file_patterns:
epics:
description: "All epics"
whole: "{output_folder}/*epic*.md"
load_strategy: "INVALID_STRATEGY"

View File

@ -0,0 +1,12 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: tags
# Error expected: array
# Error received: string
# Tests: AC20 — tags with non-array value produces error
name: bad-tags-type-test
description: "A workflow with wrong type for tags"
author: "Test"
standalone: true
web_bundle: false
tags: "qa,automation,testing"

View File

@ -0,0 +1,10 @@
# Expected: FAIL
# Error code: invalid_union
# Error path: template
# Tests: AC13 — template with number type produces error
name: bad-template-type-test
description: "A workflow with wrong type for template"
author: "Test"
standalone: true
web_bundle: false
template: 42

View File

@ -0,0 +1,9 @@
# Expected: FAIL
# Error code: too_small
# Error path: name
# Tests: AC10 — empty string name produces error
name: ""
description: "A workflow with an empty name"
author: "Test"
standalone: true
web_bundle: false

View File

@ -0,0 +1,8 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: author
# Tests: AC3 — missing author produces error
name: missing-author-test
description: "A workflow missing the author field"
standalone: true
web_bundle: false

View File

@ -0,0 +1,8 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: description
# Tests: AC2 — missing description produces error
name: missing-description-test
author: "Test"
standalone: true
web_bundle: false

View File

@ -0,0 +1,14 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: input_file_patterns.epics.load_strategy
# Tests: AC18 — input_file_patterns entry missing load_strategy produces error
name: missing-load-strategy-test
description: "A workflow with input_file_patterns missing load_strategy"
author: "Test"
standalone: true
web_bundle: false
input_file_patterns:
epics:
description: "All epics"
whole: "{output_folder}/*epic*.md"

View File

@ -0,0 +1,8 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: name
# Tests: AC1 — missing name produces error
description: "A workflow missing the name field"
author: "Test"
standalone: true
web_bundle: false

View File

@ -0,0 +1,8 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: standalone
# Tests: AC4 — missing standalone produces error
name: missing-standalone-test
description: "A workflow missing the standalone field"
author: "Test"
web_bundle: false

View File

@ -0,0 +1,8 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: web_bundle
# Tests: AC5 — missing web_bundle produces error
name: missing-web-bundle-test
description: "A workflow missing the web_bundle field"
author: "Test"
standalone: true

View File

@ -0,0 +1,11 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: name
# Error expected: string
# Error received: number
# Tests: AC7 — name with non-string value produces wrong_type error
name: 42
description: "A workflow with wrong type for name"
author: "Test"
standalone: true
web_bundle: false

View File

@ -0,0 +1,11 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: standalone
# Error expected: boolean
# Error received: string
# Tests: AC8 — standalone with non-boolean value produces wrong_type error
name: wrong-type-standalone-test
description: "A workflow with wrong type for standalone"
author: "Test"
standalone: "yes"
web_bundle: false

View File

@ -0,0 +1,11 @@
# Expected: FAIL
# Error code: invalid_type
# Error path: web_bundle
# Error expected: boolean
# Error received: string
# Tests: AC9 — web_bundle with non-boolean value produces wrong_type error
name: wrong-type-web-bundle-test
description: "A workflow with wrong type for web_bundle"
author: "Test"
standalone: true
web_bundle: "true"

View File

@ -0,0 +1,6 @@
# Expected: FAIL
# Tests: YAML parse error — malformed YAML
name: yaml-parse-error-test
description: "This file has bad YAML
indentation: [that is: broken
and: missing brackets

View File

@ -0,0 +1,23 @@
# Expected: PASS
# Tests: Excalidraw-style workflow with helpers, shared_path, library, json_validation
name: excalidraw-style-test
description: "Create diagrams in Excalidraw format"
author: "BMad"
config_source: "{project-root}/_bmad/bmm/config.yaml"
output_folder: "{config_source}:output_folder"
installed_path: "{project-root}/_bmad/bmm/workflows/excalidraw-diagrams/create-diagram"
shared_path: "{project-root}/_bmad/bmm/workflows/excalidraw-diagrams/_shared"
instructions: "{installed_path}/instructions.md"
validation: "{installed_path}/checklist.md"
helpers: "{project-root}/_bmad/core/resources/excalidraw/excalidraw-helpers.md"
json_validation: "{project-root}/_bmad/core/resources/excalidraw/validate-json-instructions.md"
templates: "{shared_path}/excalidraw-templates.yaml"
library: "{shared_path}/excalidraw-library.json"
default_output_file: "{output_folder}/excalidraw-diagrams/diagram-{timestamp}.excalidraw"
standalone: true
web_bundle: false

View File

@ -0,0 +1,25 @@
# Expected: PASS
# Tests: All common fields populated — mirrors a real implementation workflow
name: full-config-test
description: "A workflow with all common fields populated"
author: "BMad"
config_source: "{project-root}/_bmad/bmm/config.yaml"
output_folder: "{config_source}:output_folder"
user_name: "{config_source}:user_name"
communication_language: "{config_source}:communication_language"
document_output_language: "{config_source}:document_output_language"
user_skill_level: "{config_source}:user_skill_level"
date: system-generated
installed_path: "{project-root}/_bmad/bmm/workflows/test"
instructions: "{installed_path}/instructions.md"
validation: "{installed_path}/checklist.md"
template: "{installed_path}/template.md"
planning_artifacts: "{config_source}:planning_artifacts"
implementation_artifacts: "{config_source}:implementation_artifacts"
default_output_file: "{output_folder}/test-output.md"
standalone: true
web_bundle: false

View File

@ -0,0 +1,23 @@
# Expected: PASS
# Tests: Implementation-style workflow with sprint_status, story_dir, project_context
name: implementation-style-test
description: "Execute a story by implementing tasks"
author: "BMad"
config_source: "{project-root}/_bmad/bmm/config.yaml"
output_folder: "{config_source}:output_folder"
user_name: "{config_source}:user_name"
communication_language: "{config_source}:communication_language"
date: system-generated
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/dev-story"
instructions: "{installed_path}/instructions.xml"
validation: "{installed_path}/checklist.md"
story_dir: "{config_source}:implementation_artifacts"
sprint_status: "{implementation_artifacts}/sprint-status.yaml"
project_context: "**/project-context.md"
implementation_artifacts: "{config_source}:implementation_artifacts"
standalone: true
web_bundle: false

View File

@ -0,0 +1,7 @@
# Expected: PASS
# Tests: AC6 — workflow.yaml with only the 5 required fields passes
name: minimal-test
description: "A minimal workflow with only required fields"
author: "Test"
standalone: true
web_bundle: false

View File

@ -0,0 +1,8 @@
# Expected: PASS
# Tests: AC11 — template: false (boolean variant) passes
name: template-false-test
description: "A workflow with template set to false"
author: "Test"
standalone: true
web_bundle: false
template: false

View File

@ -0,0 +1,8 @@
# Expected: PASS
# Tests: AC12 — template as string path passes
name: template-path-test
description: "A workflow with template set to a path"
author: "Test"
standalone: true
web_bundle: false
template: "{installed_path}/template.md"

View File

@ -0,0 +1,12 @@
# Expected: PASS
# Tests: AC19 (positive) — execution_hints with valid boolean sub-fields
name: execution-hints-test
description: "A workflow with execution hints"
author: "Test"
standalone: true
web_bundle: false
execution_hints:
interactive: false
autonomous: true
iterative: false

View File

@ -0,0 +1,24 @@
# Expected: PASS
# Tests: AC14, AC15, AC16 — input_file_patterns with all three load strategies
name: input-patterns-test
description: "A workflow with input_file_patterns using all load strategies"
author: "Test"
standalone: true
web_bundle: false
input_file_patterns:
epics:
description: "All epics with user stories"
whole: "{output_folder}/*epic*.md"
sharded: "{output_folder}/*epic*/*.md"
load_strategy: "FULL_LOAD"
architecture:
description: "Architecture document"
whole: "{planning_artifacts}/architecture.md"
sharded: "{planning_artifacts}/architecture/*.md"
sharded_index: "{planning_artifacts}/architecture/index.md"
load_strategy: "SELECTIVE_LOAD"
retrospective:
description: "Previous retrospective"
pattern: "{implementation_artifacts}/retrospective*.md"
load_strategy: "INDEX_GUIDED"

View File

@ -0,0 +1,13 @@
# Expected: PASS
# Tests: AC21 (positive) — required_tools as array of strings
name: required-tools-test
description: "A workflow with required tools"
author: "Test"
standalone: true
web_bundle: false
required_tools:
- read_file
- write_file
- create_directory
- list_files

View File

@ -0,0 +1,12 @@
# Expected: PASS
# Tests: AC20 (positive) — tags as array of strings
name: tags-test
description: "A workflow with tags"
author: "Test"
standalone: true
web_bundle: false
tags:
- qa
- automation
- testing

View File

@ -0,0 +1,14 @@
# Expected: PASS
# Tests: AC23 — variables with arbitrary string key-value pairs
name: variables-test
description: "A workflow with variables section"
author: "Test"
standalone: true
web_bundle: false
variables:
project_context: "**/project-context.md"
test_dir: "{project-root}/tests"
source_dir: "{project-root}"
tracking_system: "file-system"
project_name: "{config_source}:project_name"

View File

@ -0,0 +1,309 @@
/**
* Workflow Schema Validation Test Runner
*
* Runs all test fixtures and verifies expected outcomes.
* Reports pass/fail for each test and overall coverage statistics.
*
* Usage: node test/test-workflow-schema.js
* Exit codes: 0 = all tests pass, 1 = test failures
*/
const fs = require('node:fs');
const path = require('node:path');
const yaml = require('yaml');
const { validateWorkflowFile } = require('../tools/schema/workflow.js');
const { glob } = require('glob');
// ANSI color codes
const colors = {
reset: '\u001B[0m',
green: '\u001B[32m',
red: '\u001B[31m',
yellow: '\u001B[33m',
blue: '\u001B[34m',
cyan: '\u001B[36m',
dim: '\u001B[2m',
};
/**
* Parse test metadata from YAML comments
* @param {string} filePath
* @returns {{shouldPass: boolean, errorExpectation?: object}}
*/
function parseTestMetadata(filePath) {
const content = fs.readFileSync(filePath, 'utf8');
const lines = content.split('\n');
let shouldPass = true;
const errorExpectation = {};
for (const line of lines) {
if (line.includes('Expected: PASS')) {
shouldPass = true;
} else if (line.includes('Expected: FAIL')) {
shouldPass = false;
}
const codeMatch = line.match(/^# Error code: (.+)$/);
if (codeMatch) {
errorExpectation.code = codeMatch[1].trim();
}
const pathMatch = line.match(/^# Error path: (.+)$/);
if (pathMatch) {
errorExpectation.path = pathMatch[1].trim();
}
const messageMatch = line.match(/^# Error message: (.+)$/);
if (messageMatch) {
errorExpectation.message = messageMatch[1].trim();
}
const expectedMatch = line.match(/^# Error expected: (.+)$/);
if (expectedMatch) {
errorExpectation.expected = expectedMatch[1].trim();
}
const receivedMatch = line.match(/^# Error received: (.+)$/);
if (receivedMatch) {
errorExpectation.received = receivedMatch[1].trim();
}
}
return {
shouldPass,
errorExpectation: Object.keys(errorExpectation).length > 0 ? errorExpectation : null,
};
}
/**
* Convert dot-notation path string to array (handles array indices)
* e.g., "input_file_patterns.epics.load_strategy" => ["input_file_patterns", "epics", "load_strategy"]
*/
function parsePathString(pathString) {
return pathString
.replaceAll(/\[(\d+)\]/g, '.$1')
.split('.')
.map((part) => {
const num = parseInt(part, 10);
return isNaN(num) ? part : num;
});
}
/**
* Validate error against expectations
* @param {object} error - Zod error issue
* @param {object} expectation - Expected error structure
* @returns {{valid: boolean, reason?: string}}
*/
function validateError(error, expectation) {
if (expectation.code && error.code !== expectation.code) {
return { valid: false, reason: `Expected code "${expectation.code}", got "${error.code}"` };
}
if (expectation.path) {
const expectedPath = parsePathString(expectation.path);
const actualPath = error.path;
if (JSON.stringify(expectedPath) !== JSON.stringify(actualPath)) {
return {
valid: false,
reason: `Expected path ${JSON.stringify(expectedPath)}, got ${JSON.stringify(actualPath)}`,
};
}
}
if (expectation.code === 'custom' && expectation.message && error.message !== expectation.message) {
return {
valid: false,
reason: `Expected message "${expectation.message}", got "${error.message}"`,
};
}
if (expectation.expected && error.expected !== expectation.expected) {
return { valid: false, reason: `Expected type "${expectation.expected}", got "${error.expected}"` };
}
if (expectation.received && error.received !== expectation.received) {
return { valid: false, reason: `Expected received "${expectation.received}", got "${error.received}"` };
}
return { valid: true };
}
/**
* Run a single test case
* @param {string} filePath
* @returns {{passed: boolean, message: string}}
*/
function runTest(filePath) {
const metadata = parseTestMetadata(filePath);
const { shouldPass, errorExpectation } = metadata;
try {
const fileContent = fs.readFileSync(filePath, 'utf8');
let workflowData;
try {
workflowData = yaml.parse(fileContent);
} catch (parseError) {
if (shouldPass) {
return {
passed: false,
message: `Expected PASS but got YAML parse error: ${parseError.message}`,
};
}
return {
passed: true,
message: 'Got expected YAML parse error',
};
}
const result = validateWorkflowFile(filePath, workflowData);
if (result.success && shouldPass) {
return {
passed: true,
message: 'Validation passed as expected',
};
}
if (!result.success && !shouldPass) {
const actualError = result.error.issues[0];
if (errorExpectation) {
const validation = validateError(actualError, errorExpectation);
if (!validation.valid) {
return {
passed: false,
message: `Error validation failed: ${validation.reason}`,
};
}
return {
passed: true,
message: `Got expected error (${errorExpectation.code}): ${actualError.message}`,
};
}
return {
passed: true,
message: `Got expected validation error: ${actualError?.message}`,
};
}
if (result.success && !shouldPass) {
return {
passed: false,
message: 'Expected validation to FAIL but it PASSED',
};
}
if (!result.success && shouldPass) {
return {
passed: false,
message: `Expected validation to PASS but it FAILED: ${result.error.issues[0]?.message}`,
};
}
return {
passed: false,
message: 'Unexpected test state',
};
} catch (error) {
return {
passed: false,
message: `Test execution error: ${error.message}`,
};
}
}
/**
* Main test runner
*/
async function main() {
console.log(`${colors.cyan}╔═══════════════════════════════════════════════════════════╗${colors.reset}`);
console.log(`${colors.cyan}║ Workflow Schema Validation Test Suite ║${colors.reset}`);
console.log(`${colors.cyan}╚═══════════════════════════════════════════════════════════╝${colors.reset}\n`);
const testFiles = await glob('test/fixtures/workflow-schema/**/*.workflow.yaml', {
cwd: path.join(__dirname, '..'),
absolute: true,
});
if (testFiles.length === 0) {
console.log(`${colors.yellow}⚠️ No test fixtures found${colors.reset}`);
process.exit(0);
}
console.log(`Found ${colors.cyan}${testFiles.length}${colors.reset} test fixture(s)\n`);
// Group tests by category
const categories = {};
for (const testFile of testFiles) {
const relativePath = path.relative(path.join(__dirname, 'fixtures/workflow-schema'), testFile);
const parts = relativePath.split(path.sep);
const validInvalid = parts[0];
const categoryKey = validInvalid;
if (!categories[categoryKey]) {
categories[categoryKey] = [];
}
categories[categoryKey].push(testFile);
}
// Run tests by category
let totalTests = 0;
let passedTests = 0;
const failures = [];
for (const [categoryKey, files] of Object.entries(categories).sort()) {
const validLabel = categoryKey === 'valid' ? '✅' : '❌';
console.log(`${colors.blue}${validLabel} ${categoryKey.toUpperCase()} FIXTURES${colors.reset}`);
for (const testFile of files.sort()) {
totalTests++;
const testName = path.basename(testFile, '.workflow.yaml');
const result = runTest(testFile);
if (result.passed) {
passedTests++;
console.log(` ${colors.green}${colors.reset} ${testName} ${colors.dim}${result.message}${colors.reset}`);
} else {
console.log(` ${colors.red}${colors.reset} ${testName} ${colors.red}${result.message}${colors.reset}`);
failures.push({
file: path.relative(process.cwd(), testFile),
message: result.message,
});
}
}
console.log('');
}
// Summary
console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}`);
console.log(`${colors.cyan}Test Results:${colors.reset}`);
console.log(` Total: ${totalTests}`);
console.log(` Passed: ${colors.green}${passedTests}${colors.reset}`);
console.log(` Failed: ${passedTests === totalTests ? colors.green : colors.red}${totalTests - passedTests}${colors.reset}`);
console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}\n`);
if (failures.length > 0) {
console.log(`${colors.red}❌ FAILED TESTS:${colors.reset}\n`);
for (const failure of failures) {
console.log(`${colors.red}${colors.reset} ${failure.file}`);
console.log(` ${failure.message}\n`);
}
process.exit(1);
}
console.log(`${colors.green}✨ All tests passed!${colors.reset}\n`);
process.exit(0);
}
main().catch((error) => {
console.error(`${colors.red}Fatal error:${colors.reset}`, error);
process.exit(1);
});

30
tools/schema/workflow.js Normal file
View File

@ -0,0 +1,30 @@
// Zod schema definition for workflow.yaml files
// STUB — implementation pending. All validations return failure.
// See: sprint/reference/mssci-12749-discovery.md for field inventory and validation rules.
/**
* Validate a workflow YAML payload against the schema.
* Exposed as the single public entry point, so callers do not reach into schema internals.
*
* @param {string} filePath Path to the workflow file.
* @param {unknown} workflowYaml Parsed YAML content.
* @returns {import('zod').SafeParseReturnType<unknown, unknown>} SafeParse result.
*/
function validateWorkflowFile(filePath, workflowYaml) {
// TODO: Implement Zod schema validation
// This stub returns failure so tests are in RED state.
return {
success: false,
error: {
issues: [
{
code: 'custom',
message: 'Schema not yet implemented',
path: [],
},
],
},
};
}
module.exports = { validateWorkflowFile };