test: add RED-phase test fixtures and runner for workflow schema validator
27 test fixtures (11 valid + 16 invalid) and a test runner that exercises the forthcoming Zod schema for workflow.yaml files. Includes a stub schema that fails all validation, confirming 26 failing / 1 passing (yaml-parse-error) — TDD red state. Follows the validate-agent-schema pattern (tools/schema/agent.js, test/test-agent-schema.js). Ref: MSSCI-12749
This commit is contained in:
parent
df176d4206
commit
0b83185a64
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: execution_hints.interactive
|
||||||
|
# Error expected: boolean
|
||||||
|
# Error received: string
|
||||||
|
# Tests: AC19 — execution_hints with non-boolean sub-fields produces error
|
||||||
|
name: bad-execution-hints-test
|
||||||
|
description: "A workflow with wrong type in execution_hints"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
||||||
|
execution_hints:
|
||||||
|
interactive: "yes"
|
||||||
|
autonomous: true
|
||||||
|
iterative: false
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: custom
|
||||||
|
# Error path: instructions
|
||||||
|
# Error message: Instructions file must end with .md or .xml
|
||||||
|
# Tests: AC22 — instructions ending in .txt produces error
|
||||||
|
name: bad-instructions-ext-test
|
||||||
|
description: "A workflow with invalid instructions file extension"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
instructions: "{installed_path}/instructions.txt"
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_enum_value
|
||||||
|
# Error path: input_file_patterns.epics.load_strategy
|
||||||
|
# Tests: AC17 — invalid load_strategy enum produces error
|
||||||
|
name: bad-load-strategy-test
|
||||||
|
description: "A workflow with invalid load_strategy"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
||||||
|
input_file_patterns:
|
||||||
|
epics:
|
||||||
|
description: "All epics"
|
||||||
|
whole: "{output_folder}/*epic*.md"
|
||||||
|
load_strategy: "INVALID_STRATEGY"
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: tags
|
||||||
|
# Error expected: array
|
||||||
|
# Error received: string
|
||||||
|
# Tests: AC20 — tags with non-array value produces error
|
||||||
|
name: bad-tags-type-test
|
||||||
|
description: "A workflow with wrong type for tags"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
tags: "qa,automation,testing"
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_union
|
||||||
|
# Error path: template
|
||||||
|
# Tests: AC13 — template with number type produces error
|
||||||
|
name: bad-template-type-test
|
||||||
|
description: "A workflow with wrong type for template"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
template: 42
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: too_small
|
||||||
|
# Error path: name
|
||||||
|
# Tests: AC10 — empty string name produces error
|
||||||
|
name: ""
|
||||||
|
description: "A workflow with an empty name"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: author
|
||||||
|
# Tests: AC3 — missing author produces error
|
||||||
|
name: missing-author-test
|
||||||
|
description: "A workflow missing the author field"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: description
|
||||||
|
# Tests: AC2 — missing description produces error
|
||||||
|
name: missing-description-test
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: input_file_patterns.epics.load_strategy
|
||||||
|
# Tests: AC18 — input_file_patterns entry missing load_strategy produces error
|
||||||
|
name: missing-load-strategy-test
|
||||||
|
description: "A workflow with input_file_patterns missing load_strategy"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
||||||
|
input_file_patterns:
|
||||||
|
epics:
|
||||||
|
description: "All epics"
|
||||||
|
whole: "{output_folder}/*epic*.md"
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: name
|
||||||
|
# Tests: AC1 — missing name produces error
|
||||||
|
description: "A workflow missing the name field"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: standalone
|
||||||
|
# Tests: AC4 — missing standalone produces error
|
||||||
|
name: missing-standalone-test
|
||||||
|
description: "A workflow missing the standalone field"
|
||||||
|
author: "Test"
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: web_bundle
|
||||||
|
# Tests: AC5 — missing web_bundle produces error
|
||||||
|
name: missing-web-bundle-test
|
||||||
|
description: "A workflow missing the web_bundle field"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: name
|
||||||
|
# Error expected: string
|
||||||
|
# Error received: number
|
||||||
|
# Tests: AC7 — name with non-string value produces wrong_type error
|
||||||
|
name: 42
|
||||||
|
description: "A workflow with wrong type for name"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: standalone
|
||||||
|
# Error expected: boolean
|
||||||
|
# Error received: string
|
||||||
|
# Tests: AC8 — standalone with non-boolean value produces wrong_type error
|
||||||
|
name: wrong-type-standalone-test
|
||||||
|
description: "A workflow with wrong type for standalone"
|
||||||
|
author: "Test"
|
||||||
|
standalone: "yes"
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Error code: invalid_type
|
||||||
|
# Error path: web_bundle
|
||||||
|
# Error expected: boolean
|
||||||
|
# Error received: string
|
||||||
|
# Tests: AC9 — web_bundle with non-boolean value produces wrong_type error
|
||||||
|
name: wrong-type-web-bundle-test
|
||||||
|
description: "A workflow with wrong type for web_bundle"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: "true"
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
# Expected: FAIL
|
||||||
|
# Tests: YAML parse error — malformed YAML
|
||||||
|
name: yaml-parse-error-test
|
||||||
|
description: "This file has bad YAML
|
||||||
|
indentation: [that is: broken
|
||||||
|
and: missing brackets
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: Excalidraw-style workflow with helpers, shared_path, library, json_validation
|
||||||
|
name: excalidraw-style-test
|
||||||
|
description: "Create diagrams in Excalidraw format"
|
||||||
|
author: "BMad"
|
||||||
|
|
||||||
|
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
||||||
|
output_folder: "{config_source}:output_folder"
|
||||||
|
|
||||||
|
installed_path: "{project-root}/_bmad/bmm/workflows/excalidraw-diagrams/create-diagram"
|
||||||
|
shared_path: "{project-root}/_bmad/bmm/workflows/excalidraw-diagrams/_shared"
|
||||||
|
instructions: "{installed_path}/instructions.md"
|
||||||
|
validation: "{installed_path}/checklist.md"
|
||||||
|
|
||||||
|
helpers: "{project-root}/_bmad/core/resources/excalidraw/excalidraw-helpers.md"
|
||||||
|
json_validation: "{project-root}/_bmad/core/resources/excalidraw/validate-json-instructions.md"
|
||||||
|
templates: "{shared_path}/excalidraw-templates.yaml"
|
||||||
|
library: "{shared_path}/excalidraw-library.json"
|
||||||
|
|
||||||
|
default_output_file: "{output_folder}/excalidraw-diagrams/diagram-{timestamp}.excalidraw"
|
||||||
|
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: All common fields populated — mirrors a real implementation workflow
|
||||||
|
name: full-config-test
|
||||||
|
description: "A workflow with all common fields populated"
|
||||||
|
author: "BMad"
|
||||||
|
|
||||||
|
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
||||||
|
output_folder: "{config_source}:output_folder"
|
||||||
|
user_name: "{config_source}:user_name"
|
||||||
|
communication_language: "{config_source}:communication_language"
|
||||||
|
document_output_language: "{config_source}:document_output_language"
|
||||||
|
user_skill_level: "{config_source}:user_skill_level"
|
||||||
|
date: system-generated
|
||||||
|
|
||||||
|
installed_path: "{project-root}/_bmad/bmm/workflows/test"
|
||||||
|
instructions: "{installed_path}/instructions.md"
|
||||||
|
validation: "{installed_path}/checklist.md"
|
||||||
|
template: "{installed_path}/template.md"
|
||||||
|
|
||||||
|
planning_artifacts: "{config_source}:planning_artifacts"
|
||||||
|
implementation_artifacts: "{config_source}:implementation_artifacts"
|
||||||
|
default_output_file: "{output_folder}/test-output.md"
|
||||||
|
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: Implementation-style workflow with sprint_status, story_dir, project_context
|
||||||
|
name: implementation-style-test
|
||||||
|
description: "Execute a story by implementing tasks"
|
||||||
|
author: "BMad"
|
||||||
|
|
||||||
|
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
||||||
|
output_folder: "{config_source}:output_folder"
|
||||||
|
user_name: "{config_source}:user_name"
|
||||||
|
communication_language: "{config_source}:communication_language"
|
||||||
|
date: system-generated
|
||||||
|
|
||||||
|
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/dev-story"
|
||||||
|
instructions: "{installed_path}/instructions.xml"
|
||||||
|
validation: "{installed_path}/checklist.md"
|
||||||
|
|
||||||
|
story_dir: "{config_source}:implementation_artifacts"
|
||||||
|
sprint_status: "{implementation_artifacts}/sprint-status.yaml"
|
||||||
|
project_context: "**/project-context.md"
|
||||||
|
implementation_artifacts: "{config_source}:implementation_artifacts"
|
||||||
|
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: AC6 — workflow.yaml with only the 5 required fields passes
|
||||||
|
name: minimal-test
|
||||||
|
description: "A minimal workflow with only required fields"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: AC11 — template: false (boolean variant) passes
|
||||||
|
name: template-false-test
|
||||||
|
description: "A workflow with template set to false"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
template: false
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: AC12 — template as string path passes
|
||||||
|
name: template-path-test
|
||||||
|
description: "A workflow with template set to a path"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
template: "{installed_path}/template.md"
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: AC19 (positive) — execution_hints with valid boolean sub-fields
|
||||||
|
name: execution-hints-test
|
||||||
|
description: "A workflow with execution hints"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
||||||
|
execution_hints:
|
||||||
|
interactive: false
|
||||||
|
autonomous: true
|
||||||
|
iterative: false
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: AC14, AC15, AC16 — input_file_patterns with all three load strategies
|
||||||
|
name: input-patterns-test
|
||||||
|
description: "A workflow with input_file_patterns using all load strategies"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
||||||
|
input_file_patterns:
|
||||||
|
epics:
|
||||||
|
description: "All epics with user stories"
|
||||||
|
whole: "{output_folder}/*epic*.md"
|
||||||
|
sharded: "{output_folder}/*epic*/*.md"
|
||||||
|
load_strategy: "FULL_LOAD"
|
||||||
|
architecture:
|
||||||
|
description: "Architecture document"
|
||||||
|
whole: "{planning_artifacts}/architecture.md"
|
||||||
|
sharded: "{planning_artifacts}/architecture/*.md"
|
||||||
|
sharded_index: "{planning_artifacts}/architecture/index.md"
|
||||||
|
load_strategy: "SELECTIVE_LOAD"
|
||||||
|
retrospective:
|
||||||
|
description: "Previous retrospective"
|
||||||
|
pattern: "{implementation_artifacts}/retrospective*.md"
|
||||||
|
load_strategy: "INDEX_GUIDED"
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: AC21 (positive) — required_tools as array of strings
|
||||||
|
name: required-tools-test
|
||||||
|
description: "A workflow with required tools"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
||||||
|
required_tools:
|
||||||
|
- read_file
|
||||||
|
- write_file
|
||||||
|
- create_directory
|
||||||
|
- list_files
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: AC20 (positive) — tags as array of strings
|
||||||
|
name: tags-test
|
||||||
|
description: "A workflow with tags"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
||||||
|
tags:
|
||||||
|
- qa
|
||||||
|
- automation
|
||||||
|
- testing
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Expected: PASS
|
||||||
|
# Tests: AC23 — variables with arbitrary string key-value pairs
|
||||||
|
name: variables-test
|
||||||
|
description: "A workflow with variables section"
|
||||||
|
author: "Test"
|
||||||
|
standalone: true
|
||||||
|
web_bundle: false
|
||||||
|
|
||||||
|
variables:
|
||||||
|
project_context: "**/project-context.md"
|
||||||
|
test_dir: "{project-root}/tests"
|
||||||
|
source_dir: "{project-root}"
|
||||||
|
tracking_system: "file-system"
|
||||||
|
project_name: "{config_source}:project_name"
|
||||||
|
|
@ -0,0 +1,309 @@
|
||||||
|
/**
|
||||||
|
* Workflow Schema Validation Test Runner
|
||||||
|
*
|
||||||
|
* Runs all test fixtures and verifies expected outcomes.
|
||||||
|
* Reports pass/fail for each test and overall coverage statistics.
|
||||||
|
*
|
||||||
|
* Usage: node test/test-workflow-schema.js
|
||||||
|
* Exit codes: 0 = all tests pass, 1 = test failures
|
||||||
|
*/
|
||||||
|
|
||||||
|
const fs = require('node:fs');
|
||||||
|
const path = require('node:path');
|
||||||
|
const yaml = require('yaml');
|
||||||
|
const { validateWorkflowFile } = require('../tools/schema/workflow.js');
|
||||||
|
const { glob } = require('glob');
|
||||||
|
|
||||||
|
// ANSI color codes
|
||||||
|
const colors = {
|
||||||
|
reset: '\u001B[0m',
|
||||||
|
green: '\u001B[32m',
|
||||||
|
red: '\u001B[31m',
|
||||||
|
yellow: '\u001B[33m',
|
||||||
|
blue: '\u001B[34m',
|
||||||
|
cyan: '\u001B[36m',
|
||||||
|
dim: '\u001B[2m',
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse test metadata from YAML comments
|
||||||
|
* @param {string} filePath
|
||||||
|
* @returns {{shouldPass: boolean, errorExpectation?: object}}
|
||||||
|
*/
|
||||||
|
function parseTestMetadata(filePath) {
|
||||||
|
const content = fs.readFileSync(filePath, 'utf8');
|
||||||
|
const lines = content.split('\n');
|
||||||
|
|
||||||
|
let shouldPass = true;
|
||||||
|
const errorExpectation = {};
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
if (line.includes('Expected: PASS')) {
|
||||||
|
shouldPass = true;
|
||||||
|
} else if (line.includes('Expected: FAIL')) {
|
||||||
|
shouldPass = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const codeMatch = line.match(/^# Error code: (.+)$/);
|
||||||
|
if (codeMatch) {
|
||||||
|
errorExpectation.code = codeMatch[1].trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
const pathMatch = line.match(/^# Error path: (.+)$/);
|
||||||
|
if (pathMatch) {
|
||||||
|
errorExpectation.path = pathMatch[1].trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
const messageMatch = line.match(/^# Error message: (.+)$/);
|
||||||
|
if (messageMatch) {
|
||||||
|
errorExpectation.message = messageMatch[1].trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
const expectedMatch = line.match(/^# Error expected: (.+)$/);
|
||||||
|
if (expectedMatch) {
|
||||||
|
errorExpectation.expected = expectedMatch[1].trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
const receivedMatch = line.match(/^# Error received: (.+)$/);
|
||||||
|
if (receivedMatch) {
|
||||||
|
errorExpectation.received = receivedMatch[1].trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
shouldPass,
|
||||||
|
errorExpectation: Object.keys(errorExpectation).length > 0 ? errorExpectation : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert dot-notation path string to array (handles array indices)
|
||||||
|
* e.g., "input_file_patterns.epics.load_strategy" => ["input_file_patterns", "epics", "load_strategy"]
|
||||||
|
*/
|
||||||
|
function parsePathString(pathString) {
|
||||||
|
return pathString
|
||||||
|
.replaceAll(/\[(\d+)\]/g, '.$1')
|
||||||
|
.split('.')
|
||||||
|
.map((part) => {
|
||||||
|
const num = parseInt(part, 10);
|
||||||
|
return isNaN(num) ? part : num;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate error against expectations
|
||||||
|
* @param {object} error - Zod error issue
|
||||||
|
* @param {object} expectation - Expected error structure
|
||||||
|
* @returns {{valid: boolean, reason?: string}}
|
||||||
|
*/
|
||||||
|
function validateError(error, expectation) {
|
||||||
|
if (expectation.code && error.code !== expectation.code) {
|
||||||
|
return { valid: false, reason: `Expected code "${expectation.code}", got "${error.code}"` };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (expectation.path) {
|
||||||
|
const expectedPath = parsePathString(expectation.path);
|
||||||
|
const actualPath = error.path;
|
||||||
|
|
||||||
|
if (JSON.stringify(expectedPath) !== JSON.stringify(actualPath)) {
|
||||||
|
return {
|
||||||
|
valid: false,
|
||||||
|
reason: `Expected path ${JSON.stringify(expectedPath)}, got ${JSON.stringify(actualPath)}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (expectation.code === 'custom' && expectation.message && error.message !== expectation.message) {
|
||||||
|
return {
|
||||||
|
valid: false,
|
||||||
|
reason: `Expected message "${expectation.message}", got "${error.message}"`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (expectation.expected && error.expected !== expectation.expected) {
|
||||||
|
return { valid: false, reason: `Expected type "${expectation.expected}", got "${error.expected}"` };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (expectation.received && error.received !== expectation.received) {
|
||||||
|
return { valid: false, reason: `Expected received "${expectation.received}", got "${error.received}"` };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { valid: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a single test case
|
||||||
|
* @param {string} filePath
|
||||||
|
* @returns {{passed: boolean, message: string}}
|
||||||
|
*/
|
||||||
|
function runTest(filePath) {
|
||||||
|
const metadata = parseTestMetadata(filePath);
|
||||||
|
const { shouldPass, errorExpectation } = metadata;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const fileContent = fs.readFileSync(filePath, 'utf8');
|
||||||
|
let workflowData;
|
||||||
|
|
||||||
|
try {
|
||||||
|
workflowData = yaml.parse(fileContent);
|
||||||
|
} catch (parseError) {
|
||||||
|
if (shouldPass) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: `Expected PASS but got YAML parse error: ${parseError.message}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
passed: true,
|
||||||
|
message: 'Got expected YAML parse error',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = validateWorkflowFile(filePath, workflowData);
|
||||||
|
|
||||||
|
if (result.success && shouldPass) {
|
||||||
|
return {
|
||||||
|
passed: true,
|
||||||
|
message: 'Validation passed as expected',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!result.success && !shouldPass) {
|
||||||
|
const actualError = result.error.issues[0];
|
||||||
|
|
||||||
|
if (errorExpectation) {
|
||||||
|
const validation = validateError(actualError, errorExpectation);
|
||||||
|
|
||||||
|
if (!validation.valid) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: `Error validation failed: ${validation.reason}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
passed: true,
|
||||||
|
message: `Got expected error (${errorExpectation.code}): ${actualError.message}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
passed: true,
|
||||||
|
message: `Got expected validation error: ${actualError?.message}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.success && !shouldPass) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: 'Expected validation to FAIL but it PASSED',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!result.success && shouldPass) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: `Expected validation to PASS but it FAILED: ${result.error.issues[0]?.message}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: 'Unexpected test state',
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: `Test execution error: ${error.message}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Main test runner
|
||||||
|
*/
|
||||||
|
async function main() {
|
||||||
|
console.log(`${colors.cyan}╔═══════════════════════════════════════════════════════════╗${colors.reset}`);
|
||||||
|
console.log(`${colors.cyan}║ Workflow Schema Validation Test Suite ║${colors.reset}`);
|
||||||
|
console.log(`${colors.cyan}╚═══════════════════════════════════════════════════════════╝${colors.reset}\n`);
|
||||||
|
|
||||||
|
const testFiles = await glob('test/fixtures/workflow-schema/**/*.workflow.yaml', {
|
||||||
|
cwd: path.join(__dirname, '..'),
|
||||||
|
absolute: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (testFiles.length === 0) {
|
||||||
|
console.log(`${colors.yellow}⚠️ No test fixtures found${colors.reset}`);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Found ${colors.cyan}${testFiles.length}${colors.reset} test fixture(s)\n`);
|
||||||
|
|
||||||
|
// Group tests by category
|
||||||
|
const categories = {};
|
||||||
|
for (const testFile of testFiles) {
|
||||||
|
const relativePath = path.relative(path.join(__dirname, 'fixtures/workflow-schema'), testFile);
|
||||||
|
const parts = relativePath.split(path.sep);
|
||||||
|
const validInvalid = parts[0];
|
||||||
|
const categoryKey = validInvalid;
|
||||||
|
|
||||||
|
if (!categories[categoryKey]) {
|
||||||
|
categories[categoryKey] = [];
|
||||||
|
}
|
||||||
|
categories[categoryKey].push(testFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run tests by category
|
||||||
|
let totalTests = 0;
|
||||||
|
let passedTests = 0;
|
||||||
|
const failures = [];
|
||||||
|
|
||||||
|
for (const [categoryKey, files] of Object.entries(categories).sort()) {
|
||||||
|
const validLabel = categoryKey === 'valid' ? '✅' : '❌';
|
||||||
|
|
||||||
|
console.log(`${colors.blue}${validLabel} ${categoryKey.toUpperCase()} FIXTURES${colors.reset}`);
|
||||||
|
|
||||||
|
for (const testFile of files.sort()) {
|
||||||
|
totalTests++;
|
||||||
|
const testName = path.basename(testFile, '.workflow.yaml');
|
||||||
|
const result = runTest(testFile);
|
||||||
|
|
||||||
|
if (result.passed) {
|
||||||
|
passedTests++;
|
||||||
|
console.log(` ${colors.green}✓${colors.reset} ${testName} ${colors.dim}${result.message}${colors.reset}`);
|
||||||
|
} else {
|
||||||
|
console.log(` ${colors.red}✗${colors.reset} ${testName} ${colors.red}${result.message}${colors.reset}`);
|
||||||
|
failures.push({
|
||||||
|
file: path.relative(process.cwd(), testFile),
|
||||||
|
message: result.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log('');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summary
|
||||||
|
console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}`);
|
||||||
|
console.log(`${colors.cyan}Test Results:${colors.reset}`);
|
||||||
|
console.log(` Total: ${totalTests}`);
|
||||||
|
console.log(` Passed: ${colors.green}${passedTests}${colors.reset}`);
|
||||||
|
console.log(` Failed: ${passedTests === totalTests ? colors.green : colors.red}${totalTests - passedTests}${colors.reset}`);
|
||||||
|
console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}\n`);
|
||||||
|
|
||||||
|
if (failures.length > 0) {
|
||||||
|
console.log(`${colors.red}❌ FAILED TESTS:${colors.reset}\n`);
|
||||||
|
for (const failure of failures) {
|
||||||
|
console.log(`${colors.red}✗${colors.reset} ${failure.file}`);
|
||||||
|
console.log(` ${failure.message}\n`);
|
||||||
|
}
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`${colors.green}✨ All tests passed!${colors.reset}\n`);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((error) => {
|
||||||
|
console.error(`${colors.red}Fatal error:${colors.reset}`, error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
// Zod schema definition for workflow.yaml files
|
||||||
|
// STUB — implementation pending. All validations return failure.
|
||||||
|
// See: sprint/reference/mssci-12749-discovery.md for field inventory and validation rules.
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate a workflow YAML payload against the schema.
|
||||||
|
* Exposed as the single public entry point, so callers do not reach into schema internals.
|
||||||
|
*
|
||||||
|
* @param {string} filePath Path to the workflow file.
|
||||||
|
* @param {unknown} workflowYaml Parsed YAML content.
|
||||||
|
* @returns {import('zod').SafeParseReturnType<unknown, unknown>} SafeParse result.
|
||||||
|
*/
|
||||||
|
function validateWorkflowFile(filePath, workflowYaml) {
|
||||||
|
// TODO: Implement Zod schema validation
|
||||||
|
// This stub returns failure so tests are in RED state.
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: {
|
||||||
|
issues: [
|
||||||
|
{
|
||||||
|
code: 'custom',
|
||||||
|
message: 'Schema not yet implemented',
|
||||||
|
path: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { validateWorkflowFile };
|
||||||
Loading…
Reference in New Issue