/** * Workflow Schema Validation Test Runner * * Runs all test fixtures and verifies expected outcomes. * Reports pass/fail for each test and overall coverage statistics. * * Usage: node test/test-workflow-schema.js * Exit codes: 0 = all tests pass, 1 = test failures * * TODO: Extract shared test utilities (parseTestMetadata, validateError, parsePathString, colors) * into a common test helper module to reduce duplication with test-agent-schema.js */ const fs = require('node:fs'); const path = require('node:path'); const yaml = require('js-yaml'); const { validateWorkflowFile } = require('../tools/schema/workflow.js'); const { glob } = require('glob'); // ANSI color codes const colors = { reset: '\u001B[0m', green: '\u001B[32m', red: '\u001B[31m', yellow: '\u001B[33m', blue: '\u001B[34m', cyan: '\u001B[36m', dim: '\u001B[2m', }; /** * Parse test metadata from YAML comments * @param {string} filePath * @returns {{shouldPass: boolean, errorExpectation?: object}} */ function parseTestMetadata(filePath) { const content = fs.readFileSync(filePath, 'utf8'); const lines = content.split('\n'); let shouldPass = true; const errorExpectation = {}; for (const line of lines) { if (line.includes('Expected: PASS')) { shouldPass = true; } else if (line.includes('Expected: FAIL')) { shouldPass = false; } // Parse error metadata const codeMatch = line.match(/^# Error code: (.+)$/); if (codeMatch) { errorExpectation.code = codeMatch[1].trim(); } const pathMatch = line.match(/^# Error path: (.+)$/); if (pathMatch) { errorExpectation.path = pathMatch[1].trim(); } const messageMatch = line.match(/^# Error message: (.+)$/); if (messageMatch) { errorExpectation.message = messageMatch[1].trim(); } } return { shouldPass, errorExpectation: Object.keys(errorExpectation).length > 0 ? errorExpectation : null, }; } /** * Convert dot-notation path string to array * e.g., "name" => ["name"] */ function parsePathString(pathString) { return pathString .replaceAll(/\[(\d+)\]/g, '.$1') .split('.') .map((part) => { const num = parseInt(part, 10); return isNaN(num) ? part : num; }); } /** * Validate error against expectations * @param {object} error - Zod error issue * @param {object} expectation - Expected error structure * @returns {{valid: boolean, reason?: string}} */ function validateError(error, expectation) { if (expectation.code && error.code !== expectation.code) { return { valid: false, reason: `Expected code "${expectation.code}", got "${error.code}"` }; } if (expectation.path) { const expectedPath = parsePathString(expectation.path); const actualPath = error.path; if (JSON.stringify(expectedPath) !== JSON.stringify(actualPath)) { return { valid: false, reason: `Expected path ${JSON.stringify(expectedPath)}, got ${JSON.stringify(actualPath)}`, }; } } if (expectation.message && !error.message.includes(expectation.message)) { return { valid: false, reason: `Expected message to include "${expectation.message}", got "${error.message}"`, }; } return { valid: true }; } /** * Run a single test case * @param {string} filePath * @returns {{passed: boolean, message: string}} */ function runTest(filePath) { const metadata = parseTestMetadata(filePath); const { shouldPass, errorExpectation } = metadata; try { const fileContent = fs.readFileSync(filePath, 'utf8'); let workflowData; try { workflowData = yaml.load(fileContent); } catch (parseError) { if (shouldPass) { return { passed: false, message: `Expected PASS but got YAML parse error: ${parseError.message}`, }; } return { passed: true, message: 'Got expected YAML parse error', }; } const result = validateWorkflowFile(filePath, workflowData); if (result.success && shouldPass) { return { passed: true, message: 'Validation passed as expected', }; } if (!result.success && !shouldPass) { const actualError = result.error.issues[0]; if (errorExpectation) { const validation = validateError(actualError, errorExpectation); if (!validation.valid) { return { passed: false, message: `Error validation failed: ${validation.reason}`, }; } return { passed: true, message: `Got expected error (${errorExpectation.code}): ${actualError.message}`, }; } return { passed: true, message: `Got expected validation error: ${actualError?.message}`, }; } if (result.success && !shouldPass) { return { passed: false, message: 'Expected validation to FAIL but it PASSED', }; } if (!result.success && shouldPass) { return { passed: false, message: `Expected validation to PASS but it FAILED: ${result.error.issues[0]?.message}`, }; } return { passed: false, message: 'Unexpected test state', }; } catch (error) { return { passed: false, message: `Test execution error: ${error.message}`, }; } } /** * Main test runner */ async function main() { console.log(`${colors.cyan}╔═══════════════════════════════════════════════════════════╗${colors.reset}`); console.log(`${colors.cyan}║ Workflow Schema Validation Test Suite ║${colors.reset}`); console.log(`${colors.cyan}╚═══════════════════════════════════════════════════════════╝${colors.reset}\n`); // Find all test fixtures const testFiles = await glob('test/fixtures/workflow-schema/**/*.workflow.yaml', { cwd: path.join(__dirname, '..'), absolute: true, }); if (testFiles.length === 0) { console.log(`${colors.yellow}⚠️ No test fixtures found${colors.reset}`); process.exit(0); } console.log(`Found ${colors.cyan}${testFiles.length}${colors.reset} test fixture(s)\n`); // Group tests by category const categories = {}; for (const testFile of testFiles) { const relativePath = path.relative(path.join(__dirname, 'fixtures/workflow-schema'), testFile); const parts = relativePath.split(path.sep); const validInvalid = parts[0]; // 'valid' or 'invalid' const category = parts[1] || 'general'; // category folder or 'general' const categoryKey = `${validInvalid}/${category}`; if (!categories[categoryKey]) { categories[categoryKey] = []; } categories[categoryKey].push(testFile); } // Run tests by category let totalTests = 0; let passedTests = 0; const failures = []; for (const [categoryKey, files] of Object.entries(categories).sort()) { const [validInvalid, category] = categoryKey.split('/'); const categoryLabel = category.replaceAll('-', ' ').toUpperCase(); const validLabel = validInvalid === 'valid' ? '✅' : '❌'; console.log(`${colors.blue}${validLabel} ${categoryLabel} (${validInvalid})${colors.reset}`); for (const testFile of files) { totalTests++; const testName = path.basename(testFile, '.workflow.yaml'); const result = runTest(testFile); if (result.passed) { passedTests++; console.log(` ${colors.green}✓${colors.reset} ${testName} ${colors.dim}${result.message}${colors.reset}`); } else { console.log(` ${colors.red}✗${colors.reset} ${testName} ${colors.red}${result.message}${colors.reset}`); failures.push({ file: path.relative(process.cwd(), testFile), message: result.message, }); } } console.log(''); } // Summary console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}`); console.log(`${colors.cyan}Test Results:${colors.reset}`); console.log(` Total: ${totalTests}`); console.log(` Passed: ${colors.green}${passedTests}${colors.reset}`); console.log(` Failed: ${passedTests === totalTests ? colors.green : colors.red}${totalTests - passedTests}${colors.reset}`); console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}\n`); // Report failures if (failures.length > 0) { console.log(`${colors.red}❌ FAILED TESTS:${colors.reset}\n`); for (const failure of failures) { console.log(`${colors.red}✗${colors.reset} ${failure.file}`); console.log(` ${failure.message}\n`); } process.exit(1); } console.log(`${colors.green}✨ All tests passed!${colors.reset}\n`); process.exit(0); } // Run main().catch((error) => { console.error(`${colors.red}Fatal error:${colors.reset}`, error); process.exit(1); });