diff --git a/test/test-agent-schema.js b/test/test-agent-schema.js new file mode 100644 index 00000000..fdfda12b --- /dev/null +++ b/test/test-agent-schema.js @@ -0,0 +1,403 @@ +/** + * Agent Schema Validation Test Runner + * + * Runs all test fixtures and verifies expected outcomes. + * Reports pass/fail for each test and overall coverage statistics. + * + * Usage: node test/test-agent-schema.js + * Exit codes: 0 = all tests pass, 1 = test failures + */ + +const fs = require('node:fs'); +const path = require('node:path'); +const yaml = require('js-yaml'); +const { validateAgentFile } = require('../tools/schema/agent.js'); +const { glob } = require('glob'); + +// ANSI color codes +const colors = { + reset: '\u001B[0m', + green: '\u001B[32m', + red: '\u001B[31m', + yellow: '\u001B[33m', + blue: '\u001B[34m', + cyan: '\u001B[36m', + dim: '\u001B[2m', +}; + +/** + * Normalize line endings in a file and split into lines. + * Handles Unix (\n), Windows (\r\n), and old Mac (\r) line endings. + * @param {string} content - File content as string + * @returns {string[]} Array of lines with normalized line endings + */ +function normalizeAndSplitLines(content) { + // First, normalize all line endings to \n + // Replace \r\n (Windows) with \n first to avoid double processing + let normalized = content.replaceAll('\r\n', '\n'); + // Then replace any remaining \r (old Mac) with \n + normalized = normalized.replaceAll('\r', '\n'); + // Split by \n and trim any remaining whitespace from line endings + return normalized.split('\n'); +} + +/** + * Parse test metadata from YAML comments + * @param {string} filePath + * @returns {{shouldPass: boolean, errorExpectation?: object, pathContext?: string}} + */ +function parseTestMetadata(filePath) { + const content = fs.readFileSync(filePath, 'utf8'); + const lines = normalizeAndSplitLines(content); + + let shouldPass = true; + let pathContext = null; + const errorExpectation = {}; + + for (const line of lines) { + if (line.includes('Expected: PASS')) { + shouldPass = true; + } else if (line.includes('Expected: FAIL')) { + shouldPass = false; + } + + // Parse error metadata + const codeMatch = line.match(/^# Error code: (.+)$/); + if (codeMatch) { + errorExpectation.code = codeMatch[1].trim(); + } + + const pathMatch = line.match(/^# Error path: (.+)$/); + if (pathMatch) { + errorExpectation.path = pathMatch[1].trim(); + } + + const messageMatch = line.match(/^# Error message: (.+)$/); + if (messageMatch) { + errorExpectation.message = messageMatch[1].trim(); + } + + const minimumMatch = line.match(/^# Error minimum: (\d+)$/); + if (minimumMatch) { + errorExpectation.minimum = parseInt(minimumMatch[1], 10); + } + + const expectedMatch = line.match(/^# Error expected: (.+)$/); + if (expectedMatch) { + errorExpectation.expected = expectedMatch[1].trim(); + } + + const receivedMatch = line.match(/^# Error received: (.+)$/); + if (receivedMatch) { + errorExpectation.received = receivedMatch[1].trim(); + } + + const keysMatch = line.match(/^# Error keys: \[(.+)\]$/); + if (keysMatch) { + errorExpectation.keys = keysMatch[1].split(',').map((k) => k.trim().replaceAll(/['"]/g, '')); + } + + const contextMatch = line.match(/^# Path context: (.+)$/); + if (contextMatch) { + pathContext = contextMatch[1].trim(); + } + } + + return { + shouldPass, + errorExpectation: Object.keys(errorExpectation).length > 0 ? errorExpectation : null, + pathContext, + }; +} + +/** + * Convert dot-notation path string to array (handles array indices) + * e.g., "agent.menu[0].trigger" => ["agent", "menu", 0, "trigger"] + */ +function parsePathString(pathString) { + return pathString + .replaceAll(/\[(\d+)\]/g, '.$1') // Convert [0] to .0 + .split('.') + .map((part) => { + const num = parseInt(part, 10); + return isNaN(num) ? part : num; + }); +} + +/** + * Validate error against expectations + * @param {object} error - Zod error issue + * @param {object} expectation - Expected error structure + * @returns {{valid: boolean, reason?: string}} + */ +function validateError(error, expectation) { + // Check error code + if (expectation.code && error.code !== expectation.code) { + return { valid: false, reason: `Expected code "${expectation.code}", got "${error.code}"` }; + } + + // Check error path + if (expectation.path) { + const expectedPath = parsePathString(expectation.path); + const actualPath = error.path; + + if (JSON.stringify(expectedPath) !== JSON.stringify(actualPath)) { + return { + valid: false, + reason: `Expected path ${JSON.stringify(expectedPath)}, got ${JSON.stringify(actualPath)}`, + }; + } + } + + // For custom errors, strictly check message + if (expectation.code === 'custom' && expectation.message && error.message !== expectation.message) { + return { + valid: false, + reason: `Expected message "${expectation.message}", got "${error.message}"`, + }; + } + + // For Zod errors, check type-specific fields + if (expectation.minimum !== undefined && error.minimum !== expectation.minimum) { + return { valid: false, reason: `Expected minimum ${expectation.minimum}, got ${error.minimum}` }; + } + + if (expectation.expected && error.expected !== expectation.expected) { + return { valid: false, reason: `Expected type "${expectation.expected}", got "${error.expected}"` }; + } + + if (expectation.received && error.received !== expectation.received) { + return { valid: false, reason: `Expected received "${expectation.received}", got "${error.received}"` }; + } + + if (expectation.keys) { + const expectedKeys = expectation.keys.sort(); + const actualKeys = (error.keys || []).sort(); + if (JSON.stringify(expectedKeys) !== JSON.stringify(actualKeys)) { + return { + valid: false, + reason: `Expected keys ${JSON.stringify(expectedKeys)}, got ${JSON.stringify(actualKeys)}`, + }; + } + } + + return { valid: true }; +} + +/** + * Run a single test case + * @param {string} filePath + * @returns {{passed: boolean, message: string}} + */ +function runTest(filePath) { + const metadata = parseTestMetadata(filePath); + const { shouldPass, errorExpectation, pathContext } = metadata; + + try { + const fileContent = fs.readFileSync(filePath, 'utf8'); + let agentData; + + try { + agentData = yaml.load(fileContent); + } catch (parseError) { + // YAML parse error + if (shouldPass) { + return { + passed: false, + message: `Expected PASS but got YAML parse error: ${parseError.message}`, + }; + } + return { + passed: true, + message: 'Got expected YAML parse error', + }; + } + + // Determine validation path + // If pathContext is specified in comments, use it; otherwise derive from fixture location + let validationPath = pathContext; + if (!validationPath) { + // Map fixture location to simulated src/ path + const relativePath = path.relative(path.join(__dirname, 'fixtures/agent-schema'), filePath); + const parts = relativePath.split(path.sep); + + if (parts.includes('metadata') && parts[0] === 'valid') { + // Valid metadata tests: check if filename suggests module or core + const filename = path.basename(filePath); + if (filename.includes('module')) { + validationPath = 'src/modules/bmm/agents/test.agent.yaml'; + } else { + validationPath = 'src/core/agents/test.agent.yaml'; + } + } else if (parts.includes('metadata') && parts[0] === 'invalid') { + // Invalid metadata tests: derive from filename + const filename = path.basename(filePath); + if (filename.includes('module') || filename.includes('wrong-module')) { + validationPath = 'src/modules/bmm/agents/test.agent.yaml'; + } else if (filename.includes('core')) { + validationPath = 'src/core/agents/test.agent.yaml'; + } else { + validationPath = 'src/core/agents/test.agent.yaml'; + } + } else { + // Default to core agent path + validationPath = 'src/core/agents/test.agent.yaml'; + } + } + + const result = validateAgentFile(validationPath, agentData); + + if (result.success && shouldPass) { + return { + passed: true, + message: 'Validation passed as expected', + }; + } + + if (!result.success && !shouldPass) { + const actualError = result.error.issues[0]; + + // If we have error expectations, validate strictly + if (errorExpectation) { + const validation = validateError(actualError, errorExpectation); + + if (!validation.valid) { + return { + passed: false, + message: `Error validation failed: ${validation.reason}`, + }; + } + + return { + passed: true, + message: `Got expected error (${errorExpectation.code}): ${actualError.message}`, + }; + } + + // No specific expectations - just check that it failed + return { + passed: true, + message: `Got expected validation error: ${actualError?.message}`, + }; + } + + if (result.success && !shouldPass) { + return { + passed: false, + message: 'Expected validation to FAIL but it PASSED', + }; + } + + if (!result.success && shouldPass) { + return { + passed: false, + message: `Expected validation to PASS but it FAILED: ${result.error.issues[0]?.message}`, + }; + } + + return { + passed: false, + message: 'Unexpected test state', + }; + } catch (error) { + return { + passed: false, + message: `Test execution error: ${error.message}`, + }; + } +} + +/** + * Main test runner + */ +async function main() { + console.log(`${colors.cyan}╔═══════════════════════════════════════════════════════════╗${colors.reset}`); + console.log(`${colors.cyan}║ Agent Schema Validation Test Suite ║${colors.reset}`); + console.log(`${colors.cyan}╚═══════════════════════════════════════════════════════════╝${colors.reset}\n`); + + // Find all test fixtures + const testFiles = await glob('test/fixtures/agent-schema/**/*.agent.yaml', { + cwd: path.join(__dirname, '..'), + absolute: true, + }); + + if (testFiles.length === 0) { + console.log(`${colors.yellow}⚠️ No test fixtures found${colors.reset}`); + process.exit(0); + } + + console.log(`Found ${colors.cyan}${testFiles.length}${colors.reset} test fixture(s)\n`); + + // Group tests by category + const categories = {}; + for (const testFile of testFiles) { + const relativePath = path.relative(path.join(__dirname, 'fixtures/agent-schema'), testFile); + const parts = relativePath.split(path.sep); + const validInvalid = parts[0]; // 'valid' or 'invalid' + const category = parts[1]; // 'top-level', 'metadata', etc. + + const categoryKey = `${validInvalid}/${category}`; + if (!categories[categoryKey]) { + categories[categoryKey] = []; + } + categories[categoryKey].push(testFile); + } + + // Run tests by category + let totalTests = 0; + let passedTests = 0; + const failures = []; + + for (const [categoryKey, files] of Object.entries(categories).sort()) { + const [validInvalid, category] = categoryKey.split('/'); + const categoryLabel = category.replaceAll('-', ' ').toUpperCase(); + const validLabel = validInvalid === 'valid' ? '✅' : '❌'; + + console.log(`${colors.blue}${validLabel} ${categoryLabel} (${validInvalid})${colors.reset}`); + + for (const testFile of files) { + totalTests++; + const testName = path.basename(testFile, '.agent.yaml'); + const result = runTest(testFile); + + if (result.passed) { + passedTests++; + console.log(` ${colors.green}✓${colors.reset} ${testName} ${colors.dim}${result.message}${colors.reset}`); + } else { + console.log(` ${colors.red}✗${colors.reset} ${testName} ${colors.red}${result.message}${colors.reset}`); + failures.push({ + file: path.relative(process.cwd(), testFile), + message: result.message, + }); + } + } + console.log(''); + } + + // Summary + console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}`); + console.log(`${colors.cyan}Test Results:${colors.reset}`); + console.log(` Total: ${totalTests}`); + console.log(` Passed: ${colors.green}${passedTests}${colors.reset}`); + console.log(` Failed: ${passedTests === totalTests ? colors.green : colors.red}${totalTests - passedTests}${colors.reset}`); + console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}\n`); + + // Report failures + if (failures.length > 0) { + console.log(`${colors.red}❌ FAILED TESTS:${colors.reset}\n`); + for (const failure of failures) { + console.log(`${colors.red}✗${colors.reset} ${failure.file}`); + console.log(` ${failure.message}\n`); + } + process.exit(1); + } + + console.log(`${colors.green}✨ All tests passed!${colors.reset}\n`); + process.exit(0); +} + +// Run +main().catch((error) => { + console.error(`${colors.red}Fatal error:${colors.reset}`, error); + process.exit(1); +});