410 lines
14 KiB
Plaintext
410 lines
14 KiB
Plaintext
diff --git a/test/test-agent-schema.js b/test/test-agent-schema.js
|
|
new file mode 100644
|
|
index 00000000..fdfda12b
|
|
--- /dev/null
|
|
+++ b/test/test-agent-schema.js
|
|
@@ -0,0 +1,403 @@
|
|
+/**
|
|
+ * Agent Schema Validation Test Runner
|
|
+ *
|
|
+ * Runs all test fixtures and verifies expected outcomes.
|
|
+ * Reports pass/fail for each test and overall coverage statistics.
|
|
+ *
|
|
+ * Usage: node test/test-agent-schema.js
|
|
+ * Exit codes: 0 = all tests pass, 1 = test failures
|
|
+ */
|
|
+
|
|
+const fs = require('node:fs');
|
|
+const path = require('node:path');
|
|
+const yaml = require('js-yaml');
|
|
+const { validateAgentFile } = require('../tools/schema/agent.js');
|
|
+const { glob } = require('glob');
|
|
+
|
|
+// ANSI color codes
|
|
+const colors = {
|
|
+ reset: '\u001B[0m',
|
|
+ green: '\u001B[32m',
|
|
+ red: '\u001B[31m',
|
|
+ yellow: '\u001B[33m',
|
|
+ blue: '\u001B[34m',
|
|
+ cyan: '\u001B[36m',
|
|
+ dim: '\u001B[2m',
|
|
+};
|
|
+
|
|
+/**
|
|
+ * Normalize line endings in a file and split into lines.
|
|
+ * Handles Unix (\n), Windows (\r\n), and old Mac (\r) line endings.
|
|
+ * @param {string} content - File content as string
|
|
+ * @returns {string[]} Array of lines with normalized line endings
|
|
+ */
|
|
+function normalizeAndSplitLines(content) {
|
|
+ // First, normalize all line endings to \n
|
|
+ // Replace \r\n (Windows) with \n first to avoid double processing
|
|
+ let normalized = content.replaceAll('\r\n', '\n');
|
|
+ // Then replace any remaining \r (old Mac) with \n
|
|
+ normalized = normalized.replaceAll('\r', '\n');
|
|
+ // Split by \n and trim any remaining whitespace from line endings
|
|
+ return normalized.split('\n');
|
|
+}
|
|
+
|
|
+/**
|
|
+ * Parse test metadata from YAML comments
|
|
+ * @param {string} filePath
|
|
+ * @returns {{shouldPass: boolean, errorExpectation?: object, pathContext?: string}}
|
|
+ */
|
|
+function parseTestMetadata(filePath) {
|
|
+ const content = fs.readFileSync(filePath, 'utf8');
|
|
+ const lines = normalizeAndSplitLines(content);
|
|
+
|
|
+ let shouldPass = true;
|
|
+ let pathContext = null;
|
|
+ const errorExpectation = {};
|
|
+
|
|
+ for (const line of lines) {
|
|
+ if (line.includes('Expected: PASS')) {
|
|
+ shouldPass = true;
|
|
+ } else if (line.includes('Expected: FAIL')) {
|
|
+ shouldPass = false;
|
|
+ }
|
|
+
|
|
+ // Parse error metadata
|
|
+ const codeMatch = line.match(/^# Error code: (.+)$/);
|
|
+ if (codeMatch) {
|
|
+ errorExpectation.code = codeMatch[1].trim();
|
|
+ }
|
|
+
|
|
+ const pathMatch = line.match(/^# Error path: (.+)$/);
|
|
+ if (pathMatch) {
|
|
+ errorExpectation.path = pathMatch[1].trim();
|
|
+ }
|
|
+
|
|
+ const messageMatch = line.match(/^# Error message: (.+)$/);
|
|
+ if (messageMatch) {
|
|
+ errorExpectation.message = messageMatch[1].trim();
|
|
+ }
|
|
+
|
|
+ const minimumMatch = line.match(/^# Error minimum: (\d+)$/);
|
|
+ if (minimumMatch) {
|
|
+ errorExpectation.minimum = parseInt(minimumMatch[1], 10);
|
|
+ }
|
|
+
|
|
+ const expectedMatch = line.match(/^# Error expected: (.+)$/);
|
|
+ if (expectedMatch) {
|
|
+ errorExpectation.expected = expectedMatch[1].trim();
|
|
+ }
|
|
+
|
|
+ const receivedMatch = line.match(/^# Error received: (.+)$/);
|
|
+ if (receivedMatch) {
|
|
+ errorExpectation.received = receivedMatch[1].trim();
|
|
+ }
|
|
+
|
|
+ const keysMatch = line.match(/^# Error keys: \[(.+)\]$/);
|
|
+ if (keysMatch) {
|
|
+ errorExpectation.keys = keysMatch[1].split(',').map((k) => k.trim().replaceAll(/['"]/g, ''));
|
|
+ }
|
|
+
|
|
+ const contextMatch = line.match(/^# Path context: (.+)$/);
|
|
+ if (contextMatch) {
|
|
+ pathContext = contextMatch[1].trim();
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return {
|
|
+ shouldPass,
|
|
+ errorExpectation: Object.keys(errorExpectation).length > 0 ? errorExpectation : null,
|
|
+ pathContext,
|
|
+ };
|
|
+}
|
|
+
|
|
+/**
|
|
+ * Convert dot-notation path string to array (handles array indices)
|
|
+ * e.g., "agent.menu[0].trigger" => ["agent", "menu", 0, "trigger"]
|
|
+ */
|
|
+function parsePathString(pathString) {
|
|
+ return pathString
|
|
+ .replaceAll(/\[(\d+)\]/g, '.$1') // Convert [0] to .0
|
|
+ .split('.')
|
|
+ .map((part) => {
|
|
+ const num = parseInt(part, 10);
|
|
+ return isNaN(num) ? part : num;
|
|
+ });
|
|
+}
|
|
+
|
|
+/**
|
|
+ * Validate error against expectations
|
|
+ * @param {object} error - Zod error issue
|
|
+ * @param {object} expectation - Expected error structure
|
|
+ * @returns {{valid: boolean, reason?: string}}
|
|
+ */
|
|
+function validateError(error, expectation) {
|
|
+ // Check error code
|
|
+ if (expectation.code && error.code !== expectation.code) {
|
|
+ return { valid: false, reason: `Expected code "${expectation.code}", got "${error.code}"` };
|
|
+ }
|
|
+
|
|
+ // Check error path
|
|
+ if (expectation.path) {
|
|
+ const expectedPath = parsePathString(expectation.path);
|
|
+ const actualPath = error.path;
|
|
+
|
|
+ if (JSON.stringify(expectedPath) !== JSON.stringify(actualPath)) {
|
|
+ return {
|
|
+ valid: false,
|
|
+ reason: `Expected path ${JSON.stringify(expectedPath)}, got ${JSON.stringify(actualPath)}`,
|
|
+ };
|
|
+ }
|
|
+ }
|
|
+
|
|
+ // For custom errors, strictly check message
|
|
+ if (expectation.code === 'custom' && expectation.message && error.message !== expectation.message) {
|
|
+ return {
|
|
+ valid: false,
|
|
+ reason: `Expected message "${expectation.message}", got "${error.message}"`,
|
|
+ };
|
|
+ }
|
|
+
|
|
+ // For Zod errors, check type-specific fields
|
|
+ if (expectation.minimum !== undefined && error.minimum !== expectation.minimum) {
|
|
+ return { valid: false, reason: `Expected minimum ${expectation.minimum}, got ${error.minimum}` };
|
|
+ }
|
|
+
|
|
+ if (expectation.expected && error.expected !== expectation.expected) {
|
|
+ return { valid: false, reason: `Expected type "${expectation.expected}", got "${error.expected}"` };
|
|
+ }
|
|
+
|
|
+ if (expectation.received && error.received !== expectation.received) {
|
|
+ return { valid: false, reason: `Expected received "${expectation.received}", got "${error.received}"` };
|
|
+ }
|
|
+
|
|
+ if (expectation.keys) {
|
|
+ const expectedKeys = expectation.keys.sort();
|
|
+ const actualKeys = (error.keys || []).sort();
|
|
+ if (JSON.stringify(expectedKeys) !== JSON.stringify(actualKeys)) {
|
|
+ return {
|
|
+ valid: false,
|
|
+ reason: `Expected keys ${JSON.stringify(expectedKeys)}, got ${JSON.stringify(actualKeys)}`,
|
|
+ };
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return { valid: true };
|
|
+}
|
|
+
|
|
+/**
|
|
+ * Run a single test case
|
|
+ * @param {string} filePath
|
|
+ * @returns {{passed: boolean, message: string}}
|
|
+ */
|
|
+function runTest(filePath) {
|
|
+ const metadata = parseTestMetadata(filePath);
|
|
+ const { shouldPass, errorExpectation, pathContext } = metadata;
|
|
+
|
|
+ try {
|
|
+ const fileContent = fs.readFileSync(filePath, 'utf8');
|
|
+ let agentData;
|
|
+
|
|
+ try {
|
|
+ agentData = yaml.load(fileContent);
|
|
+ } catch (parseError) {
|
|
+ // YAML parse error
|
|
+ if (shouldPass) {
|
|
+ return {
|
|
+ passed: false,
|
|
+ message: `Expected PASS but got YAML parse error: ${parseError.message}`,
|
|
+ };
|
|
+ }
|
|
+ return {
|
|
+ passed: true,
|
|
+ message: 'Got expected YAML parse error',
|
|
+ };
|
|
+ }
|
|
+
|
|
+ // Determine validation path
|
|
+ // If pathContext is specified in comments, use it; otherwise derive from fixture location
|
|
+ let validationPath = pathContext;
|
|
+ if (!validationPath) {
|
|
+ // Map fixture location to simulated src/ path
|
|
+ const relativePath = path.relative(path.join(__dirname, 'fixtures/agent-schema'), filePath);
|
|
+ const parts = relativePath.split(path.sep);
|
|
+
|
|
+ if (parts.includes('metadata') && parts[0] === 'valid') {
|
|
+ // Valid metadata tests: check if filename suggests module or core
|
|
+ const filename = path.basename(filePath);
|
|
+ if (filename.includes('module')) {
|
|
+ validationPath = 'src/modules/bmm/agents/test.agent.yaml';
|
|
+ } else {
|
|
+ validationPath = 'src/core/agents/test.agent.yaml';
|
|
+ }
|
|
+ } else if (parts.includes('metadata') && parts[0] === 'invalid') {
|
|
+ // Invalid metadata tests: derive from filename
|
|
+ const filename = path.basename(filePath);
|
|
+ if (filename.includes('module') || filename.includes('wrong-module')) {
|
|
+ validationPath = 'src/modules/bmm/agents/test.agent.yaml';
|
|
+ } else if (filename.includes('core')) {
|
|
+ validationPath = 'src/core/agents/test.agent.yaml';
|
|
+ } else {
|
|
+ validationPath = 'src/core/agents/test.agent.yaml';
|
|
+ }
|
|
+ } else {
|
|
+ // Default to core agent path
|
|
+ validationPath = 'src/core/agents/test.agent.yaml';
|
|
+ }
|
|
+ }
|
|
+
|
|
+ const result = validateAgentFile(validationPath, agentData);
|
|
+
|
|
+ if (result.success && shouldPass) {
|
|
+ return {
|
|
+ passed: true,
|
|
+ message: 'Validation passed as expected',
|
|
+ };
|
|
+ }
|
|
+
|
|
+ if (!result.success && !shouldPass) {
|
|
+ const actualError = result.error.issues[0];
|
|
+
|
|
+ // If we have error expectations, validate strictly
|
|
+ if (errorExpectation) {
|
|
+ const validation = validateError(actualError, errorExpectation);
|
|
+
|
|
+ if (!validation.valid) {
|
|
+ return {
|
|
+ passed: false,
|
|
+ message: `Error validation failed: ${validation.reason}`,
|
|
+ };
|
|
+ }
|
|
+
|
|
+ return {
|
|
+ passed: true,
|
|
+ message: `Got expected error (${errorExpectation.code}): ${actualError.message}`,
|
|
+ };
|
|
+ }
|
|
+
|
|
+ // No specific expectations - just check that it failed
|
|
+ return {
|
|
+ passed: true,
|
|
+ message: `Got expected validation error: ${actualError?.message}`,
|
|
+ };
|
|
+ }
|
|
+
|
|
+ if (result.success && !shouldPass) {
|
|
+ return {
|
|
+ passed: false,
|
|
+ message: 'Expected validation to FAIL but it PASSED',
|
|
+ };
|
|
+ }
|
|
+
|
|
+ if (!result.success && shouldPass) {
|
|
+ return {
|
|
+ passed: false,
|
|
+ message: `Expected validation to PASS but it FAILED: ${result.error.issues[0]?.message}`,
|
|
+ };
|
|
+ }
|
|
+
|
|
+ return {
|
|
+ passed: false,
|
|
+ message: 'Unexpected test state',
|
|
+ };
|
|
+ } catch (error) {
|
|
+ return {
|
|
+ passed: false,
|
|
+ message: `Test execution error: ${error.message}`,
|
|
+ };
|
|
+ }
|
|
+}
|
|
+
|
|
+/**
|
|
+ * Main test runner
|
|
+ */
|
|
+async function main() {
|
|
+ console.log(`${colors.cyan}╔═══════════════════════════════════════════════════════════╗${colors.reset}`);
|
|
+ console.log(`${colors.cyan}║ Agent Schema Validation Test Suite ║${colors.reset}`);
|
|
+ console.log(`${colors.cyan}╚═══════════════════════════════════════════════════════════╝${colors.reset}\n`);
|
|
+
|
|
+ // Find all test fixtures
|
|
+ const testFiles = await glob('test/fixtures/agent-schema/**/*.agent.yaml', {
|
|
+ cwd: path.join(__dirname, '..'),
|
|
+ absolute: true,
|
|
+ });
|
|
+
|
|
+ if (testFiles.length === 0) {
|
|
+ console.log(`${colors.yellow}⚠️ No test fixtures found${colors.reset}`);
|
|
+ process.exit(0);
|
|
+ }
|
|
+
|
|
+ console.log(`Found ${colors.cyan}${testFiles.length}${colors.reset} test fixture(s)\n`);
|
|
+
|
|
+ // Group tests by category
|
|
+ const categories = {};
|
|
+ for (const testFile of testFiles) {
|
|
+ const relativePath = path.relative(path.join(__dirname, 'fixtures/agent-schema'), testFile);
|
|
+ const parts = relativePath.split(path.sep);
|
|
+ const validInvalid = parts[0]; // 'valid' or 'invalid'
|
|
+ const category = parts[1]; // 'top-level', 'metadata', etc.
|
|
+
|
|
+ const categoryKey = `${validInvalid}/${category}`;
|
|
+ if (!categories[categoryKey]) {
|
|
+ categories[categoryKey] = [];
|
|
+ }
|
|
+ categories[categoryKey].push(testFile);
|
|
+ }
|
|
+
|
|
+ // Run tests by category
|
|
+ let totalTests = 0;
|
|
+ let passedTests = 0;
|
|
+ const failures = [];
|
|
+
|
|
+ for (const [categoryKey, files] of Object.entries(categories).sort()) {
|
|
+ const [validInvalid, category] = categoryKey.split('/');
|
|
+ const categoryLabel = category.replaceAll('-', ' ').toUpperCase();
|
|
+ const validLabel = validInvalid === 'valid' ? '✅' : '❌';
|
|
+
|
|
+ console.log(`${colors.blue}${validLabel} ${categoryLabel} (${validInvalid})${colors.reset}`);
|
|
+
|
|
+ for (const testFile of files) {
|
|
+ totalTests++;
|
|
+ const testName = path.basename(testFile, '.agent.yaml');
|
|
+ const result = runTest(testFile);
|
|
+
|
|
+ if (result.passed) {
|
|
+ passedTests++;
|
|
+ console.log(` ${colors.green}✓${colors.reset} ${testName} ${colors.dim}${result.message}${colors.reset}`);
|
|
+ } else {
|
|
+ console.log(` ${colors.red}✗${colors.reset} ${testName} ${colors.red}${result.message}${colors.reset}`);
|
|
+ failures.push({
|
|
+ file: path.relative(process.cwd(), testFile),
|
|
+ message: result.message,
|
|
+ });
|
|
+ }
|
|
+ }
|
|
+ console.log('');
|
|
+ }
|
|
+
|
|
+ // Summary
|
|
+ console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}`);
|
|
+ console.log(`${colors.cyan}Test Results:${colors.reset}`);
|
|
+ console.log(` Total: ${totalTests}`);
|
|
+ console.log(` Passed: ${colors.green}${passedTests}${colors.reset}`);
|
|
+ console.log(` Failed: ${passedTests === totalTests ? colors.green : colors.red}${totalTests - passedTests}${colors.reset}`);
|
|
+ console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}\n`);
|
|
+
|
|
+ // Report failures
|
|
+ if (failures.length > 0) {
|
|
+ console.log(`${colors.red}❌ FAILED TESTS:${colors.reset}\n`);
|
|
+ for (const failure of failures) {
|
|
+ console.log(`${colors.red}✗${colors.reset} ${failure.file}`);
|
|
+ console.log(` ${failure.message}\n`);
|
|
+ }
|
|
+ process.exit(1);
|
|
+ }
|
|
+
|
|
+ console.log(`${colors.green}✨ All tests passed!${colors.reset}\n`);
|
|
+ process.exit(0);
|
|
+}
|
|
+
|
|
+// Run
|
|
+main().catch((error) => {
|
|
+ console.error(`${colors.red}Fatal error:${colors.reset}`, error);
|
|
+ process.exit(1);
|
|
+});
|