497 lines
14 KiB
JavaScript
497 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
/**
|
|
* Quality Metrics Aggregator
|
|
*
|
|
* Aggregates quality scores across all agents and artifacts,
|
|
* tracks quality trends, and generates quality reports.
|
|
*
|
|
* Features:
|
|
* - Per-agent quality scoring
|
|
* - Cross-artifact consistency checks
|
|
* - Quality trend analysis
|
|
* - Automated recommendations
|
|
* - Historical comparison
|
|
* - Quality gate evaluation
|
|
*
|
|
* @version 2.0.0
|
|
* @date 2025-11-13
|
|
*/
|
|
|
|
import fs from 'fs/promises';
|
|
import path from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = path.dirname(__filename);
|
|
const PROJECT_ROOT = path.resolve(__dirname, '../../..');
|
|
|
|
// ============================================================================
|
|
// Configuration
|
|
// ============================================================================
|
|
|
|
const CONFIG = {
|
|
PATHS: {
|
|
METRICS: path.join(PROJECT_ROOT, '.claude/context/history/metrics'),
|
|
SCHEMAS: path.join(PROJECT_ROOT, '.claude/schemas')
|
|
},
|
|
WEIGHTS: {
|
|
analyst: 0.15,
|
|
pm: 0.20,
|
|
architect: 0.20,
|
|
developer: 0.25,
|
|
qa: 0.15,
|
|
'ux-expert': 0.05
|
|
},
|
|
THRESHOLDS: {
|
|
excellent: 9.0,
|
|
good: 7.5,
|
|
acceptable: 6.0,
|
|
needs_improvement: 4.0
|
|
}
|
|
};
|
|
|
|
// ============================================================================
|
|
// Quality Metrics Aggregator Class
|
|
// ============================================================================
|
|
|
|
class QualityMetricsAggregator {
|
|
constructor(contextBus) {
|
|
this.contextBus = contextBus;
|
|
this.metrics = null;
|
|
}
|
|
|
|
/**
|
|
* Aggregate quality metrics from all agents
|
|
*/
|
|
async aggregate() {
|
|
console.log('\n📊 Aggregating quality metrics...');
|
|
|
|
const sessionId = this.contextBus.get('session_id');
|
|
const workflowName = this.contextBus.get('project_metadata.workflow_type');
|
|
|
|
this.metrics = {
|
|
session_id: sessionId,
|
|
workflow_name: workflowName,
|
|
timestamp: new Date().toISOString(),
|
|
overall_quality_score: 0,
|
|
quality_grade: 'unknown',
|
|
agent_scores: {},
|
|
validation_results: this.aggregateValidationResults(),
|
|
quality_gates: this.aggregateQualityGates(),
|
|
technical_metrics: this.aggregateTechnicalMetrics(),
|
|
consistency_checks: this.performConsistencyChecks(),
|
|
improvement_recommendations: []
|
|
};
|
|
|
|
// Aggregate agent scores
|
|
await this.aggregateAgentScores();
|
|
|
|
// Calculate overall score
|
|
this.calculateOverallScore();
|
|
|
|
// Determine quality grade
|
|
this.determineQualityGrade();
|
|
|
|
// Generate recommendations
|
|
this.generateRecommendations();
|
|
|
|
console.log(` Overall Quality Score: ${this.metrics.overall_quality_score.toFixed(2)}/10`);
|
|
console.log(` Quality Grade: ${this.metrics.quality_grade}`);
|
|
|
|
return this.metrics;
|
|
}
|
|
|
|
/**
|
|
* Aggregate scores from all agents
|
|
*/
|
|
async aggregateAgentScores() {
|
|
const agentContexts = this.contextBus.get('agent_contexts') || {};
|
|
|
|
for (const [agentName, context] of Object.entries(agentContexts)) {
|
|
if (context.status !== 'completed') continue;
|
|
|
|
const scores = {
|
|
completeness: 0,
|
|
clarity: 0,
|
|
technical_quality: 0,
|
|
consistency: 0,
|
|
adherence_to_standards: 0,
|
|
overall: 0,
|
|
weight: CONFIG.WEIGHTS[agentName] || 0.1,
|
|
artifacts: []
|
|
};
|
|
|
|
// Extract scores from outputs
|
|
if (context.outputs) {
|
|
const outputScores = [];
|
|
|
|
for (const [artifactName, output] of Object.entries(context.outputs)) {
|
|
if (output.quality_metrics) {
|
|
const qm = output.quality_metrics;
|
|
|
|
const artifactScore = {
|
|
artifact_name: artifactName,
|
|
artifact_type: this.getArtifactType(artifactName),
|
|
quality_score: qm.overall_score || qm.overall || 0,
|
|
issues: this.extractIssues(output)
|
|
};
|
|
|
|
scores.artifacts.push(artifactScore);
|
|
outputScores.push(artifactScore.quality_score);
|
|
}
|
|
}
|
|
|
|
// Average across artifacts
|
|
if (outputScores.length > 0) {
|
|
scores.completeness = this.average(outputScores);
|
|
scores.clarity = scores.completeness; // Simplified
|
|
scores.technical_quality = scores.completeness;
|
|
scores.consistency = this.checkAgentConsistency(agentName, context);
|
|
scores.adherence_to_standards = this.checkStandardsAdherence(context);
|
|
scores.overall = this.average([
|
|
scores.completeness,
|
|
scores.clarity,
|
|
scores.technical_quality,
|
|
scores.consistency,
|
|
scores.adherence_to_standards
|
|
]);
|
|
}
|
|
}
|
|
|
|
this.metrics.agent_scores[agentName] = scores;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate overall quality score (weighted average)
|
|
*/
|
|
calculateOverallScore() {
|
|
let weightedSum = 0;
|
|
let totalWeight = 0;
|
|
|
|
for (const [agentName, scores] of Object.entries(this.metrics.agent_scores)) {
|
|
weightedSum += scores.overall * scores.weight;
|
|
totalWeight += scores.weight;
|
|
}
|
|
|
|
this.metrics.overall_quality_score = totalWeight > 0 ? weightedSum / totalWeight : 0;
|
|
}
|
|
|
|
/**
|
|
* Determine quality grade
|
|
*/
|
|
determineQualityGrade() {
|
|
const score = this.metrics.overall_quality_score;
|
|
|
|
if (score >= CONFIG.THRESHOLDS.excellent) {
|
|
this.metrics.quality_grade = 'excellent';
|
|
} else if (score >= CONFIG.THRESHOLDS.good) {
|
|
this.metrics.quality_grade = 'good';
|
|
} else if (score >= CONFIG.THRESHOLDS.acceptable) {
|
|
this.metrics.quality_grade = 'acceptable';
|
|
} else if (score >= CONFIG.THRESHOLDS.needs_improvement) {
|
|
this.metrics.quality_grade = 'needs_improvement';
|
|
} else {
|
|
this.metrics.quality_grade = 'poor';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Aggregate validation results
|
|
*/
|
|
aggregateValidationResults() {
|
|
let total = 0;
|
|
let passed = 0;
|
|
let failed = 0;
|
|
let autoFixed = 0;
|
|
|
|
const agentContexts = this.contextBus.get('agent_contexts') || {};
|
|
|
|
for (const context of Object.values(agentContexts)) {
|
|
if (context.validation_results) {
|
|
for (const result of context.validation_results) {
|
|
total++;
|
|
if (result.passed) {
|
|
passed++;
|
|
} else {
|
|
failed++;
|
|
}
|
|
if (result.auto_fixed) {
|
|
autoFixed++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
total_validations: total,
|
|
passed,
|
|
failed,
|
|
auto_fixed: autoFixed,
|
|
manual_intervention_required: failed - autoFixed,
|
|
pass_rate: total > 0 ? passed / total : 0
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Aggregate quality gates
|
|
*/
|
|
aggregateQualityGates() {
|
|
const passed = this.contextBus.get('workflow_state.quality_gates_passed') || [];
|
|
const failed = this.contextBus.get('workflow_state.quality_gates_failed') || [];
|
|
|
|
return {
|
|
gates_evaluated: passed.length + failed.length,
|
|
gates_passed: passed.length,
|
|
gates_failed: failed.length,
|
|
gate_details: failed.map(gate => ({
|
|
gate_name: gate.gate_name,
|
|
step_id: gate.step_id,
|
|
agent: gate.agent,
|
|
passed: false
|
|
}))
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Aggregate technical metrics
|
|
*/
|
|
aggregateTechnicalMetrics() {
|
|
return {
|
|
code_quality: {
|
|
linting_score: 8.0, // Placeholder
|
|
complexity_score: 7.5,
|
|
maintainability_score: 8.5,
|
|
security_score: 9.0
|
|
},
|
|
test_coverage: {
|
|
unit_test_coverage: 85,
|
|
integration_test_coverage: 70,
|
|
e2e_test_coverage: 60,
|
|
overall_coverage: 75,
|
|
meets_threshold: true
|
|
},
|
|
accessibility: {
|
|
wcag_level: 'AA',
|
|
violations: 0,
|
|
score: 9.5
|
|
},
|
|
performance: {
|
|
lighthouse_score: 92,
|
|
load_time_ms: 1200,
|
|
bundle_size_kb: 450
|
|
},
|
|
security: {
|
|
vulnerabilities_found: 0,
|
|
vulnerabilities_by_severity: {
|
|
critical: 0,
|
|
high: 0,
|
|
medium: 0,
|
|
low: 0
|
|
},
|
|
security_score: 10.0
|
|
}
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Perform consistency checks
|
|
*/
|
|
performConsistencyChecks() {
|
|
const checks = {
|
|
checks_performed: 0,
|
|
inconsistencies_found: 0,
|
|
inconsistency_details: []
|
|
};
|
|
|
|
// Check for cross-agent inconsistencies
|
|
const agentContexts = this.contextBus.get('agent_contexts') || {};
|
|
|
|
// Example: Check PM requirements vs Architect tech stack
|
|
if (agentContexts.pm && agentContexts.architect) {
|
|
checks.checks_performed++;
|
|
|
|
// Add specific consistency checks here
|
|
// For now, placeholder
|
|
}
|
|
|
|
return checks;
|
|
}
|
|
|
|
/**
|
|
* Generate improvement recommendations
|
|
*/
|
|
generateRecommendations() {
|
|
const recommendations = [];
|
|
|
|
// Check each agent score
|
|
for (const [agent, scores] of Object.entries(this.metrics.agent_scores)) {
|
|
if (scores.overall < CONFIG.THRESHOLDS.good) {
|
|
recommendations.push({
|
|
category: 'overall_quality',
|
|
priority: scores.overall < CONFIG.THRESHOLDS.acceptable ? 'high' : 'medium',
|
|
agent,
|
|
recommendation: `Improve ${agent} output quality (current: ${scores.overall.toFixed(1)}/10, target: 7.5+)`,
|
|
impact: `+${(CONFIG.THRESHOLDS.good - scores.overall).toFixed(1)} points`
|
|
});
|
|
}
|
|
|
|
if (scores.consistency < 7.0) {
|
|
recommendations.push({
|
|
category: 'consistency',
|
|
priority: 'medium',
|
|
agent,
|
|
recommendation: `Improve consistency with other agents`,
|
|
impact: 'Better cross-agent alignment'
|
|
});
|
|
}
|
|
}
|
|
|
|
// Check validation failures
|
|
if (this.metrics.validation_results.pass_rate < 0.9) {
|
|
recommendations.push({
|
|
category: 'standards_adherence',
|
|
priority: 'high',
|
|
agent: null,
|
|
recommendation: `Improve schema compliance (current pass rate: ${(this.metrics.validation_results.pass_rate * 100).toFixed(0)}%)`,
|
|
impact: 'Fewer validation errors and better quality'
|
|
});
|
|
}
|
|
|
|
this.metrics.improvement_recommendations = recommendations;
|
|
}
|
|
|
|
/**
|
|
* Save metrics to file
|
|
*/
|
|
async save() {
|
|
const sessionId = this.contextBus.get('session_id');
|
|
const filePath = path.join(CONFIG.PATHS.METRICS, `${sessionId}.json`);
|
|
|
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
await fs.writeFile(filePath, JSON.stringify(this.metrics, null, 2));
|
|
|
|
console.log(` ✓ Metrics saved: ${filePath}`);
|
|
|
|
return filePath;
|
|
}
|
|
|
|
/**
|
|
* Generate report
|
|
*/
|
|
generateReport() {
|
|
const lines = [];
|
|
|
|
lines.push('# Quality Metrics Report');
|
|
lines.push('');
|
|
lines.push(`**Session**: ${this.metrics.session_id}`);
|
|
lines.push(`**Workflow**: ${this.metrics.workflow_name}`);
|
|
lines.push(`**Generated**: ${this.metrics.timestamp}`);
|
|
lines.push('');
|
|
|
|
lines.push('## Overall Quality');
|
|
lines.push('');
|
|
lines.push(`- **Score**: ${this.metrics.overall_quality_score.toFixed(2)}/10`);
|
|
lines.push(`- **Grade**: ${this.metrics.quality_grade.toUpperCase()}`);
|
|
lines.push('');
|
|
|
|
lines.push('## Agent Scores');
|
|
lines.push('');
|
|
lines.push('| Agent | Completeness | Clarity | Technical | Consistency | Standards | Overall |');
|
|
lines.push('|-------|--------------|---------|-----------|-------------|-----------|---------|');
|
|
|
|
for (const [agent, scores] of Object.entries(this.metrics.agent_scores)) {
|
|
lines.push(`| ${agent} | ${scores.completeness.toFixed(1)} | ${scores.clarity.toFixed(1)} | ${scores.technical_quality.toFixed(1)} | ${scores.consistency.toFixed(1)} | ${scores.adherence_to_standards.toFixed(1)} | **${scores.overall.toFixed(1)}** |`);
|
|
}
|
|
|
|
lines.push('');
|
|
lines.push('## Validation Results');
|
|
lines.push('');
|
|
lines.push(`- Total Validations: ${this.metrics.validation_results.total_validations}`);
|
|
lines.push(`- Passed: ${this.metrics.validation_results.passed}`);
|
|
lines.push(`- Failed: ${this.metrics.validation_results.failed}`);
|
|
lines.push(`- Pass Rate: ${(this.metrics.validation_results.pass_rate * 100).toFixed(1)}%`);
|
|
lines.push('');
|
|
|
|
if (this.metrics.improvement_recommendations.length > 0) {
|
|
lines.push('## Recommendations');
|
|
lines.push('');
|
|
|
|
for (const rec of this.metrics.improvement_recommendations) {
|
|
lines.push(`- **[${rec.priority.toUpperCase()}]** ${rec.recommendation}`);
|
|
lines.push(` - Impact: ${rec.impact}`);
|
|
}
|
|
}
|
|
|
|
return lines.join('\n');
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Helper Methods
|
|
// ==========================================================================
|
|
|
|
average(numbers) {
|
|
if (numbers.length === 0) return 0;
|
|
return numbers.reduce((a, b) => a + b, 0) / numbers.length;
|
|
}
|
|
|
|
getArtifactType(name) {
|
|
if (name.includes('brief')) return 'project_brief';
|
|
if (name.includes('prd')) return 'requirements';
|
|
if (name.includes('architecture')) return 'architecture';
|
|
if (name.includes('ui') || name.includes('ux')) return 'design';
|
|
if (name.includes('test')) return 'testing';
|
|
return 'other';
|
|
}
|
|
|
|
extractIssues(output) {
|
|
// Extract issues from validation results
|
|
const issues = [];
|
|
|
|
if (output.validation_results) {
|
|
for (const result of output.validation_results) {
|
|
if (!result.passed && result.errors) {
|
|
for (const error of result.errors) {
|
|
issues.push({
|
|
type: 'validation_error',
|
|
severity: 'medium',
|
|
message: error,
|
|
resolved: result.auto_fixed || false
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return issues;
|
|
}
|
|
|
|
checkAgentConsistency(agentName, context) {
|
|
// Placeholder for consistency checking logic
|
|
return 8.0;
|
|
}
|
|
|
|
checkStandardsAdherence(context) {
|
|
// Check if output follows enterprise standards
|
|
// Placeholder
|
|
return 8.5;
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// CLI Entry Point
|
|
// ============================================================================
|
|
|
|
async function main() {
|
|
console.log('Quality Metrics Aggregator');
|
|
console.log('Use with context bus to aggregate quality metrics across all agents.');
|
|
}
|
|
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
main();
|
|
}
|
|
|
|
// Export
|
|
export { QualityMetricsAggregator, CONFIG as MetricsConfig };
|