refactor: delete old validate-* workflows (replaced by unified validate)
This commit is contained in:
Jonah Schulte
parent
323bee924e
commit
6e02497dcb
|
|
@ -1,158 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This validates EVERY epic in the project - comprehensive health check</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Discover all epics">
|
|
||||||
<action>Load {{sprint_status_file}}</action>
|
|
||||||
|
|
||||||
<check if="file not found">
|
|
||||||
<output>❌ sprint-status.yaml not found
|
|
||||||
|
|
||||||
Run /bmad:bmm:workflows:sprint-planning first.
|
|
||||||
</output>
|
|
||||||
<action>HALT</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Parse development_status section</action>
|
|
||||||
<action>Extract all epic keys (entries starting with "epic-")</action>
|
|
||||||
<action>Filter out retrospectives (ending with "-retrospective")</action>
|
|
||||||
<action>Store as {{epic_list}}</action>
|
|
||||||
|
|
||||||
<output>🔍 **Comprehensive Epic Validation**
|
|
||||||
|
|
||||||
Found {{epic_count}} epics to validate:
|
|
||||||
{{#each epic_list}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
Starting validation...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Validate each epic">
|
|
||||||
<critical>Run validate-epic-status for EACH epic</critical>
|
|
||||||
|
|
||||||
<action>Initialize counters:
|
|
||||||
- total_stories_scanned = 0
|
|
||||||
- total_valid_stories = 0
|
|
||||||
- total_invalid_stories = 0
|
|
||||||
- total_updates_applied = 0
|
|
||||||
- epics_validated = []
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<loop foreach="{{epic_list}}">
|
|
||||||
<action>Set {{current_epic}} = current loop item</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Validating {{current_epic}}...
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<!-- Use Python script for validation logic -->
|
|
||||||
<action>Execute validation script:
|
|
||||||
python3 scripts/lib/sprint-status-updater.py --epic {{current_epic}} --mode validate
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Parse script output:
|
|
||||||
- Story count
|
|
||||||
- Valid/invalid/missing counts
|
|
||||||
- Inferred statuses
|
|
||||||
- Updates needed
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<check if="{{validation_mode}} == fix">
|
|
||||||
<action>Execute fix script:
|
|
||||||
python3 scripts/lib/sprint-status-updater.py --epic {{current_epic}} --mode fix
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Count updates applied</action>
|
|
||||||
<action>Add to total_updates_applied</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Store validation results for {{current_epic}}</action>
|
|
||||||
<action>Increment totals</action>
|
|
||||||
|
|
||||||
<output>✓ {{current_epic}}: {{story_count}} stories, {{valid_count}} valid, {{updates_applied}} updates
|
|
||||||
</output>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
All Epics Validated
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Consolidate and report">
|
|
||||||
<output>
|
|
||||||
📊 **COMPREHENSIVE VALIDATION RESULTS**
|
|
||||||
|
|
||||||
**Epics Validated:** {{epic_count}}
|
|
||||||
|
|
||||||
**Stories Analyzed:** {{total_stories_scanned}}
|
|
||||||
Valid: {{total_valid_stories}} (>=10KB, >=5 tasks)
|
|
||||||
Invalid: {{total_invalid_stories}} (<10KB or <5 tasks)
|
|
||||||
Missing: {{total_missing_files}}
|
|
||||||
|
|
||||||
**Updates Applied:** {{total_updates_applied}}
|
|
||||||
|
|
||||||
**Epic Status Summary:**
|
|
||||||
{{#each_epic_with_status}}
|
|
||||||
{{epic_key}}: {{status}} ({{done_count}}/{{total_count}} done)
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Top Issues:**
|
|
||||||
{{#if_invalid_stories_exist}}
|
|
||||||
⚠️ {{total_invalid_stories}} stories need regeneration (/create-story)
|
|
||||||
{{/if}}
|
|
||||||
{{#if_missing_files_exist}}
|
|
||||||
⚠️ {{total_missing_files}} story files missing (create or remove from sprint-status.yaml)
|
|
||||||
{{/if}}
|
|
||||||
{{#if_conflicting_evidence}}
|
|
||||||
⚠️ {{conflict_count}} stories have conflicting evidence (manual review)
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Health Score:** {{health_score}}/100
|
|
||||||
(100 = perfect, all stories valid with correct status)
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<action>Write comprehensive report to {{default_output_file}}</action>
|
|
||||||
|
|
||||||
<output>💾 Full report: {{default_output_file}}</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Provide actionable recommendations">
|
|
||||||
<output>
|
|
||||||
🎯 **RECOMMENDED ACTIONS**
|
|
||||||
|
|
||||||
{{#if_health_score_lt_80}}
|
|
||||||
**Priority 1: Fix Invalid Stories ({{total_invalid_stories}})**
|
|
||||||
{{#each_invalid_story}}
|
|
||||||
/create-story-with-gap-analysis # Regenerate {{story_id}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if_missing_files_gt_0}}
|
|
||||||
**Priority 2: Create Missing Story Files ({{total_missing_files}})**
|
|
||||||
{{#each_missing}}
|
|
||||||
/create-story # Create {{story_id}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if_health_score_gte_80}}
|
|
||||||
✅ **Sprint status is healthy!**
|
|
||||||
|
|
||||||
Continue with normal development:
|
|
||||||
/sprint-status # Check what's next
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Maintenance:**
|
|
||||||
- Run /validate-all-epics weekly to catch drift
|
|
||||||
- After autonomous work, run validation
|
|
||||||
- Before sprint reviews, validate status accuracy
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
name: validate-all-epics
|
|
||||||
description: "Validate and fix sprint-status.yaml for ALL epics. Runs validate-epic-status on every epic in parallel, consolidates results, rebuilds accurate sprint-status.yaml."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-all-epics"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Variables
|
|
||||||
variables:
|
|
||||||
sprint_status_file: "{implementation_artifacts}/sprint-status.yaml"
|
|
||||||
validation_mode: "fix" # Options: "report-only", "fix"
|
|
||||||
parallel_validation: true # Validate epics in parallel for speed
|
|
||||||
|
|
||||||
# Sub-workflow
|
|
||||||
validate_epic_workflow: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-epic-status/workflow.yaml"
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.all-epics-validation-report.md"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,338 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This is the COMPREHENSIVE AUDIT - validates all stories using Haiku agents</critical>
|
|
||||||
<critical>Cost: ~$76 for 511 stories with Haiku (vs $793 with Sonnet)</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Discover all story files">
|
|
||||||
<action>Find all .md files in {{story_dir}}</action>
|
|
||||||
|
|
||||||
<action>Filter out meta-documents:
|
|
||||||
- Files starting with "EPIC-" (completion reports)
|
|
||||||
- Files starting with "." (progress files)
|
|
||||||
- Files containing: COMPLETION, SUMMARY, REPORT, SESSION-, REVIEW-, README, INDEX
|
|
||||||
- Files like "atdd-checklist-", "gap-analysis-", "review-"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<check if="{{epic_filter}} provided">
|
|
||||||
<action>Filter to stories matching: {{epic_filter}}-*.md</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Store as {{story_list}}</action>
|
|
||||||
<action>Count {{story_count}}</action>
|
|
||||||
|
|
||||||
<output>🔍 **Comprehensive Story Audit**
|
|
||||||
|
|
||||||
{{#if epic_filter}}**Epic Filter:** {{epic_filter}}{{else}}**Scope:** All epics{{/if}}
|
|
||||||
**Stories to Validate:** {{story_count}}
|
|
||||||
**Agent Model:** Haiku 4.5
|
|
||||||
**Batch Size:** {{batch_size}}
|
|
||||||
|
|
||||||
**Estimated Cost:** ~${{estimated_cost}} ({{story_count}} × $0.15/story)
|
|
||||||
**Estimated Time:** {{estimated_hours}} hours
|
|
||||||
|
|
||||||
Starting batch validation...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Batch validate all stories">
|
|
||||||
<action>Initialize counters:
|
|
||||||
- stories_validated = 0
|
|
||||||
- verified_complete = 0
|
|
||||||
- needs_rework = 0
|
|
||||||
- false_positives = 0
|
|
||||||
- in_progress = 0
|
|
||||||
- total_false_positive_tasks = 0
|
|
||||||
- total_critical_issues = 0
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Split {{story_list}} into batches of {{batch_size}}</action>
|
|
||||||
|
|
||||||
<loop foreach="{{batches}}">
|
|
||||||
<action>Set {{current_batch}} = current batch</action>
|
|
||||||
<action>Set {{batch_number}} = loop index + 1</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Batch {{batch_number}}/{{total_batches}} ({{batch_size}} stories)
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<!-- Validate each story in batch -->
|
|
||||||
<loop foreach="{{current_batch}}">
|
|
||||||
<action>Set {{story_file}} = current story path</action>
|
|
||||||
<action>Extract {{story_id}} from filename</action>
|
|
||||||
|
|
||||||
<output>{{stories_validated + 1}}/{{story_count}}: Validating {{story_id}}...</output>
|
|
||||||
|
|
||||||
<!-- Invoke validate-story-deep workflow -->
|
|
||||||
<invoke-workflow path="{{validate_story_workflow}}">
|
|
||||||
<input name="story_file" value="{{story_file}}" />
|
|
||||||
</invoke-workflow>
|
|
||||||
|
|
||||||
<action>Parse validation results:
|
|
||||||
- category (VERIFIED_COMPLETE, FALSE_POSITIVE, etc.)
|
|
||||||
- verification_score
|
|
||||||
- false_positive_count
|
|
||||||
- false_negative_count
|
|
||||||
- critical_issues_count
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Store results for {{story_id}}</action>
|
|
||||||
<action>Increment counters based on category</action>
|
|
||||||
|
|
||||||
<output> → {{category}} (Score: {{verification_score}}/100{{#if false_positives > 0}}, {{false_positives}} false positives{{/if}})</output>
|
|
||||||
|
|
||||||
<action>Increment stories_validated</action>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>Batch {{batch_number}} complete. {{stories_validated}}/{{story_count}} total validated.</output>
|
|
||||||
|
|
||||||
<!-- Save progress after each batch -->
|
|
||||||
<action>Write progress to {{progress_file}}:
|
|
||||||
- stories_validated
|
|
||||||
- current_batch
|
|
||||||
- results_so_far
|
|
||||||
</action>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
All Stories Validated
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**Total Validated:** {{story_count}}
|
|
||||||
**Total Tasks Checked:** {{total_tasks_verified}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Consolidate results and calculate platform health">
|
|
||||||
<action>Calculate platform-wide metrics:
|
|
||||||
- Overall health score: (verified_complete / story_count) × 100
|
|
||||||
- False positive rate: (false_positive_stories / story_count) × 100
|
|
||||||
- Total rework estimate: false_positive_stories × 3h + needs_rework × 2h
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Group results by epic</action>
|
|
||||||
|
|
||||||
<action>Identify worst offenders (highest false positive rates)</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
📊 **PLATFORM HEALTH ASSESSMENT**
|
|
||||||
|
|
||||||
**Overall Health Score:** {{health_score}}/100
|
|
||||||
|
|
||||||
**Story Categories:**
|
|
||||||
- ✅ VERIFIED_COMPLETE: {{verified_complete}} ({{verified_complete_pct}}%)
|
|
||||||
- ⚠️ NEEDS_REWORK: {{needs_rework}} ({{needs_rework_pct}}%)
|
|
||||||
- ❌ FALSE_POSITIVES: {{false_positives}} ({{false_positives_pct}}%)
|
|
||||||
- 🔄 IN_PROGRESS: {{in_progress}} ({{in_progress_pct}}%)
|
|
||||||
|
|
||||||
**Task-Level Issues:**
|
|
||||||
- False positive tasks: {{total_false_positive_tasks}}
|
|
||||||
- CRITICAL code quality issues: {{total_critical_issues}}
|
|
||||||
|
|
||||||
**Estimated Rework:** {{total_rework_hours}} hours
|
|
||||||
|
|
||||||
**Epic Breakdown:**
|
|
||||||
{{#each epic_summary}}
|
|
||||||
- Epic {{this.epic}}: {{this.health_score}}/100 ({{this.false_positives}} false positives)
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Worst Offenders (Most False Positives):**
|
|
||||||
{{#each worst_offenders limit=10}}
|
|
||||||
- {{this.story_id}}: {{this.false_positive_count}} tasks, score {{this.score}}/100
|
|
||||||
{{/each}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Generate comprehensive audit report">
|
|
||||||
<template-output>
|
|
||||||
# Comprehensive Platform Audit Report
|
|
||||||
|
|
||||||
**Generated:** {{date}}
|
|
||||||
**Stories Validated:** {{story_count}}
|
|
||||||
**Agent Model:** Haiku 4.5
|
|
||||||
**Total Cost:** ~${{actual_cost}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Executive Summary
|
|
||||||
|
|
||||||
**Platform Health Score:** {{health_score}}/100
|
|
||||||
|
|
||||||
{{#if health_score >= 90}}
|
|
||||||
✅ **EXCELLENT** - Platform is production-ready with high confidence
|
|
||||||
{{else if health_score >= 75}}
|
|
||||||
⚠️ **GOOD** - Minor issues to address, generally solid
|
|
||||||
{{else if health_score >= 60}}
|
|
||||||
⚠️ **NEEDS WORK** - Significant rework required before production
|
|
||||||
{{else}}
|
|
||||||
❌ **CRITICAL** - Major quality issues found, not production-ready
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Key Findings:**
|
|
||||||
- {{verified_complete}} stories verified complete ({{verified_complete_pct}}%)
|
|
||||||
- {{false_positives}} stories are false positives ({{false_positives_pct}}%)
|
|
||||||
- {{total_false_positive_tasks}} tasks claimed done but not implemented
|
|
||||||
- {{total_critical_issues}} CRITICAL code quality issues found
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ❌ False Positive Stories ({{false_positives}} total)
|
|
||||||
|
|
||||||
**These stories are marked "done" but have significant missing/stubbed code:**
|
|
||||||
|
|
||||||
{{#each false_positive_stories}}
|
|
||||||
### {{this.story_id}} (Score: {{this.score}}/100)
|
|
||||||
|
|
||||||
**Current Status:** {{this.current_status}}
|
|
||||||
**Should Be:** in-progress or ready-for-dev
|
|
||||||
|
|
||||||
**Missing/Stubbed:**
|
|
||||||
{{#each this.false_positive_tasks}}
|
|
||||||
- {{this.task}}
|
|
||||||
- {{this.evidence}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Estimated Fix:** {{this.estimated_hours}}h
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total Rework:** {{false_positive_rework_hours}} hours
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⚠️ Stories Needing Rework ({{needs_rework}} total)
|
|
||||||
|
|
||||||
{{#each needs_rework_stories}}
|
|
||||||
### {{this.story_id}} (Score: {{this.score}}/100)
|
|
||||||
|
|
||||||
**Issues:**
|
|
||||||
- {{this.false_positive_count}} incomplete tasks
|
|
||||||
- {{this.critical_issues}} CRITICAL quality issues
|
|
||||||
- {{this.high_issues}} HIGH priority issues
|
|
||||||
|
|
||||||
**Top Issues:**
|
|
||||||
{{#each this.top_issues limit=5}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total Rework:** {{needs_rework_hours}} hours
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ Verified Complete Stories ({{verified_complete}} total)
|
|
||||||
|
|
||||||
**These stories are production-ready with verified code:**
|
|
||||||
|
|
||||||
{{#each verified_complete_stories}}
|
|
||||||
- {{this.story_id}} ({{this.score}}/100)
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Epic Health Breakdown
|
|
||||||
|
|
||||||
{{#each epic_summary}}
|
|
||||||
### Epic {{this.epic}}
|
|
||||||
|
|
||||||
**Stories:** {{this.total}}
|
|
||||||
**Verified Complete:** {{this.verified}} ({{this.verified_pct}}%)
|
|
||||||
**False Positives:** {{this.false_positives}}
|
|
||||||
**Needs Rework:** {{this.needs_rework}}
|
|
||||||
|
|
||||||
**Health Score:** {{this.health_score}}/100
|
|
||||||
|
|
||||||
{{#if this.health_score < 70}}
|
|
||||||
⚠️ **ATTENTION NEEDED** - This epic has quality issues
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Top Issues:**
|
|
||||||
{{#each this.top_issues limit=3}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Recommended Action Plan
|
|
||||||
|
|
||||||
### Phase 1: Fix False Positives (CRITICAL - {{false_positive_rework_hours}}h)
|
|
||||||
|
|
||||||
{{#each false_positive_stories limit=20}}
|
|
||||||
{{@index + 1}}. **{{this.story_id}}** ({{this.estimated_hours}}h)
|
|
||||||
- {{this.false_positive_count}} tasks to implement
|
|
||||||
- Update status to in-progress
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
{{#if false_positives > 20}}
|
|
||||||
... and {{false_positives - 20}} more (see full list above)
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
### Phase 2: Address Rework Items (HIGH - {{needs_rework_hours}}h)
|
|
||||||
|
|
||||||
{{#each needs_rework_stories limit=10}}
|
|
||||||
{{@index + 1}}. **{{this.story_id}}** ({{this.estimated_hours}}h)
|
|
||||||
- Fix {{this.critical_issues}} CRITICAL issues
|
|
||||||
- Complete {{this.false_positive_count}} tasks
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
### Phase 3: Fix False Negatives (LOW - batch update)
|
|
||||||
|
|
||||||
- {{total_false_negative_tasks}} unchecked tasks that are actually complete
|
|
||||||
- Can batch update checkboxes (low priority)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 💰 Audit Cost Analysis
|
|
||||||
|
|
||||||
**This Validation Run:**
|
|
||||||
- Stories validated: {{story_count}}
|
|
||||||
- Agent sessions: {{story_count}} (one Haiku agent per story)
|
|
||||||
- Tokens used: ~{{tokens_used_millions}}M
|
|
||||||
- Cost: ~${{actual_cost}}
|
|
||||||
|
|
||||||
**Remediation Cost:**
|
|
||||||
- Estimated hours: {{total_rework_hours}}h
|
|
||||||
- At AI velocity: {{ai_velocity_days}} days of work
|
|
||||||
- Token cost: ~${{remediation_token_cost}}
|
|
||||||
|
|
||||||
**Total Investment:** ${{actual_cost}} (audit) + ${{remediation_token_cost}} (fixes) = ${{total_cost}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📅 Next Steps
|
|
||||||
|
|
||||||
1. **Immediate:** Fix {{false_positives}} false positive stories
|
|
||||||
2. **This Week:** Address {{total_critical_issues}} CRITICAL issues
|
|
||||||
3. **Next Week:** Rework {{needs_rework}} stories
|
|
||||||
4. **Ongoing:** Re-validate fixed stories to confirm
|
|
||||||
|
|
||||||
**Commands:**
|
|
||||||
```bash
|
|
||||||
# Validate specific story
|
|
||||||
/validate-story-deep docs/sprint-artifacts/16e-6-ecs-task-definitions-tier3.md
|
|
||||||
|
|
||||||
# Validate specific epic
|
|
||||||
/validate-all-stories-deep --epic 16e
|
|
||||||
|
|
||||||
# Re-run full audit (after fixes)
|
|
||||||
/validate-all-stories-deep
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Report Generated By:** validate-all-stories-deep workflow
|
|
||||||
**Validation Method:** LLM-powered (Haiku 4.5 agents read actual code)
|
|
||||||
**Confidence Level:** Very High (code-based verification, not regex patterns)
|
|
||||||
</template-output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,36 +0,0 @@
|
||||||
name: validate-all-stories-deep
|
|
||||||
description: "Comprehensive platform audit using Haiku agents. Validates ALL stories by reading actual code. The bulletproof validation for production readiness."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-all-stories-deep"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Input variables
|
|
||||||
variables:
|
|
||||||
epic_filter: "" # Optional: Only validate specific epic (e.g., "16e")
|
|
||||||
batch_size: 5 # Validate 5 stories at a time (prevents spawning 511 agents at once!)
|
|
||||||
concurrent_limit: 5 # Max 5 agents running concurrently
|
|
||||||
auto_fix: false # If true, auto-update statuses based on validation
|
|
||||||
pause_between_batches: 30 # Seconds to wait between batches (rate limiting)
|
|
||||||
|
|
||||||
# Sub-workflow
|
|
||||||
validate_story_workflow: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-story-deep/workflow.yaml"
|
|
||||||
|
|
||||||
# Agent configuration
|
|
||||||
agent_model: "haiku" # Cost: ~$66 for 511 stories vs $793 with Sonnet
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.comprehensive-audit-{date}.md"
|
|
||||||
progress_file: "{story_dir}/.validation-progress-{date}.yaml"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,411 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This is the COMPREHENSIVE AUDIT - validates every story's tasks against actual codebase</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Discover and categorize stories">
|
|
||||||
<action>Find all story files in {{story_dir}}</action>
|
|
||||||
<action>Filter out meta-documents:
|
|
||||||
- Files starting with "EPIC-" (completion reports)
|
|
||||||
- Files with "COMPLETION", "SUMMARY", "REPORT" in name
|
|
||||||
- Files starting with "." (hidden progress files)
|
|
||||||
- Files like "README", "INDEX", "SESSION-", "REVIEW-"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<check if="{{epic_filter}} provided">
|
|
||||||
<action>Filter to stories starting with {{epic_filter}}- (e.g., "16e-")</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Store as {{story_list}}</action>
|
|
||||||
<action>Count {{story_count}}</action>
|
|
||||||
|
|
||||||
<output>🔍 **Comprehensive Story Validation**
|
|
||||||
|
|
||||||
{{#if epic_filter}}
|
|
||||||
**Epic Filter:** {{epic_filter}} only
|
|
||||||
{{/if}}
|
|
||||||
**Stories to Validate:** {{story_count}}
|
|
||||||
**Validation Depth:** {{validation_depth}}
|
|
||||||
**Parallel Mode:** {{parallel_validation}}
|
|
||||||
|
|
||||||
**Estimated Time:** {{estimated_minutes}} minutes
|
|
||||||
**Estimated Cost:** ~${{estimated_cost}} ({{story_count}} × ~$0.50/story)
|
|
||||||
|
|
||||||
This will:
|
|
||||||
1. Verify all tasks against actual codebase (task-verification-engine.py)
|
|
||||||
2. Run code quality reviews on files with issues
|
|
||||||
3. Check for regressions and integration failures
|
|
||||||
4. Categorize stories: VERIFIED_COMPLETE, NEEDS_REWORK, FALSE_POSITIVE, etc.
|
|
||||||
5. Generate comprehensive audit report
|
|
||||||
|
|
||||||
Starting validation...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Run task verification on all stories">
|
|
||||||
<action>Initialize counters:
|
|
||||||
- stories_validated = 0
|
|
||||||
- verified_complete = 0
|
|
||||||
- needs_rework = 0
|
|
||||||
- false_positives = 0
|
|
||||||
- in_progress = 0
|
|
||||||
- total_false_positive_tasks = 0
|
|
||||||
- total_tasks_verified = 0
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<loop foreach="{{story_list}}">
|
|
||||||
<action>Set {{current_story}} = current story file</action>
|
|
||||||
<action>Extract {{story_id}} from filename</action>
|
|
||||||
|
|
||||||
<output>Validating {{counter}}/{{story_count}}: {{story_id}}...</output>
|
|
||||||
|
|
||||||
<!-- Run task verification engine -->
|
|
||||||
<action>Execute: python3 {{task_verification_script}} {{current_story}}</action>
|
|
||||||
|
|
||||||
<action>Parse output:
|
|
||||||
- total_tasks
|
|
||||||
- checked_tasks
|
|
||||||
- false_positives
|
|
||||||
- false_negatives
|
|
||||||
- verification_score
|
|
||||||
- task_details (with evidence)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Categorize story:
|
|
||||||
IF verification_score >= 95 AND false_positives == 0
|
|
||||||
→ category = "VERIFIED_COMPLETE"
|
|
||||||
ELSE IF verification_score >= 80 AND false_positives <= 2
|
|
||||||
→ category = "COMPLETE_WITH_MINOR_ISSUES"
|
|
||||||
ELSE IF false_positives > 5 OR verification_score < 50
|
|
||||||
→ category = "FALSE_POSITIVE" (claimed done but missing code)
|
|
||||||
ELSE IF verification_score < 80
|
|
||||||
→ category = "NEEDS_REWORK"
|
|
||||||
ELSE IF checked_tasks == 0
|
|
||||||
→ category = "NOT_STARTED"
|
|
||||||
ELSE
|
|
||||||
→ category = "IN_PROGRESS"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Store result:
|
|
||||||
- story_id
|
|
||||||
- verification_score
|
|
||||||
- category
|
|
||||||
- false_positive_count
|
|
||||||
- false_negative_count
|
|
||||||
- current_status (from sprint-status.yaml)
|
|
||||||
- recommended_status
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Increment counters based on category</action>
|
|
||||||
<action>Add false_positive_count to total</action>
|
|
||||||
<action>Add total_tasks to total_tasks_verified</action>
|
|
||||||
|
|
||||||
<output> → {{category}} ({{verification_score}}/100, {{false_positives}} false positives)</output>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Validation Complete
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**Stories Validated:** {{story_count}}
|
|
||||||
**Total Tasks Verified:** {{total_tasks_verified}}
|
|
||||||
**Total False Positives:** {{total_false_positive_tasks}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Code quality review on problem stories" if="{{validation_depth}} == deep OR comprehensive">
|
|
||||||
<action>Filter stories where:
|
|
||||||
- category = "FALSE_POSITIVE" OR
|
|
||||||
- category = "NEEDS_REWORK" OR
|
|
||||||
- false_positives > 3
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Count {{problem_story_count}}</action>
|
|
||||||
|
|
||||||
<check if="{{problem_story_count}} > 0">
|
|
||||||
<output>
|
|
||||||
🛡️ **Code Quality Review**
|
|
||||||
|
|
||||||
Found {{problem_story_count}} stories with quality issues.
|
|
||||||
Running multi-agent review on files from these stories...
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<loop foreach="{{problem_stories}}">
|
|
||||||
<action>Extract file list from story Dev Agent Record</action>
|
|
||||||
|
|
||||||
<check if="files exist">
|
|
||||||
<action>Run /multi-agent-review on files:
|
|
||||||
- Security audit
|
|
||||||
- Silent failure detection
|
|
||||||
- Architecture compliance
|
|
||||||
- Type safety check
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Categorize review findings by severity</action>
|
|
||||||
<action>Add to story's issue list</action>
|
|
||||||
</check>
|
|
||||||
</loop>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="{{problem_story_count}} == 0">
|
|
||||||
<output>✅ No problem stories found - all code quality looks good!</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Integration verification" if="{{validation_depth}} == comprehensive">
|
|
||||||
<output>
|
|
||||||
🔗 **Integration Verification**
|
|
||||||
|
|
||||||
Checking for regressions and broken dependencies...
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<action>For stories marked "VERIFIED_COMPLETE":
|
|
||||||
1. Extract service dependencies from story
|
|
||||||
2. Check if dependent services still exist
|
|
||||||
3. Run integration tests if they exist
|
|
||||||
4. Check for API contract breaking changes
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Detect overlaps:
|
|
||||||
- Multiple stories implementing same feature
|
|
||||||
- Duplicate files created
|
|
||||||
- Conflicting implementations
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
**Regressions Found:** {{regression_count}}
|
|
||||||
**Overlaps Detected:** {{overlap_count}}
|
|
||||||
**Integration Tests:** {{integration_tests_run}} ({{integration_tests_passing}} passing)
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="5" goal="Generate comprehensive report">
|
|
||||||
<template-output>
|
|
||||||
# Comprehensive Story Validation Report
|
|
||||||
|
|
||||||
**Generated:** {{date}}
|
|
||||||
**Stories Validated:** {{story_count}}
|
|
||||||
**Validation Depth:** {{validation_depth}}
|
|
||||||
**Epic Filter:** {{epic_filter}} {{#if_no_filter}}(all epics){{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Executive Summary
|
|
||||||
|
|
||||||
**Overall Health Score:** {{overall_health_score}}/100
|
|
||||||
|
|
||||||
**Story Categories:**
|
|
||||||
- ✅ **VERIFIED_COMPLETE:** {{verified_complete}} ({{verified_complete_pct}}%)
|
|
||||||
- ⚠️ **NEEDS_REWORK:** {{needs_rework}} ({{needs_rework_pct}}%)
|
|
||||||
- ❌ **FALSE_POSITIVES:** {{false_positives}} ({{false_positives_pct}}%)
|
|
||||||
- 🔄 **IN_PROGRESS:** {{in_progress}} ({{in_progress_pct}}%)
|
|
||||||
- 📋 **NOT_STARTED:** {{not_started}} ({{not_started_pct}}%)
|
|
||||||
|
|
||||||
**Task Verification:**
|
|
||||||
- Total tasks verified: {{total_tasks_verified}}
|
|
||||||
- False positive tasks: {{total_false_positive_tasks}} ({{false_positive_rate}}%)
|
|
||||||
- False negative tasks: {{total_false_negative_tasks}}
|
|
||||||
|
|
||||||
**Code Quality:**
|
|
||||||
- CRITICAL issues: {{critical_issues_total}}
|
|
||||||
- HIGH issues: {{high_issues_total}}
|
|
||||||
- Files reviewed: {{files_reviewed}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ❌ False Positive Stories (Claimed Done, Not Implemented)
|
|
||||||
|
|
||||||
{{#each false_positive_stories}}
|
|
||||||
### {{this.story_id}} (Score: {{this.verification_score}}/100)
|
|
||||||
|
|
||||||
**Current Status:** {{this.current_status}}
|
|
||||||
**Recommended:** in-progress or ready-for-dev
|
|
||||||
|
|
||||||
**Issues:**
|
|
||||||
{{#each this.false_positive_tasks}}
|
|
||||||
- [ ] {{this.task}}
|
|
||||||
- Evidence: {{this.evidence}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Action Required:**
|
|
||||||
- Uncheck {{this.false_positive_count}} tasks
|
|
||||||
- Implement missing code
|
|
||||||
- Update sprint-status.yaml to in-progress
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total:** {{false_positive_stories_count}} stories
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⚠️ Stories Needing Rework
|
|
||||||
|
|
||||||
{{#each needs_rework_stories}}
|
|
||||||
### {{this.story_id}} (Score: {{this.verification_score}}/100)
|
|
||||||
|
|
||||||
**Issues:**
|
|
||||||
- {{this.false_positive_count}} false positive tasks
|
|
||||||
- {{this.critical_issue_count}} CRITICAL code quality issues
|
|
||||||
- {{this.high_issue_count}} HIGH priority issues
|
|
||||||
|
|
||||||
**Recommended:**
|
|
||||||
1. Fix CRITICAL issues first
|
|
||||||
2. Implement {{this.false_positive_count}} missing tasks
|
|
||||||
3. Re-run validation
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total:** {{needs_rework_count}} stories
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ Verified Complete Stories
|
|
||||||
|
|
||||||
{{#each verified_complete_stories}}
|
|
||||||
- {{this.story_id}} ({{this.verification_score}}/100)
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total:** {{verified_complete_count}} stories (production-ready)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Epic Breakdown
|
|
||||||
|
|
||||||
{{#each epic_summary}}
|
|
||||||
### Epic {{this.epic_num}}
|
|
||||||
|
|
||||||
**Stories:** {{this.total_count}}
|
|
||||||
**Verified Complete:** {{this.verified_count}} ({{this.verified_pct}}%)
|
|
||||||
**False Positives:** {{this.false_positive_count}}
|
|
||||||
**Needs Rework:** {{this.needs_rework_count}}
|
|
||||||
|
|
||||||
**Health Score:** {{this.health_score}}/100
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Recommended Actions
|
|
||||||
|
|
||||||
### Immediate (CRITICAL)
|
|
||||||
|
|
||||||
{{#if false_positive_stories_count > 0}}
|
|
||||||
**Fix {{false_positive_stories_count}} False Positive Stories:**
|
|
||||||
|
|
||||||
{{#each false_positive_stories limit=10}}
|
|
||||||
1. {{this.story_id}}: Update status to in-progress, implement {{this.false_positive_count}} missing tasks
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
{{#if false_positive_stories_count > 10}}
|
|
||||||
... and {{false_positive_stories_count - 10}} more (see full list above)
|
|
||||||
{{/if}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
### Short-term (HIGH Priority)
|
|
||||||
|
|
||||||
{{#if needs_rework_count > 0}}
|
|
||||||
**Address {{needs_rework_count}} Stories Needing Rework:**
|
|
||||||
- Fix {{critical_issues_total}} CRITICAL code quality issues
|
|
||||||
- Implement missing tasks
|
|
||||||
- Re-validate after fixes
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
### Maintenance (MEDIUM Priority)
|
|
||||||
|
|
||||||
{{#if false_negative_count > 0}}
|
|
||||||
**Update {{false_negative_count}} False Negative Tasks:**
|
|
||||||
- Mark complete (code exists but checkbox unchecked)
|
|
||||||
- Low impact, can batch update
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 💰 Cost Analysis
|
|
||||||
|
|
||||||
**Validation Run:**
|
|
||||||
- Stories validated: {{story_count}}
|
|
||||||
- API tokens used: ~{{tokens_used}}K
|
|
||||||
- Cost: ~${{cost}}
|
|
||||||
|
|
||||||
**Remediation Estimate:**
|
|
||||||
- False positives: {{false_positive_stories_count}} × 3h = {{remediation_hours_fp}}h
|
|
||||||
- Needs rework: {{needs_rework_count}} × 2h = {{remediation_hours_rework}}h
|
|
||||||
- **Total:** {{total_remediation_hours}}h estimated work
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📅 Next Steps
|
|
||||||
|
|
||||||
1. **Fix false positive stories** ({{false_positive_stories_count}} stories)
|
|
||||||
2. **Address CRITICAL issues** ({{critical_issues_total}} issues)
|
|
||||||
3. **Re-run validation** on fixed stories
|
|
||||||
4. **Update sprint-status.yaml** with verified statuses
|
|
||||||
5. **Run weekly validation** to prevent future drift
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Generated by:** /validate-all-stories workflow
|
|
||||||
**Validation Engine:** task-verification-engine.py v2.0
|
|
||||||
**Multi-Agent Review:** {{multi_agent_review_enabled}}
|
|
||||||
</template-output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="6" goal="Auto-fix if enabled" if="{{fix_mode}} == true">
|
|
||||||
<output>
|
|
||||||
🔧 **Auto-Fix Mode Enabled**
|
|
||||||
|
|
||||||
Applying automatic fixes:
|
|
||||||
1. Update false negative checkboxes (code exists → mark [x])
|
|
||||||
2. Update sprint-status.yaml with verified statuses
|
|
||||||
3. Add validation scores to story files
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<loop foreach="{{false_negative_tasks_list}}">
|
|
||||||
<action>Update story file: Change [ ] to [x] for verified tasks</action>
|
|
||||||
<output> ✓ {{story_id}}: Checked {{task_count}} false negative tasks</output>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<loop foreach="{{status_updates_list}}">
|
|
||||||
<action>Update sprint-status.yaml using sprint-status-updater.py</action>
|
|
||||||
<output> ✓ {{story_id}}: {{old_status}} → {{new_status}}</output>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
✅ Auto-fix complete
|
|
||||||
- {{false_negatives_fixed}} tasks checked
|
|
||||||
- {{statuses_updated}} story statuses updated
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="7" goal="Summary and recommendations">
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
COMPREHENSIVE VALIDATION COMPLETE
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**Overall Health:** {{overall_health_score}}/100
|
|
||||||
|
|
||||||
{{#if overall_health_score >= 90}}
|
|
||||||
✅ **EXCELLENT** - Platform is production-ready
|
|
||||||
{{else if overall_health_score >= 75}}
|
|
||||||
⚠️ **GOOD** - Minor issues to address before production
|
|
||||||
{{else if overall_health_score >= 60}}
|
|
||||||
⚠️ **NEEDS WORK** - Significant rework required
|
|
||||||
{{else}}
|
|
||||||
❌ **CRITICAL** - Major quality issues found
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Top Priorities:**
|
|
||||||
1. Fix {{false_positive_stories_count}} false positive stories
|
|
||||||
2. Address {{critical_issues_total}} CRITICAL code quality issues
|
|
||||||
3. Complete {{in_progress_count}} in-progress stories
|
|
||||||
4. Re-validate after fixes
|
|
||||||
|
|
||||||
**Full Report:** {{default_output_file}}
|
|
||||||
**Summary JSON:** {{validation_summary_file}}
|
|
||||||
|
|
||||||
**Next Command:**
|
|
||||||
/validate-story <story-id> # Deep-dive on specific story
|
|
||||||
/validate-all-stories --epic 16e # Re-validate specific epic
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,36 +0,0 @@
|
||||||
name: validate-all-stories
|
|
||||||
description: "Comprehensive audit of ALL stories: verify tasks against codebase, run code quality reviews, check integrations. The bulletproof audit for production readiness."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-all-stories"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Input variables
|
|
||||||
variables:
|
|
||||||
validation_depth: "deep" # Options: "quick" (tasks only), "deep" (tasks + review), "comprehensive" (full integration)
|
|
||||||
parallel_validation: true # Run story validations in parallel for speed
|
|
||||||
fix_mode: false # If true, auto-fix false negatives and update statuses
|
|
||||||
epic_filter: "" # Optional: Only validate stories from specific epic (e.g., "16e")
|
|
||||||
|
|
||||||
# Tools
|
|
||||||
task_verification_script: "{project-root}/scripts/lib/task-verification-engine.py"
|
|
||||||
sprint_status_updater: "{project-root}/scripts/lib/sprint-status-updater.py"
|
|
||||||
|
|
||||||
# Sub-workflow
|
|
||||||
validate_story_workflow: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-story/workflow.yaml"
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.comprehensive-validation-report-{date}.md"
|
|
||||||
validation_summary_file: "{story_dir}/.validation-summary-{date}.json"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,302 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This is VALIDATION-ONLY mode - NO implementation, only status correction</critical>
|
|
||||||
<critical>Uses same logic as batch-super-dev but READS instead of WRITES code</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Validate inputs and load epic">
|
|
||||||
<action>Check if {{epic_num}} was provided</action>
|
|
||||||
|
|
||||||
<check if="{{epic_num}} is empty">
|
|
||||||
<ask>Which epic should I validate? (e.g., 19, 16d, 16e, 9b)</ask>
|
|
||||||
<action>Store response as {{epic_num}}</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Load {{sprint_status_file}}</action>
|
|
||||||
|
|
||||||
<check if="file not found">
|
|
||||||
<output>❌ sprint-status.yaml not found at: {{sprint_status_file}}
|
|
||||||
|
|
||||||
Run /bmad:bmm:workflows:sprint-planning to create it first.
|
|
||||||
</output>
|
|
||||||
<action>HALT</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Search for epic-{{epic_num}} entry in sprint_status_file</action>
|
|
||||||
<action>Extract all story entries for epic-{{epic_num}} (pattern: {{epic_num}}-*)</action>
|
|
||||||
<action>Count stories found in sprint-status.yaml for this epic</action>
|
|
||||||
|
|
||||||
<output>🔍 **Validating Epic {{epic_num}}**
|
|
||||||
|
|
||||||
Found {{story_count}} stories in sprint-status.yaml
|
|
||||||
Scanning story files for REALITY check...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Scan and validate all story files">
|
|
||||||
<critical>This is where we determine TRUTH - not from status fields, but from actual file analysis</critical>
|
|
||||||
|
|
||||||
<action>For each story in epic (from sprint-status.yaml):
|
|
||||||
1. Build story file path: {{story_dir}}/{{story_key}}.md
|
|
||||||
2. Check if file exists
|
|
||||||
3. If exists, read FULL file
|
|
||||||
4. Analyze file content
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>For each story file, extract:
|
|
||||||
- File size in KB
|
|
||||||
- Total task count (count all "- [ ]" and "- [x]" lines)
|
|
||||||
- Checked task count (count "- [x]" lines)
|
|
||||||
- Completion rate (checked / total * 100)
|
|
||||||
- Explicit Status: field (if present)
|
|
||||||
- Has proper BMAD structure (12 sections)
|
|
||||||
- Section count (count ## headings)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>📊 **Story File Quality Analysis**
|
|
||||||
|
|
||||||
Analyzing {{story_count}} story files...
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<action>For each story, classify quality:
|
|
||||||
VALID:
|
|
||||||
- File size >= 10KB
|
|
||||||
- Total tasks >= 5
|
|
||||||
- Has task list structure
|
|
||||||
|
|
||||||
INVALID:
|
|
||||||
- File size < 10KB (incomplete story)
|
|
||||||
- Total tasks < 5 (not detailed enough)
|
|
||||||
- File missing entirely
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Store results as {{story_quality_map}}</action>
|
|
||||||
|
|
||||||
<output>Quality Summary:
|
|
||||||
Valid stories: {{valid_count}}/{{story_count}}
|
|
||||||
Invalid stories: {{invalid_count}}
|
|
||||||
Missing files: {{missing_count}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Cross-reference git commits">
|
|
||||||
<action>Run git log to find commits mentioning epic stories:
|
|
||||||
Command: git log --oneline --since={{git_commit_lookback_days}} days ago
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Parse commit messages for story IDs matching pattern: {{epic_num}}-\d+[a-z]?</action>
|
|
||||||
<action>Build map of story_id → commit_count</action>
|
|
||||||
|
|
||||||
<output>Git Commit Evidence:
|
|
||||||
Stories with commits: {{stories_with_commits_count}}
|
|
||||||
Stories without commits: {{stories_without_commits_count}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Check autonomous completion reports">
|
|
||||||
<action>Search {{story_dir}} for files:
|
|
||||||
- .epic-{{epic_num}}-completion-report.md
|
|
||||||
- .batch-super-dev-{{epic_num}}-progress.yaml
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<check if="autonomous report found">
|
|
||||||
<action>Parse completed_stories list from progress file OR
|
|
||||||
Parse ✅ story entries from completion report</action>
|
|
||||||
<action>Store as {{autonomous_completed_stories}}</action>
|
|
||||||
|
|
||||||
<output>📋 Autonomous Report Found:
|
|
||||||
{{autonomous_completed_count}} stories marked complete
|
|
||||||
</output>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="no autonomous report">
|
|
||||||
<output>ℹ️ No autonomous completion report found (manual epic)</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="5" goal="Infer correct status for each story">
|
|
||||||
<critical>Use MULTIPLE sources of truth, not just Status: field</critical>
|
|
||||||
|
|
||||||
<action>For each story in epic, determine correct status using this logic:</action>
|
|
||||||
|
|
||||||
<logic>
|
|
||||||
Priority 1: Autonomous completion report
|
|
||||||
IF story in autonomous_completed_stories
|
|
||||||
→ Status = "done" (VERY HIGH confidence)
|
|
||||||
|
|
||||||
Priority 2: Task completion rate + file quality
|
|
||||||
IF completion_rate >= 90% AND file is VALID (>10KB, >5 tasks)
|
|
||||||
→ Status = "done" (HIGH confidence)
|
|
||||||
|
|
||||||
IF completion_rate 50-89% AND file is VALID
|
|
||||||
→ Status = "in-progress" (MEDIUM confidence)
|
|
||||||
|
|
||||||
IF completion_rate < 50% AND file is VALID
|
|
||||||
→ Status = "ready-for-dev" (MEDIUM confidence)
|
|
||||||
|
|
||||||
Priority 3: Explicit Status: field (if no other evidence)
|
|
||||||
IF Status: field exists AND matches above inferences
|
|
||||||
→ Use it (MEDIUM confidence)
|
|
||||||
|
|
||||||
IF Status: field conflicts with task completion
|
|
||||||
→ Prefer task completion (tasks are ground truth)
|
|
||||||
|
|
||||||
Priority 4: Git commits (supporting evidence)
|
|
||||||
IF 3+ commits + task completion >=90%
|
|
||||||
→ Upgrade confidence to VERY HIGH
|
|
||||||
|
|
||||||
IF 1-2 commits but task completion <50%
|
|
||||||
→ Status = "in-progress" (work started but not done)
|
|
||||||
|
|
||||||
Quality Gates:
|
|
||||||
IF file size < 10KB OR total tasks < 5
|
|
||||||
→ DOWNGRADE status (can't be "done" if file is incomplete)
|
|
||||||
→ Mark as "ready-for-dev" (story needs proper creation)
|
|
||||||
→ Flag for regeneration with /create-story
|
|
||||||
|
|
||||||
Missing Files:
|
|
||||||
IF story file doesn't exist
|
|
||||||
→ Status = "backlog" (story not created yet)
|
|
||||||
</logic>
|
|
||||||
|
|
||||||
<action>Build map of story_id → inferred_status with evidence and confidence</action>
|
|
||||||
|
|
||||||
<output>📊 **Status Inference Complete**
|
|
||||||
|
|
||||||
Stories to update:
|
|
||||||
{{#each_story_needing_update}}
|
|
||||||
{{story_id}}:
|
|
||||||
Current: {{current_status_in_yaml}}
|
|
||||||
Inferred: {{inferred_status}}
|
|
||||||
Confidence: {{confidence}}
|
|
||||||
Evidence: {{evidence_summary}}
|
|
||||||
Quality: {{file_size_kb}}KB, {{total_tasks}} tasks, {{completion_rate}}% done
|
|
||||||
{{/each}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="6" goal="Apply updates or report findings">
|
|
||||||
<check if="{{validation_mode}} == report-only">
|
|
||||||
<output>📝 **REPORT-ONLY MODE** - No changes will be made
|
|
||||||
|
|
||||||
Recommendations saved to: {{default_output_file}}
|
|
||||||
</output>
|
|
||||||
<action>Write detailed report to {{default_output_file}}</action>
|
|
||||||
<action>EXIT workflow</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="{{validation_mode}} == fix OR {{validation_mode}} == strict">
|
|
||||||
<output>🔧 **FIX MODE** - Updating sprint-status.yaml...
|
|
||||||
|
|
||||||
Backing up to: .sprint-status-backups/
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<action>Create backup of {{sprint_status_file}}</action>
|
|
||||||
<action>For each story needing update:
|
|
||||||
1. Find story entry in development_status section
|
|
||||||
2. Update status to inferred_status
|
|
||||||
3. Add comment: "✅ Validated {{date}} - {{evidence_summary}}"
|
|
||||||
4. Preserve all other content and structure
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Update epic-{{epic_num}} status based on story completion:
|
|
||||||
IF all stories have status "done" AND all are valid files
|
|
||||||
→ epic status = "done"
|
|
||||||
|
|
||||||
IF any stories "in-progress" OR "review"
|
|
||||||
→ epic status = "in-progress"
|
|
||||||
|
|
||||||
IF all stories "backlog" OR "ready-for-dev"
|
|
||||||
→ epic status = "backlog"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Update last_verified timestamp in header</action>
|
|
||||||
<action>Save {{sprint_status_file}}</action>
|
|
||||||
|
|
||||||
<output>✅ **sprint-status.yaml Updated**
|
|
||||||
|
|
||||||
Applied {{updates_count}} story status corrections
|
|
||||||
Epic {{epic_num}}: {{old_epic_status}} → {{new_epic_status}}
|
|
||||||
|
|
||||||
Backup: {{backup_path}}
|
|
||||||
</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="7" goal="Identify problem stories requiring action">
|
|
||||||
<action>Flag stories with issues:
|
|
||||||
- Missing story files (in sprint-status.yaml but no .md file)
|
|
||||||
- Invalid files (< 10KB or < 5 tasks)
|
|
||||||
- Conflicting evidence (Status: says done, tasks unchecked)
|
|
||||||
- Poor quality (no BMAD sections)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>⚠️ **Problem Stories Requiring Attention:**
|
|
||||||
|
|
||||||
{{#if_missing_files}}
|
|
||||||
**Missing Files ({{missing_count}}):**
|
|
||||||
{{#each_missing}}
|
|
||||||
- {{story_id}}: Referenced in sprint-status.yaml but file not found
|
|
||||||
Action: Run /create-story OR remove from sprint-status.yaml
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if_invalid_quality}}
|
|
||||||
**Invalid Quality ({{invalid_count}}):**
|
|
||||||
{{#each_invalid}}
|
|
||||||
- {{story_id}}: {{file_size_kb}}KB, {{total_tasks}} tasks
|
|
||||||
Action: Regenerate with /create-story-with-gap-analysis
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if_conflicting_evidence}}
|
|
||||||
**Conflicting Evidence ({{conflict_count}}):**
|
|
||||||
{{#each_conflict}}
|
|
||||||
- {{story_id}}: Status: says "{{status_field}}" but {{completion_rate}}% tasks checked
|
|
||||||
Action: Manual review recommended
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="8" goal="Report results and recommendations">
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Epic {{epic_num}} Validation Complete
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**Epic Status:** {{epic_status}}
|
|
||||||
|
|
||||||
**Stories:**
|
|
||||||
Done: {{done_count}}
|
|
||||||
In-Progress: {{in_progress_count}}
|
|
||||||
Review: {{review_count}}
|
|
||||||
Ready-for-Dev: {{ready_count}}
|
|
||||||
Backlog: {{backlog_count}}
|
|
||||||
|
|
||||||
**Quality:**
|
|
||||||
Valid: {{valid_count}} (>=10KB, >=5 tasks)
|
|
||||||
Invalid: {{invalid_count}} (poor quality)
|
|
||||||
Missing: {{missing_count}} (file not found)
|
|
||||||
|
|
||||||
**Updates Applied:** {{updates_count}}
|
|
||||||
|
|
||||||
**Next Steps:**
|
|
||||||
{{#if_invalid_count_gt_0}}
|
|
||||||
1. Regenerate {{invalid_count}} invalid stories with /create-story
|
|
||||||
{{/if}}
|
|
||||||
{{#if_missing_count_gt_0}}
|
|
||||||
2. Create {{missing_count}} missing story files OR remove from sprint-status.yaml
|
|
||||||
{{/if}}
|
|
||||||
{{#if_done_count_eq_story_count}}
|
|
||||||
3. Epic complete! Consider running /retrospective
|
|
||||||
{{/if}}
|
|
||||||
{{#if_in_progress_count_gt_0}}
|
|
||||||
3. Continue with in-progress stories: /dev-story {{first_in_progress}}
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<output>💾 Detailed report saved to: {{default_output_file}}</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
name: validate-epic-status
|
|
||||||
description: "Validate and fix sprint-status.yaml for a single epic. Scans story files for task completion, validates quality (>10KB, proper tasks), checks git commits, updates sprint-status.yaml to match REALITY."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-epic-status"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Inputs
|
|
||||||
variables:
|
|
||||||
epic_num: "" # User provides (e.g., "19", "16d", "16e")
|
|
||||||
sprint_status_file: "{implementation_artifacts}/sprint-status.yaml"
|
|
||||||
validation_mode: "fix" # Options: "report-only", "fix", "strict"
|
|
||||||
|
|
||||||
# Validation criteria
|
|
||||||
validation_rules:
|
|
||||||
min_story_size_kb: 10 # Stories should be >= 10KB
|
|
||||||
min_tasks_required: 5 # Stories should have >= 5 tasks
|
|
||||||
completion_threshold: 90 # 90%+ tasks checked = "done"
|
|
||||||
git_commit_lookback_days: 30 # Search last 30 days for commits
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.epic-{epic_num}-validation-report.md"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,370 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This uses HAIKU AGENTS to read actual code and verify task completion - NOT regex patterns</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Load and parse story">
|
|
||||||
<action>Load story file from {{story_file}}</action>
|
|
||||||
|
|
||||||
<check if="file not found">
|
|
||||||
<output>❌ Story file not found: {{story_file}}</output>
|
|
||||||
<action>HALT</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Extract story metadata:
|
|
||||||
- Story ID from filename
|
|
||||||
- Epic number from "Epic:" field
|
|
||||||
- Current status from "Status:" or "**Status:**" field
|
|
||||||
- Files created/modified from Dev Agent Record section
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Extract ALL tasks (pattern: "- [ ]" or "- [x]"):
|
|
||||||
- Parse checkbox state (checked/unchecked)
|
|
||||||
- Extract task text
|
|
||||||
- Count total, checked, unchecked
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>📋 **Deep Story Validation: {{story_id}}**
|
|
||||||
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Tasks:** {{checked_count}}/{{total_count}} checked
|
|
||||||
**Files Referenced:** {{file_count}}
|
|
||||||
|
|
||||||
**Validation Method:** Haiku agents read actual code
|
|
||||||
**Cost Estimate:** ~$0.13 for this story
|
|
||||||
|
|
||||||
Starting task-by-task verification...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Verify ALL tasks with single Haiku agent">
|
|
||||||
<critical>Spawn ONE Haiku agent to verify ALL tasks (avoids 50x agent startup overhead!)</critical>
|
|
||||||
|
|
||||||
<output>Spawning Haiku verification agent for {{total_count}} tasks...</output>
|
|
||||||
|
|
||||||
<!-- Spawn SINGLE Haiku agent to verify ALL tasks in this story -->
|
|
||||||
<invoke-task type="Task" model="haiku">
|
|
||||||
<description>Verify all {{total_count}} story tasks</description>
|
|
||||||
<prompt>
|
|
||||||
You are verifying ALL tasks for this user story by reading actual code.
|
|
||||||
|
|
||||||
**Story:** {{story_id}}
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Total Tasks:** {{total_count}}
|
|
||||||
|
|
||||||
**Files from Story (Dev Agent Record):**
|
|
||||||
{{#each file_list}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Tasks to Verify:**
|
|
||||||
|
|
||||||
{{#each task_list}}
|
|
||||||
{{@index}}. [{{#if this.checked}}x{{else}} {{/if}}] {{this.text}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Your Job:**
|
|
||||||
|
|
||||||
For EACH task above:
|
|
||||||
|
|
||||||
1. **Find relevant files** - Use Glob to find files mentioned in task
|
|
||||||
2. **Read the files** - Use Read tool to examine actual code
|
|
||||||
3. **Verify implementation:**
|
|
||||||
- Is code real or stubs/TODOs?
|
|
||||||
- Is there error handling?
|
|
||||||
- Multi-tenant isolation (dealerId filters)?
|
|
||||||
- Are there tests?
|
|
||||||
- Does it match task description?
|
|
||||||
|
|
||||||
4. **Make judgment for each task**
|
|
||||||
|
|
||||||
**Output Format - JSON array with one entry per task:**
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"story_id": "{{story_id}}",
|
|
||||||
"total_tasks": {{total_count}},
|
|
||||||
"tasks": [
|
|
||||||
{
|
|
||||||
"task_number": 0,
|
|
||||||
"task_text": "Implement UserService",
|
|
||||||
"is_checked": true,
|
|
||||||
"actually_complete": false,
|
|
||||||
"confidence": "high",
|
|
||||||
"evidence": "File exists but has 'TODO: Implement findById' on line 45, tests not found",
|
|
||||||
"issues_found": ["Stub implementation", "Missing tests", "No dealerId filter"],
|
|
||||||
"recommendation": "Implement real logic, add tests, add multi-tenant isolation"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"task_number": 1,
|
|
||||||
"task_text": "Add error handling",
|
|
||||||
"is_checked": true,
|
|
||||||
"actually_complete": true,
|
|
||||||
"confidence": "very_high",
|
|
||||||
"evidence": "Try-catch blocks in UserService.ts:67-89, proper error logging, tests verify error cases",
|
|
||||||
"issues_found": [],
|
|
||||||
"recommendation": "None - task complete"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Be efficient:** Read files once, verify all tasks, return comprehensive JSON.
|
|
||||||
</prompt>
|
|
||||||
<subagent_type>general-purpose</subagent_type>
|
|
||||||
</invoke-task>
|
|
||||||
|
|
||||||
<action>Parse agent response (extract JSON)</action>
|
|
||||||
|
|
||||||
<action>For each task result:
|
|
||||||
- Determine verification_status (correct/false_positive/false_negative)
|
|
||||||
- Categorize into verified_complete, false_positives, false_negatives lists
|
|
||||||
- Count totals
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Task Verification Complete
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**✅ Verified Complete:** {{verified_complete_count}}
|
|
||||||
**❌ False Positives:** {{false_positive_count}} (checked but code missing/poor)
|
|
||||||
**⚠️ False Negatives:** {{false_negative_count}} (unchecked but code exists)
|
|
||||||
**❓ Uncertain:** {{uncertain_count}}
|
|
||||||
|
|
||||||
**Verification Score:** {{verification_score}}/100
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Calculate overall story health">
|
|
||||||
<action>Calculate scores:
|
|
||||||
- Task accuracy: (correct / total) × 100
|
|
||||||
- False positive penalty: false_positive_count × -5
|
|
||||||
- Overall score: max(0, task_accuracy + penalty)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Determine story category:
|
|
||||||
IF score >= 95 AND false_positives == 0
|
|
||||||
→ VERIFIED_COMPLETE
|
|
||||||
ELSE IF score >= 80 AND false_positives <= 2
|
|
||||||
→ COMPLETE_WITH_MINOR_ISSUES
|
|
||||||
ELSE IF false_positives > 5 OR score < 50
|
|
||||||
→ FALSE_POSITIVE (story claimed done but significant missing code)
|
|
||||||
ELSE IF false_positives > 0
|
|
||||||
→ NEEDS_REWORK
|
|
||||||
ELSE
|
|
||||||
→ IN_PROGRESS
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Determine recommended status:
|
|
||||||
VERIFIED_COMPLETE → "done"
|
|
||||||
COMPLETE_WITH_MINOR_ISSUES → "review"
|
|
||||||
FALSE_POSITIVE → "in-progress" or "ready-for-dev"
|
|
||||||
NEEDS_REWORK → "in-progress"
|
|
||||||
IN_PROGRESS → "in-progress"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
📊 **STORY HEALTH ASSESSMENT**
|
|
||||||
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Recommended Status:** {{recommended_status}}
|
|
||||||
**Overall Score:** {{overall_score}}/100
|
|
||||||
|
|
||||||
**Category:** {{category}}
|
|
||||||
|
|
||||||
{{#if category == "VERIFIED_COMPLETE"}}
|
|
||||||
✅ **Story is production-ready**
|
|
||||||
- All tasks verified complete
|
|
||||||
- Code quality confirmed
|
|
||||||
- No significant issues found
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "FALSE_POSITIVE"}}
|
|
||||||
❌ **Story claimed done but has significant missing code**
|
|
||||||
- {{false_positive_count}} tasks checked but not implemented
|
|
||||||
- Verification score: {{overall_score}}/100 (< 50% = false positive)
|
|
||||||
- Action: Update status to in-progress, implement missing tasks
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "NEEDS_REWORK"}}
|
|
||||||
⚠️ **Story needs rework before marking complete**
|
|
||||||
- {{false_positive_count}} tasks with missing/poor code
|
|
||||||
- Issues found in verification
|
|
||||||
- Action: Fix issues, re-verify
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Generate detailed validation report">
|
|
||||||
<template-output>
|
|
||||||
# Story Validation Report: {{story_id}}
|
|
||||||
|
|
||||||
**Generated:** {{date}}
|
|
||||||
**Validation Method:** LLM-powered deep verification (Haiku 4.5)
|
|
||||||
**Overall Score:** {{overall_score}}/100
|
|
||||||
**Category:** {{category}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
**Story:** {{story_id}}
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Recommended Status:** {{recommended_status}}
|
|
||||||
|
|
||||||
**Task Verification:**
|
|
||||||
- Total: {{total_count}}
|
|
||||||
- Checked: {{checked_count}}
|
|
||||||
- Verified Complete: {{verified_complete_count}}
|
|
||||||
- False Positives: {{false_positive_count}}
|
|
||||||
- False Negatives: {{false_negative_count}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Verification Details
|
|
||||||
|
|
||||||
{{#if false_positive_count > 0}}
|
|
||||||
### ❌ False Positives (CRITICAL - Code Claims vs Reality)
|
|
||||||
|
|
||||||
{{#each false_positives}}
|
|
||||||
**Task {{@index + 1}}:** {{this.task}}
|
|
||||||
**Claimed:** [x] Complete
|
|
||||||
**Reality:** Code missing or stub implementation
|
|
||||||
|
|
||||||
**Evidence:**
|
|
||||||
{{this.evidence}}
|
|
||||||
|
|
||||||
**Issues Found:**
|
|
||||||
{{#each this.issues_found}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Recommendation:** {{this.recommendation}}
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if false_negative_count > 0}}
|
|
||||||
### ⚠️ False Negatives (Unchecked But Working)
|
|
||||||
|
|
||||||
{{#each false_negatives}}
|
|
||||||
**Task {{@index + 1}}:** {{this.task}}
|
|
||||||
**Status:** [ ] Unchecked
|
|
||||||
**Reality:** Code exists and working
|
|
||||||
|
|
||||||
**Evidence:**
|
|
||||||
{{this.evidence}}
|
|
||||||
|
|
||||||
**Recommendation:** Mark task as complete [x]
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if verified_complete_count > 0}}
|
|
||||||
### ✅ Verified Complete Tasks
|
|
||||||
|
|
||||||
{{verified_complete_count}} tasks verified with actual code review.
|
|
||||||
|
|
||||||
{{#if show_all_verified}}
|
|
||||||
{{#each verified_complete}}
|
|
||||||
- {{this.task}} ({{this.confidence}} confidence)
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Final Verdict
|
|
||||||
|
|
||||||
**Overall Score:** {{overall_score}}/100
|
|
||||||
|
|
||||||
{{#if category == "VERIFIED_COMPLETE"}}
|
|
||||||
✅ **VERIFIED COMPLETE**
|
|
||||||
|
|
||||||
This story is production-ready:
|
|
||||||
- All {{total_count}} tasks verified complete
|
|
||||||
- Code quality confirmed through review
|
|
||||||
- No significant issues found
|
|
||||||
- Status "done" is accurate
|
|
||||||
|
|
||||||
**Action:** None needed - story is solid
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "FALSE_POSITIVE"}}
|
|
||||||
❌ **FALSE POSITIVE - Story NOT Actually Complete**
|
|
||||||
|
|
||||||
**Problems:**
|
|
||||||
- {{false_positive_count}} tasks checked but code missing/stubbed
|
|
||||||
- Verification score: {{overall_score}}/100 (< 50%)
|
|
||||||
- Story marked "{{current_status}}" but significant work remains
|
|
||||||
|
|
||||||
**Required Actions:**
|
|
||||||
1. Update sprint-status.yaml: {{story_id}} → in-progress
|
|
||||||
2. Uncheck {{false_positive_count}} false positive tasks
|
|
||||||
3. Implement missing code
|
|
||||||
4. Re-run validation after implementation
|
|
||||||
|
|
||||||
**Estimated Rework:** {{estimated_rework_hours}} hours
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "NEEDS_REWORK"}}
|
|
||||||
⚠️ **NEEDS REWORK**
|
|
||||||
|
|
||||||
**Problems:**
|
|
||||||
- {{false_positive_count}} tasks with quality issues
|
|
||||||
- Some code exists but has problems (TODOs, missing features, poor quality)
|
|
||||||
|
|
||||||
**Required Actions:**
|
|
||||||
{{#each action_items}}
|
|
||||||
- [ ] {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Estimated Fix Time:** {{estimated_fix_hours}} hours
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "IN_PROGRESS"}}
|
|
||||||
🔄 **IN PROGRESS** (accurate status)
|
|
||||||
|
|
||||||
- {{checked_count}}/{{total_count}} tasks complete
|
|
||||||
- {{unchecked_count}} tasks remaining
|
|
||||||
- Current status reflects reality
|
|
||||||
|
|
||||||
**No action needed** - continue implementation
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Validation Cost:** ~${{validation_cost}}
|
|
||||||
**Agent Model:** {{agent_model}}
|
|
||||||
**Tasks Verified:** {{total_count}}
|
|
||||||
</template-output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="5" goal="Update sprint-status if needed">
|
|
||||||
<check if="{{recommended_status}} != {{current_status}}">
|
|
||||||
<ask>Story status should be updated from "{{current_status}}" to "{{recommended_status}}". Update sprint-status.yaml? (y/n)</ask>
|
|
||||||
|
|
||||||
<check if="user says yes">
|
|
||||||
<action>Update sprint-status.yaml:
|
|
||||||
python3 scripts/lib/sprint-status-updater.py --epic {{epic_num}} --mode fix
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Add validation note to story file Dev Agent Record</action>
|
|
||||||
|
|
||||||
<output>✅ Updated {{story_id}}: {{current_status}} → {{recommended_status}}</output>
|
|
||||||
</check>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="{{recommended_status}} == {{current_status}}">
|
|
||||||
<output>✅ Story status is accurate - no changes needed</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
name: validate-story-deep
|
|
||||||
description: "Deep story validation using Haiku agents to read and verify actual code. Each task gets micro code review to verify implementation quality."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-story-deep"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Input variables
|
|
||||||
variables:
|
|
||||||
story_file: "" # Path to story file to validate
|
|
||||||
|
|
||||||
# Agent configuration
|
|
||||||
agent_model: "haiku" # Use Haiku 4.5 for cost efficiency ($0.13/story vs $1.50)
|
|
||||||
parallel_tasks: true # Validate tasks in parallel (faster)
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.validation-{story_id}-{date}.md"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,395 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This performs DEEP validation - not just checkbox counting, but verifying code actually exists and works</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Load and parse story file">
|
|
||||||
<action>Load story file from {{story_file}}</action>
|
|
||||||
|
|
||||||
<check if="file not found">
|
|
||||||
<output>❌ Story file not found: {{story_file}}
|
|
||||||
|
|
||||||
Please provide a valid story file path.
|
|
||||||
</output>
|
|
||||||
<action>HALT</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Extract story metadata:
|
|
||||||
- Story ID (from filename)
|
|
||||||
- Epic number
|
|
||||||
- Current status from Status: field
|
|
||||||
- Priority
|
|
||||||
- Estimated effort
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Extract all tasks:
|
|
||||||
- Pattern: "- [ ]" or "- [x]"
|
|
||||||
- Count total tasks
|
|
||||||
- Count checked tasks
|
|
||||||
- Count unchecked tasks
|
|
||||||
- Calculate completion percentage
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Extract file references from Dev Agent Record:
|
|
||||||
- Files created
|
|
||||||
- Files modified
|
|
||||||
- Files deleted
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>📋 **Story Validation: {{story_id}}**
|
|
||||||
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Tasks:** {{checked_count}}/{{total_count}} complete ({{completion_pct}}%)
|
|
||||||
**Files Referenced:** {{file_count}}
|
|
||||||
|
|
||||||
Starting deep validation...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Task-based verification (Deep)">
|
|
||||||
<critical>Use task-verification-engine.py for DEEP verification (not just file existence)</critical>
|
|
||||||
|
|
||||||
<action>For each task in story:
|
|
||||||
1. Extract task text
|
|
||||||
2. Note if checked [x] or unchecked [ ]
|
|
||||||
3. Pass to task-verification-engine.py
|
|
||||||
4. Receive verification result with:
|
|
||||||
- should_be_checked: true/false
|
|
||||||
- confidence: very high/high/medium/low
|
|
||||||
- evidence: list of findings
|
|
||||||
- verification_status: correct/false_positive/false_negative/uncertain
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Categorize tasks by verification status:
|
|
||||||
- ✅ CORRECT: Checkbox matches reality
|
|
||||||
- ❌ FALSE POSITIVE: Checked but code missing/stubbed
|
|
||||||
- ⚠️ FALSE NEGATIVE: Unchecked but code exists
|
|
||||||
- ❓ UNCERTAIN: Cannot verify (low confidence)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Calculate verification score:
|
|
||||||
- (correct_tasks / total_tasks) × 100
|
|
||||||
- Penalize false positives heavily (-5 points each)
|
|
||||||
- Penalize false negatives lightly (-2 points each)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
🔍 **Task Verification Results**
|
|
||||||
|
|
||||||
**Total Tasks:** {{total_count}}
|
|
||||||
|
|
||||||
**✅ CORRECT:** {{correct_count}} tasks (checkbox matches reality)
|
|
||||||
**❌ FALSE POSITIVES:** {{false_positive_count}} tasks (checked but code missing/stubbed)
|
|
||||||
**⚠️ FALSE NEGATIVES:** {{false_negative_count}} tasks (unchecked but code exists)
|
|
||||||
**❓ UNCERTAIN:** {{uncertain_count}} tasks (cannot verify)
|
|
||||||
|
|
||||||
**Verification Score:** {{verification_score}}/100
|
|
||||||
|
|
||||||
{{#if false_positive_count > 0}}
|
|
||||||
### ❌ False Positives (CRITICAL - Code Claims vs Reality)
|
|
||||||
|
|
||||||
{{#each false_positives}}
|
|
||||||
**Task:** {{this.task}}
|
|
||||||
**Claimed:** [x] Complete
|
|
||||||
**Reality:** {{this.evidence}}
|
|
||||||
**Action Required:** {{this.recommended_action}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if false_negative_count > 0}}
|
|
||||||
### ⚠️ False Negatives (Unchecked but Working)
|
|
||||||
|
|
||||||
{{#each false_negatives}}
|
|
||||||
**Task:** {{this.task}}
|
|
||||||
**Status:** [ ] Unchecked
|
|
||||||
**Reality:** {{this.evidence}}
|
|
||||||
**Recommendation:** Mark as complete [x]
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Code quality review" if="{{validation_depth}} == deep OR comprehensive">
|
|
||||||
<action>Extract all files from Dev Agent Record file list</action>
|
|
||||||
|
|
||||||
<check if="no files listed">
|
|
||||||
<output>⚠️ No files listed in Dev Agent Record - cannot perform code review</output>
|
|
||||||
<action>Skip to step 4</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>For each file:
|
|
||||||
1. Check if file exists
|
|
||||||
2. Read file content
|
|
||||||
3. Check for quality issues:
|
|
||||||
- TODO/FIXME comments without GitHub issues
|
|
||||||
- any types in TypeScript
|
|
||||||
- Hardcoded values (siteId, dealerId, API keys)
|
|
||||||
- Missing error handling
|
|
||||||
- Missing multi-tenant isolation (dealerId filters)
|
|
||||||
- Missing audit logging on mutations
|
|
||||||
- Security vulnerabilities (SQL injection, XSS)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Run multi-agent review if files exist:
|
|
||||||
- Security audit
|
|
||||||
- Silent failure detection
|
|
||||||
- Architecture compliance
|
|
||||||
- Performance analysis
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Categorize issues by severity:
|
|
||||||
- CRITICAL: Security, data loss, breaking changes
|
|
||||||
- HIGH: Missing features, poor quality, technical debt
|
|
||||||
- MEDIUM: Code smells, minor violations
|
|
||||||
- LOW: Style issues, nice-to-haves
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
🛡️ **Code Quality Review**
|
|
||||||
|
|
||||||
**Files Reviewed:** {{files_reviewed}}
|
|
||||||
**Files Missing:** {{files_missing}}
|
|
||||||
|
|
||||||
**Issues Found:** {{total_issues}}
|
|
||||||
CRITICAL: {{critical_count}}
|
|
||||||
HIGH: {{high_count}}
|
|
||||||
MEDIUM: {{medium_count}}
|
|
||||||
LOW: {{low_count}}
|
|
||||||
|
|
||||||
{{#if critical_count > 0}}
|
|
||||||
### 🚨 CRITICAL Issues (Must Fix)
|
|
||||||
|
|
||||||
{{#each critical_issues}}
|
|
||||||
**File:** {{this.file}}
|
|
||||||
**Issue:** {{this.description}}
|
|
||||||
**Impact:** {{this.impact}}
|
|
||||||
**Fix:** {{this.recommended_fix}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if high_count > 0}}
|
|
||||||
### ⚠️ HIGH Priority Issues
|
|
||||||
|
|
||||||
{{#each high_issues}}
|
|
||||||
**File:** {{this.file}}
|
|
||||||
**Issue:** {{this.description}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Code Quality Score:** {{quality_score}}/100
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Integration verification" if="{{validation_depth}} == comprehensive">
|
|
||||||
<action>Extract dependencies from story:
|
|
||||||
- Services called
|
|
||||||
- APIs consumed
|
|
||||||
- Database tables used
|
|
||||||
- Cache keys accessed
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>For each dependency:
|
|
||||||
1. Check if dependency still exists
|
|
||||||
2. Check if API contract is still valid
|
|
||||||
3. Run integration tests if they exist
|
|
||||||
4. Check for breaking changes in dependent stories
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
🔗 **Integration Verification**
|
|
||||||
|
|
||||||
**Dependencies Checked:** {{dependency_count}}
|
|
||||||
|
|
||||||
{{#if broken_integrations}}
|
|
||||||
### ❌ Broken Integrations
|
|
||||||
|
|
||||||
{{#each broken_integrations}}
|
|
||||||
**Dependency:** {{this.name}}
|
|
||||||
**Issue:** {{this.problem}}
|
|
||||||
**Likely Cause:** {{this.cause}}
|
|
||||||
**Fix:** {{this.fix}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if all_integrations_ok}}
|
|
||||||
✅ All integrations verified working
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="5" goal="Determine final story status">
|
|
||||||
<action>Calculate overall story health:
|
|
||||||
- Task verification score (0-100)
|
|
||||||
- Code quality score (0-100)
|
|
||||||
- Integration score (0-100)
|
|
||||||
- Overall score = weighted average
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Determine recommended status:
|
|
||||||
IF verification_score >= 95 AND quality_score >= 90 AND no CRITICAL issues
|
|
||||||
→ VERIFIED_COMPLETE
|
|
||||||
ELSE IF verification_score >= 80 AND quality_score >= 70
|
|
||||||
→ COMPLETE_WITH_ISSUES (document issues)
|
|
||||||
ELSE IF false_positives > 0 OR critical_issues > 0
|
|
||||||
→ NEEDS_REWORK (code missing or broken)
|
|
||||||
ELSE IF verification_score < 50
|
|
||||||
→ FALSE_POSITIVE (claimed done but not implemented)
|
|
||||||
ELSE
|
|
||||||
→ IN_PROGRESS (partially complete)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
📊 **FINAL VERDICT**
|
|
||||||
|
|
||||||
**Story:** {{story_id}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Recommended Status:** {{recommended_status}}
|
|
||||||
|
|
||||||
**Scores:**
|
|
||||||
Task Verification: {{verification_score}}/100
|
|
||||||
Code Quality: {{quality_score}}/100
|
|
||||||
Integration: {{integration_score}}/100
|
|
||||||
**Overall: {{overall_score}}/100**
|
|
||||||
|
|
||||||
**Confidence:** {{confidence_level}}
|
|
||||||
|
|
||||||
{{#if recommended_status != current_status}}
|
|
||||||
### ⚠️ Status Change Recommended
|
|
||||||
|
|
||||||
**Current:** {{current_status}}
|
|
||||||
**Should Be:** {{recommended_status}}
|
|
||||||
|
|
||||||
**Reason:**
|
|
||||||
{{status_change_reason}}
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="6" goal="Generate actionable report">
|
|
||||||
<template-output>
|
|
||||||
# Story Validation Report: {{story_id}}
|
|
||||||
|
|
||||||
**Validation Date:** {{date}}
|
|
||||||
**Validation Depth:** {{validation_depth}}
|
|
||||||
**Overall Score:** {{overall_score}}/100
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
**Story:** {{story_id}} - {{story_title}}
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Recommended Status:** {{recommended_status}}
|
|
||||||
|
|
||||||
**Task Completion:** {{checked_count}}/{{total_count}} ({{completion_pct}}%)
|
|
||||||
**Verification Score:** {{verification_score}}/100
|
|
||||||
**Code Quality Score:** {{quality_score}}/100
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task Verification Details
|
|
||||||
|
|
||||||
{{task_verification_output}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Code Quality Review
|
|
||||||
|
|
||||||
{{code_quality_output}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Integration Verification
|
|
||||||
|
|
||||||
{{integration_output}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Recommended Actions
|
|
||||||
|
|
||||||
{{#if critical_issues}}
|
|
||||||
### Priority 1: Fix Critical Issues (BLOCKING)
|
|
||||||
{{#each critical_issues}}
|
|
||||||
- [ ] {{this.file}}: {{this.description}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if false_positives}}
|
|
||||||
### Priority 2: Fix False Positives (Code Claims vs Reality)
|
|
||||||
{{#each false_positives}}
|
|
||||||
- [ ] {{this.task}} - {{this.evidence}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if high_issues}}
|
|
||||||
### Priority 3: Address High Priority Issues
|
|
||||||
{{#each high_issues}}
|
|
||||||
- [ ] {{this.file}}: {{this.description}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if false_negatives}}
|
|
||||||
### Priority 4: Update Task Checkboxes (Low Impact)
|
|
||||||
{{#each false_negatives}}
|
|
||||||
- [ ] Mark complete: {{this.task}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Next Steps
|
|
||||||
|
|
||||||
{{#if recommended_status == "VERIFIED_COMPLETE"}}
|
|
||||||
✅ **Story is verified complete and production-ready**
|
|
||||||
- Update sprint-status.yaml: {{story_id}} = done
|
|
||||||
- No further action required
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if recommended_status == "NEEDS_REWORK"}}
|
|
||||||
⚠️ **Story requires rework before marking complete**
|
|
||||||
- Fix {{critical_count}} CRITICAL issues
|
|
||||||
- Address {{false_positive_count}} false positive tasks
|
|
||||||
- Re-run validation after fixes
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if recommended_status == "FALSE_POSITIVE"}}
|
|
||||||
❌ **Story is marked done but not actually implemented**
|
|
||||||
- Verification score: {{verification_score}}/100 (< 50%)
|
|
||||||
- Update sprint-status.yaml: {{story_id}} = in-progress or ready-for-dev
|
|
||||||
- Implement missing tasks before claiming done
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Generated by:** /validate-story workflow
|
|
||||||
**Validation Engine:** task-verification-engine.py v2.0
|
|
||||||
</template-output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="7" goal="Update story file and sprint-status">
|
|
||||||
<ask>Apply recommended status change to sprint-status.yaml? (y/n)</ask>
|
|
||||||
|
|
||||||
<check if="user says yes">
|
|
||||||
<action>Update sprint-status.yaml:
|
|
||||||
- Use sprint-status-updater.py
|
|
||||||
- Update {{story_id}} to {{recommended_status}}
|
|
||||||
- Add comment: "Validated {{date}}, score {{overall_score}}/100"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Update story file:
|
|
||||||
- Add validation report link to Dev Agent Record
|
|
||||||
- Add validation score to completion notes
|
|
||||||
- Update Status: field if changed
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>✅ Updated {{story_id}} status: {{current_status}} → {{recommended_status}}</output>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="user says no">
|
|
||||||
<output>ℹ️ Status not updated. Validation report saved for reference.</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
name: validate-story
|
|
||||||
description: "Deep validation of a single story: verify tasks against codebase, run code quality review, check for regressions. Produces verification report with actionable findings."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-story"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Input variables
|
|
||||||
variables:
|
|
||||||
story_file: "" # Path to story file (e.g., docs/sprint-artifacts/16e-6-ecs-task-definitions-tier3.md)
|
|
||||||
validation_depth: "deep" # Options: "quick" (tasks only), "deep" (tasks + code review), "comprehensive" (tasks + review + integration tests)
|
|
||||||
|
|
||||||
# Tools
|
|
||||||
task_verification_script: "{project-root}/scripts/lib/task-verification-engine.py"
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.validation-{story_id}-{date}.md"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,158 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This validates EVERY epic in the project - comprehensive health check</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Discover all epics">
|
|
||||||
<action>Load {{sprint_status_file}}</action>
|
|
||||||
|
|
||||||
<check if="file not found">
|
|
||||||
<output>❌ sprint-status.yaml not found
|
|
||||||
|
|
||||||
Run /bmad:bmm:workflows:sprint-planning first.
|
|
||||||
</output>
|
|
||||||
<action>HALT</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Parse development_status section</action>
|
|
||||||
<action>Extract all epic keys (entries starting with "epic-")</action>
|
|
||||||
<action>Filter out retrospectives (ending with "-retrospective")</action>
|
|
||||||
<action>Store as {{epic_list}}</action>
|
|
||||||
|
|
||||||
<output>🔍 **Comprehensive Epic Validation**
|
|
||||||
|
|
||||||
Found {{epic_count}} epics to validate:
|
|
||||||
{{#each epic_list}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
Starting validation...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Validate each epic">
|
|
||||||
<critical>Run validate-epic-status for EACH epic</critical>
|
|
||||||
|
|
||||||
<action>Initialize counters:
|
|
||||||
- total_stories_scanned = 0
|
|
||||||
- total_valid_stories = 0
|
|
||||||
- total_invalid_stories = 0
|
|
||||||
- total_updates_applied = 0
|
|
||||||
- epics_validated = []
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<loop foreach="{{epic_list}}">
|
|
||||||
<action>Set {{current_epic}} = current loop item</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Validating {{current_epic}}...
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<!-- Use Python script for validation logic -->
|
|
||||||
<action>Execute validation script:
|
|
||||||
python3 scripts/lib/sprint-status-updater.py --epic {{current_epic}} --mode validate
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Parse script output:
|
|
||||||
- Story count
|
|
||||||
- Valid/invalid/missing counts
|
|
||||||
- Inferred statuses
|
|
||||||
- Updates needed
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<check if="{{validation_mode}} == fix">
|
|
||||||
<action>Execute fix script:
|
|
||||||
python3 scripts/lib/sprint-status-updater.py --epic {{current_epic}} --mode fix
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Count updates applied</action>
|
|
||||||
<action>Add to total_updates_applied</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Store validation results for {{current_epic}}</action>
|
|
||||||
<action>Increment totals</action>
|
|
||||||
|
|
||||||
<output>✓ {{current_epic}}: {{story_count}} stories, {{valid_count}} valid, {{updates_applied}} updates
|
|
||||||
</output>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
All Epics Validated
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Consolidate and report">
|
|
||||||
<output>
|
|
||||||
📊 **COMPREHENSIVE VALIDATION RESULTS**
|
|
||||||
|
|
||||||
**Epics Validated:** {{epic_count}}
|
|
||||||
|
|
||||||
**Stories Analyzed:** {{total_stories_scanned}}
|
|
||||||
Valid: {{total_valid_stories}} (>=10KB, >=5 tasks)
|
|
||||||
Invalid: {{total_invalid_stories}} (<10KB or <5 tasks)
|
|
||||||
Missing: {{total_missing_files}}
|
|
||||||
|
|
||||||
**Updates Applied:** {{total_updates_applied}}
|
|
||||||
|
|
||||||
**Epic Status Summary:**
|
|
||||||
{{#each_epic_with_status}}
|
|
||||||
{{epic_key}}: {{status}} ({{done_count}}/{{total_count}} done)
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Top Issues:**
|
|
||||||
{{#if_invalid_stories_exist}}
|
|
||||||
⚠️ {{total_invalid_stories}} stories need regeneration (/create-story)
|
|
||||||
{{/if}}
|
|
||||||
{{#if_missing_files_exist}}
|
|
||||||
⚠️ {{total_missing_files}} story files missing (create or remove from sprint-status.yaml)
|
|
||||||
{{/if}}
|
|
||||||
{{#if_conflicting_evidence}}
|
|
||||||
⚠️ {{conflict_count}} stories have conflicting evidence (manual review)
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Health Score:** {{health_score}}/100
|
|
||||||
(100 = perfect, all stories valid with correct status)
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<action>Write comprehensive report to {{default_output_file}}</action>
|
|
||||||
|
|
||||||
<output>💾 Full report: {{default_output_file}}</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Provide actionable recommendations">
|
|
||||||
<output>
|
|
||||||
🎯 **RECOMMENDED ACTIONS**
|
|
||||||
|
|
||||||
{{#if_health_score_lt_80}}
|
|
||||||
**Priority 1: Fix Invalid Stories ({{total_invalid_stories}})**
|
|
||||||
{{#each_invalid_story}}
|
|
||||||
/create-story-with-gap-analysis # Regenerate {{story_id}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if_missing_files_gt_0}}
|
|
||||||
**Priority 2: Create Missing Story Files ({{total_missing_files}})**
|
|
||||||
{{#each_missing}}
|
|
||||||
/create-story # Create {{story_id}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if_health_score_gte_80}}
|
|
||||||
✅ **Sprint status is healthy!**
|
|
||||||
|
|
||||||
Continue with normal development:
|
|
||||||
/sprint-status # Check what's next
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Maintenance:**
|
|
||||||
- Run /validate-all-epics weekly to catch drift
|
|
||||||
- After autonomous work, run validation
|
|
||||||
- Before sprint reviews, validate status accuracy
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
name: validate-all-epics
|
|
||||||
description: "Validate and fix sprint-status.yaml for ALL epics. Runs validate-epic-status on every epic in parallel, consolidates results, rebuilds accurate sprint-status.yaml."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-all-epics"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Variables
|
|
||||||
variables:
|
|
||||||
sprint_status_file: "{implementation_artifacts}/sprint-status.yaml"
|
|
||||||
validation_mode: "fix" # Options: "report-only", "fix"
|
|
||||||
parallel_validation: true # Validate epics in parallel for speed
|
|
||||||
|
|
||||||
# Sub-workflow
|
|
||||||
validate_epic_workflow: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-epic-status/workflow.yaml"
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.all-epics-validation-report.md"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,338 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This is the COMPREHENSIVE AUDIT - validates all stories using Haiku agents</critical>
|
|
||||||
<critical>Cost: ~$76 for 511 stories with Haiku (vs $793 with Sonnet)</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Discover all story files">
|
|
||||||
<action>Find all .md files in {{story_dir}}</action>
|
|
||||||
|
|
||||||
<action>Filter out meta-documents:
|
|
||||||
- Files starting with "EPIC-" (completion reports)
|
|
||||||
- Files starting with "." (progress files)
|
|
||||||
- Files containing: COMPLETION, SUMMARY, REPORT, SESSION-, REVIEW-, README, INDEX
|
|
||||||
- Files like "atdd-checklist-", "gap-analysis-", "review-"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<check if="{{epic_filter}} provided">
|
|
||||||
<action>Filter to stories matching: {{epic_filter}}-*.md</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Store as {{story_list}}</action>
|
|
||||||
<action>Count {{story_count}}</action>
|
|
||||||
|
|
||||||
<output>🔍 **Comprehensive Story Audit**
|
|
||||||
|
|
||||||
{{#if epic_filter}}**Epic Filter:** {{epic_filter}}{{else}}**Scope:** All epics{{/if}}
|
|
||||||
**Stories to Validate:** {{story_count}}
|
|
||||||
**Agent Model:** Haiku 4.5
|
|
||||||
**Batch Size:** {{batch_size}}
|
|
||||||
|
|
||||||
**Estimated Cost:** ~${{estimated_cost}} ({{story_count}} × $0.15/story)
|
|
||||||
**Estimated Time:** {{estimated_hours}} hours
|
|
||||||
|
|
||||||
Starting batch validation...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Batch validate all stories">
|
|
||||||
<action>Initialize counters:
|
|
||||||
- stories_validated = 0
|
|
||||||
- verified_complete = 0
|
|
||||||
- needs_rework = 0
|
|
||||||
- false_positives = 0
|
|
||||||
- in_progress = 0
|
|
||||||
- total_false_positive_tasks = 0
|
|
||||||
- total_critical_issues = 0
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Split {{story_list}} into batches of {{batch_size}}</action>
|
|
||||||
|
|
||||||
<loop foreach="{{batches}}">
|
|
||||||
<action>Set {{current_batch}} = current batch</action>
|
|
||||||
<action>Set {{batch_number}} = loop index + 1</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Batch {{batch_number}}/{{total_batches}} ({{batch_size}} stories)
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<!-- Validate each story in batch -->
|
|
||||||
<loop foreach="{{current_batch}}">
|
|
||||||
<action>Set {{story_file}} = current story path</action>
|
|
||||||
<action>Extract {{story_id}} from filename</action>
|
|
||||||
|
|
||||||
<output>{{stories_validated + 1}}/{{story_count}}: Validating {{story_id}}...</output>
|
|
||||||
|
|
||||||
<!-- Invoke validate-story-deep workflow -->
|
|
||||||
<invoke-workflow path="{{validate_story_workflow}}">
|
|
||||||
<input name="story_file" value="{{story_file}}" />
|
|
||||||
</invoke-workflow>
|
|
||||||
|
|
||||||
<action>Parse validation results:
|
|
||||||
- category (VERIFIED_COMPLETE, FALSE_POSITIVE, etc.)
|
|
||||||
- verification_score
|
|
||||||
- false_positive_count
|
|
||||||
- false_negative_count
|
|
||||||
- critical_issues_count
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Store results for {{story_id}}</action>
|
|
||||||
<action>Increment counters based on category</action>
|
|
||||||
|
|
||||||
<output> → {{category}} (Score: {{verification_score}}/100{{#if false_positives > 0}}, {{false_positives}} false positives{{/if}})</output>
|
|
||||||
|
|
||||||
<action>Increment stories_validated</action>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>Batch {{batch_number}} complete. {{stories_validated}}/{{story_count}} total validated.</output>
|
|
||||||
|
|
||||||
<!-- Save progress after each batch -->
|
|
||||||
<action>Write progress to {{progress_file}}:
|
|
||||||
- stories_validated
|
|
||||||
- current_batch
|
|
||||||
- results_so_far
|
|
||||||
</action>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
All Stories Validated
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**Total Validated:** {{story_count}}
|
|
||||||
**Total Tasks Checked:** {{total_tasks_verified}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Consolidate results and calculate platform health">
|
|
||||||
<action>Calculate platform-wide metrics:
|
|
||||||
- Overall health score: (verified_complete / story_count) × 100
|
|
||||||
- False positive rate: (false_positive_stories / story_count) × 100
|
|
||||||
- Total rework estimate: false_positive_stories × 3h + needs_rework × 2h
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Group results by epic</action>
|
|
||||||
|
|
||||||
<action>Identify worst offenders (highest false positive rates)</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
📊 **PLATFORM HEALTH ASSESSMENT**
|
|
||||||
|
|
||||||
**Overall Health Score:** {{health_score}}/100
|
|
||||||
|
|
||||||
**Story Categories:**
|
|
||||||
- ✅ VERIFIED_COMPLETE: {{verified_complete}} ({{verified_complete_pct}}%)
|
|
||||||
- ⚠️ NEEDS_REWORK: {{needs_rework}} ({{needs_rework_pct}}%)
|
|
||||||
- ❌ FALSE_POSITIVES: {{false_positives}} ({{false_positives_pct}}%)
|
|
||||||
- 🔄 IN_PROGRESS: {{in_progress}} ({{in_progress_pct}}%)
|
|
||||||
|
|
||||||
**Task-Level Issues:**
|
|
||||||
- False positive tasks: {{total_false_positive_tasks}}
|
|
||||||
- CRITICAL code quality issues: {{total_critical_issues}}
|
|
||||||
|
|
||||||
**Estimated Rework:** {{total_rework_hours}} hours
|
|
||||||
|
|
||||||
**Epic Breakdown:**
|
|
||||||
{{#each epic_summary}}
|
|
||||||
- Epic {{this.epic}}: {{this.health_score}}/100 ({{this.false_positives}} false positives)
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Worst Offenders (Most False Positives):**
|
|
||||||
{{#each worst_offenders limit=10}}
|
|
||||||
- {{this.story_id}}: {{this.false_positive_count}} tasks, score {{this.score}}/100
|
|
||||||
{{/each}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Generate comprehensive audit report">
|
|
||||||
<template-output>
|
|
||||||
# Comprehensive Platform Audit Report
|
|
||||||
|
|
||||||
**Generated:** {{date}}
|
|
||||||
**Stories Validated:** {{story_count}}
|
|
||||||
**Agent Model:** Haiku 4.5
|
|
||||||
**Total Cost:** ~${{actual_cost}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Executive Summary
|
|
||||||
|
|
||||||
**Platform Health Score:** {{health_score}}/100
|
|
||||||
|
|
||||||
{{#if health_score >= 90}}
|
|
||||||
✅ **EXCELLENT** - Platform is production-ready with high confidence
|
|
||||||
{{else if health_score >= 75}}
|
|
||||||
⚠️ **GOOD** - Minor issues to address, generally solid
|
|
||||||
{{else if health_score >= 60}}
|
|
||||||
⚠️ **NEEDS WORK** - Significant rework required before production
|
|
||||||
{{else}}
|
|
||||||
❌ **CRITICAL** - Major quality issues found, not production-ready
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Key Findings:**
|
|
||||||
- {{verified_complete}} stories verified complete ({{verified_complete_pct}}%)
|
|
||||||
- {{false_positives}} stories are false positives ({{false_positives_pct}}%)
|
|
||||||
- {{total_false_positive_tasks}} tasks claimed done but not implemented
|
|
||||||
- {{total_critical_issues}} CRITICAL code quality issues found
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ❌ False Positive Stories ({{false_positives}} total)
|
|
||||||
|
|
||||||
**These stories are marked "done" but have significant missing/stubbed code:**
|
|
||||||
|
|
||||||
{{#each false_positive_stories}}
|
|
||||||
### {{this.story_id}} (Score: {{this.score}}/100)
|
|
||||||
|
|
||||||
**Current Status:** {{this.current_status}}
|
|
||||||
**Should Be:** in-progress or ready-for-dev
|
|
||||||
|
|
||||||
**Missing/Stubbed:**
|
|
||||||
{{#each this.false_positive_tasks}}
|
|
||||||
- {{this.task}}
|
|
||||||
- {{this.evidence}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Estimated Fix:** {{this.estimated_hours}}h
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total Rework:** {{false_positive_rework_hours}} hours
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⚠️ Stories Needing Rework ({{needs_rework}} total)
|
|
||||||
|
|
||||||
{{#each needs_rework_stories}}
|
|
||||||
### {{this.story_id}} (Score: {{this.score}}/100)
|
|
||||||
|
|
||||||
**Issues:**
|
|
||||||
- {{this.false_positive_count}} incomplete tasks
|
|
||||||
- {{this.critical_issues}} CRITICAL quality issues
|
|
||||||
- {{this.high_issues}} HIGH priority issues
|
|
||||||
|
|
||||||
**Top Issues:**
|
|
||||||
{{#each this.top_issues limit=5}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total Rework:** {{needs_rework_hours}} hours
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ Verified Complete Stories ({{verified_complete}} total)
|
|
||||||
|
|
||||||
**These stories are production-ready with verified code:**
|
|
||||||
|
|
||||||
{{#each verified_complete_stories}}
|
|
||||||
- {{this.story_id}} ({{this.score}}/100)
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Epic Health Breakdown
|
|
||||||
|
|
||||||
{{#each epic_summary}}
|
|
||||||
### Epic {{this.epic}}
|
|
||||||
|
|
||||||
**Stories:** {{this.total}}
|
|
||||||
**Verified Complete:** {{this.verified}} ({{this.verified_pct}}%)
|
|
||||||
**False Positives:** {{this.false_positives}}
|
|
||||||
**Needs Rework:** {{this.needs_rework}}
|
|
||||||
|
|
||||||
**Health Score:** {{this.health_score}}/100
|
|
||||||
|
|
||||||
{{#if this.health_score < 70}}
|
|
||||||
⚠️ **ATTENTION NEEDED** - This epic has quality issues
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Top Issues:**
|
|
||||||
{{#each this.top_issues limit=3}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Recommended Action Plan
|
|
||||||
|
|
||||||
### Phase 1: Fix False Positives (CRITICAL - {{false_positive_rework_hours}}h)
|
|
||||||
|
|
||||||
{{#each false_positive_stories limit=20}}
|
|
||||||
{{@index + 1}}. **{{this.story_id}}** ({{this.estimated_hours}}h)
|
|
||||||
- {{this.false_positive_count}} tasks to implement
|
|
||||||
- Update status to in-progress
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
{{#if false_positives > 20}}
|
|
||||||
... and {{false_positives - 20}} more (see full list above)
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
### Phase 2: Address Rework Items (HIGH - {{needs_rework_hours}}h)
|
|
||||||
|
|
||||||
{{#each needs_rework_stories limit=10}}
|
|
||||||
{{@index + 1}}. **{{this.story_id}}** ({{this.estimated_hours}}h)
|
|
||||||
- Fix {{this.critical_issues}} CRITICAL issues
|
|
||||||
- Complete {{this.false_positive_count}} tasks
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
### Phase 3: Fix False Negatives (LOW - batch update)
|
|
||||||
|
|
||||||
- {{total_false_negative_tasks}} unchecked tasks that are actually complete
|
|
||||||
- Can batch update checkboxes (low priority)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 💰 Audit Cost Analysis
|
|
||||||
|
|
||||||
**This Validation Run:**
|
|
||||||
- Stories validated: {{story_count}}
|
|
||||||
- Agent sessions: {{story_count}} (one Haiku agent per story)
|
|
||||||
- Tokens used: ~{{tokens_used_millions}}M
|
|
||||||
- Cost: ~${{actual_cost}}
|
|
||||||
|
|
||||||
**Remediation Cost:**
|
|
||||||
- Estimated hours: {{total_rework_hours}}h
|
|
||||||
- At AI velocity: {{ai_velocity_days}} days of work
|
|
||||||
- Token cost: ~${{remediation_token_cost}}
|
|
||||||
|
|
||||||
**Total Investment:** ${{actual_cost}} (audit) + ${{remediation_token_cost}} (fixes) = ${{total_cost}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📅 Next Steps
|
|
||||||
|
|
||||||
1. **Immediate:** Fix {{false_positives}} false positive stories
|
|
||||||
2. **This Week:** Address {{total_critical_issues}} CRITICAL issues
|
|
||||||
3. **Next Week:** Rework {{needs_rework}} stories
|
|
||||||
4. **Ongoing:** Re-validate fixed stories to confirm
|
|
||||||
|
|
||||||
**Commands:**
|
|
||||||
```bash
|
|
||||||
# Validate specific story
|
|
||||||
/validate-story-deep docs/sprint-artifacts/16e-6-ecs-task-definitions-tier3.md
|
|
||||||
|
|
||||||
# Validate specific epic
|
|
||||||
/validate-all-stories-deep --epic 16e
|
|
||||||
|
|
||||||
# Re-run full audit (after fixes)
|
|
||||||
/validate-all-stories-deep
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Report Generated By:** validate-all-stories-deep workflow
|
|
||||||
**Validation Method:** LLM-powered (Haiku 4.5 agents read actual code)
|
|
||||||
**Confidence Level:** Very High (code-based verification, not regex patterns)
|
|
||||||
</template-output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,36 +0,0 @@
|
||||||
name: validate-all-stories-deep
|
|
||||||
description: "Comprehensive platform audit using Haiku agents. Validates ALL stories by reading actual code. The bulletproof validation for production readiness."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-all-stories-deep"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Input variables
|
|
||||||
variables:
|
|
||||||
epic_filter: "" # Optional: Only validate specific epic (e.g., "16e")
|
|
||||||
batch_size: 5 # Validate 5 stories at a time (prevents spawning 511 agents at once!)
|
|
||||||
concurrent_limit: 5 # Max 5 agents running concurrently
|
|
||||||
auto_fix: false # If true, auto-update statuses based on validation
|
|
||||||
pause_between_batches: 30 # Seconds to wait between batches (rate limiting)
|
|
||||||
|
|
||||||
# Sub-workflow
|
|
||||||
validate_story_workflow: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-story-deep/workflow.yaml"
|
|
||||||
|
|
||||||
# Agent configuration
|
|
||||||
agent_model: "haiku" # Cost: ~$66 for 511 stories vs $793 with Sonnet
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.comprehensive-audit-{date}.md"
|
|
||||||
progress_file: "{story_dir}/.validation-progress-{date}.yaml"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,411 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This is the COMPREHENSIVE AUDIT - validates every story's tasks against actual codebase</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Discover and categorize stories">
|
|
||||||
<action>Find all story files in {{story_dir}}</action>
|
|
||||||
<action>Filter out meta-documents:
|
|
||||||
- Files starting with "EPIC-" (completion reports)
|
|
||||||
- Files with "COMPLETION", "SUMMARY", "REPORT" in name
|
|
||||||
- Files starting with "." (hidden progress files)
|
|
||||||
- Files like "README", "INDEX", "SESSION-", "REVIEW-"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<check if="{{epic_filter}} provided">
|
|
||||||
<action>Filter to stories starting with {{epic_filter}}- (e.g., "16e-")</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Store as {{story_list}}</action>
|
|
||||||
<action>Count {{story_count}}</action>
|
|
||||||
|
|
||||||
<output>🔍 **Comprehensive Story Validation**
|
|
||||||
|
|
||||||
{{#if epic_filter}}
|
|
||||||
**Epic Filter:** {{epic_filter}} only
|
|
||||||
{{/if}}
|
|
||||||
**Stories to Validate:** {{story_count}}
|
|
||||||
**Validation Depth:** {{validation_depth}}
|
|
||||||
**Parallel Mode:** {{parallel_validation}}
|
|
||||||
|
|
||||||
**Estimated Time:** {{estimated_minutes}} minutes
|
|
||||||
**Estimated Cost:** ~${{estimated_cost}} ({{story_count}} × ~$0.50/story)
|
|
||||||
|
|
||||||
This will:
|
|
||||||
1. Verify all tasks against actual codebase (task-verification-engine.py)
|
|
||||||
2. Run code quality reviews on files with issues
|
|
||||||
3. Check for regressions and integration failures
|
|
||||||
4. Categorize stories: VERIFIED_COMPLETE, NEEDS_REWORK, FALSE_POSITIVE, etc.
|
|
||||||
5. Generate comprehensive audit report
|
|
||||||
|
|
||||||
Starting validation...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Run task verification on all stories">
|
|
||||||
<action>Initialize counters:
|
|
||||||
- stories_validated = 0
|
|
||||||
- verified_complete = 0
|
|
||||||
- needs_rework = 0
|
|
||||||
- false_positives = 0
|
|
||||||
- in_progress = 0
|
|
||||||
- total_false_positive_tasks = 0
|
|
||||||
- total_tasks_verified = 0
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<loop foreach="{{story_list}}">
|
|
||||||
<action>Set {{current_story}} = current story file</action>
|
|
||||||
<action>Extract {{story_id}} from filename</action>
|
|
||||||
|
|
||||||
<output>Validating {{counter}}/{{story_count}}: {{story_id}}...</output>
|
|
||||||
|
|
||||||
<!-- Run task verification engine -->
|
|
||||||
<action>Execute: python3 {{task_verification_script}} {{current_story}}</action>
|
|
||||||
|
|
||||||
<action>Parse output:
|
|
||||||
- total_tasks
|
|
||||||
- checked_tasks
|
|
||||||
- false_positives
|
|
||||||
- false_negatives
|
|
||||||
- verification_score
|
|
||||||
- task_details (with evidence)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Categorize story:
|
|
||||||
IF verification_score >= 95 AND false_positives == 0
|
|
||||||
→ category = "VERIFIED_COMPLETE"
|
|
||||||
ELSE IF verification_score >= 80 AND false_positives <= 2
|
|
||||||
→ category = "COMPLETE_WITH_MINOR_ISSUES"
|
|
||||||
ELSE IF false_positives > 5 OR verification_score < 50
|
|
||||||
→ category = "FALSE_POSITIVE" (claimed done but missing code)
|
|
||||||
ELSE IF verification_score < 80
|
|
||||||
→ category = "NEEDS_REWORK"
|
|
||||||
ELSE IF checked_tasks == 0
|
|
||||||
→ category = "NOT_STARTED"
|
|
||||||
ELSE
|
|
||||||
→ category = "IN_PROGRESS"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Store result:
|
|
||||||
- story_id
|
|
||||||
- verification_score
|
|
||||||
- category
|
|
||||||
- false_positive_count
|
|
||||||
- false_negative_count
|
|
||||||
- current_status (from sprint-status.yaml)
|
|
||||||
- recommended_status
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Increment counters based on category</action>
|
|
||||||
<action>Add false_positive_count to total</action>
|
|
||||||
<action>Add total_tasks to total_tasks_verified</action>
|
|
||||||
|
|
||||||
<output> → {{category}} ({{verification_score}}/100, {{false_positives}} false positives)</output>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Validation Complete
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**Stories Validated:** {{story_count}}
|
|
||||||
**Total Tasks Verified:** {{total_tasks_verified}}
|
|
||||||
**Total False Positives:** {{total_false_positive_tasks}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Code quality review on problem stories" if="{{validation_depth}} == deep OR comprehensive">
|
|
||||||
<action>Filter stories where:
|
|
||||||
- category = "FALSE_POSITIVE" OR
|
|
||||||
- category = "NEEDS_REWORK" OR
|
|
||||||
- false_positives > 3
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Count {{problem_story_count}}</action>
|
|
||||||
|
|
||||||
<check if="{{problem_story_count}} > 0">
|
|
||||||
<output>
|
|
||||||
🛡️ **Code Quality Review**
|
|
||||||
|
|
||||||
Found {{problem_story_count}} stories with quality issues.
|
|
||||||
Running multi-agent review on files from these stories...
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<loop foreach="{{problem_stories}}">
|
|
||||||
<action>Extract file list from story Dev Agent Record</action>
|
|
||||||
|
|
||||||
<check if="files exist">
|
|
||||||
<action>Run /multi-agent-review on files:
|
|
||||||
- Security audit
|
|
||||||
- Silent failure detection
|
|
||||||
- Architecture compliance
|
|
||||||
- Type safety check
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Categorize review findings by severity</action>
|
|
||||||
<action>Add to story's issue list</action>
|
|
||||||
</check>
|
|
||||||
</loop>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="{{problem_story_count}} == 0">
|
|
||||||
<output>✅ No problem stories found - all code quality looks good!</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Integration verification" if="{{validation_depth}} == comprehensive">
|
|
||||||
<output>
|
|
||||||
🔗 **Integration Verification**
|
|
||||||
|
|
||||||
Checking for regressions and broken dependencies...
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<action>For stories marked "VERIFIED_COMPLETE":
|
|
||||||
1. Extract service dependencies from story
|
|
||||||
2. Check if dependent services still exist
|
|
||||||
3. Run integration tests if they exist
|
|
||||||
4. Check for API contract breaking changes
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Detect overlaps:
|
|
||||||
- Multiple stories implementing same feature
|
|
||||||
- Duplicate files created
|
|
||||||
- Conflicting implementations
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
**Regressions Found:** {{regression_count}}
|
|
||||||
**Overlaps Detected:** {{overlap_count}}
|
|
||||||
**Integration Tests:** {{integration_tests_run}} ({{integration_tests_passing}} passing)
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="5" goal="Generate comprehensive report">
|
|
||||||
<template-output>
|
|
||||||
# Comprehensive Story Validation Report
|
|
||||||
|
|
||||||
**Generated:** {{date}}
|
|
||||||
**Stories Validated:** {{story_count}}
|
|
||||||
**Validation Depth:** {{validation_depth}}
|
|
||||||
**Epic Filter:** {{epic_filter}} {{#if_no_filter}}(all epics){{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Executive Summary
|
|
||||||
|
|
||||||
**Overall Health Score:** {{overall_health_score}}/100
|
|
||||||
|
|
||||||
**Story Categories:**
|
|
||||||
- ✅ **VERIFIED_COMPLETE:** {{verified_complete}} ({{verified_complete_pct}}%)
|
|
||||||
- ⚠️ **NEEDS_REWORK:** {{needs_rework}} ({{needs_rework_pct}}%)
|
|
||||||
- ❌ **FALSE_POSITIVES:** {{false_positives}} ({{false_positives_pct}}%)
|
|
||||||
- 🔄 **IN_PROGRESS:** {{in_progress}} ({{in_progress_pct}}%)
|
|
||||||
- 📋 **NOT_STARTED:** {{not_started}} ({{not_started_pct}}%)
|
|
||||||
|
|
||||||
**Task Verification:**
|
|
||||||
- Total tasks verified: {{total_tasks_verified}}
|
|
||||||
- False positive tasks: {{total_false_positive_tasks}} ({{false_positive_rate}}%)
|
|
||||||
- False negative tasks: {{total_false_negative_tasks}}
|
|
||||||
|
|
||||||
**Code Quality:**
|
|
||||||
- CRITICAL issues: {{critical_issues_total}}
|
|
||||||
- HIGH issues: {{high_issues_total}}
|
|
||||||
- Files reviewed: {{files_reviewed}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ❌ False Positive Stories (Claimed Done, Not Implemented)
|
|
||||||
|
|
||||||
{{#each false_positive_stories}}
|
|
||||||
### {{this.story_id}} (Score: {{this.verification_score}}/100)
|
|
||||||
|
|
||||||
**Current Status:** {{this.current_status}}
|
|
||||||
**Recommended:** in-progress or ready-for-dev
|
|
||||||
|
|
||||||
**Issues:**
|
|
||||||
{{#each this.false_positive_tasks}}
|
|
||||||
- [ ] {{this.task}}
|
|
||||||
- Evidence: {{this.evidence}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Action Required:**
|
|
||||||
- Uncheck {{this.false_positive_count}} tasks
|
|
||||||
- Implement missing code
|
|
||||||
- Update sprint-status.yaml to in-progress
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total:** {{false_positive_stories_count}} stories
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⚠️ Stories Needing Rework
|
|
||||||
|
|
||||||
{{#each needs_rework_stories}}
|
|
||||||
### {{this.story_id}} (Score: {{this.verification_score}}/100)
|
|
||||||
|
|
||||||
**Issues:**
|
|
||||||
- {{this.false_positive_count}} false positive tasks
|
|
||||||
- {{this.critical_issue_count}} CRITICAL code quality issues
|
|
||||||
- {{this.high_issue_count}} HIGH priority issues
|
|
||||||
|
|
||||||
**Recommended:**
|
|
||||||
1. Fix CRITICAL issues first
|
|
||||||
2. Implement {{this.false_positive_count}} missing tasks
|
|
||||||
3. Re-run validation
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total:** {{needs_rework_count}} stories
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ Verified Complete Stories
|
|
||||||
|
|
||||||
{{#each verified_complete_stories}}
|
|
||||||
- {{this.story_id}} ({{this.verification_score}}/100)
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Total:** {{verified_complete_count}} stories (production-ready)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Epic Breakdown
|
|
||||||
|
|
||||||
{{#each epic_summary}}
|
|
||||||
### Epic {{this.epic_num}}
|
|
||||||
|
|
||||||
**Stories:** {{this.total_count}}
|
|
||||||
**Verified Complete:** {{this.verified_count}} ({{this.verified_pct}}%)
|
|
||||||
**False Positives:** {{this.false_positive_count}}
|
|
||||||
**Needs Rework:** {{this.needs_rework_count}}
|
|
||||||
|
|
||||||
**Health Score:** {{this.health_score}}/100
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Recommended Actions
|
|
||||||
|
|
||||||
### Immediate (CRITICAL)
|
|
||||||
|
|
||||||
{{#if false_positive_stories_count > 0}}
|
|
||||||
**Fix {{false_positive_stories_count}} False Positive Stories:**
|
|
||||||
|
|
||||||
{{#each false_positive_stories limit=10}}
|
|
||||||
1. {{this.story_id}}: Update status to in-progress, implement {{this.false_positive_count}} missing tasks
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
{{#if false_positive_stories_count > 10}}
|
|
||||||
... and {{false_positive_stories_count - 10}} more (see full list above)
|
|
||||||
{{/if}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
### Short-term (HIGH Priority)
|
|
||||||
|
|
||||||
{{#if needs_rework_count > 0}}
|
|
||||||
**Address {{needs_rework_count}} Stories Needing Rework:**
|
|
||||||
- Fix {{critical_issues_total}} CRITICAL code quality issues
|
|
||||||
- Implement missing tasks
|
|
||||||
- Re-validate after fixes
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
### Maintenance (MEDIUM Priority)
|
|
||||||
|
|
||||||
{{#if false_negative_count > 0}}
|
|
||||||
**Update {{false_negative_count}} False Negative Tasks:**
|
|
||||||
- Mark complete (code exists but checkbox unchecked)
|
|
||||||
- Low impact, can batch update
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 💰 Cost Analysis
|
|
||||||
|
|
||||||
**Validation Run:**
|
|
||||||
- Stories validated: {{story_count}}
|
|
||||||
- API tokens used: ~{{tokens_used}}K
|
|
||||||
- Cost: ~${{cost}}
|
|
||||||
|
|
||||||
**Remediation Estimate:**
|
|
||||||
- False positives: {{false_positive_stories_count}} × 3h = {{remediation_hours_fp}}h
|
|
||||||
- Needs rework: {{needs_rework_count}} × 2h = {{remediation_hours_rework}}h
|
|
||||||
- **Total:** {{total_remediation_hours}}h estimated work
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📅 Next Steps
|
|
||||||
|
|
||||||
1. **Fix false positive stories** ({{false_positive_stories_count}} stories)
|
|
||||||
2. **Address CRITICAL issues** ({{critical_issues_total}} issues)
|
|
||||||
3. **Re-run validation** on fixed stories
|
|
||||||
4. **Update sprint-status.yaml** with verified statuses
|
|
||||||
5. **Run weekly validation** to prevent future drift
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Generated by:** /validate-all-stories workflow
|
|
||||||
**Validation Engine:** task-verification-engine.py v2.0
|
|
||||||
**Multi-Agent Review:** {{multi_agent_review_enabled}}
|
|
||||||
</template-output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="6" goal="Auto-fix if enabled" if="{{fix_mode}} == true">
|
|
||||||
<output>
|
|
||||||
🔧 **Auto-Fix Mode Enabled**
|
|
||||||
|
|
||||||
Applying automatic fixes:
|
|
||||||
1. Update false negative checkboxes (code exists → mark [x])
|
|
||||||
2. Update sprint-status.yaml with verified statuses
|
|
||||||
3. Add validation scores to story files
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<loop foreach="{{false_negative_tasks_list}}">
|
|
||||||
<action>Update story file: Change [ ] to [x] for verified tasks</action>
|
|
||||||
<output> ✓ {{story_id}}: Checked {{task_count}} false negative tasks</output>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<loop foreach="{{status_updates_list}}">
|
|
||||||
<action>Update sprint-status.yaml using sprint-status-updater.py</action>
|
|
||||||
<output> ✓ {{story_id}}: {{old_status}} → {{new_status}}</output>
|
|
||||||
</loop>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
✅ Auto-fix complete
|
|
||||||
- {{false_negatives_fixed}} tasks checked
|
|
||||||
- {{statuses_updated}} story statuses updated
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="7" goal="Summary and recommendations">
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
COMPREHENSIVE VALIDATION COMPLETE
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**Overall Health:** {{overall_health_score}}/100
|
|
||||||
|
|
||||||
{{#if overall_health_score >= 90}}
|
|
||||||
✅ **EXCELLENT** - Platform is production-ready
|
|
||||||
{{else if overall_health_score >= 75}}
|
|
||||||
⚠️ **GOOD** - Minor issues to address before production
|
|
||||||
{{else if overall_health_score >= 60}}
|
|
||||||
⚠️ **NEEDS WORK** - Significant rework required
|
|
||||||
{{else}}
|
|
||||||
❌ **CRITICAL** - Major quality issues found
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Top Priorities:**
|
|
||||||
1. Fix {{false_positive_stories_count}} false positive stories
|
|
||||||
2. Address {{critical_issues_total}} CRITICAL code quality issues
|
|
||||||
3. Complete {{in_progress_count}} in-progress stories
|
|
||||||
4. Re-validate after fixes
|
|
||||||
|
|
||||||
**Full Report:** {{default_output_file}}
|
|
||||||
**Summary JSON:** {{validation_summary_file}}
|
|
||||||
|
|
||||||
**Next Command:**
|
|
||||||
/validate-story <story-id> # Deep-dive on specific story
|
|
||||||
/validate-all-stories --epic 16e # Re-validate specific epic
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,36 +0,0 @@
|
||||||
name: validate-all-stories
|
|
||||||
description: "Comprehensive audit of ALL stories: verify tasks against codebase, run code quality reviews, check integrations. The bulletproof audit for production readiness."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-all-stories"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Input variables
|
|
||||||
variables:
|
|
||||||
validation_depth: "deep" # Options: "quick" (tasks only), "deep" (tasks + review), "comprehensive" (full integration)
|
|
||||||
parallel_validation: true # Run story validations in parallel for speed
|
|
||||||
fix_mode: false # If true, auto-fix false negatives and update statuses
|
|
||||||
epic_filter: "" # Optional: Only validate stories from specific epic (e.g., "16e")
|
|
||||||
|
|
||||||
# Tools
|
|
||||||
task_verification_script: "{project-root}/scripts/lib/task-verification-engine.py"
|
|
||||||
sprint_status_updater: "{project-root}/scripts/lib/sprint-status-updater.py"
|
|
||||||
|
|
||||||
# Sub-workflow
|
|
||||||
validate_story_workflow: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-story/workflow.yaml"
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.comprehensive-validation-report-{date}.md"
|
|
||||||
validation_summary_file: "{story_dir}/.validation-summary-{date}.json"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,302 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This is VALIDATION-ONLY mode - NO implementation, only status correction</critical>
|
|
||||||
<critical>Uses same logic as batch-super-dev but READS instead of WRITES code</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Validate inputs and load epic">
|
|
||||||
<action>Check if {{epic_num}} was provided</action>
|
|
||||||
|
|
||||||
<check if="{{epic_num}} is empty">
|
|
||||||
<ask>Which epic should I validate? (e.g., 19, 16d, 16e, 9b)</ask>
|
|
||||||
<action>Store response as {{epic_num}}</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Load {{sprint_status_file}}</action>
|
|
||||||
|
|
||||||
<check if="file not found">
|
|
||||||
<output>❌ sprint-status.yaml not found at: {{sprint_status_file}}
|
|
||||||
|
|
||||||
Run /bmad:bmm:workflows:sprint-planning to create it first.
|
|
||||||
</output>
|
|
||||||
<action>HALT</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Search for epic-{{epic_num}} entry in sprint_status_file</action>
|
|
||||||
<action>Extract all story entries for epic-{{epic_num}} (pattern: {{epic_num}}-*)</action>
|
|
||||||
<action>Count stories found in sprint-status.yaml for this epic</action>
|
|
||||||
|
|
||||||
<output>🔍 **Validating Epic {{epic_num}}**
|
|
||||||
|
|
||||||
Found {{story_count}} stories in sprint-status.yaml
|
|
||||||
Scanning story files for REALITY check...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Scan and validate all story files">
|
|
||||||
<critical>This is where we determine TRUTH - not from status fields, but from actual file analysis</critical>
|
|
||||||
|
|
||||||
<action>For each story in epic (from sprint-status.yaml):
|
|
||||||
1. Build story file path: {{story_dir}}/{{story_key}}.md
|
|
||||||
2. Check if file exists
|
|
||||||
3. If exists, read FULL file
|
|
||||||
4. Analyze file content
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>For each story file, extract:
|
|
||||||
- File size in KB
|
|
||||||
- Total task count (count all "- [ ]" and "- [x]" lines)
|
|
||||||
- Checked task count (count "- [x]" lines)
|
|
||||||
- Completion rate (checked / total * 100)
|
|
||||||
- Explicit Status: field (if present)
|
|
||||||
- Has proper BMAD structure (12 sections)
|
|
||||||
- Section count (count ## headings)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>📊 **Story File Quality Analysis**
|
|
||||||
|
|
||||||
Analyzing {{story_count}} story files...
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<action>For each story, classify quality:
|
|
||||||
VALID:
|
|
||||||
- File size >= 10KB
|
|
||||||
- Total tasks >= 5
|
|
||||||
- Has task list structure
|
|
||||||
|
|
||||||
INVALID:
|
|
||||||
- File size < 10KB (incomplete story)
|
|
||||||
- Total tasks < 5 (not detailed enough)
|
|
||||||
- File missing entirely
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Store results as {{story_quality_map}}</action>
|
|
||||||
|
|
||||||
<output>Quality Summary:
|
|
||||||
Valid stories: {{valid_count}}/{{story_count}}
|
|
||||||
Invalid stories: {{invalid_count}}
|
|
||||||
Missing files: {{missing_count}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Cross-reference git commits">
|
|
||||||
<action>Run git log to find commits mentioning epic stories:
|
|
||||||
Command: git log --oneline --since={{git_commit_lookback_days}} days ago
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Parse commit messages for story IDs matching pattern: {{epic_num}}-\d+[a-z]?</action>
|
|
||||||
<action>Build map of story_id → commit_count</action>
|
|
||||||
|
|
||||||
<output>Git Commit Evidence:
|
|
||||||
Stories with commits: {{stories_with_commits_count}}
|
|
||||||
Stories without commits: {{stories_without_commits_count}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Check autonomous completion reports">
|
|
||||||
<action>Search {{story_dir}} for files:
|
|
||||||
- .epic-{{epic_num}}-completion-report.md
|
|
||||||
- .batch-super-dev-{{epic_num}}-progress.yaml
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<check if="autonomous report found">
|
|
||||||
<action>Parse completed_stories list from progress file OR
|
|
||||||
Parse ✅ story entries from completion report</action>
|
|
||||||
<action>Store as {{autonomous_completed_stories}}</action>
|
|
||||||
|
|
||||||
<output>📋 Autonomous Report Found:
|
|
||||||
{{autonomous_completed_count}} stories marked complete
|
|
||||||
</output>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="no autonomous report">
|
|
||||||
<output>ℹ️ No autonomous completion report found (manual epic)</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="5" goal="Infer correct status for each story">
|
|
||||||
<critical>Use MULTIPLE sources of truth, not just Status: field</critical>
|
|
||||||
|
|
||||||
<action>For each story in epic, determine correct status using this logic:</action>
|
|
||||||
|
|
||||||
<logic>
|
|
||||||
Priority 1: Autonomous completion report
|
|
||||||
IF story in autonomous_completed_stories
|
|
||||||
→ Status = "done" (VERY HIGH confidence)
|
|
||||||
|
|
||||||
Priority 2: Task completion rate + file quality
|
|
||||||
IF completion_rate >= 90% AND file is VALID (>10KB, >5 tasks)
|
|
||||||
→ Status = "done" (HIGH confidence)
|
|
||||||
|
|
||||||
IF completion_rate 50-89% AND file is VALID
|
|
||||||
→ Status = "in-progress" (MEDIUM confidence)
|
|
||||||
|
|
||||||
IF completion_rate < 50% AND file is VALID
|
|
||||||
→ Status = "ready-for-dev" (MEDIUM confidence)
|
|
||||||
|
|
||||||
Priority 3: Explicit Status: field (if no other evidence)
|
|
||||||
IF Status: field exists AND matches above inferences
|
|
||||||
→ Use it (MEDIUM confidence)
|
|
||||||
|
|
||||||
IF Status: field conflicts with task completion
|
|
||||||
→ Prefer task completion (tasks are ground truth)
|
|
||||||
|
|
||||||
Priority 4: Git commits (supporting evidence)
|
|
||||||
IF 3+ commits + task completion >=90%
|
|
||||||
→ Upgrade confidence to VERY HIGH
|
|
||||||
|
|
||||||
IF 1-2 commits but task completion <50%
|
|
||||||
→ Status = "in-progress" (work started but not done)
|
|
||||||
|
|
||||||
Quality Gates:
|
|
||||||
IF file size < 10KB OR total tasks < 5
|
|
||||||
→ DOWNGRADE status (can't be "done" if file is incomplete)
|
|
||||||
→ Mark as "ready-for-dev" (story needs proper creation)
|
|
||||||
→ Flag for regeneration with /create-story
|
|
||||||
|
|
||||||
Missing Files:
|
|
||||||
IF story file doesn't exist
|
|
||||||
→ Status = "backlog" (story not created yet)
|
|
||||||
</logic>
|
|
||||||
|
|
||||||
<action>Build map of story_id → inferred_status with evidence and confidence</action>
|
|
||||||
|
|
||||||
<output>📊 **Status Inference Complete**
|
|
||||||
|
|
||||||
Stories to update:
|
|
||||||
{{#each_story_needing_update}}
|
|
||||||
{{story_id}}:
|
|
||||||
Current: {{current_status_in_yaml}}
|
|
||||||
Inferred: {{inferred_status}}
|
|
||||||
Confidence: {{confidence}}
|
|
||||||
Evidence: {{evidence_summary}}
|
|
||||||
Quality: {{file_size_kb}}KB, {{total_tasks}} tasks, {{completion_rate}}% done
|
|
||||||
{{/each}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="6" goal="Apply updates or report findings">
|
|
||||||
<check if="{{validation_mode}} == report-only">
|
|
||||||
<output>📝 **REPORT-ONLY MODE** - No changes will be made
|
|
||||||
|
|
||||||
Recommendations saved to: {{default_output_file}}
|
|
||||||
</output>
|
|
||||||
<action>Write detailed report to {{default_output_file}}</action>
|
|
||||||
<action>EXIT workflow</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="{{validation_mode}} == fix OR {{validation_mode}} == strict">
|
|
||||||
<output>🔧 **FIX MODE** - Updating sprint-status.yaml...
|
|
||||||
|
|
||||||
Backing up to: .sprint-status-backups/
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<action>Create backup of {{sprint_status_file}}</action>
|
|
||||||
<action>For each story needing update:
|
|
||||||
1. Find story entry in development_status section
|
|
||||||
2. Update status to inferred_status
|
|
||||||
3. Add comment: "✅ Validated {{date}} - {{evidence_summary}}"
|
|
||||||
4. Preserve all other content and structure
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Update epic-{{epic_num}} status based on story completion:
|
|
||||||
IF all stories have status "done" AND all are valid files
|
|
||||||
→ epic status = "done"
|
|
||||||
|
|
||||||
IF any stories "in-progress" OR "review"
|
|
||||||
→ epic status = "in-progress"
|
|
||||||
|
|
||||||
IF all stories "backlog" OR "ready-for-dev"
|
|
||||||
→ epic status = "backlog"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Update last_verified timestamp in header</action>
|
|
||||||
<action>Save {{sprint_status_file}}</action>
|
|
||||||
|
|
||||||
<output>✅ **sprint-status.yaml Updated**
|
|
||||||
|
|
||||||
Applied {{updates_count}} story status corrections
|
|
||||||
Epic {{epic_num}}: {{old_epic_status}} → {{new_epic_status}}
|
|
||||||
|
|
||||||
Backup: {{backup_path}}
|
|
||||||
</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="7" goal="Identify problem stories requiring action">
|
|
||||||
<action>Flag stories with issues:
|
|
||||||
- Missing story files (in sprint-status.yaml but no .md file)
|
|
||||||
- Invalid files (< 10KB or < 5 tasks)
|
|
||||||
- Conflicting evidence (Status: says done, tasks unchecked)
|
|
||||||
- Poor quality (no BMAD sections)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>⚠️ **Problem Stories Requiring Attention:**
|
|
||||||
|
|
||||||
{{#if_missing_files}}
|
|
||||||
**Missing Files ({{missing_count}}):**
|
|
||||||
{{#each_missing}}
|
|
||||||
- {{story_id}}: Referenced in sprint-status.yaml but file not found
|
|
||||||
Action: Run /create-story OR remove from sprint-status.yaml
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if_invalid_quality}}
|
|
||||||
**Invalid Quality ({{invalid_count}}):**
|
|
||||||
{{#each_invalid}}
|
|
||||||
- {{story_id}}: {{file_size_kb}}KB, {{total_tasks}} tasks
|
|
||||||
Action: Regenerate with /create-story-with-gap-analysis
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if_conflicting_evidence}}
|
|
||||||
**Conflicting Evidence ({{conflict_count}}):**
|
|
||||||
{{#each_conflict}}
|
|
||||||
- {{story_id}}: Status: says "{{status_field}}" but {{completion_rate}}% tasks checked
|
|
||||||
Action: Manual review recommended
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="8" goal="Report results and recommendations">
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Epic {{epic_num}} Validation Complete
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**Epic Status:** {{epic_status}}
|
|
||||||
|
|
||||||
**Stories:**
|
|
||||||
Done: {{done_count}}
|
|
||||||
In-Progress: {{in_progress_count}}
|
|
||||||
Review: {{review_count}}
|
|
||||||
Ready-for-Dev: {{ready_count}}
|
|
||||||
Backlog: {{backlog_count}}
|
|
||||||
|
|
||||||
**Quality:**
|
|
||||||
Valid: {{valid_count}} (>=10KB, >=5 tasks)
|
|
||||||
Invalid: {{invalid_count}} (poor quality)
|
|
||||||
Missing: {{missing_count}} (file not found)
|
|
||||||
|
|
||||||
**Updates Applied:** {{updates_count}}
|
|
||||||
|
|
||||||
**Next Steps:**
|
|
||||||
{{#if_invalid_count_gt_0}}
|
|
||||||
1. Regenerate {{invalid_count}} invalid stories with /create-story
|
|
||||||
{{/if}}
|
|
||||||
{{#if_missing_count_gt_0}}
|
|
||||||
2. Create {{missing_count}} missing story files OR remove from sprint-status.yaml
|
|
||||||
{{/if}}
|
|
||||||
{{#if_done_count_eq_story_count}}
|
|
||||||
3. Epic complete! Consider running /retrospective
|
|
||||||
{{/if}}
|
|
||||||
{{#if_in_progress_count_gt_0}}
|
|
||||||
3. Continue with in-progress stories: /dev-story {{first_in_progress}}
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
|
|
||||||
<output>💾 Detailed report saved to: {{default_output_file}}</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
name: validate-epic-status
|
|
||||||
description: "Validate and fix sprint-status.yaml for a single epic. Scans story files for task completion, validates quality (>10KB, proper tasks), checks git commits, updates sprint-status.yaml to match REALITY."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-epic-status"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Inputs
|
|
||||||
variables:
|
|
||||||
epic_num: "" # User provides (e.g., "19", "16d", "16e")
|
|
||||||
sprint_status_file: "{implementation_artifacts}/sprint-status.yaml"
|
|
||||||
validation_mode: "fix" # Options: "report-only", "fix", "strict"
|
|
||||||
|
|
||||||
# Validation criteria
|
|
||||||
validation_rules:
|
|
||||||
min_story_size_kb: 10 # Stories should be >= 10KB
|
|
||||||
min_tasks_required: 5 # Stories should have >= 5 tasks
|
|
||||||
completion_threshold: 90 # 90%+ tasks checked = "done"
|
|
||||||
git_commit_lookback_days: 30 # Search last 30 days for commits
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.epic-{epic_num}-validation-report.md"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,370 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This uses HAIKU AGENTS to read actual code and verify task completion - NOT regex patterns</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Load and parse story">
|
|
||||||
<action>Load story file from {{story_file}}</action>
|
|
||||||
|
|
||||||
<check if="file not found">
|
|
||||||
<output>❌ Story file not found: {{story_file}}</output>
|
|
||||||
<action>HALT</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Extract story metadata:
|
|
||||||
- Story ID from filename
|
|
||||||
- Epic number from "Epic:" field
|
|
||||||
- Current status from "Status:" or "**Status:**" field
|
|
||||||
- Files created/modified from Dev Agent Record section
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Extract ALL tasks (pattern: "- [ ]" or "- [x]"):
|
|
||||||
- Parse checkbox state (checked/unchecked)
|
|
||||||
- Extract task text
|
|
||||||
- Count total, checked, unchecked
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>📋 **Deep Story Validation: {{story_id}}**
|
|
||||||
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Tasks:** {{checked_count}}/{{total_count}} checked
|
|
||||||
**Files Referenced:** {{file_count}}
|
|
||||||
|
|
||||||
**Validation Method:** Haiku agents read actual code
|
|
||||||
**Cost Estimate:** ~$0.13 for this story
|
|
||||||
|
|
||||||
Starting task-by-task verification...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Verify ALL tasks with single Haiku agent">
|
|
||||||
<critical>Spawn ONE Haiku agent to verify ALL tasks (avoids 50x agent startup overhead!)</critical>
|
|
||||||
|
|
||||||
<output>Spawning Haiku verification agent for {{total_count}} tasks...</output>
|
|
||||||
|
|
||||||
<!-- Spawn SINGLE Haiku agent to verify ALL tasks in this story -->
|
|
||||||
<invoke-task type="Task" model="haiku">
|
|
||||||
<description>Verify all {{total_count}} story tasks</description>
|
|
||||||
<prompt>
|
|
||||||
You are verifying ALL tasks for this user story by reading actual code.
|
|
||||||
|
|
||||||
**Story:** {{story_id}}
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Total Tasks:** {{total_count}}
|
|
||||||
|
|
||||||
**Files from Story (Dev Agent Record):**
|
|
||||||
{{#each file_list}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Tasks to Verify:**
|
|
||||||
|
|
||||||
{{#each task_list}}
|
|
||||||
{{@index}}. [{{#if this.checked}}x{{else}} {{/if}}] {{this.text}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Your Job:**
|
|
||||||
|
|
||||||
For EACH task above:
|
|
||||||
|
|
||||||
1. **Find relevant files** - Use Glob to find files mentioned in task
|
|
||||||
2. **Read the files** - Use Read tool to examine actual code
|
|
||||||
3. **Verify implementation:**
|
|
||||||
- Is code real or stubs/TODOs?
|
|
||||||
- Is there error handling?
|
|
||||||
- Multi-tenant isolation (dealerId filters)?
|
|
||||||
- Are there tests?
|
|
||||||
- Does it match task description?
|
|
||||||
|
|
||||||
4. **Make judgment for each task**
|
|
||||||
|
|
||||||
**Output Format - JSON array with one entry per task:**
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"story_id": "{{story_id}}",
|
|
||||||
"total_tasks": {{total_count}},
|
|
||||||
"tasks": [
|
|
||||||
{
|
|
||||||
"task_number": 0,
|
|
||||||
"task_text": "Implement UserService",
|
|
||||||
"is_checked": true,
|
|
||||||
"actually_complete": false,
|
|
||||||
"confidence": "high",
|
|
||||||
"evidence": "File exists but has 'TODO: Implement findById' on line 45, tests not found",
|
|
||||||
"issues_found": ["Stub implementation", "Missing tests", "No dealerId filter"],
|
|
||||||
"recommendation": "Implement real logic, add tests, add multi-tenant isolation"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"task_number": 1,
|
|
||||||
"task_text": "Add error handling",
|
|
||||||
"is_checked": true,
|
|
||||||
"actually_complete": true,
|
|
||||||
"confidence": "very_high",
|
|
||||||
"evidence": "Try-catch blocks in UserService.ts:67-89, proper error logging, tests verify error cases",
|
|
||||||
"issues_found": [],
|
|
||||||
"recommendation": "None - task complete"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Be efficient:** Read files once, verify all tasks, return comprehensive JSON.
|
|
||||||
</prompt>
|
|
||||||
<subagent_type>general-purpose</subagent_type>
|
|
||||||
</invoke-task>
|
|
||||||
|
|
||||||
<action>Parse agent response (extract JSON)</action>
|
|
||||||
|
|
||||||
<action>For each task result:
|
|
||||||
- Determine verification_status (correct/false_positive/false_negative)
|
|
||||||
- Categorize into verified_complete, false_positives, false_negatives lists
|
|
||||||
- Count totals
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
Task Verification Complete
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
**✅ Verified Complete:** {{verified_complete_count}}
|
|
||||||
**❌ False Positives:** {{false_positive_count}} (checked but code missing/poor)
|
|
||||||
**⚠️ False Negatives:** {{false_negative_count}} (unchecked but code exists)
|
|
||||||
**❓ Uncertain:** {{uncertain_count}}
|
|
||||||
|
|
||||||
**Verification Score:** {{verification_score}}/100
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Calculate overall story health">
|
|
||||||
<action>Calculate scores:
|
|
||||||
- Task accuracy: (correct / total) × 100
|
|
||||||
- False positive penalty: false_positive_count × -5
|
|
||||||
- Overall score: max(0, task_accuracy + penalty)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Determine story category:
|
|
||||||
IF score >= 95 AND false_positives == 0
|
|
||||||
→ VERIFIED_COMPLETE
|
|
||||||
ELSE IF score >= 80 AND false_positives <= 2
|
|
||||||
→ COMPLETE_WITH_MINOR_ISSUES
|
|
||||||
ELSE IF false_positives > 5 OR score < 50
|
|
||||||
→ FALSE_POSITIVE (story claimed done but significant missing code)
|
|
||||||
ELSE IF false_positives > 0
|
|
||||||
→ NEEDS_REWORK
|
|
||||||
ELSE
|
|
||||||
→ IN_PROGRESS
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Determine recommended status:
|
|
||||||
VERIFIED_COMPLETE → "done"
|
|
||||||
COMPLETE_WITH_MINOR_ISSUES → "review"
|
|
||||||
FALSE_POSITIVE → "in-progress" or "ready-for-dev"
|
|
||||||
NEEDS_REWORK → "in-progress"
|
|
||||||
IN_PROGRESS → "in-progress"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
📊 **STORY HEALTH ASSESSMENT**
|
|
||||||
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Recommended Status:** {{recommended_status}}
|
|
||||||
**Overall Score:** {{overall_score}}/100
|
|
||||||
|
|
||||||
**Category:** {{category}}
|
|
||||||
|
|
||||||
{{#if category == "VERIFIED_COMPLETE"}}
|
|
||||||
✅ **Story is production-ready**
|
|
||||||
- All tasks verified complete
|
|
||||||
- Code quality confirmed
|
|
||||||
- No significant issues found
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "FALSE_POSITIVE"}}
|
|
||||||
❌ **Story claimed done but has significant missing code**
|
|
||||||
- {{false_positive_count}} tasks checked but not implemented
|
|
||||||
- Verification score: {{overall_score}}/100 (< 50% = false positive)
|
|
||||||
- Action: Update status to in-progress, implement missing tasks
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "NEEDS_REWORK"}}
|
|
||||||
⚠️ **Story needs rework before marking complete**
|
|
||||||
- {{false_positive_count}} tasks with missing/poor code
|
|
||||||
- Issues found in verification
|
|
||||||
- Action: Fix issues, re-verify
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Generate detailed validation report">
|
|
||||||
<template-output>
|
|
||||||
# Story Validation Report: {{story_id}}
|
|
||||||
|
|
||||||
**Generated:** {{date}}
|
|
||||||
**Validation Method:** LLM-powered deep verification (Haiku 4.5)
|
|
||||||
**Overall Score:** {{overall_score}}/100
|
|
||||||
**Category:** {{category}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
**Story:** {{story_id}}
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Recommended Status:** {{recommended_status}}
|
|
||||||
|
|
||||||
**Task Verification:**
|
|
||||||
- Total: {{total_count}}
|
|
||||||
- Checked: {{checked_count}}
|
|
||||||
- Verified Complete: {{verified_complete_count}}
|
|
||||||
- False Positives: {{false_positive_count}}
|
|
||||||
- False Negatives: {{false_negative_count}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Verification Details
|
|
||||||
|
|
||||||
{{#if false_positive_count > 0}}
|
|
||||||
### ❌ False Positives (CRITICAL - Code Claims vs Reality)
|
|
||||||
|
|
||||||
{{#each false_positives}}
|
|
||||||
**Task {{@index + 1}}:** {{this.task}}
|
|
||||||
**Claimed:** [x] Complete
|
|
||||||
**Reality:** Code missing or stub implementation
|
|
||||||
|
|
||||||
**Evidence:**
|
|
||||||
{{this.evidence}}
|
|
||||||
|
|
||||||
**Issues Found:**
|
|
||||||
{{#each this.issues_found}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Recommendation:** {{this.recommendation}}
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if false_negative_count > 0}}
|
|
||||||
### ⚠️ False Negatives (Unchecked But Working)
|
|
||||||
|
|
||||||
{{#each false_negatives}}
|
|
||||||
**Task {{@index + 1}}:** {{this.task}}
|
|
||||||
**Status:** [ ] Unchecked
|
|
||||||
**Reality:** Code exists and working
|
|
||||||
|
|
||||||
**Evidence:**
|
|
||||||
{{this.evidence}}
|
|
||||||
|
|
||||||
**Recommendation:** Mark task as complete [x]
|
|
||||||
|
|
||||||
---
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if verified_complete_count > 0}}
|
|
||||||
### ✅ Verified Complete Tasks
|
|
||||||
|
|
||||||
{{verified_complete_count}} tasks verified with actual code review.
|
|
||||||
|
|
||||||
{{#if show_all_verified}}
|
|
||||||
{{#each verified_complete}}
|
|
||||||
- {{this.task}} ({{this.confidence}} confidence)
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Final Verdict
|
|
||||||
|
|
||||||
**Overall Score:** {{overall_score}}/100
|
|
||||||
|
|
||||||
{{#if category == "VERIFIED_COMPLETE"}}
|
|
||||||
✅ **VERIFIED COMPLETE**
|
|
||||||
|
|
||||||
This story is production-ready:
|
|
||||||
- All {{total_count}} tasks verified complete
|
|
||||||
- Code quality confirmed through review
|
|
||||||
- No significant issues found
|
|
||||||
- Status "done" is accurate
|
|
||||||
|
|
||||||
**Action:** None needed - story is solid
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "FALSE_POSITIVE"}}
|
|
||||||
❌ **FALSE POSITIVE - Story NOT Actually Complete**
|
|
||||||
|
|
||||||
**Problems:**
|
|
||||||
- {{false_positive_count}} tasks checked but code missing/stubbed
|
|
||||||
- Verification score: {{overall_score}}/100 (< 50%)
|
|
||||||
- Story marked "{{current_status}}" but significant work remains
|
|
||||||
|
|
||||||
**Required Actions:**
|
|
||||||
1. Update sprint-status.yaml: {{story_id}} → in-progress
|
|
||||||
2. Uncheck {{false_positive_count}} false positive tasks
|
|
||||||
3. Implement missing code
|
|
||||||
4. Re-run validation after implementation
|
|
||||||
|
|
||||||
**Estimated Rework:** {{estimated_rework_hours}} hours
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "NEEDS_REWORK"}}
|
|
||||||
⚠️ **NEEDS REWORK**
|
|
||||||
|
|
||||||
**Problems:**
|
|
||||||
- {{false_positive_count}} tasks with quality issues
|
|
||||||
- Some code exists but has problems (TODOs, missing features, poor quality)
|
|
||||||
|
|
||||||
**Required Actions:**
|
|
||||||
{{#each action_items}}
|
|
||||||
- [ ] {{this}}
|
|
||||||
{{/each}}
|
|
||||||
|
|
||||||
**Estimated Fix Time:** {{estimated_fix_hours}} hours
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if category == "IN_PROGRESS"}}
|
|
||||||
🔄 **IN PROGRESS** (accurate status)
|
|
||||||
|
|
||||||
- {{checked_count}}/{{total_count}} tasks complete
|
|
||||||
- {{unchecked_count}} tasks remaining
|
|
||||||
- Current status reflects reality
|
|
||||||
|
|
||||||
**No action needed** - continue implementation
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Validation Cost:** ~${{validation_cost}}
|
|
||||||
**Agent Model:** {{agent_model}}
|
|
||||||
**Tasks Verified:** {{total_count}}
|
|
||||||
</template-output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="5" goal="Update sprint-status if needed">
|
|
||||||
<check if="{{recommended_status}} != {{current_status}}">
|
|
||||||
<ask>Story status should be updated from "{{current_status}}" to "{{recommended_status}}". Update sprint-status.yaml? (y/n)</ask>
|
|
||||||
|
|
||||||
<check if="user says yes">
|
|
||||||
<action>Update sprint-status.yaml:
|
|
||||||
python3 scripts/lib/sprint-status-updater.py --epic {{epic_num}} --mode fix
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Add validation note to story file Dev Agent Record</action>
|
|
||||||
|
|
||||||
<output>✅ Updated {{story_id}}: {{current_status}} → {{recommended_status}}</output>
|
|
||||||
</check>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="{{recommended_status}} == {{current_status}}">
|
|
||||||
<output>✅ Story status is accurate - no changes needed</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
name: validate-story-deep
|
|
||||||
description: "Deep story validation using Haiku agents to read and verify actual code. Each task gets micro code review to verify implementation quality."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-story-deep"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Input variables
|
|
||||||
variables:
|
|
||||||
story_file: "" # Path to story file to validate
|
|
||||||
|
|
||||||
# Agent configuration
|
|
||||||
agent_model: "haiku" # Use Haiku 4.5 for cost efficiency ($0.13/story vs $1.50)
|
|
||||||
parallel_tasks: true # Validate tasks in parallel (faster)
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.validation-{story_id}-{date}.md"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
|
|
@ -1,395 +0,0 @@
|
||||||
<workflow>
|
|
||||||
<critical>The workflow execution engine is governed by: {project-root}/_bmad/core/tasks/workflow.xml</critical>
|
|
||||||
<critical>You MUST have already loaded and processed: {installed_path}/workflow.yaml</critical>
|
|
||||||
<critical>This performs DEEP validation - not just checkbox counting, but verifying code actually exists and works</critical>
|
|
||||||
|
|
||||||
<step n="1" goal="Load and parse story file">
|
|
||||||
<action>Load story file from {{story_file}}</action>
|
|
||||||
|
|
||||||
<check if="file not found">
|
|
||||||
<output>❌ Story file not found: {{story_file}}
|
|
||||||
|
|
||||||
Please provide a valid story file path.
|
|
||||||
</output>
|
|
||||||
<action>HALT</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>Extract story metadata:
|
|
||||||
- Story ID (from filename)
|
|
||||||
- Epic number
|
|
||||||
- Current status from Status: field
|
|
||||||
- Priority
|
|
||||||
- Estimated effort
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Extract all tasks:
|
|
||||||
- Pattern: "- [ ]" or "- [x]"
|
|
||||||
- Count total tasks
|
|
||||||
- Count checked tasks
|
|
||||||
- Count unchecked tasks
|
|
||||||
- Calculate completion percentage
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Extract file references from Dev Agent Record:
|
|
||||||
- Files created
|
|
||||||
- Files modified
|
|
||||||
- Files deleted
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>📋 **Story Validation: {{story_id}}**
|
|
||||||
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Tasks:** {{checked_count}}/{{total_count}} complete ({{completion_pct}}%)
|
|
||||||
**Files Referenced:** {{file_count}}
|
|
||||||
|
|
||||||
Starting deep validation...
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="2" goal="Task-based verification (Deep)">
|
|
||||||
<critical>Use task-verification-engine.py for DEEP verification (not just file existence)</critical>
|
|
||||||
|
|
||||||
<action>For each task in story:
|
|
||||||
1. Extract task text
|
|
||||||
2. Note if checked [x] or unchecked [ ]
|
|
||||||
3. Pass to task-verification-engine.py
|
|
||||||
4. Receive verification result with:
|
|
||||||
- should_be_checked: true/false
|
|
||||||
- confidence: very high/high/medium/low
|
|
||||||
- evidence: list of findings
|
|
||||||
- verification_status: correct/false_positive/false_negative/uncertain
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Categorize tasks by verification status:
|
|
||||||
- ✅ CORRECT: Checkbox matches reality
|
|
||||||
- ❌ FALSE POSITIVE: Checked but code missing/stubbed
|
|
||||||
- ⚠️ FALSE NEGATIVE: Unchecked but code exists
|
|
||||||
- ❓ UNCERTAIN: Cannot verify (low confidence)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Calculate verification score:
|
|
||||||
- (correct_tasks / total_tasks) × 100
|
|
||||||
- Penalize false positives heavily (-5 points each)
|
|
||||||
- Penalize false negatives lightly (-2 points each)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
🔍 **Task Verification Results**
|
|
||||||
|
|
||||||
**Total Tasks:** {{total_count}}
|
|
||||||
|
|
||||||
**✅ CORRECT:** {{correct_count}} tasks (checkbox matches reality)
|
|
||||||
**❌ FALSE POSITIVES:** {{false_positive_count}} tasks (checked but code missing/stubbed)
|
|
||||||
**⚠️ FALSE NEGATIVES:** {{false_negative_count}} tasks (unchecked but code exists)
|
|
||||||
**❓ UNCERTAIN:** {{uncertain_count}} tasks (cannot verify)
|
|
||||||
|
|
||||||
**Verification Score:** {{verification_score}}/100
|
|
||||||
|
|
||||||
{{#if false_positive_count > 0}}
|
|
||||||
### ❌ False Positives (CRITICAL - Code Claims vs Reality)
|
|
||||||
|
|
||||||
{{#each false_positives}}
|
|
||||||
**Task:** {{this.task}}
|
|
||||||
**Claimed:** [x] Complete
|
|
||||||
**Reality:** {{this.evidence}}
|
|
||||||
**Action Required:** {{this.recommended_action}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if false_negative_count > 0}}
|
|
||||||
### ⚠️ False Negatives (Unchecked but Working)
|
|
||||||
|
|
||||||
{{#each false_negatives}}
|
|
||||||
**Task:** {{this.task}}
|
|
||||||
**Status:** [ ] Unchecked
|
|
||||||
**Reality:** {{this.evidence}}
|
|
||||||
**Recommendation:** Mark as complete [x]
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="3" goal="Code quality review" if="{{validation_depth}} == deep OR comprehensive">
|
|
||||||
<action>Extract all files from Dev Agent Record file list</action>
|
|
||||||
|
|
||||||
<check if="no files listed">
|
|
||||||
<output>⚠️ No files listed in Dev Agent Record - cannot perform code review</output>
|
|
||||||
<action>Skip to step 4</action>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<action>For each file:
|
|
||||||
1. Check if file exists
|
|
||||||
2. Read file content
|
|
||||||
3. Check for quality issues:
|
|
||||||
- TODO/FIXME comments without GitHub issues
|
|
||||||
- any types in TypeScript
|
|
||||||
- Hardcoded values (siteId, dealerId, API keys)
|
|
||||||
- Missing error handling
|
|
||||||
- Missing multi-tenant isolation (dealerId filters)
|
|
||||||
- Missing audit logging on mutations
|
|
||||||
- Security vulnerabilities (SQL injection, XSS)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Run multi-agent review if files exist:
|
|
||||||
- Security audit
|
|
||||||
- Silent failure detection
|
|
||||||
- Architecture compliance
|
|
||||||
- Performance analysis
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Categorize issues by severity:
|
|
||||||
- CRITICAL: Security, data loss, breaking changes
|
|
||||||
- HIGH: Missing features, poor quality, technical debt
|
|
||||||
- MEDIUM: Code smells, minor violations
|
|
||||||
- LOW: Style issues, nice-to-haves
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
🛡️ **Code Quality Review**
|
|
||||||
|
|
||||||
**Files Reviewed:** {{files_reviewed}}
|
|
||||||
**Files Missing:** {{files_missing}}
|
|
||||||
|
|
||||||
**Issues Found:** {{total_issues}}
|
|
||||||
CRITICAL: {{critical_count}}
|
|
||||||
HIGH: {{high_count}}
|
|
||||||
MEDIUM: {{medium_count}}
|
|
||||||
LOW: {{low_count}}
|
|
||||||
|
|
||||||
{{#if critical_count > 0}}
|
|
||||||
### 🚨 CRITICAL Issues (Must Fix)
|
|
||||||
|
|
||||||
{{#each critical_issues}}
|
|
||||||
**File:** {{this.file}}
|
|
||||||
**Issue:** {{this.description}}
|
|
||||||
**Impact:** {{this.impact}}
|
|
||||||
**Fix:** {{this.recommended_fix}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if high_count > 0}}
|
|
||||||
### ⚠️ HIGH Priority Issues
|
|
||||||
|
|
||||||
{{#each high_issues}}
|
|
||||||
**File:** {{this.file}}
|
|
||||||
**Issue:** {{this.description}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
**Code Quality Score:** {{quality_score}}/100
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="4" goal="Integration verification" if="{{validation_depth}} == comprehensive">
|
|
||||||
<action>Extract dependencies from story:
|
|
||||||
- Services called
|
|
||||||
- APIs consumed
|
|
||||||
- Database tables used
|
|
||||||
- Cache keys accessed
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>For each dependency:
|
|
||||||
1. Check if dependency still exists
|
|
||||||
2. Check if API contract is still valid
|
|
||||||
3. Run integration tests if they exist
|
|
||||||
4. Check for breaking changes in dependent stories
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
🔗 **Integration Verification**
|
|
||||||
|
|
||||||
**Dependencies Checked:** {{dependency_count}}
|
|
||||||
|
|
||||||
{{#if broken_integrations}}
|
|
||||||
### ❌ Broken Integrations
|
|
||||||
|
|
||||||
{{#each broken_integrations}}
|
|
||||||
**Dependency:** {{this.name}}
|
|
||||||
**Issue:** {{this.problem}}
|
|
||||||
**Likely Cause:** {{this.cause}}
|
|
||||||
**Fix:** {{this.fix}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if all_integrations_ok}}
|
|
||||||
✅ All integrations verified working
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="5" goal="Determine final story status">
|
|
||||||
<action>Calculate overall story health:
|
|
||||||
- Task verification score (0-100)
|
|
||||||
- Code quality score (0-100)
|
|
||||||
- Integration score (0-100)
|
|
||||||
- Overall score = weighted average
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Determine recommended status:
|
|
||||||
IF verification_score >= 95 AND quality_score >= 90 AND no CRITICAL issues
|
|
||||||
→ VERIFIED_COMPLETE
|
|
||||||
ELSE IF verification_score >= 80 AND quality_score >= 70
|
|
||||||
→ COMPLETE_WITH_ISSUES (document issues)
|
|
||||||
ELSE IF false_positives > 0 OR critical_issues > 0
|
|
||||||
→ NEEDS_REWORK (code missing or broken)
|
|
||||||
ELSE IF verification_score < 50
|
|
||||||
→ FALSE_POSITIVE (claimed done but not implemented)
|
|
||||||
ELSE
|
|
||||||
→ IN_PROGRESS (partially complete)
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>
|
|
||||||
📊 **FINAL VERDICT**
|
|
||||||
|
|
||||||
**Story:** {{story_id}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Recommended Status:** {{recommended_status}}
|
|
||||||
|
|
||||||
**Scores:**
|
|
||||||
Task Verification: {{verification_score}}/100
|
|
||||||
Code Quality: {{quality_score}}/100
|
|
||||||
Integration: {{integration_score}}/100
|
|
||||||
**Overall: {{overall_score}}/100**
|
|
||||||
|
|
||||||
**Confidence:** {{confidence_level}}
|
|
||||||
|
|
||||||
{{#if recommended_status != current_status}}
|
|
||||||
### ⚠️ Status Change Recommended
|
|
||||||
|
|
||||||
**Current:** {{current_status}}
|
|
||||||
**Should Be:** {{recommended_status}}
|
|
||||||
|
|
||||||
**Reason:**
|
|
||||||
{{status_change_reason}}
|
|
||||||
{{/if}}
|
|
||||||
</output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="6" goal="Generate actionable report">
|
|
||||||
<template-output>
|
|
||||||
# Story Validation Report: {{story_id}}
|
|
||||||
|
|
||||||
**Validation Date:** {{date}}
|
|
||||||
**Validation Depth:** {{validation_depth}}
|
|
||||||
**Overall Score:** {{overall_score}}/100
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
**Story:** {{story_id}} - {{story_title}}
|
|
||||||
**Epic:** {{epic_num}}
|
|
||||||
**Current Status:** {{current_status}}
|
|
||||||
**Recommended Status:** {{recommended_status}}
|
|
||||||
|
|
||||||
**Task Completion:** {{checked_count}}/{{total_count}} ({{completion_pct}}%)
|
|
||||||
**Verification Score:** {{verification_score}}/100
|
|
||||||
**Code Quality Score:** {{quality_score}}/100
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task Verification Details
|
|
||||||
|
|
||||||
{{task_verification_output}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Code Quality Review
|
|
||||||
|
|
||||||
{{code_quality_output}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Integration Verification
|
|
||||||
|
|
||||||
{{integration_output}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Recommended Actions
|
|
||||||
|
|
||||||
{{#if critical_issues}}
|
|
||||||
### Priority 1: Fix Critical Issues (BLOCKING)
|
|
||||||
{{#each critical_issues}}
|
|
||||||
- [ ] {{this.file}}: {{this.description}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if false_positives}}
|
|
||||||
### Priority 2: Fix False Positives (Code Claims vs Reality)
|
|
||||||
{{#each false_positives}}
|
|
||||||
- [ ] {{this.task}} - {{this.evidence}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if high_issues}}
|
|
||||||
### Priority 3: Address High Priority Issues
|
|
||||||
{{#each high_issues}}
|
|
||||||
- [ ] {{this.file}}: {{this.description}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if false_negatives}}
|
|
||||||
### Priority 4: Update Task Checkboxes (Low Impact)
|
|
||||||
{{#each false_negatives}}
|
|
||||||
- [ ] Mark complete: {{this.task}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Next Steps
|
|
||||||
|
|
||||||
{{#if recommended_status == "VERIFIED_COMPLETE"}}
|
|
||||||
✅ **Story is verified complete and production-ready**
|
|
||||||
- Update sprint-status.yaml: {{story_id}} = done
|
|
||||||
- No further action required
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if recommended_status == "NEEDS_REWORK"}}
|
|
||||||
⚠️ **Story requires rework before marking complete**
|
|
||||||
- Fix {{critical_count}} CRITICAL issues
|
|
||||||
- Address {{false_positive_count}} false positive tasks
|
|
||||||
- Re-run validation after fixes
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if recommended_status == "FALSE_POSITIVE"}}
|
|
||||||
❌ **Story is marked done but not actually implemented**
|
|
||||||
- Verification score: {{verification_score}}/100 (< 50%)
|
|
||||||
- Update sprint-status.yaml: {{story_id}} = in-progress or ready-for-dev
|
|
||||||
- Implement missing tasks before claiming done
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Generated by:** /validate-story workflow
|
|
||||||
**Validation Engine:** task-verification-engine.py v2.0
|
|
||||||
</template-output>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
<step n="7" goal="Update story file and sprint-status">
|
|
||||||
<ask>Apply recommended status change to sprint-status.yaml? (y/n)</ask>
|
|
||||||
|
|
||||||
<check if="user says yes">
|
|
||||||
<action>Update sprint-status.yaml:
|
|
||||||
- Use sprint-status-updater.py
|
|
||||||
- Update {{story_id}} to {{recommended_status}}
|
|
||||||
- Add comment: "Validated {{date}}, score {{overall_score}}/100"
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action>Update story file:
|
|
||||||
- Add validation report link to Dev Agent Record
|
|
||||||
- Add validation score to completion notes
|
|
||||||
- Update Status: field if changed
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<output>✅ Updated {{story_id}} status: {{current_status}} → {{recommended_status}}</output>
|
|
||||||
</check>
|
|
||||||
|
|
||||||
<check if="user says no">
|
|
||||||
<output>ℹ️ Status not updated. Validation report saved for reference.</output>
|
|
||||||
</check>
|
|
||||||
</step>
|
|
||||||
|
|
||||||
</workflow>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
name: validate-story
|
|
||||||
description: "Deep validation of a single story: verify tasks against codebase, run code quality review, check for regressions. Produces verification report with actionable findings."
|
|
||||||
author: "BMad"
|
|
||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
# Critical variables from config
|
|
||||||
config_source: "{project-root}/_bmad/bmm/config.yaml"
|
|
||||||
user_name: "{config_source}:user_name"
|
|
||||||
communication_language: "{config_source}:communication_language"
|
|
||||||
implementation_artifacts: "{config_source}:implementation_artifacts"
|
|
||||||
story_dir: "{implementation_artifacts}"
|
|
||||||
|
|
||||||
# Workflow components
|
|
||||||
installed_path: "{project-root}/_bmad/bmm/workflows/4-implementation/validate-story"
|
|
||||||
instructions: "{installed_path}/instructions.xml"
|
|
||||||
|
|
||||||
# Input variables
|
|
||||||
variables:
|
|
||||||
story_file: "" # Path to story file (e.g., docs/sprint-artifacts/16e-6-ecs-task-definitions-tier3.md)
|
|
||||||
validation_depth: "deep" # Options: "quick" (tasks only), "deep" (tasks + code review), "comprehensive" (tasks + review + integration tests)
|
|
||||||
|
|
||||||
# Tools
|
|
||||||
task_verification_script: "{project-root}/scripts/lib/task-verification-engine.py"
|
|
||||||
|
|
||||||
# Output
|
|
||||||
default_output_file: "{story_dir}/.validation-{story_id}-{date}.md"
|
|
||||||
|
|
||||||
standalone: true
|
|
||||||
web_bundle: false
|
|
||||||
Loading…
Reference in New Issue