ldy26098
/
BMAD-METHOD
mirror of https://github.com/bmad-code-org/BMAD-METHOD.git
1
0
Fork 0
Code Packages Projects Releases Wiki Activity

feat(uat-validate): add human intervention detection and enhanced fix context 

- Add human intervention detection with BLOCKING/WARNING pattern matching
  - Detects .env issues, API keys, permissions, connection failures
  - Categorizes issues for Barry to handle appropriately

- Add story context extraction for fix documents
  - Extracts acceptance criteria from story files
  - Includes Dev Agent Record (implementation notes)

- Enhance fix context generation
  - Includes human intervention items section
  - Adds root cause hints for each failure
  - Appends story context with AC mapping

- Add human actions file generation
  - Creates epic-{id}-human-actions.md with actionable checklist
  - Provides specific guidance per issue type

- Add timing metrics to track UAT evaluation duration
  - Captures start_time, end_time, duration_seconds
  - Shows duration in summary output

- Update output signals with human action flags
  - UAT_HUMAN_ACTION_REQUIRED: true/false
  - UAT_HUMAN_ACTION_COUNT: N
  - UAT_HUMAN_ACTION_FILE: path

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Caleb 2026-01-19 04:37:29 -06:00
parent 2326f72803
commit 9f532eff65
2 changed files with 515 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

513
scripts/uat-validate.sh
View File

@ -288,6 +288,181 @@ classify_single_scenario() {
fi fi
} }
# =============================================================================
# Section 5.5: Human Intervention Detection
# =============================================================================
# Arrays to store human intervention items
declare -a HUMAN_INTERVENTION_BLOCKING
declare -a HUMAN_INTERVENTION_WARNING
# Patterns that indicate human intervention is required
BLOCKING_PATTERNS=(
"EACCES|permission denied"
"\.env|environment variable|not set|undefined|not defined"
"API[_-]?KEY|SECRET|TOKEN.*required|missing|invalid"
"authentication failed|unauthorized|401|403"
"license|subscription|quota exceeded"
"EPERM|operation not permitted"
"credentials.*required|credentials.*missing"
)
WARNING_PATTERNS=(
"connection refused|ECONNREFUSED|ETIMEDOUT|timeout"
"check.*inbox|verify.*email|manual.*verification"
"relation.*does not exist|migration|table.*not found"
"deprecated|warning:"
"rate limit|throttl"
"service unavailable|503"
"could not connect|connection failed"
)
detect_human_intervention() {
local error_output="$1"
local scenario_id="$2"
local scenario_name="$3"
# Check for BLOCKING patterns
for pattern in "${BLOCKING_PATTERNS[@]}"; do
if echo "$error_output" | grep -qiE "$pattern"; then
local matched_line
matched_line=$(echo "$error_output" | grep -iE "$pattern" | head -1)
HUMAN_INTERVENTION_BLOCKING+=("$scenario_id|$scenario_name|$matched_line")
[ "$VERBOSE" = true ] && log_warn " [BLOCKING] Detected: $matched_line"
fi
done
# Check for WARNING patterns
for pattern in "${WARNING_PATTERNS[@]}"; do
if echo "$error_output" | grep -qiE "$pattern"; then
local matched_line
matched_line=$(echo "$error_output" | grep -iE "$pattern" | head -1)
HUMAN_INTERVENTION_WARNING+=("$scenario_id|$scenario_name|$matched_line")
[ "$VERBOSE" = true ] && log_warn " [WARNING] Detected: $matched_line"
fi
done
}
analyze_root_cause() {
local error_output="$1"
local exit_code="$2"
# Analyze error patterns and return a hint
if echo "$error_output" | grep -qiE "\.env|environment variable|not set"; then
echo "Missing environment configuration. Check .env file or .env.example for required variables."
elif echo "$error_output" | grep -qiE "API[_-]?KEY|SECRET|TOKEN"; then
echo "Missing or invalid API credentials. Verify API keys are correctly configured."
elif echo "$error_output" | grep -qiE "connection refused|ECONNREFUSED"; then
echo "Service connection failed. Ensure the required service is running (database, redis, etc.)."
elif echo "$error_output" | grep -qiE "relation.*does not exist|table.*not found"; then
echo "Database schema issue. Run migrations or check database setup."
elif echo "$error_output" | grep -qiE "permission denied|EACCES|EPERM"; then
echo "Permission issue. Check file/directory permissions or run with appropriate privileges."
elif echo "$error_output" | grep -qiE "timeout|ETIMEDOUT"; then
echo "Operation timed out. Check network connectivity or increase timeout."
elif [ "$exit_code" -eq 1 ]; then
echo "Command failed with exit code 1. Check the error output for specific details."
elif [ "$exit_code" -eq 124 ]; then
echo "Command timed out. Consider increasing timeout or checking for blocking operations."
else
echo "Analyze error output above. Exit code: $exit_code"
fi
}
# =============================================================================
# Section 5.6: Story Context Extraction
# =============================================================================
extract_story_context() {
local epic_id="$1"
local output_file="$2"
log "Extracting story context for Epic $epic_id..."
# Find story files for this epic
local story_files=()
for pattern in "${epic_id}-" "epic-${epic_id}-" "0${epic_id}-"; do
while IFS= read -r -d '' file; do
story_files+=("$file")
done < <(find "$STORIES_DIR" -name "${pattern}*.md" -print0 2>/dev/null)
done
if [ ${#story_files[@]} -eq 0 ]; then
log_warn "No story files found for Epic $epic_id in $STORIES_DIR"
echo "No story files found for this epic." >> "$output_file"
return 1
fi
log "Found ${#story_files[@]} story file(s)"
echo "## Story Context" >> "$output_file"
echo "" >> "$output_file"
for story_file in "${story_files[@]}"; do
local story_name
story_name=$(basename "$story_file" .md)
echo "### $story_name" >> "$output_file"
echo "" >> "$output_file"
# Extract acceptance criteria section
local in_ac_section=false
local ac_content=""
while IFS= read -r line; do
# Detect start of acceptance criteria section
if echo "$line" | grep -qiE "^##.*[Aa]cceptance [Cc]riteria|^##.*AC"; then
in_ac_section=true
ac_content="**Acceptance Criteria:**"$'\n'
continue
fi
# Detect end of section (next ## header)
if [ "$in_ac_section" = true ] && echo "$line" | grep -qE "^##[^#]"; then
in_ac_section=false
fi
# Accumulate content
if [ "$in_ac_section" = true ]; then
ac_content+="$line"$'\n'
fi
done < "$story_file"
if [ -n "$ac_content" ]; then
echo "$ac_content" >> "$output_file"
else
echo "*No acceptance criteria section found in this story.*" >> "$output_file"
fi
echo "" >> "$output_file"
# Extract Dev Agent Record section (implementation notes)
local in_dar_section=false
local dar_content=""
while IFS= read -r line; do
# Detect start of Dev Agent Record section
if echo "$line" | grep -qiE "^##.*[Dd]ev [Aa]gent [Rr]ecord|^##.*Implementation [Nn]otes"; then
in_dar_section=true
dar_content="**Dev Agent Record (Implementation Notes):**"$'\n'
continue
fi
# Detect end of section (next ## header)
if [ "$in_dar_section" = true ] && echo "$line" | grep -qE "^##[^#]"; then
in_dar_section=false
fi
# Accumulate content
if [ "$in_dar_section" = true ]; then
dar_content+="$line"$'\n'
fi
done < "$story_file"
if [ -n "$dar_content" ]; then
echo "$dar_content" >> "$output_file"
fi
echo "" >> "$output_file"
echo "---" >> "$output_file"
echo "" >> "$output_file"
done
return 0
}
# ============================================================================= # =============================================================================
# Section 6: Scenario Execution # Section 6: Scenario Execution
# ============================================================================= # =============================================================================
@ -415,6 +590,8 @@ execute_single_scenario() {
log_error " Scenario $scenario_id: FAIL (timeout after ${TIMEOUT_SECONDS}s)" log_error " Scenario $scenario_id: FAIL (timeout after ${TIMEOUT_SECONDS}s)"
FAILED_SCENARIOS+=("$scenario_id") FAILED_SCENARIOS+=("$scenario_id")
FAILED_DETAILS+=("$scenario_id|$scenario_name|$scenario_cmd|timeout|$exit_code|$output|$stderr") FAILED_DETAILS+=("$scenario_id|$scenario_name|$scenario_cmd|timeout|$exit_code|$output|$stderr")
# Detect human intervention needs from output
detect_human_intervention "$output$stderr" "$scenario_id" "$scenario_name"
else else
log_error " Scenario $scenario_id: FAIL (exit code $exit_code)" log_error " Scenario $scenario_id: FAIL (exit code $exit_code)"
if [ -n "$stderr" ] && [ "$VERBOSE" = true ]; then if [ -n "$stderr" ] && [ "$VERBOSE" = true ]; then
@ -422,6 +599,8 @@ execute_single_scenario() {
fi fi
FAILED_SCENARIOS+=("$scenario_id") FAILED_SCENARIOS+=("$scenario_id")
FAILED_DETAILS+=("$scenario_id|$scenario_name|$scenario_cmd|error|$exit_code|$output|$stderr") FAILED_DETAILS+=("$scenario_id|$scenario_name|$scenario_cmd|error|$exit_code|$output|$stderr")
# Detect human intervention needs from error output
detect_human_intervention "$output$stderr" "$scenario_id" "$scenario_name"
fi fi
return $exit_code return $exit_code
@ -496,7 +675,41 @@ This document contains the context needed to fix UAT failures for Epic $epic_id.
EOF EOF
fi fi
# Append failed scenarios details # Add human intervention section
echo "" >> "$fix_file"
echo "## Human Intervention Items" >> "$fix_file"
echo "" >> "$fix_file"
if [ ${#HUMAN_INTERVENTION_BLOCKING[@]} -gt 0 ]; then
echo "### BLOCKING (likely requires human action)" >> "$fix_file"
echo "" >> "$fix_file"
for item in "${HUMAN_INTERVENTION_BLOCKING[@]}"; do
IFS='|' read -r scenario_id scenario_name matched_line <<< "$item"
echo "- [ ] **Scenario $scenario_id ($scenario_name):** $matched_line" >> "$fix_file"
done
echo "" >> "$fix_file"
fi
if [ ${#HUMAN_INTERVENTION_WARNING[@]} -gt 0 ]; then
echo "### WARNING (may need attention)" >> "$fix_file"
echo "" >> "$fix_file"
for item in "${HUMAN_INTERVENTION_WARNING[@]}"; do
IFS='|' read -r scenario_id scenario_name matched_line <<< "$item"
echo "- [ ] **Scenario $scenario_id ($scenario_name):** $matched_line" >> "$fix_file"
done
echo "" >> "$fix_file"
fi
if [ ${#HUMAN_INTERVENTION_BLOCKING[@]} -eq 0 ] && [ ${#HUMAN_INTERVENTION_WARNING[@]} -eq 0 ]; then
echo "*No human intervention items detected. All failures appear to be code-fixable.*" >> "$fix_file"
echo "" >> "$fix_file"
fi
echo "**Instructions for Barry:** Attempt to fix what you can. For items you cannot resolve programmatically, document them clearly in the fix commit message and update the human-actions file." >> "$fix_file"
echo "" >> "$fix_file"
echo "---" >> "$fix_file"
# Append failed scenarios details with root cause hints
echo "" >> "$fix_file" echo "" >> "$fix_file"
echo "## Failed Scenarios" >> "$fix_file" echo "## Failed Scenarios" >> "$fix_file"
echo "" >> "$fix_file" echo "" >> "$fix_file"
@ -504,6 +717,10 @@ EOF
for detail in "${FAILED_DETAILS[@]}"; do for detail in "${FAILED_DETAILS[@]}"; do
IFS='|' read -r scenario_id scenario_name cmd error_type exit_code output stderr <<< "$detail" IFS='|' read -r scenario_id scenario_name cmd error_type exit_code output stderr <<< "$detail"
# Generate root cause hint for this failure
local root_cause_hint
root_cause_hint=$(analyze_root_cause "$output$stderr" "$exit_code")
cat >> "$fix_file" << EOF cat >> "$fix_file" << EOF
### Scenario $scenario_id: $scenario_name ### Scenario $scenario_id: $scenario_name
@ -525,11 +742,17 @@ $output
$stderr $stderr
\`\`\` \`\`\`
**Root Cause Hint:** $root_cause_hint
--- ---
EOF EOF
done done
# Extract and append story context (acceptance criteria + dev agent record)
echo "" >> "$fix_file"
extract_story_context "$epic_id" "$fix_file"
# Add context references section # Add context references section
cat >> "$fix_file" << EOF cat >> "$fix_file" << EOF
@ -586,26 +809,47 @@ run_quick_dev_fix() {
log "Spawning quick-dev fix session (attempt $attempt/$MAX_RETRIES)" log "Spawning quick-dev fix session (attempt $attempt/$MAX_RETRIES)"
# Build human intervention summary for prompt
local human_intervention_note=""
if [ ${#HUMAN_INTERVENTION_BLOCKING[@]} -gt 0 ] || [ ${#HUMAN_INTERVENTION_WARNING[@]} -gt 0 ]; then
human_intervention_note="
IMPORTANT: Some failures may require human intervention (marked in the fix context).
- For items you CANNOT fix programmatically (missing API keys, .env configuration, etc.):
Document them clearly and proceed with what you CAN fix.
- Do NOT attempt to create fake credentials or placeholder values.
- Focus on code-level fixes that don't require external configuration."
fi
local fix_prompt="You are Barry, the Quick Flow Solo Dev. local fix_prompt="You are Barry, the Quick Flow Solo Dev.
Load and process this fix context document: FIRST: Read the fix context document at:
$fix_context_file $fix_context_file
Your task: This document contains:
1. Read the failed scenarios and error details from the fix context 1. Human Intervention Items - issues that may require human action
2. Analyze root cause for each failure 2. Failed Scenarios - with commands, errors, and root cause hints
3. Implement targeted fixes 3. Story Context - acceptance criteria and implementation notes from the original stories
4. Run the failing commands to verify fixes
5. Stage changes: git add -A
6. Commit with message: fix(epic-${epic_id}): UAT fix #${attempt}
Your task:
1. Read the fix context document completely before starting
2. Review the Human Intervention Items section - note which issues you CAN vs CANNOT fix
3. For each failed scenario:
a. Check the root cause hint
b. Review the related acceptance criteria
c. Implement targeted fixes for code-level issues
4. Run the failing commands to verify your fixes work
5. Stage changes: git add -A
6. Commit with message: fix(epic-${epic_id}): UAT fix #${attempt} - {brief description}
$human_intervention_note
Constraints: Constraints:
- Only fix the identified failures - Only fix the identified failures - do not refactor unrelated code
- Do not refactor unrelated code - Run the specific failing commands to verify each fix
- Run tests after fixes - Run project tests after all fixes: npm test
- If a fix requires external configuration (API keys, .env), document it but don't block on it
When done, output exactly: When done, output exactly:
FIX_COMPLETE: {number_fixed}/${#FAILED_SCENARIOS[@]}" FIX_COMPLETE: {number_fixed}/${#FAILED_SCENARIOS[@]}
HUMAN_ACTION_NEEDED: {yes/no}"
if [ "$DRY_RUN" = true ]; then if [ "$DRY_RUN" = true ]; then
echo "[DRY RUN] Would spawn Claude for fixes with prompt:" echo "[DRY RUN] Would spawn Claude for fixes with prompt:"
@ -621,6 +865,10 @@ FIX_COMPLETE: {number_fixed}/${#FAILED_SCENARIOS[@]}"
if echo "$result" | grep -q "FIX_COMPLETE"; then if echo "$result" | grep -q "FIX_COMPLETE"; then
log_success "Quick-dev fix session completed" log_success "Quick-dev fix session completed"
# Check if human action was flagged
if echo "$result" | grep -q "HUMAN_ACTION_NEEDED: yes"; then
log_warn "Barry indicated human action is needed for some issues"
fi
return 0 return 0
else else
log_warn "Quick-dev fix session may not have completed cleanly" log_warn "Quick-dev fix session may not have completed cleanly"
@ -628,6 +876,140 @@ FIX_COMPLETE: {number_fixed}/${#FAILED_SCENARIOS[@]}"
fi fi
} }
generate_human_actions_file() {
local epic_id="$1"
local final_attempt="$2"
local human_actions_file="$FIX_DIR/epic-${epic_id}-human-actions.md"
local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
# Only generate if there are human intervention items
if [ ${#HUMAN_INTERVENTION_BLOCKING[@]} -eq 0 ] && [ ${#HUMAN_INTERVENTION_WARNING[@]} -eq 0 ]; then
log "No human actions required"
return 0
fi
log "Generating human actions file: $human_actions_file"
cat > "$human_actions_file" << EOF
# Human Actions Required - Epic $epic_id
**Generated:** $timestamp
**After:** Fix attempt $final_attempt of $MAX_RETRIES
**UAT Result:** FAIL (${#PASSED_SCENARIOS[@]}/${#AUTOMATABLE_SCENARIOS[@]} scenarios passed)
---
## Required Actions
The following items could not be automatically fixed and require human intervention.
EOF
local action_num=0
# Add BLOCKING items
if [ ${#HUMAN_INTERVENTION_BLOCKING[@]} -gt 0 ]; then
for item in "${HUMAN_INTERVENTION_BLOCKING[@]}"; do
((action_num++))
IFS='|' read -r scenario_id scenario_name matched_line <<< "$item"
cat >> "$human_actions_file" << EOF
### $action_num. $scenario_name (Scenario $scenario_id)
**Priority:** High (BLOCKING)
**Issue:** $matched_line
**Suggested Action:**
EOF
# Add specific guidance based on pattern
if echo "$matched_line" | grep -qiE "\.env|environment variable"; then
cat >> "$human_actions_file" << EOF
Check your \`.env\` file and ensure all required environment variables are set.
Reference \`.env.example\` if available.
EOF
elif echo "$matched_line" | grep -qiE "API[_-]?KEY|SECRET|TOKEN"; then
cat >> "$human_actions_file" << EOF
Verify your API credentials are correctly configured.
Check the service dashboard for valid keys.
EOF
elif echo "$matched_line" | grep -qiE "permission denied|EACCES"; then
cat >> "$human_actions_file" << EOF
Check file/directory permissions. You may need to run with elevated privileges
or adjust ownership/permissions on the affected files.
EOF
else
echo "Review the error message and take appropriate action." >> "$human_actions_file"
echo "" >> "$human_actions_file"
fi
done
fi
# Add WARNING items
if [ ${#HUMAN_INTERVENTION_WARNING[@]} -gt 0 ]; then
for item in "${HUMAN_INTERVENTION_WARNING[@]}"; do
((action_num++))
IFS='|' read -r scenario_id scenario_name matched_line <<< "$item"
cat >> "$human_actions_file" << EOF
### $action_num. $scenario_name (Scenario $scenario_id)
**Priority:** Medium (WARNING)
**Issue:** $matched_line
**Suggested Action:**
EOF
# Add specific guidance based on pattern
if echo "$matched_line" | grep -qiE "connection refused|ECONNREFUSED"; then
cat >> "$human_actions_file" << EOF
Ensure the required service is running (database, Redis, etc.).
Check service logs for startup errors.
EOF
elif echo "$matched_line" | grep -qiE "inbox|email|manual.*verification"; then
cat >> "$human_actions_file" << EOF
Manual verification required. Check the relevant inbox or UI to confirm
the expected behavior.
EOF
elif echo "$matched_line" | grep -qiE "migration|relation.*does not exist"; then
cat >> "$human_actions_file" << EOF
Database schema may need updating. Run migrations:
\`\`\`bash
npm run db:migrate # or your project's migration command
\`\`\`
EOF
else
echo "Review the warning and take appropriate action if needed." >> "$human_actions_file"
echo "" >> "$human_actions_file"
fi
done
fi
cat >> "$human_actions_file" << EOF
---
## After Completing Actions
Re-run UAT validation:
\`\`\`bash
./scripts/uat-validate.sh $epic_id --gate-mode=$UAT_GATE_MODE
\`\`\`
---
*Generated by UAT Validate Workflow*
*BMAD Method - Epic Chain Self-Healing*
EOF
echo "$human_actions_file"
}
self_healing_loop() { self_healing_loop() {
local epic_id="$1" local epic_id="$1"
local attempt=0 local attempt=0
@ -637,10 +1019,22 @@ self_healing_loop() {
log_section "Self-Healing Fix Loop (Attempt $attempt/$MAX_RETRIES)" log_section "Self-Healing Fix Loop (Attempt $attempt/$MAX_RETRIES)"
# Generate fix context # Reset human intervention arrays for this attempt
HUMAN_INTERVENTION_BLOCKING=()
HUMAN_INTERVENTION_WARNING=()
# Generate fix context (this will detect human intervention items)
local fix_file local fix_file
fix_file=$(generate_fix_context "$epic_id" "$attempt") fix_file=$(generate_fix_context "$epic_id" "$attempt")
# Log human intervention summary
if [ ${#HUMAN_INTERVENTION_BLOCKING[@]} -gt 0 ]; then
log_warn "Detected ${#HUMAN_INTERVENTION_BLOCKING[@]} BLOCKING human intervention item(s)"
fi
if [ ${#HUMAN_INTERVENTION_WARNING[@]} -gt 0 ]; then
log_warn "Detected ${#HUMAN_INTERVENTION_WARNING[@]} WARNING human intervention item(s)"
fi
# Run quick-dev fix # Run quick-dev fix
if ! run_quick_dev_fix "$fix_file" "$epic_id" "$attempt"; then if ! run_quick_dev_fix "$fix_file" "$epic_id" "$attempt"; then
log_warn "Fix attempt $attempt may have issues" log_warn "Fix attempt $attempt may have issues"
@ -649,7 +1043,7 @@ self_healing_loop() {
# Re-run validation # Re-run validation
log "Re-validating after fix attempt $attempt..." log "Re-validating after fix attempt $attempt..."
# Reset and re-execute # Reset scenario results but preserve human intervention items
PASSED_SCENARIOS=() PASSED_SCENARIOS=()
FAILED_SCENARIOS=() FAILED_SCENARIOS=()
FAILED_DETAILS=() FAILED_DETAILS=()
@ -662,6 +1056,9 @@ self_healing_loop() {
log_warn "UAT still failing after attempt $attempt" log_warn "UAT still failing after attempt $attempt"
done done
# Generate human actions file for remaining issues
generate_human_actions_file "$epic_id" "$MAX_RETRIES"
log_error "Max retries ($MAX_RETRIES) exceeded" log_error "Max retries ($MAX_RETRIES) exceeded"
return 2 return 2
} }
@ -678,7 +1075,15 @@ update_metrics() {
mkdir -p "$METRICS_DIR" mkdir -p "$METRICS_DIR"
local metrics_file="$METRICS_DIR/epic-${epic_id}-metrics.yaml" local metrics_file="$METRICS_DIR/epic-${epic_id}-metrics.yaml"
local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
# Calculate timing
local end_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
local end_epoch=$(date +%s)
local duration_seconds=$((end_epoch - UAT_START_EPOCH))
# Calculate human intervention counts
local blocking_count=${#HUMAN_INTERVENTION_BLOCKING[@]}
local warning_count=${#HUMAN_INTERVENTION_WARNING[@]}
# Check if yq is available for YAML manipulation # Check if yq is available for YAML manipulation
if command -v yq >/dev/null 2>&1; then if command -v yq >/dev/null 2>&1; then
@ -688,7 +1093,11 @@ update_metrics() {
yq -i ".validation.fix_attempts = $fix_attempts" "$metrics_file" yq -i ".validation.fix_attempts = $fix_attempts" "$metrics_file"
yq -i ".validation.scenarios_passed = ${#PASSED_SCENARIOS[@]}" "$metrics_file" yq -i ".validation.scenarios_passed = ${#PASSED_SCENARIOS[@]}" "$metrics_file"
yq -i ".validation.scenarios_failed = ${#FAILED_SCENARIOS[@]}" "$metrics_file" yq -i ".validation.scenarios_failed = ${#FAILED_SCENARIOS[@]}" "$metrics_file"
yq -i ".validation.timestamp = \"$timestamp\"" "$metrics_file" yq -i ".validation.start_time = \"$UAT_START_TIME\"" "$metrics_file"
yq -i ".validation.end_time = \"$end_time\"" "$metrics_file"
yq -i ".validation.duration_seconds = $duration_seconds" "$metrics_file"
yq -i ".validation.human_intervention.blocking = $blocking_count" "$metrics_file"
yq -i ".validation.human_intervention.warning = $warning_count" "$metrics_file"
else else
# Create new metrics file # Create new metrics file
cat > "$metrics_file" << EOF cat > "$metrics_file" << EOF
@ -699,7 +1108,12 @@ validation:
fix_attempts: $fix_attempts fix_attempts: $fix_attempts
scenarios_passed: ${#PASSED_SCENARIOS[@]} scenarios_passed: ${#PASSED_SCENARIOS[@]}
scenarios_failed: ${#FAILED_SCENARIOS[@]} scenarios_failed: ${#FAILED_SCENARIOS[@]}
timestamp: "$timestamp" start_time: "$UAT_START_TIME"
end_time: "$end_time"
duration_seconds: $duration_seconds
human_intervention:
blocking: $blocking_count
warning: $warning_count
EOF EOF
fi fi
else else
@ -713,7 +1127,12 @@ validation:
fix_attempts: $fix_attempts fix_attempts: $fix_attempts
scenarios_passed: ${#PASSED_SCENARIOS[@]} scenarios_passed: ${#PASSED_SCENARIOS[@]}
scenarios_failed: ${#FAILED_SCENARIOS[@]} scenarios_failed: ${#FAILED_SCENARIOS[@]}
timestamp: "$timestamp" start_time: "$UAT_START_TIME"
end_time: "$end_time"
duration_seconds: $duration_seconds
human_intervention:
blocking: $blocking_count
warning: $warning_count
EOF EOF
else else
# Simple append for validation section # Simple append for validation section
@ -722,6 +1141,7 @@ EOF
fi fi
log "Metrics updated: $metrics_file" log "Metrics updated: $metrics_file"
log " Duration: ${duration_seconds}s"
} }
output_signals() { output_signals() {
@ -730,22 +1150,43 @@ output_signals() {
local total=${#AUTOMATABLE_SCENARIOS[@]} local total=${#AUTOMATABLE_SCENARIOS[@]}
local passed=${#PASSED_SCENARIOS[@]} local passed=${#PASSED_SCENARIOS[@]}
local human_action_count=$((${#HUMAN_INTERVENTION_BLOCKING[@]} + ${#HUMAN_INTERVENTION_WARNING[@]}))
local human_action_required="false"
[ $human_action_count -gt 0 ] && human_action_required="true"
echo "" echo ""
echo "UAT_GATE_RESULT: $gate_status" echo "UAT_GATE_RESULT: $gate_status"
echo "UAT_FIX_ATTEMPTS: $fix_attempts" echo "UAT_FIX_ATTEMPTS: $fix_attempts"
echo "UAT_SCENARIOS_PASSED: $passed/$total" echo "UAT_SCENARIOS_PASSED: $passed/$total"
echo "UAT_HUMAN_ACTION_REQUIRED: $human_action_required"
echo "UAT_HUMAN_ACTION_COUNT: $human_action_count"
if [ "$human_action_required" = "true" ]; then
echo "UAT_HUMAN_ACTION_FILE: $FIX_DIR/epic-${EPIC_ID}-human-actions.md"
fi
} }
print_summary() { print_summary() {
local gate_status="$1" local gate_status="$1"
local fix_attempts="$2" local fix_attempts="$2"
local human_action_count=$((${#HUMAN_INTERVENTION_BLOCKING[@]} + ${#HUMAN_INTERVENTION_WARNING[@]}))
# Calculate duration
local end_epoch=$(date +%s)
local duration_seconds=$((end_epoch - UAT_START_EPOCH))
local duration_display="${duration_seconds}s"
if [ $duration_seconds -ge 60 ]; then
local minutes=$((duration_seconds / 60))
local seconds=$((duration_seconds % 60))
duration_display="${minutes}m ${seconds}s"
fi
log_header "UAT VALIDATION COMPLETE" log_header "UAT VALIDATION COMPLETE"
echo " Epic: $EPIC_ID" echo " Epic: $EPIC_ID"
echo " Gate Mode: $UAT_GATE_MODE" echo " Gate Mode: $UAT_GATE_MODE"
echo " Gate Result: $gate_status" echo " Gate Result: $gate_status"
echo " Duration: $duration_display"
echo "" echo ""
echo " Scenarios:" echo " Scenarios:"
echo " Automatable: ${#AUTOMATABLE_SCENARIOS[@]}" echo " Automatable: ${#AUTOMATABLE_SCENARIOS[@]}"
@ -757,23 +1198,57 @@ print_summary() {
echo " Failed: ${#FAILED_SCENARIOS[@]}" echo " Failed: ${#FAILED_SCENARIOS[@]}"
echo " Fix Attempts: $fix_attempts" echo " Fix Attempts: $fix_attempts"
echo "" echo ""
echo " Human Intervention:"
echo " Blocking Items: ${#HUMAN_INTERVENTION_BLOCKING[@]}"
echo " Warning Items: ${#HUMAN_INTERVENTION_WARNING[@]}"
echo ""
echo " Artifacts:" echo " Artifacts:"
echo " Log: $LOG_FILE" echo " Log: $LOG_FILE"
echo " UAT Document: $UAT_FILE" echo " UAT Document: $UAT_FILE"
if [ ${#FAILED_SCENARIOS[@]} -gt 0 ] && [ -d "$FIX_DIR" ]; then if [ ${#FAILED_SCENARIOS[@]} -gt 0 ] && [ -d "$FIX_DIR" ]; then
echo " Fix Contexts: $FIX_DIR/" echo " Fix Contexts: $FIX_DIR/"
fi fi
if [ $human_action_count -gt 0 ]; then
echo " Human Actions: $FIX_DIR/epic-${EPIC_ID}-human-actions.md"
fi
echo "" echo ""
# Print human intervention summary if any
if [ $human_action_count -gt 0 ]; then
echo -e "${YELLOW}${BOLD} ⚠ Human Action Required:${NC}"
if [ ${#HUMAN_INTERVENTION_BLOCKING[@]} -gt 0 ]; then
echo -e " ${RED}BLOCKING:${NC}"
for item in "${HUMAN_INTERVENTION_BLOCKING[@]}"; do
IFS='|' read -r scenario_id scenario_name matched_line <<< "$item"
echo " - Scenario $scenario_id: $matched_line"
done
fi
if [ ${#HUMAN_INTERVENTION_WARNING[@]} -gt 0 ]; then
echo -e " ${YELLOW}WARNING:${NC}"
for item in "${HUMAN_INTERVENTION_WARNING[@]}"; do
IFS='|' read -r scenario_id scenario_name matched_line <<< "$item"
echo " - Scenario $scenario_id: $matched_line"
done
fi
echo ""
echo " See $FIX_DIR/epic-${EPIC_ID}-human-actions.md for details."
echo ""
fi
} }
# ============================================================================= # =============================================================================
# Main Execution # Main Execution
# ============================================================================= # =============================================================================
# Capture UAT evaluation start time
UAT_START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
UAT_START_EPOCH=$(date +%s)
log_header "UAT VALIDATION: Epic $EPIC_ID" log_header "UAT VALIDATION: Epic $EPIC_ID"
log "Gate mode: $UAT_GATE_MODE" log "Gate mode: $UAT_GATE_MODE"
log "Max retries: $MAX_RETRIES" log "Max retries: $MAX_RETRIES"
log "Timeout: ${TIMEOUT_SECONDS}s" log "Timeout: ${TIMEOUT_SECONDS}s"
log "Started: $UAT_START_TIME"
# Ensure directories exist # Ensure directories exist
mkdir -p "$METRICS_DIR" mkdir -p "$METRICS_DIR"

26
src/modules/bmm/workflows/5-validation/uat-validate/uat-fix-context-template.md
View File

@ -18,10 +18,26 @@ This document contains the context needed to fix UAT failures for Epic {epic_id}
## Quick Start ## Quick Start
1. Read each failed scenario below 1. Review the **Human Intervention Items** section first
2. Identify root cause from error output 2. Read each **Failed Scenario** with its root cause hint
3. Locate and fix the relevant code 3. Check the **Story Context** for acceptance criteria and implementation notes
4. Verify fix: run the failing command 4. Implement targeted fixes for code-level issues
5. Commit: `fix(epic-{epic_id}): {description}` 5. Verify: run the failing command
6. Commit: `fix(epic-{epic_id}): {description}`
---
## Document Structure
This fix context includes:
1. **Human Intervention Items** - Issues that may require human action (env vars, API keys, etc.)
2. **Failed Scenarios** - Detailed failure information with root cause hints
3. **Story Context** - Acceptance criteria and Dev Agent Record from original stories
For items marked as BLOCKING or WARNING that require human configuration:
- Document what configuration is needed
- Proceed with code fixes you CAN make
- Do not create placeholder credentials or fake values
--- ---