diff --git a/src/core/pipeline/orchestrator.xml b/src/core/pipeline/orchestrator.xml new file mode 100644 index 00000000..32a2780b --- /dev/null +++ b/src/core/pipeline/orchestrator.xml @@ -0,0 +1,194 @@ + + + Orchestrate multi-stage agent pipelines with dependency management + + + {project-root}/_bmad/core/pipeline/pipeline-config.yaml + {project-root}/_bmad-output/pipelines + + + + + + + + + + + + + + + Ask user for pipeline source: + 1. Use template (show available templates) + 2. Define custom pipeline + + + + List templates from config: full_sdlc, quick_flow, analysis_only, design_review, test_suite + User selects template + Copy template stages to new pipeline definition + + + + Ask for pipeline name + Ask for stages (agent, parallel, depends_on, outputs) + Build pipeline definition + + + Generate pipeline_id: "PIPE-{YYYYMMDD}-{name}" + Create pipeline file: {pipeline_dir}/{pipeline_id}.yaml + Initialize all stages as "pending" + + + Pipeline created: {pipeline_id} + Stages: {stage_count} + Estimated duration: {estimate} + + Run with: PIPELINE run {pipeline_id} + + + + + + Load pipeline from {pipeline_id}.yaml + Validate pipeline definition (agents exist, dependencies valid) + Set pipeline status to "running" + Initialize execution state tracking + + + Find stages where: + - Status == "pending" AND + - All depends_on stages are "completed" + + Mark found stages as "queued" + + + + + Launch agents in parallel using Task tool with run_in_background=true + Track agent_ids for each stage + + + + Launch agent using Task tool with run_in_background=false + Wait for completion + + + + + For parallel stages: Use TaskOutput to collect results + Update stage status based on result (completed/failed) + Store outputs in {pipeline_dir}/outputs/{stage_name}/ + + + + Mark pipeline as "failed" + Mark dependent stages as "skipped" + HALT: "Pipeline failed at stage: {stage_name}" + + + + Mark dependent stages as "skipped" + Continue with non-dependent stages + + + Update pipeline file with current state + + + Mark pipeline as "completed" + + Pipeline completed: {pipeline_id} + + Results: + {for each stage} + - {stage_name}: {status} ({duration}) + {end for} + + Outputs saved to: {pipeline_dir}/outputs/ + + + + + + Load pipeline from {pipeline_id}.yaml (or active-pipeline.yaml) + Calculate progress percentage + + + Pipeline: {pipeline_id} + Status: {overall_status} + Progress: [{progress_bar}] {percentage}% + + Stages: + {for each stage} + [{status_icon}] {stage_name} + Agents: {agents} + Duration: {duration or "pending"} + Output: {output_path or "n/a"} + {end for} + + {if running} + Currently executing: {current_stage} + Estimated remaining: {remaining_estimate} + {end if} + + + + + + Scan {pipeline_dir} for pipeline files + Load templates from config + + + Available Templates: + {for each template} + - {template_name}: {description} + Stages: {stage_names} + {end for} + + Existing Pipelines: + {for each pipeline} + - {pipeline_id}: {status} ({created_date}) + {end for} + + + + + + Load running pipeline + Send abort signal to running agents + Mark running stages as "aborted" + Mark pipeline as "aborted" + + + Pipeline {pipeline_id} aborted. + Completed stages: {completed_count} + Aborted stages: {aborted_count} + + + + + + Load failed/aborted pipeline + Identify failed/aborted stages + Ask user: Retry failed stages or skip? + + + Reset failed stages to "pending" + Continue with run logic + + + + Mark failed stages as "skipped" + Continue with remaining stages + + + + + + + + + + + diff --git a/src/core/pipeline/pipeline-config.yaml b/src/core/pipeline/pipeline-config.yaml new file mode 100644 index 00000000..bcf7a9a5 --- /dev/null +++ b/src/core/pipeline/pipeline-config.yaml @@ -0,0 +1,157 @@ +# Pipeline Orchestrator Configuration +# Enables multi-agent pipelines with parallel and sequential execution + +name: pipeline-orchestrator +version: "1.0.0" +description: "Orchestrate multi-stage agent pipelines with dependency management" + +# Pipeline storage +pipeline_dir: "{project-root}/_bmad-output/pipelines" +active_pipeline_file: "{pipeline_dir}/active-pipeline.yaml" +pipeline_archive_dir: "{pipeline_dir}/archive" + +# Execution modes +execution_modes: + sequential: + description: "Execute stages one after another" + use_when: "Each stage depends on previous output" + parallel: + description: "Execute independent stages simultaneously" + use_when: "Stages have no dependencies on each other" + hybrid: + description: "Mix of parallel and sequential based on dependencies" + use_when: "Complex pipelines with partial dependencies" + +# Pipeline templates +templates: + full_sdlc: + name: "Full SDLC Pipeline" + description: "Complete software development lifecycle" + stages: + - name: analysis + agents: [analyst] + parallel: false + outputs: [product_brief] + - name: requirements + agents: [pm] + parallel: false + depends_on: [analysis] + outputs: [prd] + - name: design + agents: [architect, ux-designer] + parallel: true + depends_on: [requirements] + outputs: [architecture, ux_design] + - name: planning + agents: [sm] + parallel: false + depends_on: [design] + outputs: [epics_and_stories] + - name: implementation + agents: [dev] + parallel: false + depends_on: [planning] + outputs: [code, tests] + - name: review + agents: [tea] + parallel: false + depends_on: [implementation] + outputs: [review_report] + + quick_flow: + name: "Quick Flow Pipeline" + description: "Rapid development with minimal ceremony" + stages: + - name: spec + agents: [quick-flow-solo-dev] + parallel: false + outputs: [tech_spec] + - name: implement + agents: [quick-flow-solo-dev] + parallel: false + depends_on: [spec] + outputs: [code, tests] + + analysis_only: + name: "Analysis Pipeline" + description: "Product analysis and requirements" + stages: + - name: research + agents: [analyst] + parallel: false + outputs: [research_findings] + - name: brief + agents: [analyst] + parallel: false + depends_on: [research] + outputs: [product_brief] + - name: requirements + agents: [pm] + parallel: false + depends_on: [brief] + outputs: [prd] + + design_review: + name: "Design Review Pipeline" + description: "Architecture and UX design with review" + stages: + - name: architecture + agents: [architect] + parallel: false + outputs: [architecture] + - name: ux + agents: [ux-designer] + parallel: false + depends_on: [architecture] + outputs: [ux_design] + - name: review + agents: [analyst, pm] + parallel: true + depends_on: [architecture, ux] + outputs: [design_review] + + test_suite: + name: "Test Suite Pipeline" + description: "Comprehensive testing workflow" + stages: + - name: test_design + agents: [tea] + parallel: false + outputs: [test_plan] + - name: test_impl + agents: [tea] + parallel: false + depends_on: [test_design] + outputs: [test_suite] + - name: security + agents: [tea] + parallel: false + depends_on: [test_impl] + outputs: [security_report] + - name: trace + agents: [tea] + parallel: false + depends_on: [test_impl] + outputs: [traceability_matrix] + +# Stage status values +status_values: + - pending # Not yet started + - queued # Ready to start (dependencies met) + - running # Currently executing + - completed # Finished successfully + - failed # Finished with errors + - skipped # Skipped (dependency failed) + - blocked # Waiting for dependencies + +# Error handling +error_handling: + on_stage_failure: "halt" # halt, skip_dependents, retry + max_retries: 2 + retry_delay_seconds: 30 + +# Output management +output_management: + intermediate_outputs_dir: "{pipeline_dir}/outputs" + preserve_intermediate: true + compress_on_complete: false diff --git a/src/core/pipeline/pipeline.md b/src/core/pipeline/pipeline.md new file mode 100644 index 00000000..d7d4e345 --- /dev/null +++ b/src/core/pipeline/pipeline.md @@ -0,0 +1,249 @@ +# Pipeline Orchestrator + +## Overview + +The Pipeline Orchestrator enables multi-stage agent pipelines with automatic dependency management. It supports sequential, parallel, and hybrid execution modes. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ PIPELINE ORCHESTRATOR │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ +│ │ Stage 1 │───▶│ Stage 2 │───▶│ Stage 3 │ (Sequential) │ +│ └─────────┘ └─────────┘ └─────────┘ │ +│ │ +│ ┌─────────┐ │ +│ │ Stage A │─┐ │ +│ └─────────┘ │ ┌─────────┐ │ +│ ├─▶│ Stage D │ (Parallel → Sequential) │ +│ ┌─────────┐ │ └─────────┘ │ +│ │ Stage B │─┘ │ +│ └─────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +## Commands + +### PIPELINE create + +Create a new pipeline from template or custom definition. + +``` +PIPELINE create +PIPELINE create --template full_sdlc +PIPELINE create --name "Custom Pipeline" +``` + +### PIPELINE run + +Execute a pipeline. + +``` +PIPELINE run PIPE-20250115-myproject +PIPELINE run --active +``` + +### PIPELINE status + +Show pipeline status. + +``` +PIPELINE status +PIPELINE status PIPE-20250115-myproject +``` + +**Example output:** +``` +Pipeline: PIPE-20250115-myproject +Status: running +Progress: [████████░░░░░░░░░░░░] 40% + +Stages: +[✓] analysis + Agents: analyst + Duration: 12m 34s + Output: pipelines/outputs/analysis/ + +[✓] requirements + Agents: pm + Duration: 18m 22s + Output: pipelines/outputs/requirements/ + +[►] design + Agents: architect, ux-designer (parallel) + Duration: 8m 15s (running) + Output: pending + +[○] planning + Agents: sm + Duration: pending + Output: n/a + +Currently executing: design +Estimated remaining: ~45 minutes +``` + +### PIPELINE list + +List available pipelines and templates. + +``` +PIPELINE list +``` + +### PIPELINE abort + +Abort a running pipeline. + +``` +PIPELINE abort PIPE-20250115-myproject +``` + +### PIPELINE resume + +Resume a failed or paused pipeline. + +``` +PIPELINE resume PIPE-20250115-myproject +``` + +## Pipeline Templates + +### full_sdlc +Complete software development lifecycle. + +```yaml +stages: + - analysis (analyst) + - requirements (pm) → depends on analysis + - design (architect + ux-designer, parallel) → depends on requirements + - planning (sm) → depends on design + - implementation (dev) → depends on planning + - review (tea) → depends on implementation +``` + +### quick_flow +Rapid development with minimal ceremony. + +```yaml +stages: + - spec (quick-flow-solo-dev) + - implement (quick-flow-solo-dev) → depends on spec +``` + +### analysis_only +Product analysis and requirements. + +```yaml +stages: + - research (analyst) + - brief (analyst) → depends on research + - requirements (pm) → depends on brief +``` + +### design_review +Architecture and UX design with review. + +```yaml +stages: + - architecture (architect) + - ux (ux-designer) → depends on architecture + - review (analyst + pm, parallel) → depends on architecture, ux +``` + +### test_suite +Comprehensive testing workflow. + +```yaml +stages: + - test_design (tea) + - test_impl (tea) → depends on test_design + - security (tea) → depends on test_impl + - trace (tea) → depends on test_impl +``` + +## Custom Pipeline Definition + +```yaml +pipeline_id: "PIPE-20250115-custom" +name: "Custom Pipeline" +description: "My custom pipeline" +created: "2025-01-15T10:00:00Z" +status: "pending" + +stages: + - name: "stage1" + agents: ["analyst"] + parallel: false + depends_on: [] + outputs: ["analysis_report"] + status: "pending" + + - name: "stage2" + agents: ["architect", "ux-designer"] + parallel: true + depends_on: ["stage1"] + outputs: ["architecture", "ux_design"] + status: "pending" + + - name: "stage3" + agents: ["dev"] + parallel: false + depends_on: ["stage2"] + outputs: ["implementation"] + status: "pending" +``` + +## Execution Flow + +1. **Initialize**: Load pipeline definition, validate agents exist +2. **Queue**: Find stages with all dependencies completed +3. **Execute**: Run queued stages (parallel or sequential based on config) +4. **Collect**: Gather outputs from completed stages +5. **Update**: Update pipeline state file +6. **Repeat**: Continue until all stages complete or failure + +## Error Handling + +| Mode | Behavior | +|------|----------| +| halt | Stop pipeline on first failure | +| skip_dependents | Skip stages that depend on failed stage | +| retry | Retry failed stage (up to max_retries) | + +## Output Management + +Pipeline outputs are stored in: +``` +_bmad-output/ +└── pipelines/ + ├── PIPE-20250115-myproject.yaml # Pipeline state + └── outputs/ + ├── analysis/ + │ └── product-brief.md + ├── requirements/ + │ └── prd.md + └── design/ + ├── architecture.md + └── ux-design.md +``` + +## Integration with Token Isolation + +When executing stages: +1. Each agent runs in isolated subprocess (via Task tool) +2. Outputs written to pipeline output directory +3. Only status summaries return to orchestrator +4. Token budget preserved across multi-stage pipelines + +## Best Practices + +1. **Use templates** for common workflows +2. **Define dependencies explicitly** for correct execution order +3. **Enable parallel** only for truly independent stages +4. **Monitor progress** with status command +5. **Archive completed** pipelines to maintain clean state