ldy26098
/
BMAD-METHOD
mirror of https://github.com/bmad-code-org/BMAD-METHOD.git
1
0
Fork 0
Code Packages Projects Releases Wiki Activity

Complete shard-doc replay evidence and compatibility closure

This commit is contained in:
Dicky Moore 2026-03-04 19:39:47 +00:00
parent a770fa5808
commit c7680ab1a8
2 changed files with 917 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
test/test-installation-components.js
View File

@ -4265,6 +4265,24 @@ async function runTests() {
path.join(tempConfigDir, 'bmad-help.csv'),
[...HELP_CATALOG_COMPATIBILITY_PREFIX_COLUMNS, ...HELP_CATALOG_CANONICAL_ADDITIVE_COLUMNS],
[
{
module: 'core',
phase: 'anytime',
name: 'Help',
code: 'BH',
sequence: '',
'workflow-file': '_bmad/core/tasks/help.md',
command: 'bmad-help',
required: 'false',
'agent-name': '',
'agent-command': '',
'agent-display-name': '',
'agent-title': '',
options: '',
description: 'Show BMAD help and available resources.',
'output-location': '',
outputs: '',
},
{
module: 'core',
phase: 'anytime',
@ -4396,6 +4414,60 @@ async function runTests() {
}
assert(deterministicOutputs, 'Shard-doc validation harness outputs are byte-stable across unchanged repeated runs');
try {
await harness.executeIsolatedReplay({
artifactPath: '_bmad/_config/task-manifest.csv',
componentPath: 'bmad-fork/tools/cli/installers/lib/core/manifest-generator.js',
rowIdentity: '',
runtimeFolder: '_bmad',
});
assert(false, 'Shard-doc replay evidence generation rejects missing claimed rowIdentity');
} catch (error) {
assert(
error.code === SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING,
'Shard-doc replay evidence generation emits deterministic missing-claimed-rowIdentity error code',
);
}
try {
await harness.executeIsolatedReplay({
artifactPath: '_bmad/_config/task-manifest.csv',
componentPath: 'bmad-fork/tools/cli/installers/lib/core/installer.js::mergeModuleHelpCatalogs()',
rowIdentity: 'issued-artifact:_bmad-_config-task-manifest.csv',
runtimeFolder: '_bmad',
});
assert(false, 'Shard-doc replay evidence generation rejects issuing-component contract mismatch');
} catch (error) {
assert(
error.code === SHARD_DOC_VALIDATION_ERROR_CODES.BINDING_EVIDENCE_INVALID,
'Shard-doc replay evidence generation emits deterministic issuing-component contract mismatch code',
);
}
const artifactElevenPath = artifactPathsById.get(11);
const artifactElevenRows = csv.parse(await fs.readFile(artifactElevenPath, 'utf8'), {
columns: true,
skip_empty_lines: true,
});
artifactElevenRows[0].baselineArtifactSha256 = 'not-a-sha';
await writeCsv(artifactElevenPath, SHARD_DOC_VALIDATION_ARTIFACT_REGISTRY[10].columns, artifactElevenRows);
try {
await harness.validateGeneratedArtifacts({ projectDir: tempProjectRoot });
assert(false, 'Shard-doc validation harness rejects malformed replay-evidence payloads');
} catch (error) {
assert(
error.code === SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID,
'Shard-doc validation harness emits deterministic replay-evidence validation error code',
);
}
await harness.generateAndValidate({
projectDir: tempProjectRoot,
bmadDir: tempBmadDir,
bmadFolderName: '_bmad',
shardDocAuthorityRecords: authorityRecords,
});
await fs.remove(artifactPathsById.get(8));
try {
await harness.validateGeneratedArtifacts({ projectDir: tempProjectRoot });

850
tools/cli/installers/lib/core/shard-doc-validation-harness.js
View File

@ -1,18 +1,31 @@
const path = require('node:path');
const crypto = require('node:crypto');
const os = require('node:os');
const fs = require('fs-extra');
const yaml = require('yaml');
const csv = require('csv-parse/sync');
const { getSourcePath } = require('../../../lib/project-root');
const { normalizeDisplayedCommandLabel } = require('./help-catalog-generator');
const { ManifestGenerator } = require('./manifest-generator');
const {
ProjectionCompatibilityError,
validateTaskManifestCompatibilitySurface,
validateHelpCatalogLoaderEntries,
validateGithubCopilotHelpLoaderEntries,
} = require('./projection-compatibility-validator');
const SHARD_DOC_SIDECAR_SOURCE_PATH = 'bmad-fork/src/core/tasks/shard-doc.artifact.yaml';
const SHARD_DOC_SOURCE_XML_SOURCE_PATH = 'bmad-fork/src/core/tasks/shard-doc.xml';
const SHARD_DOC_EVIDENCE_ISSUER_COMPONENT = 'bmad-fork/tools/cli/installers/lib/core/shard-doc-validation-harness.js';
const SHARD_DOC_VALIDATION_ERROR_CODES = Object.freeze({
REQUIRED_ARTIFACT_MISSING: 'ERR_SHARD_DOC_VALIDATION_REQUIRED_ARTIFACT_MISSING',
CSV_SCHEMA_MISMATCH: 'ERR_SHARD_DOC_VALIDATION_CSV_SCHEMA_MISMATCH',
REQUIRED_ROW_MISSING: 'ERR_SHARD_DOC_VALIDATION_REQUIRED_ROW_MISSING',
YAML_SCHEMA_MISMATCH: 'ERR_SHARD_DOC_VALIDATION_YAML_SCHEMA_MISMATCH',
BINDING_EVIDENCE_INVALID: 'ERR_SHARD_DOC_VALIDATION_BINDING_EVIDENCE_INVALID',
COMPATIBILITY_GATE_FAILED: 'ERR_SHARD_DOC_VALIDATION_COMPATIBILITY_GATE_FAILED',
REPLAY_EVIDENCE_INVALID: 'ERR_SHARD_DOC_VALIDATION_REPLAY_EVIDENCE_INVALID',
});
const SHARD_DOC_VALIDATION_ARTIFACT_REGISTRY = Object.freeze([
@ -107,6 +120,61 @@ const SHARD_DOC_VALIDATION_ARTIFACT_REGISTRY = Object.freeze([
columns: ['rowIdentity', 'artifactId', 'artifactPath', 'artifactType', 'required', 'rowCount', 'exists', 'schemaVersion', 'status'],
requiredRowIdentityFields: ['rowIdentity'],
}),
Object.freeze({
artifactId: 9,
relativePath: path.join('validation', 'shard-doc', 'shard-doc-compatibility-gates.csv'),
type: 'csv',
columns: ['gateId', 'surface', 'sourcePath', 'status', 'failureCode', 'failureDetail'],
requiredRowIdentityFields: ['gateId'],
}),
Object.freeze({
artifactId: 10,
relativePath: path.join('validation', 'shard-doc', 'shard-doc-issued-artifact-provenance.csv'),
type: 'csv',
columns: [
'rowIdentity',
'artifactPath',
'canonicalId',
'issuerOwnerClass',
'evidenceIssuerComponent',
'evidenceMethod',
'issuingComponent',
'issuingComponentBindingBasis',
'issuingComponentBindingEvidence',
'claimScope',
'status',
],
requiredRowIdentityFields: ['rowIdentity'],
}),
Object.freeze({
artifactId: 11,
relativePath: path.join('validation', 'shard-doc', 'shard-doc-replay-evidence.csv'),
type: 'csv',
columns: [
'rowIdentity',
'provenanceRowIdentity',
'artifactPath',
'issuingComponent',
'targetedRowLocator',
'baselineArtifactSha256',
'mutatedArtifactSha256',
'rowLevelDiffSha256',
'perturbationApplied',
'baselineTargetRowCount',
'mutatedTargetRowCount',
'mutationKind',
'evidenceIssuerClass',
'status',
],
requiredRowIdentityFields: ['rowIdentity', 'provenanceRowIdentity'],
}),
Object.freeze({
artifactId: 12,
relativePath: path.join('validation', 'shard-doc', 'shard-doc-gate-summary.csv'),
type: 'csv',
columns: ['gateId', 'status', 'detail', 'sourcePath'],
requiredRowIdentityFields: ['gateId'],
}),
]);
class ShardDocValidationHarnessError extends Error {
@ -170,6 +238,55 @@ function sortRowsDeterministically(rows, columns) {
});
}
function computeSha256(value) {
return crypto
.createHash('sha256')
.update(String(value || ''), 'utf8')
.digest('hex');
}
function sortObjectKeysDeep(value) {
if (Array.isArray(value)) return value.map((item) => sortObjectKeysDeep(item));
if (!value || typeof value !== 'object') return value;
const sorted = {};
for (const key of Object.keys(value).sort()) {
sorted[key] = sortObjectKeysDeep(value[key]);
}
return sorted;
}
function canonicalJsonStringify(value) {
return JSON.stringify(sortObjectKeysDeep(value));
}
function isSha256(value) {
return /^[a-f0-9]{64}$/.test(String(value || ''));
}
function buildIssuedArtifactRowIdentity(artifactPath) {
return `issued-artifact:${String(artifactPath || '').replaceAll('/', '-')}`;
}
function countShardDocManifestClaimRows(csvContent, runtimeFolder) {
const expectedPath = normalizePath(`${runtimeFolder}/core/tasks/shard-doc.xml`).toLowerCase();
return parseCsvRows(csvContent).filter((row) => {
return (
normalizeValue(row.canonicalId) === 'bmad-shard-doc' &&
normalizeValue(row.name).toLowerCase() === 'shard-doc' &&
normalizeValue(row.module).toLowerCase() === 'core' &&
normalizePath(normalizeValue(row.path)).toLowerCase() === expectedPath
);
}).length;
}
function countShardDocHelpCatalogClaimRows(csvContent) {
return parseCsvRows(csvContent).filter((row) => {
const command = normalizeValue(row.command).replace(/^\/+/, '').toLowerCase();
const workflowFile = normalizePath(normalizeValue(row['workflow-file'])).toLowerCase();
return command === 'bmad-shard-doc' && workflowFile.endsWith('/core/tasks/shard-doc.xml');
}).length;
}
class ShardDocValidationHarness {
constructor() {
this.registry = SHARD_DOC_VALIDATION_ARTIFACT_REGISTRY;
@ -234,6 +351,522 @@ class ShardDocValidationHarness {
});
}
resolveReplayContract({ artifactPath, componentPath, rowIdentity, runtimeFolder }) {
const claimedRowIdentity = normalizeValue(rowIdentity);
if (!claimedRowIdentity) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING,
detail: 'Claimed replay rowIdentity is required',
artifactId: 11,
fieldPath: 'rowIdentity',
sourcePath: normalizePath(artifactPath),
observedValue: '<empty>',
expectedValue: 'non-empty rowIdentity',
});
}
const expectedRowIdentity = buildIssuedArtifactRowIdentity(artifactPath);
if (claimedRowIdentity !== expectedRowIdentity) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING,
detail: 'Claimed replay rowIdentity does not match issued-artifact contract',
artifactId: 11,
fieldPath: 'rowIdentity',
sourcePath: normalizePath(artifactPath),
observedValue: claimedRowIdentity,
expectedValue: expectedRowIdentity,
});
}
const contractsByRowIdentity = new Map([
[
buildIssuedArtifactRowIdentity(`${runtimeFolder}/_config/task-manifest.csv`),
{
artifactPath: `${runtimeFolder}/_config/task-manifest.csv`,
componentPathIncludes: 'manifest-generator.js',
mutationKind: 'component-input-perturbation:manifest-generator/tasks',
run: ({ workspaceRoot, perturbed }) => this.runManifestGeneratorReplay({ workspaceRoot, runtimeFolder, perturbed }),
},
],
[
buildIssuedArtifactRowIdentity(`${runtimeFolder}/_config/bmad-help.csv`),
{
artifactPath: `${runtimeFolder}/_config/bmad-help.csv`,
componentPathIncludes: 'installer.js::mergemodulehelpcatalogs',
mutationKind: 'component-input-perturbation:installer/module-help-command',
run: ({ workspaceRoot, perturbed }) => this.runInstallerMergeReplay({ workspaceRoot, runtimeFolder, perturbed }),
},
],
]);
const contract = contractsByRowIdentity.get(claimedRowIdentity);
if (!contract) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING,
detail: 'Claimed replay rowIdentity is not mapped to a replay contract',
artifactId: 11,
fieldPath: 'rowIdentity',
sourcePath: normalizePath(artifactPath),
observedValue: claimedRowIdentity,
expectedValue: 'known issued-artifact rowIdentity',
});
}
const normalizedComponentPath = normalizeValue(componentPath).toLowerCase();
if (
normalizeValue(artifactPath) !== normalizeValue(contract.artifactPath) ||
!normalizedComponentPath.includes(String(contract.componentPathIncludes || '').toLowerCase())
) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.BINDING_EVIDENCE_INVALID,
detail: 'Claimed issuingComponent does not match replay contract mapping',
artifactId: 11,
fieldPath: 'issuingComponent',
sourcePath: normalizePath(artifactPath),
observedValue: canonicalJsonStringify({
artifactPath,
componentPath,
rowIdentity: claimedRowIdentity,
}),
expectedValue: canonicalJsonStringify({
artifactPath: contract.artifactPath,
componentPathIncludes: contract.componentPathIncludes,
rowIdentity: claimedRowIdentity,
}),
});
}
return contract;
}
async runManifestGeneratorReplay({ workspaceRoot, runtimeFolder, perturbed }) {
const bmadDir = path.join(workspaceRoot, runtimeFolder);
const cfgDir = path.join(bmadDir, '_config');
await fs.ensureDir(cfgDir);
const generator = new ManifestGenerator();
generator.bmadFolderName = runtimeFolder;
generator.helpAuthorityRecords = [];
generator.taskAuthorityRecords = [
{
recordType: 'metadata-authority',
canonicalId: 'bmad-shard-doc',
authoritySourceType: 'sidecar',
authoritySourcePath: SHARD_DOC_SIDECAR_SOURCE_PATH,
sourcePath: SHARD_DOC_SOURCE_XML_SOURCE_PATH,
},
];
generator.tasks = perturbed
? []
: [
{
name: 'shard-doc',
displayName: 'Shard Document',
description: 'Split large markdown documents into smaller files by section with an index.',
module: 'core',
path: `${runtimeFolder}/core/tasks/shard-doc.xml`,
standalone: 'true',
},
];
await generator.writeTaskManifest(cfgDir);
const outputPath = path.join(cfgDir, 'task-manifest.csv');
const content = await fs.readFile(outputPath, 'utf8');
return {
content,
targetRowCount: countShardDocManifestClaimRows(content, runtimeFolder),
};
}
async runInstallerMergeReplay({ workspaceRoot, runtimeFolder, perturbed }) {
const { Installer } = require('./installer');
const bmadDir = path.join(workspaceRoot, runtimeFolder);
const coreDir = path.join(bmadDir, 'core');
const cfgDir = path.join(bmadDir, '_config');
await fs.ensureDir(coreDir);
await fs.ensureDir(cfgDir);
const buildCsvLine = (values) =>
values
.map((value) => {
const text = String(value ?? '');
return text.includes(',') || text.includes('"') ? `"${text.replaceAll('"', '""')}"` : text;
})
.join(',');
const writeCsv = async (filePath, columns, rows) => {
const lines = [columns.join(','), ...rows.map((row) => buildCsvLine(columns.map((column) => row[column] ?? '')))];
await fs.writeFile(filePath, `${lines.join('\n')}\n`, 'utf8');
};
await writeCsv(
path.join(coreDir, 'module-help.csv'),
[
'module',
'phase',
'name',
'code',
'sequence',
'workflow-file',
'command',
'required',
'agent',
'options',
'description',
'output-location',
'outputs',
],
[
{
module: 'core',
phase: 'anytime',
name: 'help',
code: 'BH',
sequence: '',
'workflow-file': `${runtimeFolder}/core/tasks/help.md`,
command: 'bmad-help',
required: 'false',
agent: '',
options: '',
description: 'Show BMAD help',
'output-location': '',
outputs: '',
},
{
module: 'core',
phase: 'anytime',
name: 'Shard Document',
code: 'SD',
sequence: '',
'workflow-file': `${runtimeFolder}/core/tasks/shard-doc.xml`,
command: perturbed ? 'shard-doc' : 'bmad-shard-doc',
required: 'false',
agent: '',
options: '',
description: 'Split large markdown documents into smaller files by section with an index.',
'output-location': '',
outputs: '',
},
],
);
await fs.writeFile(
path.join(cfgDir, 'agent-manifest.csv'),
'name,displayName,title,icon,capabilities,role,identity,communicationStyle,principles,module,path\n',
'utf8',
);
const installer = new Installer();
installer.bmadFolderName = runtimeFolder;
installer.installedFiles = new Set();
installer.helpAuthorityRecords = [];
installer.shardDocAuthorityRecords = [
{
recordType: 'metadata-authority',
canonicalId: 'bmad-shard-doc',
authoritySourceType: 'sidecar',
authoritySourcePath: SHARD_DOC_SIDECAR_SOURCE_PATH,
sourcePath: SHARD_DOC_SOURCE_XML_SOURCE_PATH,
},
];
try {
await installer.mergeModuleHelpCatalogs(bmadDir);
const outputPath = path.join(cfgDir, 'bmad-help.csv');
const content = await fs.readFile(outputPath, 'utf8');
return {
content,
targetRowCount: countShardDocHelpCatalogClaimRows(content),
};
} catch (error) {
if (perturbed && normalizeValue(error?.code) === 'ERR_HELP_COMMAND_LABEL_CONTRACT_FAILED') {
return {
content: `PERTURBED_COMPONENT_FAILURE:${normalizeValue(error.code)}:${normalizeValue(error.detail || error.message)}`,
targetRowCount: 0,
};
}
throw error;
}
}
async executeIsolatedReplay({ artifactPath, componentPath, rowIdentity, runtimeFolder }) {
const contract = this.resolveReplayContract({
artifactPath,
componentPath,
rowIdentity,
runtimeFolder,
});
const baselineWorkspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'shard-doc-replay-baseline-'));
const perturbedWorkspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'shard-doc-replay-perturbed-'));
try {
const baseline = await contract.run({ workspaceRoot: baselineWorkspaceRoot, perturbed: false });
if (Number(baseline.targetRowCount) <= 0) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING,
detail: 'Claimed replay rowIdentity target is absent in baseline component output',
artifactId: 11,
fieldPath: 'rowIdentity',
sourcePath: normalizePath(artifactPath),
observedValue: String(baseline.targetRowCount),
expectedValue: `at least one row for ${normalizeValue(rowIdentity)}`,
});
}
const mutated = await contract.run({ workspaceRoot: perturbedWorkspaceRoot, perturbed: true });
return {
baselineContent: baseline.content,
mutatedContent: mutated.content,
baselineTargetRowCount: Number(baseline.targetRowCount),
mutatedTargetRowCount: Number(mutated.targetRowCount),
perturbationApplied: true,
mutationKind: contract.mutationKind,
targetedRowLocator: normalizeValue(rowIdentity),
};
} finally {
await fs.remove(baselineWorkspaceRoot);
await fs.remove(perturbedWorkspaceRoot);
}
}
async buildObservedBindingEvidence({ artifactPath, absolutePath, componentPath, rowIdentity, runtimeFolder }) {
await this.assertRequiredInputSurfaceExists({
artifactId: 10,
absolutePath,
sourcePath: artifactPath,
description: 'issued-artifact replay target surface',
});
const mutationResult = await this.executeIsolatedReplay({
artifactPath,
componentPath,
rowIdentity,
runtimeFolder: normalizeValue(runtimeFolder || '_bmad'),
});
const baselineArtifactSha256 = computeSha256(mutationResult.baselineContent);
const mutatedArtifactSha256 = computeSha256(mutationResult.mutatedContent);
const diffPayload = {
artifactPath,
componentPath,
rowIdentity,
mutationKind: mutationResult.mutationKind,
targetedRowLocator: mutationResult.targetedRowLocator,
baselineTargetRowCount: mutationResult.baselineTargetRowCount,
mutatedTargetRowCount: mutationResult.mutatedTargetRowCount,
baselineArtifactSha256,
mutatedArtifactSha256,
};
const rowLevelDiffSha256 = computeSha256(canonicalJsonStringify(diffPayload));
const evidencePayload = canonicalJsonStringify({
evidenceVersion: 1,
observationMethod: 'validator-observed-baseline-plus-isolated-single-component-perturbation',
observationOutcome:
mutationResult.baselineTargetRowCount > mutationResult.mutatedTargetRowCount ? 'observed-impact' : 'no-impact-observed',
artifactPath,
componentPath,
targetedRowLocator: mutationResult.targetedRowLocator,
mutationKind: mutationResult.mutationKind,
baselineTargetRowCount: mutationResult.baselineTargetRowCount,
mutatedTargetRowCount: mutationResult.mutatedTargetRowCount,
baselineArtifactSha256,
mutatedArtifactSha256,
rowLevelDiffSha256,
perturbationApplied: true,
serializationFormat: 'json-canonical-v1',
encoding: 'utf-8',
lineEndings: 'lf',
worktreePath: 'isolated-replay-temp-workspaces',
commitSha: 'not-applicable',
timestampUtc: '1970-01-01T00:00:00Z',
});
return {
evidenceMethod: 'validator-observed-baseline-plus-isolated-single-component-perturbation',
issuingComponentBindingBasis: 'validator-observed-baseline-plus-isolated-single-component-perturbation',
issuingComponentBindingEvidence: evidencePayload,
targetedRowLocator: mutationResult.targetedRowLocator,
baselineArtifactSha256,
mutatedArtifactSha256,
rowLevelDiffSha256,
perturbationApplied: true,
baselineTargetRowCount: mutationResult.baselineTargetRowCount,
mutatedTargetRowCount: mutationResult.mutatedTargetRowCount,
mutationKind: mutationResult.mutationKind,
status: mutationResult.baselineTargetRowCount > mutationResult.mutatedTargetRowCount ? 'PASS' : 'FAIL',
};
}
async createIssuedArtifactEvidenceRows({ runtimeFolder, bmadDir }) {
const bindings = [
{
artifactPath: `${runtimeFolder}/_config/task-manifest.csv`,
absolutePath: path.join(bmadDir, '_config', 'task-manifest.csv'),
issuingComponent: 'bmad-fork/tools/cli/installers/lib/core/manifest-generator.js',
},
{
artifactPath: `${runtimeFolder}/_config/bmad-help.csv`,
absolutePath: path.join(bmadDir, '_config', 'bmad-help.csv'),
issuingComponent: 'bmad-fork/tools/cli/installers/lib/core/installer.js::mergeModuleHelpCatalogs()',
},
];
const provenanceRows = [];
const replayEvidenceRows = [];
for (const binding of bindings) {
const rowIdentity = buildIssuedArtifactRowIdentity(binding.artifactPath);
const evidence = await this.buildObservedBindingEvidence({
artifactPath: binding.artifactPath,
absolutePath: binding.absolutePath,
componentPath: binding.issuingComponent,
rowIdentity,
runtimeFolder,
});
provenanceRows.push({
rowIdentity,
artifactPath: binding.artifactPath,
canonicalId: 'bmad-shard-doc',
issuerOwnerClass: 'independent-validator',
evidenceIssuerComponent: SHARD_DOC_EVIDENCE_ISSUER_COMPONENT,
evidenceMethod: evidence.evidenceMethod,
issuingComponent: binding.issuingComponent,
issuingComponentBindingBasis: evidence.issuingComponentBindingBasis,
issuingComponentBindingEvidence: evidence.issuingComponentBindingEvidence,
claimScope: binding.artifactPath,
status: evidence.status,
});
replayEvidenceRows.push({
rowIdentity: `replay-evidence:${rowIdentity}`,
provenanceRowIdentity: rowIdentity,
artifactPath: binding.artifactPath,
issuingComponent: binding.issuingComponent,
targetedRowLocator: evidence.targetedRowLocator,
baselineArtifactSha256: evidence.baselineArtifactSha256,
mutatedArtifactSha256: evidence.mutatedArtifactSha256,
rowLevelDiffSha256: evidence.rowLevelDiffSha256,
perturbationApplied: evidence.perturbationApplied ? 'true' : 'false',
baselineTargetRowCount: String(evidence.baselineTargetRowCount),
mutatedTargetRowCount: String(evidence.mutatedTargetRowCount),
mutationKind: evidence.mutationKind,
evidenceIssuerClass: 'independent-validator',
status: evidence.status,
});
}
return {
provenanceRows,
replayEvidenceRows,
};
}
runCompatibilityGate({ gateId, surface, sourcePath, runner }) {
try {
runner();
return {
gateId,
surface,
sourcePath,
status: 'PASS',
failureCode: '',
failureDetail: '',
};
} catch (error) {
if (error instanceof ProjectionCompatibilityError) {
return {
gateId,
surface,
sourcePath,
status: 'FAIL',
failureCode: normalizeValue(error.code || 'ERR_COMPATIBILITY_GATE_FAILED'),
failureDetail: normalizeValue(error.detail || error.message || 'compatibility gate failure'),
};
}
throw error;
}
}
generateCompatibilityGateRows({ taskManifestCsvContent, helpCatalogCsvContent, runtimeFolder }) {
const helpRows = parseCsvRows(helpCatalogCsvContent);
const helpHeaderColumns = parseCsvHeader(helpCatalogCsvContent);
return [
this.runCompatibilityGate({
gateId: 'task-manifest-loader',
surface: 'task-manifest-loader',
sourcePath: `${runtimeFolder}/_config/task-manifest.csv`,
runner: () => {
validateTaskManifestCompatibilitySurface(taskManifestCsvContent, {
surface: 'task-manifest-loader',
sourcePath: `${runtimeFolder}/_config/task-manifest.csv`,
});
},
}),
this.runCompatibilityGate({
gateId: 'bmad-help-catalog-loader',
surface: 'bmad-help-catalog-loader',
sourcePath: `${runtimeFolder}/_config/bmad-help.csv`,
runner: () => {
validateHelpCatalogLoaderEntries(helpRows, {
surface: 'bmad-help-catalog-loader',
sourcePath: `${runtimeFolder}/_config/bmad-help.csv`,
headerColumns: helpHeaderColumns,
});
},
}),
this.runCompatibilityGate({
gateId: 'github-copilot-help-loader',
surface: 'github-copilot-help-loader',
sourcePath: `${runtimeFolder}/_config/bmad-help.csv`,
runner: () => {
validateGithubCopilotHelpLoaderEntries(helpRows, {
surface: 'github-copilot-help-loader',
sourcePath: `${runtimeFolder}/_config/bmad-help.csv`,
headerColumns: helpHeaderColumns,
});
},
}),
];
}
buildGateSummaryRows({ compatibilityRows, provenanceRows, replayRows, runtimeFolder }) {
const compatibilityPass = compatibilityRows.every((row) => normalizeValue(row.status) === 'PASS');
const provenancePass = provenanceRows.every((row) => normalizeValue(row.status) === 'PASS');
const replayPass = replayRows.every((row) => normalizeValue(row.status) === 'PASS');
return [
{
gateId: 'compatibility-gates',
status: compatibilityPass ? 'PASS' : 'FAIL',
detail: compatibilityPass ? 'task/help/copilot compatibility gates passed' : 'one or more compatibility gates failed',
sourcePath: `${runtimeFolder}/_config/task-manifest.csv|${runtimeFolder}/_config/bmad-help.csv`,
},
{
gateId: 'issued-artifact-provenance',
status: provenancePass ? 'PASS' : 'FAIL',
detail: provenancePass ? 'all issued-artifact provenance claims validated' : 'one or more provenance claims failed replay binding',
sourcePath: 'validation/shard-doc/shard-doc-issued-artifact-provenance.csv',
},
{
gateId: 'replay-evidence',
status: replayPass ? 'PASS' : 'FAIL',
detail: replayPass ? 'row-targeted isolated replay evidence validated' : 'replay evidence is missing or invalid',
sourcePath: 'validation/shard-doc/shard-doc-replay-evidence.csv',
},
{
gateId: 'required-test-commands',
status: compatibilityPass && provenancePass && replayPass ? 'PASS' : 'FAIL',
detail:
compatibilityPass && provenancePass && replayPass
? 'harness prerequisites satisfied; CI/local test commands must also pass'
: 'harness prerequisites failed; required test command gate is blocked',
sourcePath: 'npm run test:install|npm test',
},
];
}
async generateValidationArtifacts(options = {}) {
const outputPaths = this.resolveOutputPaths(options);
const runtimeFolder = normalizeValue(options.bmadFolderName || '_bmad');
@ -284,9 +917,12 @@ class ShardDocValidationHarness {
});
const sidecarMetadata = yaml.parse(await fs.readFile(sidecarPath, 'utf8'));
const taskManifestRows = parseCsvRows(await fs.readFile(path.join(bmadDir, '_config', 'task-manifest.csv'), 'utf8'));
const helpCatalogRows = parseCsvRows(await fs.readFile(path.join(bmadDir, '_config', 'bmad-help.csv'), 'utf8'));
const aliasRows = parseCsvRows(await fs.readFile(path.join(bmadDir, '_config', 'canonical-aliases.csv'), 'utf8'));
const taskManifestCsvContent = await fs.readFile(path.join(bmadDir, '_config', 'task-manifest.csv'), 'utf8');
const helpCatalogCsvContent = await fs.readFile(path.join(bmadDir, '_config', 'bmad-help.csv'), 'utf8');
const aliasCsvContent = await fs.readFile(path.join(bmadDir, '_config', 'canonical-aliases.csv'), 'utf8');
const taskManifestRows = parseCsvRows(taskManifestCsvContent);
const helpCatalogRows = parseCsvRows(helpCatalogCsvContent);
const aliasRows = parseCsvRows(aliasCsvContent);
const commandLabelReportPath = path.join(bmadDir, '_config', 'bmad-help-command-label-report.csv');
let commandLabelRows = [];
if (Array.isArray(options.helpCatalogCommandLabelReportRows) && options.helpCatalogCommandLabelReportRows.length > 0) {
@ -490,7 +1126,32 @@ class ShardDocValidationHarness {
];
await this.writeCsvArtifact(artifactPaths.get(7), this.registry[6].columns, duplicateRows);
// Artifact 8
// Artifact 9
const compatibilityRows = this.generateCompatibilityGateRows({
taskManifestCsvContent,
helpCatalogCsvContent,
runtimeFolder,
});
await this.writeCsvArtifact(artifactPaths.get(9), this.registry[8].columns, compatibilityRows);
// Artifact 10 + 11
const { provenanceRows, replayEvidenceRows } = await this.createIssuedArtifactEvidenceRows({
runtimeFolder,
bmadDir,
});
await this.writeCsvArtifact(artifactPaths.get(10), this.registry[9].columns, provenanceRows);
await this.writeCsvArtifact(artifactPaths.get(11), this.registry[10].columns, replayEvidenceRows);
// Artifact 12
const gateSummaryRows = this.buildGateSummaryRows({
compatibilityRows,
provenanceRows,
replayRows: replayEvidenceRows,
runtimeFolder,
});
await this.writeCsvArtifact(artifactPaths.get(12), this.registry[11].columns, gateSummaryRows);
// Artifact 8 (after all other artifacts exist)
const inventoryRows = [];
for (const artifact of this.registry) {
const artifactPath = normalizePath(artifact.relativePath);
@ -505,7 +1166,6 @@ class ShardDocValidationHarness {
} else if (exists && artifact.type === 'yaml') {
rowCount = 1;
}
inventoryRows.push({
rowIdentity: `artifact-inventory-row:${artifact.artifactId}`,
artifactId: String(artifact.artifactId),
@ -529,6 +1189,55 @@ class ShardDocValidationHarness {
};
}
validateReplayEvidenceRow(row, sourcePath) {
if (!isSha256(row.baselineArtifactSha256)) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID,
detail: 'Replay evidence baselineArtifactSha256 must be a valid sha256 hex digest',
artifactId: 11,
fieldPath: 'rows[*].baselineArtifactSha256',
sourcePath,
observedValue: normalizeValue(row.baselineArtifactSha256),
expectedValue: '64-char lowercase sha256 hex',
});
}
if (!isSha256(row.mutatedArtifactSha256)) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID,
detail: 'Replay evidence mutatedArtifactSha256 must be a valid sha256 hex digest',
artifactId: 11,
fieldPath: 'rows[*].mutatedArtifactSha256',
sourcePath,
observedValue: normalizeValue(row.mutatedArtifactSha256),
expectedValue: '64-char lowercase sha256 hex',
});
}
if (!isSha256(row.rowLevelDiffSha256)) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID,
detail: 'Replay evidence rowLevelDiffSha256 must be a valid sha256 hex digest',
artifactId: 11,
fieldPath: 'rows[*].rowLevelDiffSha256',
sourcePath,
observedValue: normalizeValue(row.rowLevelDiffSha256),
expectedValue: '64-char lowercase sha256 hex',
});
}
const perturbationApplied = normalizeValue(row.perturbationApplied).toLowerCase();
if (perturbationApplied !== 'true') {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID,
detail: 'Replay evidence must prove perturbationApplied=true from isolated component replay',
artifactId: 11,
fieldPath: 'rows[*].perturbationApplied',
sourcePath,
observedValue: normalizeValue(row.perturbationApplied),
expectedValue: 'true',
});
}
}
async validateGeneratedArtifacts(options = {}) {
const outputPaths = this.resolveOutputPaths(options);
const artifactDataById = new Map();
@ -662,6 +1371,137 @@ class ShardDocValidationHarness {
detail: 'Source-body authority record for shard-doc is missing',
});
const compatibilityRows = artifactDataById.get(9)?.rows || [];
for (const gateId of ['task-manifest-loader', 'bmad-help-catalog-loader', 'github-copilot-help-loader']) {
const gateRow = this.requireRow({
rows: compatibilityRows,
predicate: (row) => normalizeValue(row.gateId) === gateId,
artifactId: 9,
fieldPath: 'rows[*].gateId',
sourcePath: normalizePath(this.registry[8].relativePath),
detail: `Required compatibility gate row is missing (${gateId})`,
});
if (normalizeValue(gateRow.status) !== 'PASS') {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.COMPATIBILITY_GATE_FAILED,
detail: `Compatibility gate failed (${gateId})`,
artifactId: 9,
fieldPath: `rows[gateId=${gateId}].status`,
sourcePath: normalizePath(this.registry[8].relativePath),
observedValue: normalizeValue(gateRow.status),
expectedValue: 'PASS',
});
}
}
const provenanceRows = artifactDataById.get(10)?.rows || [];
for (const artifactPath of ['_bmad/_config/task-manifest.csv', '_bmad/_config/bmad-help.csv']) {
const rowIdentity = buildIssuedArtifactRowIdentity(artifactPath);
const provenanceRow = this.requireRow({
rows: provenanceRows,
predicate: (row) => normalizeValue(row.rowIdentity) === rowIdentity,
artifactId: 10,
fieldPath: 'rows[*].rowIdentity',
sourcePath: normalizePath(this.registry[9].relativePath),
detail: `Required issued-artifact provenance row is missing (${rowIdentity})`,
});
if (
normalizeValue(provenanceRow.status) !== 'PASS' ||
normalizeValue(provenanceRow.issuerOwnerClass) !== 'independent-validator' ||
normalizeValue(provenanceRow.evidenceIssuerComponent) !== SHARD_DOC_EVIDENCE_ISSUER_COMPONENT
) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.BINDING_EVIDENCE_INVALID,
detail: 'Issued-artifact provenance row failed deterministic issuer binding contract',
artifactId: 10,
fieldPath: `rows[rowIdentity=${rowIdentity}]`,
sourcePath: normalizePath(this.registry[9].relativePath),
observedValue: canonicalJsonStringify({
status: normalizeValue(provenanceRow.status),
issuerOwnerClass: normalizeValue(provenanceRow.issuerOwnerClass),
evidenceIssuerComponent: normalizeValue(provenanceRow.evidenceIssuerComponent),
}),
expectedValue: canonicalJsonStringify({
status: 'PASS',
issuerOwnerClass: 'independent-validator',
evidenceIssuerComponent: SHARD_DOC_EVIDENCE_ISSUER_COMPONENT,
}),
});
}
if (!normalizeValue(provenanceRow.issuingComponentBindingEvidence)) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.BINDING_EVIDENCE_INVALID,
detail: 'Issued-artifact provenance row is missing binding evidence payload',
artifactId: 10,
fieldPath: `rows[rowIdentity=${rowIdentity}].issuingComponentBindingEvidence`,
sourcePath: normalizePath(this.registry[9].relativePath),
observedValue: '<empty>',
expectedValue: 'non-empty canonical JSON payload',
});
}
}
const replayRows = artifactDataById.get(11)?.rows || [];
for (const replayRow of replayRows) {
this.validateReplayEvidenceRow(replayRow, normalizePath(this.registry[10].relativePath));
const provenanceRow = this.requireRow({
rows: provenanceRows,
predicate: (row) => normalizeValue(row.rowIdentity) === normalizeValue(replayRow.provenanceRowIdentity),
artifactId: 11,
fieldPath: 'rows[*].provenanceRowIdentity',
sourcePath: normalizePath(this.registry[10].relativePath),
detail: 'Replay evidence row references missing issued-artifact provenance rowIdentity',
});
if (normalizeValue(replayRow.targetedRowLocator) !== normalizeValue(provenanceRow.rowIdentity)) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID,
detail: 'Replay evidence targetedRowLocator must equal provenance rowIdentity',
artifactId: 11,
fieldPath: 'rows[*].targetedRowLocator',
sourcePath: normalizePath(this.registry[10].relativePath),
observedValue: normalizeValue(replayRow.targetedRowLocator),
expectedValue: normalizeValue(provenanceRow.rowIdentity),
});
}
if (
Number.parseInt(normalizeValue(replayRow.baselineTargetRowCount), 10) <=
Number.parseInt(normalizeValue(replayRow.mutatedTargetRowCount), 10)
) {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID,
detail: 'Replay evidence must show baseline target count greater than mutated target count',
artifactId: 11,
fieldPath: 'rows[*].baselineTargetRowCount',
sourcePath: normalizePath(this.registry[10].relativePath),
observedValue: `${normalizeValue(replayRow.baselineTargetRowCount)}<=${normalizeValue(replayRow.mutatedTargetRowCount)}`,
expectedValue: 'baselineTargetRowCount > mutatedTargetRowCount',
});
}
}
const gateSummaryRows = artifactDataById.get(12)?.rows || [];
for (const gateId of ['compatibility-gates', 'issued-artifact-provenance', 'replay-evidence']) {
const summaryRow = this.requireRow({
rows: gateSummaryRows,
predicate: (row) => normalizeValue(row.gateId) === gateId,
artifactId: 12,
fieldPath: 'rows[*].gateId',
sourcePath: normalizePath(this.registry[11].relativePath),
detail: `Required gate summary row is missing (${gateId})`,
});
if (normalizeValue(summaryRow.status) !== 'PASS') {
throw new ShardDocValidationHarnessError({
code: SHARD_DOC_VALIDATION_ERROR_CODES.COMPATIBILITY_GATE_FAILED,
detail: `Gate summary failed (${gateId})`,
artifactId: 12,
fieldPath: `rows[gateId=${gateId}].status`,
sourcePath: normalizePath(this.registry[11].relativePath),
observedValue: normalizeValue(summaryRow.status),
expectedValue: 'PASS',
});
}
}
const inventoryRows = artifactDataById.get(8)?.rows || [];
if (inventoryRows.length !== this.registry.length) {
throw new ShardDocValidationHarnessError({