diff --git a/test/test-installation-components.js b/test/test-installation-components.js index e3ac496d1..208b48cb0 100644 --- a/test/test-installation-components.js +++ b/test/test-installation-components.js @@ -4265,6 +4265,24 @@ async function runTests() { path.join(tempConfigDir, 'bmad-help.csv'), [...HELP_CATALOG_COMPATIBILITY_PREFIX_COLUMNS, ...HELP_CATALOG_CANONICAL_ADDITIVE_COLUMNS], [ + { + module: 'core', + phase: 'anytime', + name: 'Help', + code: 'BH', + sequence: '', + 'workflow-file': '_bmad/core/tasks/help.md', + command: 'bmad-help', + required: 'false', + 'agent-name': '', + 'agent-command': '', + 'agent-display-name': '', + 'agent-title': '', + options: '', + description: 'Show BMAD help and available resources.', + 'output-location': '', + outputs: '', + }, { module: 'core', phase: 'anytime', @@ -4396,6 +4414,60 @@ async function runTests() { } assert(deterministicOutputs, 'Shard-doc validation harness outputs are byte-stable across unchanged repeated runs'); + try { + await harness.executeIsolatedReplay({ + artifactPath: '_bmad/_config/task-manifest.csv', + componentPath: 'bmad-fork/tools/cli/installers/lib/core/manifest-generator.js', + rowIdentity: '', + runtimeFolder: '_bmad', + }); + assert(false, 'Shard-doc replay evidence generation rejects missing claimed rowIdentity'); + } catch (error) { + assert( + error.code === SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING, + 'Shard-doc replay evidence generation emits deterministic missing-claimed-rowIdentity error code', + ); + } + + try { + await harness.executeIsolatedReplay({ + artifactPath: '_bmad/_config/task-manifest.csv', + componentPath: 'bmad-fork/tools/cli/installers/lib/core/installer.js::mergeModuleHelpCatalogs()', + rowIdentity: 'issued-artifact:_bmad-_config-task-manifest.csv', + runtimeFolder: '_bmad', + }); + assert(false, 'Shard-doc replay evidence generation rejects issuing-component contract mismatch'); + } catch (error) { + assert( + error.code === SHARD_DOC_VALIDATION_ERROR_CODES.BINDING_EVIDENCE_INVALID, + 'Shard-doc replay evidence generation emits deterministic issuing-component contract mismatch code', + ); + } + + const artifactElevenPath = artifactPathsById.get(11); + const artifactElevenRows = csv.parse(await fs.readFile(artifactElevenPath, 'utf8'), { + columns: true, + skip_empty_lines: true, + }); + artifactElevenRows[0].baselineArtifactSha256 = 'not-a-sha'; + await writeCsv(artifactElevenPath, SHARD_DOC_VALIDATION_ARTIFACT_REGISTRY[10].columns, artifactElevenRows); + try { + await harness.validateGeneratedArtifacts({ projectDir: tempProjectRoot }); + assert(false, 'Shard-doc validation harness rejects malformed replay-evidence payloads'); + } catch (error) { + assert( + error.code === SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID, + 'Shard-doc validation harness emits deterministic replay-evidence validation error code', + ); + } + + await harness.generateAndValidate({ + projectDir: tempProjectRoot, + bmadDir: tempBmadDir, + bmadFolderName: '_bmad', + shardDocAuthorityRecords: authorityRecords, + }); + await fs.remove(artifactPathsById.get(8)); try { await harness.validateGeneratedArtifacts({ projectDir: tempProjectRoot }); diff --git a/tools/cli/installers/lib/core/shard-doc-validation-harness.js b/tools/cli/installers/lib/core/shard-doc-validation-harness.js index 932ed4db3..7cbebd42e 100644 --- a/tools/cli/installers/lib/core/shard-doc-validation-harness.js +++ b/tools/cli/installers/lib/core/shard-doc-validation-harness.js @@ -1,18 +1,31 @@ const path = require('node:path'); +const crypto = require('node:crypto'); +const os = require('node:os'); const fs = require('fs-extra'); const yaml = require('yaml'); const csv = require('csv-parse/sync'); const { getSourcePath } = require('../../../lib/project-root'); const { normalizeDisplayedCommandLabel } = require('./help-catalog-generator'); +const { ManifestGenerator } = require('./manifest-generator'); +const { + ProjectionCompatibilityError, + validateTaskManifestCompatibilitySurface, + validateHelpCatalogLoaderEntries, + validateGithubCopilotHelpLoaderEntries, +} = require('./projection-compatibility-validator'); const SHARD_DOC_SIDECAR_SOURCE_PATH = 'bmad-fork/src/core/tasks/shard-doc.artifact.yaml'; const SHARD_DOC_SOURCE_XML_SOURCE_PATH = 'bmad-fork/src/core/tasks/shard-doc.xml'; +const SHARD_DOC_EVIDENCE_ISSUER_COMPONENT = 'bmad-fork/tools/cli/installers/lib/core/shard-doc-validation-harness.js'; const SHARD_DOC_VALIDATION_ERROR_CODES = Object.freeze({ REQUIRED_ARTIFACT_MISSING: 'ERR_SHARD_DOC_VALIDATION_REQUIRED_ARTIFACT_MISSING', CSV_SCHEMA_MISMATCH: 'ERR_SHARD_DOC_VALIDATION_CSV_SCHEMA_MISMATCH', REQUIRED_ROW_MISSING: 'ERR_SHARD_DOC_VALIDATION_REQUIRED_ROW_MISSING', YAML_SCHEMA_MISMATCH: 'ERR_SHARD_DOC_VALIDATION_YAML_SCHEMA_MISMATCH', + BINDING_EVIDENCE_INVALID: 'ERR_SHARD_DOC_VALIDATION_BINDING_EVIDENCE_INVALID', + COMPATIBILITY_GATE_FAILED: 'ERR_SHARD_DOC_VALIDATION_COMPATIBILITY_GATE_FAILED', + REPLAY_EVIDENCE_INVALID: 'ERR_SHARD_DOC_VALIDATION_REPLAY_EVIDENCE_INVALID', }); const SHARD_DOC_VALIDATION_ARTIFACT_REGISTRY = Object.freeze([ @@ -107,6 +120,61 @@ const SHARD_DOC_VALIDATION_ARTIFACT_REGISTRY = Object.freeze([ columns: ['rowIdentity', 'artifactId', 'artifactPath', 'artifactType', 'required', 'rowCount', 'exists', 'schemaVersion', 'status'], requiredRowIdentityFields: ['rowIdentity'], }), + Object.freeze({ + artifactId: 9, + relativePath: path.join('validation', 'shard-doc', 'shard-doc-compatibility-gates.csv'), + type: 'csv', + columns: ['gateId', 'surface', 'sourcePath', 'status', 'failureCode', 'failureDetail'], + requiredRowIdentityFields: ['gateId'], + }), + Object.freeze({ + artifactId: 10, + relativePath: path.join('validation', 'shard-doc', 'shard-doc-issued-artifact-provenance.csv'), + type: 'csv', + columns: [ + 'rowIdentity', + 'artifactPath', + 'canonicalId', + 'issuerOwnerClass', + 'evidenceIssuerComponent', + 'evidenceMethod', + 'issuingComponent', + 'issuingComponentBindingBasis', + 'issuingComponentBindingEvidence', + 'claimScope', + 'status', + ], + requiredRowIdentityFields: ['rowIdentity'], + }), + Object.freeze({ + artifactId: 11, + relativePath: path.join('validation', 'shard-doc', 'shard-doc-replay-evidence.csv'), + type: 'csv', + columns: [ + 'rowIdentity', + 'provenanceRowIdentity', + 'artifactPath', + 'issuingComponent', + 'targetedRowLocator', + 'baselineArtifactSha256', + 'mutatedArtifactSha256', + 'rowLevelDiffSha256', + 'perturbationApplied', + 'baselineTargetRowCount', + 'mutatedTargetRowCount', + 'mutationKind', + 'evidenceIssuerClass', + 'status', + ], + requiredRowIdentityFields: ['rowIdentity', 'provenanceRowIdentity'], + }), + Object.freeze({ + artifactId: 12, + relativePath: path.join('validation', 'shard-doc', 'shard-doc-gate-summary.csv'), + type: 'csv', + columns: ['gateId', 'status', 'detail', 'sourcePath'], + requiredRowIdentityFields: ['gateId'], + }), ]); class ShardDocValidationHarnessError extends Error { @@ -170,6 +238,55 @@ function sortRowsDeterministically(rows, columns) { }); } +function computeSha256(value) { + return crypto + .createHash('sha256') + .update(String(value || ''), 'utf8') + .digest('hex'); +} + +function sortObjectKeysDeep(value) { + if (Array.isArray(value)) return value.map((item) => sortObjectKeysDeep(item)); + if (!value || typeof value !== 'object') return value; + const sorted = {}; + for (const key of Object.keys(value).sort()) { + sorted[key] = sortObjectKeysDeep(value[key]); + } + return sorted; +} + +function canonicalJsonStringify(value) { + return JSON.stringify(sortObjectKeysDeep(value)); +} + +function isSha256(value) { + return /^[a-f0-9]{64}$/.test(String(value || '')); +} + +function buildIssuedArtifactRowIdentity(artifactPath) { + return `issued-artifact:${String(artifactPath || '').replaceAll('/', '-')}`; +} + +function countShardDocManifestClaimRows(csvContent, runtimeFolder) { + const expectedPath = normalizePath(`${runtimeFolder}/core/tasks/shard-doc.xml`).toLowerCase(); + return parseCsvRows(csvContent).filter((row) => { + return ( + normalizeValue(row.canonicalId) === 'bmad-shard-doc' && + normalizeValue(row.name).toLowerCase() === 'shard-doc' && + normalizeValue(row.module).toLowerCase() === 'core' && + normalizePath(normalizeValue(row.path)).toLowerCase() === expectedPath + ); + }).length; +} + +function countShardDocHelpCatalogClaimRows(csvContent) { + return parseCsvRows(csvContent).filter((row) => { + const command = normalizeValue(row.command).replace(/^\/+/, '').toLowerCase(); + const workflowFile = normalizePath(normalizeValue(row['workflow-file'])).toLowerCase(); + return command === 'bmad-shard-doc' && workflowFile.endsWith('/core/tasks/shard-doc.xml'); + }).length; +} + class ShardDocValidationHarness { constructor() { this.registry = SHARD_DOC_VALIDATION_ARTIFACT_REGISTRY; @@ -234,6 +351,522 @@ class ShardDocValidationHarness { }); } + resolveReplayContract({ artifactPath, componentPath, rowIdentity, runtimeFolder }) { + const claimedRowIdentity = normalizeValue(rowIdentity); + if (!claimedRowIdentity) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING, + detail: 'Claimed replay rowIdentity is required', + artifactId: 11, + fieldPath: 'rowIdentity', + sourcePath: normalizePath(artifactPath), + observedValue: '', + expectedValue: 'non-empty rowIdentity', + }); + } + + const expectedRowIdentity = buildIssuedArtifactRowIdentity(artifactPath); + if (claimedRowIdentity !== expectedRowIdentity) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING, + detail: 'Claimed replay rowIdentity does not match issued-artifact contract', + artifactId: 11, + fieldPath: 'rowIdentity', + sourcePath: normalizePath(artifactPath), + observedValue: claimedRowIdentity, + expectedValue: expectedRowIdentity, + }); + } + + const contractsByRowIdentity = new Map([ + [ + buildIssuedArtifactRowIdentity(`${runtimeFolder}/_config/task-manifest.csv`), + { + artifactPath: `${runtimeFolder}/_config/task-manifest.csv`, + componentPathIncludes: 'manifest-generator.js', + mutationKind: 'component-input-perturbation:manifest-generator/tasks', + run: ({ workspaceRoot, perturbed }) => this.runManifestGeneratorReplay({ workspaceRoot, runtimeFolder, perturbed }), + }, + ], + [ + buildIssuedArtifactRowIdentity(`${runtimeFolder}/_config/bmad-help.csv`), + { + artifactPath: `${runtimeFolder}/_config/bmad-help.csv`, + componentPathIncludes: 'installer.js::mergemodulehelpcatalogs', + mutationKind: 'component-input-perturbation:installer/module-help-command', + run: ({ workspaceRoot, perturbed }) => this.runInstallerMergeReplay({ workspaceRoot, runtimeFolder, perturbed }), + }, + ], + ]); + + const contract = contractsByRowIdentity.get(claimedRowIdentity); + if (!contract) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING, + detail: 'Claimed replay rowIdentity is not mapped to a replay contract', + artifactId: 11, + fieldPath: 'rowIdentity', + sourcePath: normalizePath(artifactPath), + observedValue: claimedRowIdentity, + expectedValue: 'known issued-artifact rowIdentity', + }); + } + + const normalizedComponentPath = normalizeValue(componentPath).toLowerCase(); + if ( + normalizeValue(artifactPath) !== normalizeValue(contract.artifactPath) || + !normalizedComponentPath.includes(String(contract.componentPathIncludes || '').toLowerCase()) + ) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.BINDING_EVIDENCE_INVALID, + detail: 'Claimed issuingComponent does not match replay contract mapping', + artifactId: 11, + fieldPath: 'issuingComponent', + sourcePath: normalizePath(artifactPath), + observedValue: canonicalJsonStringify({ + artifactPath, + componentPath, + rowIdentity: claimedRowIdentity, + }), + expectedValue: canonicalJsonStringify({ + artifactPath: contract.artifactPath, + componentPathIncludes: contract.componentPathIncludes, + rowIdentity: claimedRowIdentity, + }), + }); + } + + return contract; + } + + async runManifestGeneratorReplay({ workspaceRoot, runtimeFolder, perturbed }) { + const bmadDir = path.join(workspaceRoot, runtimeFolder); + const cfgDir = path.join(bmadDir, '_config'); + await fs.ensureDir(cfgDir); + + const generator = new ManifestGenerator(); + generator.bmadFolderName = runtimeFolder; + generator.helpAuthorityRecords = []; + generator.taskAuthorityRecords = [ + { + recordType: 'metadata-authority', + canonicalId: 'bmad-shard-doc', + authoritySourceType: 'sidecar', + authoritySourcePath: SHARD_DOC_SIDECAR_SOURCE_PATH, + sourcePath: SHARD_DOC_SOURCE_XML_SOURCE_PATH, + }, + ]; + generator.tasks = perturbed + ? [] + : [ + { + name: 'shard-doc', + displayName: 'Shard Document', + description: 'Split large markdown documents into smaller files by section with an index.', + module: 'core', + path: `${runtimeFolder}/core/tasks/shard-doc.xml`, + standalone: 'true', + }, + ]; + + await generator.writeTaskManifest(cfgDir); + const outputPath = path.join(cfgDir, 'task-manifest.csv'); + const content = await fs.readFile(outputPath, 'utf8'); + return { + content, + targetRowCount: countShardDocManifestClaimRows(content, runtimeFolder), + }; + } + + async runInstallerMergeReplay({ workspaceRoot, runtimeFolder, perturbed }) { + const { Installer } = require('./installer'); + + const bmadDir = path.join(workspaceRoot, runtimeFolder); + const coreDir = path.join(bmadDir, 'core'); + const cfgDir = path.join(bmadDir, '_config'); + await fs.ensureDir(coreDir); + await fs.ensureDir(cfgDir); + + const buildCsvLine = (values) => + values + .map((value) => { + const text = String(value ?? ''); + return text.includes(',') || text.includes('"') ? `"${text.replaceAll('"', '""')}"` : text; + }) + .join(','); + const writeCsv = async (filePath, columns, rows) => { + const lines = [columns.join(','), ...rows.map((row) => buildCsvLine(columns.map((column) => row[column] ?? '')))]; + await fs.writeFile(filePath, `${lines.join('\n')}\n`, 'utf8'); + }; + + await writeCsv( + path.join(coreDir, 'module-help.csv'), + [ + 'module', + 'phase', + 'name', + 'code', + 'sequence', + 'workflow-file', + 'command', + 'required', + 'agent', + 'options', + 'description', + 'output-location', + 'outputs', + ], + [ + { + module: 'core', + phase: 'anytime', + name: 'help', + code: 'BH', + sequence: '', + 'workflow-file': `${runtimeFolder}/core/tasks/help.md`, + command: 'bmad-help', + required: 'false', + agent: '', + options: '', + description: 'Show BMAD help', + 'output-location': '', + outputs: '', + }, + { + module: 'core', + phase: 'anytime', + name: 'Shard Document', + code: 'SD', + sequence: '', + 'workflow-file': `${runtimeFolder}/core/tasks/shard-doc.xml`, + command: perturbed ? 'shard-doc' : 'bmad-shard-doc', + required: 'false', + agent: '', + options: '', + description: 'Split large markdown documents into smaller files by section with an index.', + 'output-location': '', + outputs: '', + }, + ], + ); + + await fs.writeFile( + path.join(cfgDir, 'agent-manifest.csv'), + 'name,displayName,title,icon,capabilities,role,identity,communicationStyle,principles,module,path\n', + 'utf8', + ); + + const installer = new Installer(); + installer.bmadFolderName = runtimeFolder; + installer.installedFiles = new Set(); + installer.helpAuthorityRecords = []; + installer.shardDocAuthorityRecords = [ + { + recordType: 'metadata-authority', + canonicalId: 'bmad-shard-doc', + authoritySourceType: 'sidecar', + authoritySourcePath: SHARD_DOC_SIDECAR_SOURCE_PATH, + sourcePath: SHARD_DOC_SOURCE_XML_SOURCE_PATH, + }, + ]; + + try { + await installer.mergeModuleHelpCatalogs(bmadDir); + const outputPath = path.join(cfgDir, 'bmad-help.csv'); + const content = await fs.readFile(outputPath, 'utf8'); + return { + content, + targetRowCount: countShardDocHelpCatalogClaimRows(content), + }; + } catch (error) { + if (perturbed && normalizeValue(error?.code) === 'ERR_HELP_COMMAND_LABEL_CONTRACT_FAILED') { + return { + content: `PERTURBED_COMPONENT_FAILURE:${normalizeValue(error.code)}:${normalizeValue(error.detail || error.message)}`, + targetRowCount: 0, + }; + } + throw error; + } + } + + async executeIsolatedReplay({ artifactPath, componentPath, rowIdentity, runtimeFolder }) { + const contract = this.resolveReplayContract({ + artifactPath, + componentPath, + rowIdentity, + runtimeFolder, + }); + const baselineWorkspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'shard-doc-replay-baseline-')); + const perturbedWorkspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'shard-doc-replay-perturbed-')); + + try { + const baseline = await contract.run({ workspaceRoot: baselineWorkspaceRoot, perturbed: false }); + if (Number(baseline.targetRowCount) <= 0) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REQUIRED_ROW_MISSING, + detail: 'Claimed replay rowIdentity target is absent in baseline component output', + artifactId: 11, + fieldPath: 'rowIdentity', + sourcePath: normalizePath(artifactPath), + observedValue: String(baseline.targetRowCount), + expectedValue: `at least one row for ${normalizeValue(rowIdentity)}`, + }); + } + + const mutated = await contract.run({ workspaceRoot: perturbedWorkspaceRoot, perturbed: true }); + return { + baselineContent: baseline.content, + mutatedContent: mutated.content, + baselineTargetRowCount: Number(baseline.targetRowCount), + mutatedTargetRowCount: Number(mutated.targetRowCount), + perturbationApplied: true, + mutationKind: contract.mutationKind, + targetedRowLocator: normalizeValue(rowIdentity), + }; + } finally { + await fs.remove(baselineWorkspaceRoot); + await fs.remove(perturbedWorkspaceRoot); + } + } + + async buildObservedBindingEvidence({ artifactPath, absolutePath, componentPath, rowIdentity, runtimeFolder }) { + await this.assertRequiredInputSurfaceExists({ + artifactId: 10, + absolutePath, + sourcePath: artifactPath, + description: 'issued-artifact replay target surface', + }); + + const mutationResult = await this.executeIsolatedReplay({ + artifactPath, + componentPath, + rowIdentity, + runtimeFolder: normalizeValue(runtimeFolder || '_bmad'), + }); + + const baselineArtifactSha256 = computeSha256(mutationResult.baselineContent); + const mutatedArtifactSha256 = computeSha256(mutationResult.mutatedContent); + const diffPayload = { + artifactPath, + componentPath, + rowIdentity, + mutationKind: mutationResult.mutationKind, + targetedRowLocator: mutationResult.targetedRowLocator, + baselineTargetRowCount: mutationResult.baselineTargetRowCount, + mutatedTargetRowCount: mutationResult.mutatedTargetRowCount, + baselineArtifactSha256, + mutatedArtifactSha256, + }; + const rowLevelDiffSha256 = computeSha256(canonicalJsonStringify(diffPayload)); + const evidencePayload = canonicalJsonStringify({ + evidenceVersion: 1, + observationMethod: 'validator-observed-baseline-plus-isolated-single-component-perturbation', + observationOutcome: + mutationResult.baselineTargetRowCount > mutationResult.mutatedTargetRowCount ? 'observed-impact' : 'no-impact-observed', + artifactPath, + componentPath, + targetedRowLocator: mutationResult.targetedRowLocator, + mutationKind: mutationResult.mutationKind, + baselineTargetRowCount: mutationResult.baselineTargetRowCount, + mutatedTargetRowCount: mutationResult.mutatedTargetRowCount, + baselineArtifactSha256, + mutatedArtifactSha256, + rowLevelDiffSha256, + perturbationApplied: true, + serializationFormat: 'json-canonical-v1', + encoding: 'utf-8', + lineEndings: 'lf', + worktreePath: 'isolated-replay-temp-workspaces', + commitSha: 'not-applicable', + timestampUtc: '1970-01-01T00:00:00Z', + }); + + return { + evidenceMethod: 'validator-observed-baseline-plus-isolated-single-component-perturbation', + issuingComponentBindingBasis: 'validator-observed-baseline-plus-isolated-single-component-perturbation', + issuingComponentBindingEvidence: evidencePayload, + targetedRowLocator: mutationResult.targetedRowLocator, + baselineArtifactSha256, + mutatedArtifactSha256, + rowLevelDiffSha256, + perturbationApplied: true, + baselineTargetRowCount: mutationResult.baselineTargetRowCount, + mutatedTargetRowCount: mutationResult.mutatedTargetRowCount, + mutationKind: mutationResult.mutationKind, + status: mutationResult.baselineTargetRowCount > mutationResult.mutatedTargetRowCount ? 'PASS' : 'FAIL', + }; + } + + async createIssuedArtifactEvidenceRows({ runtimeFolder, bmadDir }) { + const bindings = [ + { + artifactPath: `${runtimeFolder}/_config/task-manifest.csv`, + absolutePath: path.join(bmadDir, '_config', 'task-manifest.csv'), + issuingComponent: 'bmad-fork/tools/cli/installers/lib/core/manifest-generator.js', + }, + { + artifactPath: `${runtimeFolder}/_config/bmad-help.csv`, + absolutePath: path.join(bmadDir, '_config', 'bmad-help.csv'), + issuingComponent: 'bmad-fork/tools/cli/installers/lib/core/installer.js::mergeModuleHelpCatalogs()', + }, + ]; + + const provenanceRows = []; + const replayEvidenceRows = []; + + for (const binding of bindings) { + const rowIdentity = buildIssuedArtifactRowIdentity(binding.artifactPath); + const evidence = await this.buildObservedBindingEvidence({ + artifactPath: binding.artifactPath, + absolutePath: binding.absolutePath, + componentPath: binding.issuingComponent, + rowIdentity, + runtimeFolder, + }); + + provenanceRows.push({ + rowIdentity, + artifactPath: binding.artifactPath, + canonicalId: 'bmad-shard-doc', + issuerOwnerClass: 'independent-validator', + evidenceIssuerComponent: SHARD_DOC_EVIDENCE_ISSUER_COMPONENT, + evidenceMethod: evidence.evidenceMethod, + issuingComponent: binding.issuingComponent, + issuingComponentBindingBasis: evidence.issuingComponentBindingBasis, + issuingComponentBindingEvidence: evidence.issuingComponentBindingEvidence, + claimScope: binding.artifactPath, + status: evidence.status, + }); + + replayEvidenceRows.push({ + rowIdentity: `replay-evidence:${rowIdentity}`, + provenanceRowIdentity: rowIdentity, + artifactPath: binding.artifactPath, + issuingComponent: binding.issuingComponent, + targetedRowLocator: evidence.targetedRowLocator, + baselineArtifactSha256: evidence.baselineArtifactSha256, + mutatedArtifactSha256: evidence.mutatedArtifactSha256, + rowLevelDiffSha256: evidence.rowLevelDiffSha256, + perturbationApplied: evidence.perturbationApplied ? 'true' : 'false', + baselineTargetRowCount: String(evidence.baselineTargetRowCount), + mutatedTargetRowCount: String(evidence.mutatedTargetRowCount), + mutationKind: evidence.mutationKind, + evidenceIssuerClass: 'independent-validator', + status: evidence.status, + }); + } + + return { + provenanceRows, + replayEvidenceRows, + }; + } + + runCompatibilityGate({ gateId, surface, sourcePath, runner }) { + try { + runner(); + return { + gateId, + surface, + sourcePath, + status: 'PASS', + failureCode: '', + failureDetail: '', + }; + } catch (error) { + if (error instanceof ProjectionCompatibilityError) { + return { + gateId, + surface, + sourcePath, + status: 'FAIL', + failureCode: normalizeValue(error.code || 'ERR_COMPATIBILITY_GATE_FAILED'), + failureDetail: normalizeValue(error.detail || error.message || 'compatibility gate failure'), + }; + } + throw error; + } + } + + generateCompatibilityGateRows({ taskManifestCsvContent, helpCatalogCsvContent, runtimeFolder }) { + const helpRows = parseCsvRows(helpCatalogCsvContent); + const helpHeaderColumns = parseCsvHeader(helpCatalogCsvContent); + + return [ + this.runCompatibilityGate({ + gateId: 'task-manifest-loader', + surface: 'task-manifest-loader', + sourcePath: `${runtimeFolder}/_config/task-manifest.csv`, + runner: () => { + validateTaskManifestCompatibilitySurface(taskManifestCsvContent, { + surface: 'task-manifest-loader', + sourcePath: `${runtimeFolder}/_config/task-manifest.csv`, + }); + }, + }), + this.runCompatibilityGate({ + gateId: 'bmad-help-catalog-loader', + surface: 'bmad-help-catalog-loader', + sourcePath: `${runtimeFolder}/_config/bmad-help.csv`, + runner: () => { + validateHelpCatalogLoaderEntries(helpRows, { + surface: 'bmad-help-catalog-loader', + sourcePath: `${runtimeFolder}/_config/bmad-help.csv`, + headerColumns: helpHeaderColumns, + }); + }, + }), + this.runCompatibilityGate({ + gateId: 'github-copilot-help-loader', + surface: 'github-copilot-help-loader', + sourcePath: `${runtimeFolder}/_config/bmad-help.csv`, + runner: () => { + validateGithubCopilotHelpLoaderEntries(helpRows, { + surface: 'github-copilot-help-loader', + sourcePath: `${runtimeFolder}/_config/bmad-help.csv`, + headerColumns: helpHeaderColumns, + }); + }, + }), + ]; + } + + buildGateSummaryRows({ compatibilityRows, provenanceRows, replayRows, runtimeFolder }) { + const compatibilityPass = compatibilityRows.every((row) => normalizeValue(row.status) === 'PASS'); + const provenancePass = provenanceRows.every((row) => normalizeValue(row.status) === 'PASS'); + const replayPass = replayRows.every((row) => normalizeValue(row.status) === 'PASS'); + + return [ + { + gateId: 'compatibility-gates', + status: compatibilityPass ? 'PASS' : 'FAIL', + detail: compatibilityPass ? 'task/help/copilot compatibility gates passed' : 'one or more compatibility gates failed', + sourcePath: `${runtimeFolder}/_config/task-manifest.csv|${runtimeFolder}/_config/bmad-help.csv`, + }, + { + gateId: 'issued-artifact-provenance', + status: provenancePass ? 'PASS' : 'FAIL', + detail: provenancePass ? 'all issued-artifact provenance claims validated' : 'one or more provenance claims failed replay binding', + sourcePath: 'validation/shard-doc/shard-doc-issued-artifact-provenance.csv', + }, + { + gateId: 'replay-evidence', + status: replayPass ? 'PASS' : 'FAIL', + detail: replayPass ? 'row-targeted isolated replay evidence validated' : 'replay evidence is missing or invalid', + sourcePath: 'validation/shard-doc/shard-doc-replay-evidence.csv', + }, + { + gateId: 'required-test-commands', + status: compatibilityPass && provenancePass && replayPass ? 'PASS' : 'FAIL', + detail: + compatibilityPass && provenancePass && replayPass + ? 'harness prerequisites satisfied; CI/local test commands must also pass' + : 'harness prerequisites failed; required test command gate is blocked', + sourcePath: 'npm run test:install|npm test', + }, + ]; + } + async generateValidationArtifacts(options = {}) { const outputPaths = this.resolveOutputPaths(options); const runtimeFolder = normalizeValue(options.bmadFolderName || '_bmad'); @@ -284,9 +917,12 @@ class ShardDocValidationHarness { }); const sidecarMetadata = yaml.parse(await fs.readFile(sidecarPath, 'utf8')); - const taskManifestRows = parseCsvRows(await fs.readFile(path.join(bmadDir, '_config', 'task-manifest.csv'), 'utf8')); - const helpCatalogRows = parseCsvRows(await fs.readFile(path.join(bmadDir, '_config', 'bmad-help.csv'), 'utf8')); - const aliasRows = parseCsvRows(await fs.readFile(path.join(bmadDir, '_config', 'canonical-aliases.csv'), 'utf8')); + const taskManifestCsvContent = await fs.readFile(path.join(bmadDir, '_config', 'task-manifest.csv'), 'utf8'); + const helpCatalogCsvContent = await fs.readFile(path.join(bmadDir, '_config', 'bmad-help.csv'), 'utf8'); + const aliasCsvContent = await fs.readFile(path.join(bmadDir, '_config', 'canonical-aliases.csv'), 'utf8'); + const taskManifestRows = parseCsvRows(taskManifestCsvContent); + const helpCatalogRows = parseCsvRows(helpCatalogCsvContent); + const aliasRows = parseCsvRows(aliasCsvContent); const commandLabelReportPath = path.join(bmadDir, '_config', 'bmad-help-command-label-report.csv'); let commandLabelRows = []; if (Array.isArray(options.helpCatalogCommandLabelReportRows) && options.helpCatalogCommandLabelReportRows.length > 0) { @@ -490,7 +1126,32 @@ class ShardDocValidationHarness { ]; await this.writeCsvArtifact(artifactPaths.get(7), this.registry[6].columns, duplicateRows); - // Artifact 8 + // Artifact 9 + const compatibilityRows = this.generateCompatibilityGateRows({ + taskManifestCsvContent, + helpCatalogCsvContent, + runtimeFolder, + }); + await this.writeCsvArtifact(artifactPaths.get(9), this.registry[8].columns, compatibilityRows); + + // Artifact 10 + 11 + const { provenanceRows, replayEvidenceRows } = await this.createIssuedArtifactEvidenceRows({ + runtimeFolder, + bmadDir, + }); + await this.writeCsvArtifact(artifactPaths.get(10), this.registry[9].columns, provenanceRows); + await this.writeCsvArtifact(artifactPaths.get(11), this.registry[10].columns, replayEvidenceRows); + + // Artifact 12 + const gateSummaryRows = this.buildGateSummaryRows({ + compatibilityRows, + provenanceRows, + replayRows: replayEvidenceRows, + runtimeFolder, + }); + await this.writeCsvArtifact(artifactPaths.get(12), this.registry[11].columns, gateSummaryRows); + + // Artifact 8 (after all other artifacts exist) const inventoryRows = []; for (const artifact of this.registry) { const artifactPath = normalizePath(artifact.relativePath); @@ -505,7 +1166,6 @@ class ShardDocValidationHarness { } else if (exists && artifact.type === 'yaml') { rowCount = 1; } - inventoryRows.push({ rowIdentity: `artifact-inventory-row:${artifact.artifactId}`, artifactId: String(artifact.artifactId), @@ -529,6 +1189,55 @@ class ShardDocValidationHarness { }; } + validateReplayEvidenceRow(row, sourcePath) { + if (!isSha256(row.baselineArtifactSha256)) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID, + detail: 'Replay evidence baselineArtifactSha256 must be a valid sha256 hex digest', + artifactId: 11, + fieldPath: 'rows[*].baselineArtifactSha256', + sourcePath, + observedValue: normalizeValue(row.baselineArtifactSha256), + expectedValue: '64-char lowercase sha256 hex', + }); + } + if (!isSha256(row.mutatedArtifactSha256)) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID, + detail: 'Replay evidence mutatedArtifactSha256 must be a valid sha256 hex digest', + artifactId: 11, + fieldPath: 'rows[*].mutatedArtifactSha256', + sourcePath, + observedValue: normalizeValue(row.mutatedArtifactSha256), + expectedValue: '64-char lowercase sha256 hex', + }); + } + if (!isSha256(row.rowLevelDiffSha256)) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID, + detail: 'Replay evidence rowLevelDiffSha256 must be a valid sha256 hex digest', + artifactId: 11, + fieldPath: 'rows[*].rowLevelDiffSha256', + sourcePath, + observedValue: normalizeValue(row.rowLevelDiffSha256), + expectedValue: '64-char lowercase sha256 hex', + }); + } + + const perturbationApplied = normalizeValue(row.perturbationApplied).toLowerCase(); + if (perturbationApplied !== 'true') { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID, + detail: 'Replay evidence must prove perturbationApplied=true from isolated component replay', + artifactId: 11, + fieldPath: 'rows[*].perturbationApplied', + sourcePath, + observedValue: normalizeValue(row.perturbationApplied), + expectedValue: 'true', + }); + } + } + async validateGeneratedArtifacts(options = {}) { const outputPaths = this.resolveOutputPaths(options); const artifactDataById = new Map(); @@ -662,6 +1371,137 @@ class ShardDocValidationHarness { detail: 'Source-body authority record for shard-doc is missing', }); + const compatibilityRows = artifactDataById.get(9)?.rows || []; + for (const gateId of ['task-manifest-loader', 'bmad-help-catalog-loader', 'github-copilot-help-loader']) { + const gateRow = this.requireRow({ + rows: compatibilityRows, + predicate: (row) => normalizeValue(row.gateId) === gateId, + artifactId: 9, + fieldPath: 'rows[*].gateId', + sourcePath: normalizePath(this.registry[8].relativePath), + detail: `Required compatibility gate row is missing (${gateId})`, + }); + if (normalizeValue(gateRow.status) !== 'PASS') { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.COMPATIBILITY_GATE_FAILED, + detail: `Compatibility gate failed (${gateId})`, + artifactId: 9, + fieldPath: `rows[gateId=${gateId}].status`, + sourcePath: normalizePath(this.registry[8].relativePath), + observedValue: normalizeValue(gateRow.status), + expectedValue: 'PASS', + }); + } + } + + const provenanceRows = artifactDataById.get(10)?.rows || []; + for (const artifactPath of ['_bmad/_config/task-manifest.csv', '_bmad/_config/bmad-help.csv']) { + const rowIdentity = buildIssuedArtifactRowIdentity(artifactPath); + const provenanceRow = this.requireRow({ + rows: provenanceRows, + predicate: (row) => normalizeValue(row.rowIdentity) === rowIdentity, + artifactId: 10, + fieldPath: 'rows[*].rowIdentity', + sourcePath: normalizePath(this.registry[9].relativePath), + detail: `Required issued-artifact provenance row is missing (${rowIdentity})`, + }); + if ( + normalizeValue(provenanceRow.status) !== 'PASS' || + normalizeValue(provenanceRow.issuerOwnerClass) !== 'independent-validator' || + normalizeValue(provenanceRow.evidenceIssuerComponent) !== SHARD_DOC_EVIDENCE_ISSUER_COMPONENT + ) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.BINDING_EVIDENCE_INVALID, + detail: 'Issued-artifact provenance row failed deterministic issuer binding contract', + artifactId: 10, + fieldPath: `rows[rowIdentity=${rowIdentity}]`, + sourcePath: normalizePath(this.registry[9].relativePath), + observedValue: canonicalJsonStringify({ + status: normalizeValue(provenanceRow.status), + issuerOwnerClass: normalizeValue(provenanceRow.issuerOwnerClass), + evidenceIssuerComponent: normalizeValue(provenanceRow.evidenceIssuerComponent), + }), + expectedValue: canonicalJsonStringify({ + status: 'PASS', + issuerOwnerClass: 'independent-validator', + evidenceIssuerComponent: SHARD_DOC_EVIDENCE_ISSUER_COMPONENT, + }), + }); + } + if (!normalizeValue(provenanceRow.issuingComponentBindingEvidence)) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.BINDING_EVIDENCE_INVALID, + detail: 'Issued-artifact provenance row is missing binding evidence payload', + artifactId: 10, + fieldPath: `rows[rowIdentity=${rowIdentity}].issuingComponentBindingEvidence`, + sourcePath: normalizePath(this.registry[9].relativePath), + observedValue: '', + expectedValue: 'non-empty canonical JSON payload', + }); + } + } + + const replayRows = artifactDataById.get(11)?.rows || []; + for (const replayRow of replayRows) { + this.validateReplayEvidenceRow(replayRow, normalizePath(this.registry[10].relativePath)); + const provenanceRow = this.requireRow({ + rows: provenanceRows, + predicate: (row) => normalizeValue(row.rowIdentity) === normalizeValue(replayRow.provenanceRowIdentity), + artifactId: 11, + fieldPath: 'rows[*].provenanceRowIdentity', + sourcePath: normalizePath(this.registry[10].relativePath), + detail: 'Replay evidence row references missing issued-artifact provenance rowIdentity', + }); + if (normalizeValue(replayRow.targetedRowLocator) !== normalizeValue(provenanceRow.rowIdentity)) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID, + detail: 'Replay evidence targetedRowLocator must equal provenance rowIdentity', + artifactId: 11, + fieldPath: 'rows[*].targetedRowLocator', + sourcePath: normalizePath(this.registry[10].relativePath), + observedValue: normalizeValue(replayRow.targetedRowLocator), + expectedValue: normalizeValue(provenanceRow.rowIdentity), + }); + } + if ( + Number.parseInt(normalizeValue(replayRow.baselineTargetRowCount), 10) <= + Number.parseInt(normalizeValue(replayRow.mutatedTargetRowCount), 10) + ) { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.REPLAY_EVIDENCE_INVALID, + detail: 'Replay evidence must show baseline target count greater than mutated target count', + artifactId: 11, + fieldPath: 'rows[*].baselineTargetRowCount', + sourcePath: normalizePath(this.registry[10].relativePath), + observedValue: `${normalizeValue(replayRow.baselineTargetRowCount)}<=${normalizeValue(replayRow.mutatedTargetRowCount)}`, + expectedValue: 'baselineTargetRowCount > mutatedTargetRowCount', + }); + } + } + + const gateSummaryRows = artifactDataById.get(12)?.rows || []; + for (const gateId of ['compatibility-gates', 'issued-artifact-provenance', 'replay-evidence']) { + const summaryRow = this.requireRow({ + rows: gateSummaryRows, + predicate: (row) => normalizeValue(row.gateId) === gateId, + artifactId: 12, + fieldPath: 'rows[*].gateId', + sourcePath: normalizePath(this.registry[11].relativePath), + detail: `Required gate summary row is missing (${gateId})`, + }); + if (normalizeValue(summaryRow.status) !== 'PASS') { + throw new ShardDocValidationHarnessError({ + code: SHARD_DOC_VALIDATION_ERROR_CODES.COMPATIBILITY_GATE_FAILED, + detail: `Gate summary failed (${gateId})`, + artifactId: 12, + fieldPath: `rows[gateId=${gateId}].status`, + sourcePath: normalizePath(this.registry[11].relativePath), + observedValue: normalizeValue(summaryRow.status), + expectedValue: 'PASS', + }); + } + } + const inventoryRows = artifactDataById.get(8)?.rows || []; if (inventoryRows.length !== this.registry.length) { throw new ShardDocValidationHarnessError({