fix(build): use file glob instead of sidebar parsing for llms-full.txt
Replace brittle sidebar.js regex parsing with recursive file glob. The old approach captured non-file strings like 'autogenerated' and category labels, resulting in only 5 files being processed. Now correctly processes all 86+ markdown files (~95k tokens). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
42cc21e4cb
commit
1fc1b03107
|
|
@ -224,7 +224,7 @@ function generateLlmsFullTxt(outputDir) {
|
||||||
console.log(' → Generating llms-full.txt...');
|
console.log(' → Generating llms-full.txt...');
|
||||||
|
|
||||||
const date = new Date().toISOString().split('T')[0];
|
const date = new Date().toISOString().split('T')[0];
|
||||||
const files = getDocsFromSidebar();
|
const files = getAllMarkdownFiles(outputDir);
|
||||||
|
|
||||||
const output = [
|
const output = [
|
||||||
'# BMAD Method Documentation (Full)',
|
'# BMAD Method Documentation (Full)',
|
||||||
|
|
@ -266,32 +266,22 @@ function generateLlmsFullTxt(outputDir) {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function getDocsFromSidebar() {
|
function getAllMarkdownFiles(dir, baseDir = dir) {
|
||||||
const sidebarsPath = path.join(PROJECT_ROOT, 'website', 'sidebars.js');
|
const files = [];
|
||||||
|
|
||||||
try {
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||||
const sidebarContent = fs.readFileSync(sidebarsPath, 'utf-8');
|
const fullPath = path.join(dir, entry.name);
|
||||||
const matches = sidebarContent.matchAll(/'([a-zA-Z0-9\-_/]+)'/g);
|
|
||||||
const files = [];
|
|
||||||
|
|
||||||
for (const match of matches) {
|
if (entry.isDirectory()) {
|
||||||
const docId = match[1];
|
files.push(...getAllMarkdownFiles(fullPath, baseDir));
|
||||||
// Skip Docusaurus keywords
|
} else if (entry.name.endsWith('.md')) {
|
||||||
if (docId.includes('Sidebar') || docId === 'doc' || docId === 'category') {
|
// Return relative path from baseDir
|
||||||
continue;
|
const relativePath = path.relative(baseDir, fullPath);
|
||||||
}
|
files.push(relativePath);
|
||||||
// Skip category labels (Title Case words without slashes like 'Workflows', 'Reference')
|
|
||||||
if (!docId.includes('/') && /^[A-Z][a-z]/.test(docId)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
files.push(docId + '.md');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return files;
|
|
||||||
} catch {
|
|
||||||
console.log(' Warning: Could not parse sidebars');
|
|
||||||
return [];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return files;
|
||||||
}
|
}
|
||||||
|
|
||||||
function shouldExcludeFromLlm(filePath) {
|
function shouldExcludeFromLlm(filePath) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue