feat(flattener): improve file processing with progress indicators
- Add spinner support for file processing progress - Simplify XML output format to focus on text files - Add statistics calculation and summary display - Improve error handling and user feedback
This commit is contained in:
manjaroblack
parent
6aa7539aea
commit
7fb2ad8c93
|
|
@ -37,3 +37,4 @@ tests/
|
|||
custom-output.xml
|
||||
flattened-codebase.xml
|
||||
biome.json
|
||||
__tests__/
|
||||
|
|
@ -86,6 +86,10 @@ Switch back to your IDE for document management:
|
|||
|
||||
Follow the SM → Dev cycle for systematic story development:
|
||||
|
||||
#### Create new Branch
|
||||
|
||||
1. **Start new branch**
|
||||
|
||||
#### Story Creation (Scrum Master)
|
||||
|
||||
1. **Start new chat/conversation**
|
||||
|
|
@ -98,7 +102,7 @@ Follow the SM → Dev cycle for systematic story development:
|
|||
|
||||
1. **Start new chat/conversation**
|
||||
2. **Load Dev agent**
|
||||
3. **Execute**: `{selected-story}` (runs execute-checklist task)
|
||||
3. **Execute**: `*develop-story {selected-story}` (runs execute-checklist task)
|
||||
4. **Review generated report** in `{selected-story}`
|
||||
|
||||
#### Story Review (Quality Assurance)
|
||||
|
|
@ -108,11 +112,18 @@ Follow the SM → Dev cycle for systematic story development:
|
|||
3. **Execute**: `*review {selected-story}` (runs review-story task)
|
||||
4. **Review generated report** in `{selected-story}`
|
||||
|
||||
#### Commit Changes and Push
|
||||
|
||||
1. **Commit changes**
|
||||
2. **Push to remote**
|
||||
|
||||
#### Repeat Until Complete
|
||||
|
||||
- **SM**: Create next story → Review → Approve
|
||||
- **Dev**: Implement story → Complete → Mark Ready for Review
|
||||
- **QA**: Review story → Mark done
|
||||
- **Commit**: All changes
|
||||
- **Push**: To remote
|
||||
- **Continue**: Until all features implemented
|
||||
|
||||
## IDE-Specific Syntax
|
||||
|
|
|
|||
|
|
@ -120,9 +120,10 @@ async function isBinaryFile(filePath) {
|
|||
* Read and aggregate content from text files
|
||||
* @param {string[]} files - Array of file paths
|
||||
* @param {string} rootDir - The root directory
|
||||
* @param {Object} spinner - Optional spinner instance for progress display
|
||||
* @returns {Promise<Object>} Object containing file contents and metadata
|
||||
*/
|
||||
async function aggregateFileContents(files, rootDir) {
|
||||
async function aggregateFileContents(files, rootDir, spinner = null) {
|
||||
const results = {
|
||||
textFiles: [],
|
||||
binaryFiles: [],
|
||||
|
|
@ -134,6 +135,12 @@ async function aggregateFileContents(files, rootDir) {
|
|||
for (const filePath of files) {
|
||||
try {
|
||||
const relativePath = path.relative(rootDir, filePath);
|
||||
|
||||
// Update progress indicator
|
||||
if (spinner) {
|
||||
spinner.text = `Processing file ${results.processedFiles + 1}/${results.totalFiles}: ${relativePath}`;
|
||||
}
|
||||
|
||||
const isBinary = await isBinaryFile(filePath);
|
||||
|
||||
if (isBinary) {
|
||||
|
|
@ -164,7 +171,14 @@ async function aggregateFileContents(files, rootDir) {
|
|||
};
|
||||
|
||||
results.errors.push(errorInfo);
|
||||
|
||||
// Log warning without interfering with spinner
|
||||
if (spinner) {
|
||||
spinner.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
||||
} else {
|
||||
console.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
||||
}
|
||||
|
||||
results.processedFiles++;
|
||||
}
|
||||
}
|
||||
|
|
@ -179,91 +193,27 @@ async function aggregateFileContents(files, rootDir) {
|
|||
* @returns {string} XML content
|
||||
*/
|
||||
function generateXMLOutput(aggregatedContent, projectRoot) {
|
||||
const { textFiles, binaryFiles, errors, totalFiles, processedFiles } = aggregatedContent;
|
||||
const timestamp = new Date().toISOString();
|
||||
const { textFiles } = aggregatedContent;
|
||||
|
||||
let xml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
`;
|
||||
xml += `<codebase>
|
||||
`;
|
||||
xml += ` <metadata>
|
||||
`;
|
||||
xml += ` <generated>${timestamp}</generated>
|
||||
`;
|
||||
xml += ` <project_root>${escapeXml(projectRoot)}</project_root>
|
||||
`;
|
||||
xml += ` <total_files>${totalFiles}</total_files>
|
||||
`;
|
||||
xml += ` <processed_files>${processedFiles}</processed_files>
|
||||
`;
|
||||
xml += ` <text_files>${textFiles.length}</text_files>
|
||||
`;
|
||||
xml += ` <binary_files>${binaryFiles.length}</binary_files>
|
||||
`;
|
||||
xml += ` <errors>${errors.length}</errors>
|
||||
`;
|
||||
xml += ` </metadata>
|
||||
xml += `<files>
|
||||
`;
|
||||
|
||||
// Add text files with content
|
||||
if (textFiles.length > 0) {
|
||||
xml += ` <text_files>
|
||||
`;
|
||||
// Add text files with content (only text files as per story requirements)
|
||||
for (const file of textFiles) {
|
||||
xml += ` <file>
|
||||
`;
|
||||
xml += ` <path>${escapeXml(file.path)}</path>
|
||||
`;
|
||||
xml += ` <size>${file.size}</size>
|
||||
`;
|
||||
xml += ` <lines>${file.lines}</lines>
|
||||
`;
|
||||
xml += ` <content><![CDATA[${file.content}]]></content>
|
||||
`;
|
||||
xml += ` <file path="${escapeXml(file.path)}">`;
|
||||
|
||||
// Use CDATA for code content to preserve formatting and handle special characters
|
||||
if (file.content.trim()) {
|
||||
xml += `<![CDATA[${file.content}]]>`;
|
||||
}
|
||||
|
||||
xml += `</file>
|
||||
`;
|
||||
}
|
||||
xml += ` </text_files>
|
||||
`;
|
||||
}
|
||||
|
||||
// Add binary files (paths only)
|
||||
if (binaryFiles.length > 0) {
|
||||
xml += ` <binary_files>
|
||||
`;
|
||||
for (const file of binaryFiles) {
|
||||
xml += ` <file>
|
||||
`;
|
||||
xml += ` <path>${escapeXml(file.path)}</path>
|
||||
`;
|
||||
xml += ` <size>${file.size}</size>
|
||||
`;
|
||||
xml += ` </file>
|
||||
`;
|
||||
}
|
||||
xml += ` </binary_files>
|
||||
`;
|
||||
}
|
||||
|
||||
// Add errors if any
|
||||
if (errors.length > 0) {
|
||||
xml += ` <errors>
|
||||
`;
|
||||
for (const error of errors) {
|
||||
xml += ` <error>
|
||||
`;
|
||||
xml += ` <path>${escapeXml(error.path)}</path>
|
||||
`;
|
||||
xml += ` <message>${escapeXml(error.error)}</message>
|
||||
`;
|
||||
xml += ` </error>
|
||||
`;
|
||||
}
|
||||
xml += ` </errors>
|
||||
`;
|
||||
}
|
||||
|
||||
xml += `</codebase>`;
|
||||
xml += `</files>`;
|
||||
return xml;
|
||||
}
|
||||
|
||||
|
|
@ -284,6 +234,45 @@ function escapeXml(str) {
|
|||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate statistics for the processed files
|
||||
* @param {Object} aggregatedContent - The aggregated content object
|
||||
* @param {string} xmlContent - The generated XML content
|
||||
* @returns {Object} Statistics object
|
||||
*/
|
||||
function calculateStatistics(aggregatedContent, xmlContent) {
|
||||
const { textFiles, binaryFiles, errors } = aggregatedContent;
|
||||
|
||||
// Calculate total file size in bytes
|
||||
const totalTextSize = textFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
const totalBinarySize = binaryFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
const totalSize = totalTextSize + totalBinarySize;
|
||||
|
||||
// Calculate total lines of code
|
||||
const totalLines = textFiles.reduce((sum, file) => sum + file.lines, 0);
|
||||
|
||||
// Estimate token count (rough approximation: 1 token ≈ 4 characters)
|
||||
const estimatedTokens = Math.ceil(xmlContent.length / 4);
|
||||
|
||||
// Format file size
|
||||
const formatSize = (bytes) => {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
};
|
||||
|
||||
return {
|
||||
totalFiles: textFiles.length + binaryFiles.length,
|
||||
textFiles: textFiles.length,
|
||||
binaryFiles: binaryFiles.length,
|
||||
errorFiles: errors.length,
|
||||
totalSize: formatSize(totalSize),
|
||||
xmlSize: formatSize(xmlContent.length),
|
||||
totalLines,
|
||||
estimatedTokens: estimatedTokens.toLocaleString()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter files based on .gitignore patterns
|
||||
* @param {string[]} files - Array of file paths
|
||||
|
|
@ -346,44 +335,56 @@ program
|
|||
.version('1.0.0')
|
||||
.option('-o, --output <path>', 'Output file path', 'flattened-codebase.xml')
|
||||
.action(async (options) => {
|
||||
try {
|
||||
console.log(`Flattening codebase to: ${options.output}`);
|
||||
|
||||
const projectRoot = process.cwd();
|
||||
const outputPath = path.resolve(options.output);
|
||||
try {
|
||||
// Import ora dynamically
|
||||
const { default: ora } = await import('ora');
|
||||
|
||||
// Discover and filter files
|
||||
const discoveredFiles = await discoverFiles(projectRoot);
|
||||
const filteredFiles = await filterFiles(discoveredFiles, projectRoot);
|
||||
// Start file discovery with spinner
|
||||
const discoverySpinner = ora('🔍 Discovering files...').start();
|
||||
const files = await discoverFiles(process.cwd());
|
||||
const filteredFiles = await filterFiles(files, process.cwd());
|
||||
discoverySpinner.succeed(`📁 Found ${filteredFiles.length} files to include`);
|
||||
|
||||
console.log(`Found ${filteredFiles.length} files to include`);
|
||||
// Process files with progress tracking
|
||||
console.log('Reading file contents');
|
||||
const processingSpinner = ora('📄 Processing files...').start();
|
||||
const aggregatedContent = await aggregateFileContents(filteredFiles, process.cwd(), processingSpinner);
|
||||
processingSpinner.succeed(`✅ Processed ${aggregatedContent.processedFiles}/${filteredFiles.length} files`);
|
||||
|
||||
// Debug: log the files being included (only in debug mode)
|
||||
if (process.env.DEBUG_FLATTENER) {
|
||||
console.log('Files to include:');
|
||||
filteredFiles.forEach(file => {
|
||||
console.log(` - ${path.relative(projectRoot, file)}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Aggregate file contents
|
||||
console.log('Reading file contents...');
|
||||
const aggregatedContent = await aggregateFileContents(filteredFiles, projectRoot);
|
||||
|
||||
console.log(`Processed ${aggregatedContent.processedFiles}/${aggregatedContent.totalFiles} files`);
|
||||
console.log(`Text files: ${aggregatedContent.textFiles.length}`);
|
||||
console.log(`Binary files: ${aggregatedContent.binaryFiles.length}`);
|
||||
// Log processing results for test validation
|
||||
console.log(`Processed ${aggregatedContent.processedFiles}/${filteredFiles.length} files`);
|
||||
if (aggregatedContent.errors.length > 0) {
|
||||
console.log(`Errors: ${aggregatedContent.errors.length}`);
|
||||
}
|
||||
console.log(`Text files: ${aggregatedContent.textFiles.length}`);
|
||||
if (aggregatedContent.binaryFiles.length > 0) {
|
||||
console.log(`Binary files: ${aggregatedContent.binaryFiles.length}`);
|
||||
}
|
||||
|
||||
// Generate XML content with file contents
|
||||
const xmlContent = generateXMLOutput(aggregatedContent, projectRoot);
|
||||
// Generate XML output
|
||||
const xmlSpinner = ora('🔧 Generating XML output...').start();
|
||||
const xmlOutput = generateXMLOutput(aggregatedContent, process.cwd());
|
||||
await fs.writeFile(options.output, xmlOutput);
|
||||
xmlSpinner.succeed('📝 XML generation completed');
|
||||
|
||||
// Calculate and display statistics
|
||||
const stats = calculateStatistics(aggregatedContent, xmlOutput);
|
||||
|
||||
// Display completion summary
|
||||
console.log('\n📊 Completion Summary:');
|
||||
console.log(`✅ Successfully processed ${filteredFiles.length} files into ${options.output}`);
|
||||
console.log(`📁 Output file: ${path.resolve(options.output)}`);
|
||||
console.log(`📏 Total source size: ${stats.totalSize}`);
|
||||
console.log(`📄 Generated XML size: ${stats.xmlSize}`);
|
||||
console.log(`📝 Total lines of code: ${stats.totalLines.toLocaleString()}`);
|
||||
console.log(`🔢 Estimated tokens: ${stats.estimatedTokens}`);
|
||||
console.log(`📊 File breakdown: ${stats.textFiles} text, ${stats.binaryFiles} binary, ${stats.errorFiles} errors`);
|
||||
|
||||
await fs.writeFile(outputPath, xmlContent);
|
||||
console.log(`Codebase flattened successfully to: ${outputPath}`);
|
||||
} catch (error) {
|
||||
console.error('Flattening failed:', error.message);
|
||||
console.error('❌ Critical error:', error.message);
|
||||
console.error('An unexpected error occurred.');
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue