From 61b6c3b2ec261e65d27be56016bd55f5c64d24ce Mon Sep 17 00:00:00 2001 From: manjaroblack Date: Fri, 18 Jul 2025 21:55:53 -0500 Subject: [PATCH] feat(flattener): implement file discovery and filtering logic Add file discovery and filtering functionality to the codebase flattener tool - Use glob and minimatch for pattern matching - Support .gitignore patterns - Generate basic XML output with file count --- .gitignore | 3 +- package.json | 1 + tools/flattener/main.js | 139 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 042d1861..90ac1ab7 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,5 @@ docs/prd/ docs/stories/ docs/project-architecture.md tests/ -custom-output.xml \ No newline at end of file +custom-output.xml +flattened-codebase.xml \ No newline at end of file diff --git a/package.json b/package.json index e9eb3121..160a7ac3 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "glob": "^11.0.3", "inquirer": "^12.6.3", "js-yaml": "^4.1.0", + "minimatch": "^10.0.3", "ora": "^8.2.0" }, "keywords": [ diff --git a/tools/flattener/main.js b/tools/flattener/main.js index bda22a2d..5a13fec3 100644 --- a/tools/flattener/main.js +++ b/tools/flattener/main.js @@ -3,6 +3,128 @@ const { Command } = require('commander'); const fs = require('fs-extra'); const path = require('path'); +const { glob } = require('glob'); +const { minimatch } = require('minimatch'); + +/** + * Recursively discover all files in a directory + * @param {string} rootDir - The root directory to scan + * @returns {Promise} Array of file paths + */ +async function discoverFiles(rootDir) { + try { + // Use glob to recursively find all files, excluding common ignore patterns + const files = await glob('**/*', { + cwd: rootDir, + nodir: true, // Only files, not directories + dot: true, // Include hidden files + follow: false, // Don't follow symbolic links + ignore: [ + // Standard ignore patterns + 'node_modules/**', + '.git/**', + 'build/**', + 'dist/**', + '.next/**', + 'coverage/**', + '.nyc_output/**', + 'tmp/**', + 'temp/**', + '.gitignore', + '.gitattributes', + '.gitmodules' + ] + }); + + return files.map(file => path.resolve(rootDir, file)); + } catch (error) { + console.error('Error discovering files:', error.message); + return []; + } +} + +/** + * Parse .gitignore file and return ignore patterns + * @param {string} gitignorePath - Path to .gitignore file + * @returns {Promise} Array of ignore patterns + */ +async function parseGitignore(gitignorePath) { + try { + if (!await fs.pathExists(gitignorePath)) { + return []; + } + + const content = await fs.readFile(gitignorePath, 'utf8'); + return content + .split('\n') + .map(line => line.trim()) + .filter(line => line && !line.startsWith('#')) // Remove empty lines and comments + .map(pattern => { + // Convert gitignore patterns to glob patterns + if (pattern.endsWith('/')) { + return pattern + '**'; + } + return pattern; + }); + } catch (error) { + console.error('Error parsing .gitignore:', error.message); + return []; + } +} + +/** + * Filter files based on .gitignore patterns + * @param {string[]} files - Array of file paths + * @param {string} rootDir - The root directory + * @returns {Promise} Filtered array of file paths + */ +async function filterFiles(files, rootDir) { + const gitignorePath = path.join(rootDir, '.gitignore'); + const ignorePatterns = await parseGitignore(gitignorePath); + + if (ignorePatterns.length === 0) { + return files; + } + + // Convert absolute paths to relative for pattern matching + const relativeFiles = files.map(file => path.relative(rootDir, file)); + + // Separate positive and negative patterns + const positivePatterns = ignorePatterns.filter(p => !p.startsWith('!')); + const negativePatterns = ignorePatterns.filter(p => p.startsWith('!')).map(p => p.slice(1)); + + // Filter out files that match ignore patterns + const filteredRelative = []; + + for (const file of relativeFiles) { + let shouldIgnore = false; + + // First check positive patterns (ignore these files) + for (const pattern of positivePatterns) { + if (minimatch(file, pattern)) { + shouldIgnore = true; + break; + } + } + + // Then check negative patterns (don't ignore these files even if they match positive patterns) + if (shouldIgnore) { + for (const pattern of negativePatterns) { + if (minimatch(file, pattern)) { + shouldIgnore = false; + break; + } + } + } + + if (!shouldIgnore) { + filteredRelative.push(file); + } + } + + // Convert back to absolute paths + return filteredRelative.map(file => path.resolve(rootDir, file)); +} const program = new Command(); @@ -15,13 +137,28 @@ program try { console.log(`Flattening codebase to: ${options.output}`); - // TODO: Implement actual flattening logic + const projectRoot = process.cwd(); const outputPath = path.resolve(options.output); + // Discover and filter files + const discoveredFiles = await discoverFiles(projectRoot); + const filteredFiles = await filterFiles(discoveredFiles, projectRoot); + + console.log(`Found ${filteredFiles.length} files to include`); + + // Debug: log the files being included (only in debug mode) + if (process.env.DEBUG_FLATTENER) { + console.log('Files to include:'); + filteredFiles.forEach(file => { + console.log(` - ${path.relative(projectRoot, file)}`); + }); + } + // Create basic XML structure for now const xmlContent = ` + `; await fs.writeFile(outputPath, xmlContent);