From 7c58c0bab163141264cee6ca68ff76f54a2c8979 Mon Sep 17 00:00:00 2001 From: Brian Madison Date: Tue, 7 Apr 2026 23:58:52 -0500 Subject: [PATCH] feat(installer): add community module browser and custom URL support Three-tier module selection: official, community (category drill-down with featured/search), and custom GitHub URL. - Add RegistryClient shared fetch utility - Add CommunityModuleManager with SHA-pinned cloning (refuses install if approved SHA cannot be reached; uses HEAD when no SHA set) - Add CustomModuleManager for arbitrary GitHub repo installation - Extend findModuleSource chain with community and custom fallthrough - Extend manifest to detect community and custom source types - Add Config.customModulesMeta for custom module metadata --- tools/installer/core/config.js | 4 +- tools/installer/core/manifest.js | 28 ++ tools/installer/modules/community-manager.js | 377 ++++++++++++++++++ .../modules/custom-module-manager.js | 308 ++++++++++++++ tools/installer/modules/external-manager.js | 35 +- tools/installer/modules/official-modules.js | 16 + tools/installer/modules/registry-client.js | 66 +++ tools/installer/ui.js | 279 ++++++++++++- 8 files changed, 1072 insertions(+), 41 deletions(-) create mode 100644 tools/installer/modules/community-manager.js create mode 100644 tools/installer/modules/custom-module-manager.js create mode 100644 tools/installer/modules/registry-client.js diff --git a/tools/installer/core/config.js b/tools/installer/core/config.js index c844e2d00..03ffbe279 100644 --- a/tools/installer/core/config.js +++ b/tools/installer/core/config.js @@ -3,7 +3,7 @@ * User input comes from either UI answers or headless CLI flags. */ class Config { - constructor({ directory, modules, ides, skipPrompts, verbose, actionType, coreConfig, moduleConfigs, quickUpdate }) { + constructor({ directory, modules, ides, skipPrompts, verbose, actionType, coreConfig, moduleConfigs, quickUpdate, customModulesMeta }) { this.directory = directory; this.modules = Object.freeze([...modules]); this.ides = Object.freeze([...ides]); @@ -13,6 +13,7 @@ class Config { this.coreConfig = coreConfig; this.moduleConfigs = moduleConfigs; this._quickUpdate = quickUpdate; + this.customModulesMeta = Object.freeze(customModulesMeta || []); Object.freeze(this); } @@ -37,6 +38,7 @@ class Config { coreConfig: userInput.coreConfig || {}, moduleConfigs: userInput.moduleConfigs || null, quickUpdate: userInput._quickUpdate || false, + customModulesMeta: userInput.customModulesMeta || [], }); } diff --git a/tools/installer/core/manifest.js b/tools/installer/core/manifest.js index f70482f43..d810ec1d3 100644 --- a/tools/installer/core/manifest.js +++ b/tools/installer/core/manifest.js @@ -818,6 +818,34 @@ class Manifest { }; } + // Check if this is a community module + const { CommunityModuleManager } = require('../modules/community-manager'); + const communityMgr = new CommunityModuleManager(); + const communityInfo = await communityMgr.getModuleByCode(moduleName); + if (communityInfo) { + const communityVersion = await this._readMarketplaceVersion(moduleName, moduleSourcePath); + return { + version: communityVersion || communityInfo.version, + source: 'community', + npmPackage: communityInfo.npmPackage || null, + repoUrl: communityInfo.url || null, + }; + } + + // Check if this is a custom module (from user-provided URL) + const { CustomModuleManager } = require('../modules/custom-module-manager'); + const customMgr = new CustomModuleManager(); + const customSource = await customMgr.findModuleSourceByCode(moduleName); + if (customSource) { + const customVersion = await this._readMarketplaceVersion(moduleName, moduleSourcePath); + return { + version: customVersion, + source: 'custom', + npmPackage: null, + repoUrl: null, + }; + } + // Unknown module const version = await this._readMarketplaceVersion(moduleName, moduleSourcePath); return { diff --git a/tools/installer/modules/community-manager.js b/tools/installer/modules/community-manager.js new file mode 100644 index 000000000..ea8a9e866 --- /dev/null +++ b/tools/installer/modules/community-manager.js @@ -0,0 +1,377 @@ +const fs = require('fs-extra'); +const os = require('node:os'); +const path = require('node:path'); +const { execSync } = require('node:child_process'); +const prompts = require('../prompts'); +const { RegistryClient } = require('./registry-client'); + +const MARKETPLACE_BASE = 'https://raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main'; +const COMMUNITY_INDEX_URL = `${MARKETPLACE_BASE}/registry/community-index.yaml`; +const CATEGORIES_URL = `${MARKETPLACE_BASE}/categories.yaml`; + +/** + * Manages community modules from the BMad marketplace registry. + * Fetches community-index.yaml and categories.yaml from GitHub. + * Returns empty results when the registry is unreachable. + * Community modules are pinned to approved tags (not HEAD). + */ +class CommunityModuleManager { + constructor() { + this._client = new RegistryClient(); + this._cachedIndex = null; + this._cachedCategories = null; + } + + // ─── Data Loading ────────────────────────────────────────────────────────── + + /** + * Load the community module index from the marketplace repo. + * Returns empty when the registry is unreachable. + * @returns {Object} Parsed YAML with modules array + */ + async loadCommunityIndex() { + if (this._cachedIndex) return this._cachedIndex; + + try { + const config = await this._client.fetchYaml(COMMUNITY_INDEX_URL); + if (config?.modules?.length) { + this._cachedIndex = config; + return config; + } + } catch { + // Registry unreachable - no community modules available + } + + return { modules: [] }; + } + + /** + * Load categories from the marketplace repo. + * Returns empty when the registry is unreachable. + * @returns {Object} Parsed categories.yaml content + */ + async loadCategories() { + if (this._cachedCategories) return this._cachedCategories; + + try { + const config = await this._client.fetchYaml(CATEGORIES_URL); + if (config?.categories) { + this._cachedCategories = config; + return config; + } + } catch { + // Registry unreachable - no categories available + } + + return { categories: {} }; + } + + // ─── Listing & Filtering ────────────────────────────────────────────────── + + /** + * Get all community modules, normalized. + * @returns {Array} Normalized community modules + */ + async listAll() { + const index = await this.loadCommunityIndex(); + return (index.modules || []).map((mod) => this._normalizeCommunityModule(mod)); + } + + /** + * Get community modules filtered to a category. + * @param {string} categorySlug - Category slug (e.g., 'design-and-creative') + * @returns {Array} Filtered modules + */ + async listByCategory(categorySlug) { + const all = await this.listAll(); + return all.filter((mod) => mod.category === categorySlug); + } + + /** + * Get promoted/featured community modules, sorted by rank. + * @returns {Array} Featured modules + */ + async listFeatured() { + const all = await this.listAll(); + return all.filter((mod) => mod.promoted === true).sort((a, b) => (a.promotedRank || 999) - (b.promotedRank || 999)); + } + + /** + * Search community modules by keyword. + * Matches against name, display name, description, and keywords array. + * @param {string} query - Search query + * @returns {Array} Matching modules + */ + async searchByKeyword(query) { + const all = await this.listAll(); + const q = query.toLowerCase(); + return all.filter((mod) => { + const searchable = [mod.name, mod.displayName, mod.description, ...(mod.keywords || [])].join(' ').toLowerCase(); + return searchable.includes(q); + }); + } + + /** + * Get categories with module counts for UI display. + * Only returns categories that have at least one community module. + * @returns {Array} Array of { slug, name, moduleCount } + */ + async getCategoryList() { + const all = await this.listAll(); + const categoriesData = await this.loadCategories(); + const categories = categoriesData.categories || {}; + + // Count modules per category + const counts = {}; + for (const mod of all) { + counts[mod.category] = (counts[mod.category] || 0) + 1; + } + + // Build list with display names from categories.yaml + const result = []; + for (const [slug, count] of Object.entries(counts)) { + const catInfo = categories[slug]; + result.push({ + slug, + name: catInfo?.name || slug, + moduleCount: count, + }); + } + + // Sort alphabetically by name + result.sort((a, b) => a.name.localeCompare(b.name)); + return result; + } + + // ─── Module Lookup ──────────────────────────────────────────────────────── + + /** + * Get a community module by its code. + * @param {string} code - Module code (e.g., 'wds') + * @returns {Object|null} Normalized module or null + */ + async getModuleByCode(code) { + const all = await this.listAll(); + return all.find((m) => m.code === code) || null; + } + + // ─── Clone with Tag Pinning ─────────────────────────────────────────────── + + /** + * Get the cache directory for community modules. + * @returns {string} Path to the community modules cache directory + */ + getCacheDir() { + return path.join(os.homedir(), '.bmad', 'cache', 'community-modules'); + } + + /** + * Clone a community module repository, pinned to its approved tag. + * @param {string} moduleCode - Module code + * @param {Object} [options] - Clone options + * @param {boolean} [options.silent] - Suppress spinner output + * @returns {string} Path to the cloned repository + */ + async cloneModule(moduleCode, options = {}) { + const moduleInfo = await this.getModuleByCode(moduleCode); + if (!moduleInfo) { + throw new Error(`Community module '${moduleCode}' not found in the registry`); + } + + const cacheDir = this.getCacheDir(); + const moduleCacheDir = path.join(cacheDir, moduleCode); + const silent = options.silent || false; + + await fs.ensureDir(cacheDir); + + const createSpinner = async () => { + if (silent) { + return { start() {}, stop() {}, error() {}, message() {} }; + } + return await prompts.spinner(); + }; + + const sha = moduleInfo.approvedSha; + let needsDependencyInstall = false; + let wasNewClone = false; + + if (await fs.pathExists(moduleCacheDir)) { + // Already cloned - update to latest HEAD + const fetchSpinner = await createSpinner(); + fetchSpinner.start(`Checking ${moduleInfo.displayName}...`); + try { + const currentRef = execSync('git rev-parse HEAD', { cwd: moduleCacheDir, stdio: 'pipe' }).toString().trim(); + execSync('git fetch origin --depth 1', { + cwd: moduleCacheDir, + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, GIT_TERMINAL_PROMPT: '0' }, + }); + execSync('git reset --hard origin/HEAD', { + cwd: moduleCacheDir, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const newRef = execSync('git rev-parse HEAD', { cwd: moduleCacheDir, stdio: 'pipe' }).toString().trim(); + if (currentRef !== newRef) needsDependencyInstall = true; + fetchSpinner.stop(`Verified ${moduleInfo.displayName}`); + } catch { + fetchSpinner.error(`Fetch failed, re-downloading ${moduleInfo.displayName}`); + await fs.remove(moduleCacheDir); + wasNewClone = true; + } + } else { + wasNewClone = true; + } + + if (wasNewClone) { + const fetchSpinner = await createSpinner(); + fetchSpinner.start(`Fetching ${moduleInfo.displayName}...`); + try { + execSync(`git clone --depth 1 "${moduleInfo.url}" "${moduleCacheDir}"`, { + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, GIT_TERMINAL_PROMPT: '0' }, + }); + fetchSpinner.stop(`Fetched ${moduleInfo.displayName}`); + needsDependencyInstall = true; + } catch (error) { + fetchSpinner.error(`Failed to fetch ${moduleInfo.displayName}`); + throw new Error(`Failed to clone community module '${moduleCode}': ${error.message}`); + } + } + + // If pinned to a specific SHA, check out that exact commit. + // Refuse to install if the approved SHA cannot be reached - security requirement. + if (sha) { + const headSha = execSync('git rev-parse HEAD', { cwd: moduleCacheDir, stdio: 'pipe' }).toString().trim(); + if (headSha !== sha) { + try { + execSync(`git fetch --depth 1 origin ${sha}`, { + cwd: moduleCacheDir, + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, GIT_TERMINAL_PROMPT: '0' }, + }); + execSync(`git checkout ${sha}`, { + cwd: moduleCacheDir, + stdio: ['ignore', 'pipe', 'pipe'], + }); + needsDependencyInstall = true; + } catch { + await fs.remove(moduleCacheDir); + throw new Error( + `Community module '${moduleCode}' could not be pinned to its approved commit (${sha}). ` + + `Installation refused for security. The module registry entry may need updating.`, + ); + } + } + } + + // Install dependencies if needed + const packageJsonPath = path.join(moduleCacheDir, 'package.json'); + if ((needsDependencyInstall || wasNewClone) && (await fs.pathExists(packageJsonPath))) { + const installSpinner = await createSpinner(); + installSpinner.start(`Installing dependencies for ${moduleInfo.displayName}...`); + try { + execSync('npm install --omit=dev --no-audit --no-fund --no-progress --legacy-peer-deps', { + cwd: moduleCacheDir, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }); + installSpinner.stop(`Installed dependencies for ${moduleInfo.displayName}`); + } catch (error) { + installSpinner.error(`Failed to install dependencies for ${moduleInfo.displayName}`); + if (!silent) await prompts.log.warn(` ${error.message}`); + } + } + + return moduleCacheDir; + } + + // ─── Source Finding ─────────────────────────────────────────────────────── + + /** + * Find the source path for a community module (clone + locate module.yaml). + * @param {string} moduleCode - Module code + * @param {Object} [options] - Options passed to cloneModule + * @returns {string|null} Path to the module source or null + */ + async findModuleSource(moduleCode, options = {}) { + const moduleInfo = await this.getModuleByCode(moduleCode); + if (!moduleInfo) return null; + + const cloneDir = await this.cloneModule(moduleCode, options); + + // Check configured module_definition path first + if (moduleInfo.moduleDefinition) { + const configuredPath = path.join(cloneDir, moduleInfo.moduleDefinition); + if (await fs.pathExists(configuredPath)) { + return path.dirname(configuredPath); + } + } + + // Fallback: search skills/ and src/ directories + for (const dir of ['skills', 'src']) { + const rootCandidate = path.join(cloneDir, dir, 'module.yaml'); + if (await fs.pathExists(rootCandidate)) { + return path.dirname(rootCandidate); + } + const dirPath = path.join(cloneDir, dir); + if (await fs.pathExists(dirPath)) { + const entries = await fs.readdir(dirPath, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isDirectory()) { + const subCandidate = path.join(dirPath, entry.name, 'module.yaml'); + if (await fs.pathExists(subCandidate)) { + return path.dirname(subCandidate); + } + } + } + } + } + + // Check repo root + const rootCandidate = path.join(cloneDir, 'module.yaml'); + if (await fs.pathExists(rootCandidate)) { + return path.dirname(rootCandidate); + } + + return moduleInfo.moduleDefinition ? path.dirname(path.join(cloneDir, moduleInfo.moduleDefinition)) : null; + } + + // ─── Normalization ──────────────────────────────────────────────────────── + + /** + * Normalize a community module entry to a consistent shape. + * @param {Object} mod - Raw module from community-index.yaml + * @returns {Object} Normalized module info + */ + _normalizeCommunityModule(mod) { + return { + key: mod.name, + code: mod.code, + name: mod.display_name || mod.name, + displayName: mod.display_name || mod.name, + description: mod.description || '', + url: mod.repository || mod.url, + moduleDefinition: mod.module_definition || mod['module-definition'], + npmPackage: mod.npm_package || mod.npmPackage || null, + author: mod.author || '', + license: mod.license || '', + type: 'community', + category: mod.category || '', + subcategory: mod.subcategory || '', + keywords: mod.keywords || [], + version: mod.version || null, + approvedTag: mod.approved_tag || null, + approvedSha: mod.approved_sha || null, + approvedDate: mod.approved_date || null, + reviewer: mod.reviewer || null, + trustTier: mod.trust_tier || 'unverified', + promoted: mod.promoted === true, + promotedRank: mod.promoted_rank || null, + defaultSelected: false, + builtIn: false, + isExternal: true, + }; + } +} + +module.exports = { CommunityModuleManager }; diff --git a/tools/installer/modules/custom-module-manager.js b/tools/installer/modules/custom-module-manager.js new file mode 100644 index 000000000..18a631a29 --- /dev/null +++ b/tools/installer/modules/custom-module-manager.js @@ -0,0 +1,308 @@ +const fs = require('fs-extra'); +const os = require('node:os'); +const path = require('node:path'); +const { execSync } = require('node:child_process'); +const prompts = require('../prompts'); +const { RegistryClient } = require('./registry-client'); + +/** + * Manages custom modules installed from user-provided GitHub URLs. + * Validates URLs, fetches .claude-plugin/marketplace.json, clones repos. + */ +class CustomModuleManager { + constructor() { + this._client = new RegistryClient(); + } + + // ─── URL Validation ─────────────────────────────────────────────────────── + + /** + * Parse and validate a GitHub repository URL. + * Supports HTTPS and SSH formats. + * @param {string} url - GitHub URL to validate + * @returns {Object} { owner, repo, isValid, error } + */ + validateGitHubUrl(url) { + if (!url || typeof url !== 'string') { + return { owner: null, repo: null, isValid: false, error: 'URL is required' }; + } + + const trimmed = url.trim(); + + // HTTPS format: https://github.com/owner/repo[.git] + const httpsMatch = trimmed.match(/^https?:\/\/github\.com\/([^/]+)\/([^/.]+?)(?:\.git)?$/); + if (httpsMatch) { + return { owner: httpsMatch[1], repo: httpsMatch[2], isValid: true, error: null }; + } + + // SSH format: git@github.com:owner/repo.git + const sshMatch = trimmed.match(/^git@github\.com:([^/]+)\/([^/.]+?)(?:\.git)?$/); + if (sshMatch) { + return { owner: sshMatch[1], repo: sshMatch[2], isValid: true, error: null }; + } + + return { owner: null, repo: null, isValid: false, error: 'Not a valid GitHub URL (expected https://github.com/owner/repo)' }; + } + + // ─── Discovery ──────────────────────────────────────────────────────────── + + /** + * Fetch .claude-plugin/marketplace.json from a GitHub repository. + * @param {string} repoUrl - GitHub repository URL + * @returns {Object} Parsed marketplace.json content + */ + async fetchMarketplaceJson(repoUrl) { + const { owner, repo, isValid, error } = this.validateGitHubUrl(repoUrl); + if (!isValid) throw new Error(error); + + const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/HEAD/.claude-plugin/marketplace.json`; + + try { + return await this._client.fetchJson(rawUrl); + } catch (error_) { + if (error_.message.includes('404')) { + throw new Error(`No .claude-plugin/marketplace.json found in ${owner}/${repo}. This repository may not be a BMad module.`); + } + if (error_.message.includes('403')) { + throw new Error(`Repository ${owner}/${repo} is not accessible. Make sure it is public.`); + } + throw new Error(`Failed to fetch marketplace.json from ${owner}/${repo}: ${error_.message}`); + } + } + + /** + * Discover modules from a GitHub repository's marketplace.json. + * @param {string} repoUrl - GitHub repository URL + * @returns {Array} Normalized plugin list + */ + async discoverModules(repoUrl) { + const data = await this.fetchMarketplaceJson(repoUrl); + const plugins = data?.plugins; + + if (!Array.isArray(plugins) || plugins.length === 0) { + throw new Error('marketplace.json contains no plugins'); + } + + return plugins.map((plugin) => this._normalizeCustomModule(plugin, repoUrl, data)); + } + + // ─── Clone ──────────────────────────────────────────────────────────────── + + /** + * Get the cache directory for custom modules. + * @returns {string} Path to the custom modules cache directory + */ + getCacheDir() { + return path.join(os.homedir(), '.bmad', 'cache', 'custom-modules'); + } + + /** + * Clone a custom module repository to cache. + * @param {string} repoUrl - GitHub repository URL + * @param {Object} [options] - Clone options + * @param {boolean} [options.silent] - Suppress spinner output + * @returns {string} Path to the cloned repository + */ + async cloneRepo(repoUrl, options = {}) { + const { owner, repo, isValid, error } = this.validateGitHubUrl(repoUrl); + if (!isValid) throw new Error(error); + + const cacheDir = this.getCacheDir(); + const repoCacheDir = path.join(cacheDir, owner, repo); + const silent = options.silent || false; + + await fs.ensureDir(path.join(cacheDir, owner)); + + const createSpinner = async () => { + if (silent) { + return { start() {}, stop() {}, error() {} }; + } + return await prompts.spinner(); + }; + + if (await fs.pathExists(repoCacheDir)) { + // Update existing clone + const fetchSpinner = await createSpinner(); + fetchSpinner.start(`Updating ${owner}/${repo}...`); + try { + execSync('git fetch origin --depth 1', { + cwd: repoCacheDir, + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, GIT_TERMINAL_PROMPT: '0' }, + }); + execSync('git reset --hard origin/HEAD', { + cwd: repoCacheDir, + stdio: ['ignore', 'pipe', 'pipe'], + }); + fetchSpinner.stop(`Updated ${owner}/${repo}`); + } catch { + fetchSpinner.error(`Update failed, re-downloading ${owner}/${repo}`); + await fs.remove(repoCacheDir); + } + } + + if (!(await fs.pathExists(repoCacheDir))) { + const fetchSpinner = await createSpinner(); + fetchSpinner.start(`Cloning ${owner}/${repo}...`); + try { + execSync(`git clone --depth 1 "${repoUrl}" "${repoCacheDir}"`, { + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, GIT_TERMINAL_PROMPT: '0' }, + }); + fetchSpinner.stop(`Cloned ${owner}/${repo}`); + } catch (error_) { + fetchSpinner.error(`Failed to clone ${owner}/${repo}`); + throw new Error(`Failed to clone ${repoUrl}: ${error_.message}`); + } + } + + // Install dependencies if package.json exists + const packageJsonPath = path.join(repoCacheDir, 'package.json'); + if (await fs.pathExists(packageJsonPath)) { + const installSpinner = await createSpinner(); + installSpinner.start(`Installing dependencies for ${owner}/${repo}...`); + try { + execSync('npm install --omit=dev --no-audit --no-fund --no-progress --legacy-peer-deps', { + cwd: repoCacheDir, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }); + installSpinner.stop(`Installed dependencies for ${owner}/${repo}`); + } catch (error_) { + installSpinner.error(`Failed to install dependencies for ${owner}/${repo}`); + if (!silent) await prompts.log.warn(` ${error_.message}`); + } + } + + return repoCacheDir; + } + + // ─── Source Finding ─────────────────────────────────────────────────────── + + /** + * Find the module source path within a cloned custom repo. + * @param {string} repoUrl - GitHub repository URL (for cache location) + * @param {string} [pluginSource] - Plugin source path from marketplace.json + * @returns {string|null} Path to directory containing module.yaml + */ + async findModuleSource(repoUrl, pluginSource) { + const { owner, repo } = this.validateGitHubUrl(repoUrl); + const repoCacheDir = path.join(this.getCacheDir(), owner, repo); + + if (!(await fs.pathExists(repoCacheDir))) return null; + + // Try plugin source path first (e.g., "./src/pro-skills") + if (pluginSource) { + const sourcePath = path.join(repoCacheDir, pluginSource); + const moduleYaml = path.join(sourcePath, 'module.yaml'); + if (await fs.pathExists(moduleYaml)) { + return sourcePath; + } + } + + // Fallback: search skills/ and src/ directories + for (const dir of ['skills', 'src']) { + const rootCandidate = path.join(repoCacheDir, dir, 'module.yaml'); + if (await fs.pathExists(rootCandidate)) { + return path.dirname(rootCandidate); + } + const dirPath = path.join(repoCacheDir, dir); + if (await fs.pathExists(dirPath)) { + const entries = await fs.readdir(dirPath, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isDirectory()) { + const subCandidate = path.join(dirPath, entry.name, 'module.yaml'); + if (await fs.pathExists(subCandidate)) { + return path.dirname(subCandidate); + } + } + } + } + } + + // Check repo root + const rootCandidate = path.join(repoCacheDir, 'module.yaml'); + if (await fs.pathExists(rootCandidate)) { + return repoCacheDir; + } + + return null; + } + + /** + * Find module source by module code, searching the custom cache. + * @param {string} moduleCode - Module code to search for + * @param {Object} [options] - Options + * @returns {string|null} Path to the module source or null + */ + async findModuleSourceByCode(moduleCode, options = {}) { + const cacheDir = this.getCacheDir(); + if (!(await fs.pathExists(cacheDir))) return null; + + // Search through all custom repo caches + try { + const owners = await fs.readdir(cacheDir, { withFileTypes: true }); + for (const ownerEntry of owners) { + if (!ownerEntry.isDirectory()) continue; + const ownerPath = path.join(cacheDir, ownerEntry.name); + const repos = await fs.readdir(ownerPath, { withFileTypes: true }); + for (const repoEntry of repos) { + if (!repoEntry.isDirectory()) continue; + const repoPath = path.join(ownerPath, repoEntry.name); + + // Check marketplace.json for matching module code + const marketplacePath = path.join(repoPath, '.claude-plugin', 'marketplace.json'); + if (await fs.pathExists(marketplacePath)) { + try { + const data = JSON.parse(await fs.readFile(marketplacePath, 'utf8')); + for (const plugin of data.plugins || []) { + if (plugin.name === moduleCode) { + // Found the module - find its source + const sourcePath = plugin.source ? path.join(repoPath, plugin.source) : repoPath; + const moduleYaml = path.join(sourcePath, 'module.yaml'); + if (await fs.pathExists(moduleYaml)) { + return sourcePath; + } + } + } + } catch { + // Skip malformed marketplace.json + } + } + } + } + } catch { + // Cache doesn't exist or is inaccessible + } + + return null; + } + + // ─── Normalization ──────────────────────────────────────────────────────── + + /** + * Normalize a plugin from marketplace.json to a consistent shape. + * @param {Object} plugin - Plugin object from marketplace.json + * @param {string} repoUrl - Source repository URL + * @param {Object} data - Full marketplace.json data + * @returns {Object} Normalized module info + */ + _normalizeCustomModule(plugin, repoUrl, data) { + return { + code: plugin.name, + name: plugin.name, + displayName: plugin.name, + description: plugin.description || '', + version: plugin.version || null, + author: plugin.author || data.owner || '', + url: repoUrl, + source: plugin.source || null, + type: 'custom', + trustTier: 'unverified', + builtIn: false, + isExternal: true, + }; + } +} + +module.exports = { CustomModuleManager }; diff --git a/tools/installer/modules/external-manager.js b/tools/installer/modules/external-manager.js index db70a6678..f9f9ff06e 100644 --- a/tools/installer/modules/external-manager.js +++ b/tools/installer/modules/external-manager.js @@ -1,10 +1,10 @@ const fs = require('fs-extra'); const os = require('node:os'); const path = require('node:path'); -const https = require('node:https'); const { execSync } = require('node:child_process'); const yaml = require('yaml'); const prompts = require('../prompts'); +const { RegistryClient } = require('./registry-client'); const REGISTRY_RAW_URL = 'https://raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main/registry/official.yaml'; const FALLBACK_CONFIG_PATH = path.join(__dirname, 'registry-fallback.yaml'); @@ -17,35 +17,8 @@ const FALLBACK_CONFIG_PATH = path.join(__dirname, 'registry-fallback.yaml'); * @class ExternalModuleManager */ class ExternalModuleManager { - constructor() {} - - /** - * Fetch a URL and return the response body as a string. - * @param {string} url - URL to fetch - * @param {number} timeout - Timeout in ms (default 10s) - * @returns {Promise} Response body - */ - _fetch(url, timeout = 10_000) { - return new Promise((resolve, reject) => { - const req = https - .get(url, { timeout }, (res) => { - // Follow one redirect (GitHub sometimes 301s) - if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { - return this._fetch(res.headers.location, timeout).then(resolve, reject); - } - if (res.statusCode !== 200) { - return reject(new Error(`HTTP ${res.statusCode}`)); - } - let data = ''; - res.on('data', (chunk) => (data += chunk)); - res.on('end', () => resolve(data)); - }) - .on('error', reject) - .on('timeout', () => { - req.destroy(); - reject(new Error('Request timed out')); - }); - }); + constructor() { + this._client = new RegistryClient(); } /** @@ -60,7 +33,7 @@ class ExternalModuleManager { // Try remote registry first try { - const content = await this._fetch(REGISTRY_RAW_URL); + const content = await this._client.fetch(REGISTRY_RAW_URL); const config = yaml.parse(content); if (config?.modules?.length) { this.cachedModules = config; diff --git a/tools/installer/modules/official-modules.js b/tools/installer/modules/official-modules.js index 0effc86b8..aa709ee45 100644 --- a/tools/installer/modules/official-modules.js +++ b/tools/installer/modules/official-modules.js @@ -202,6 +202,22 @@ class OfficialModules { return externalSource; } + // Check community modules + const { CommunityModuleManager } = require('./community-manager'); + const communityMgr = new CommunityModuleManager(); + const communitySource = await communityMgr.findModuleSource(moduleCode, options); + if (communitySource) { + return communitySource; + } + + // Check custom modules (from user-provided URLs, already cloned to cache) + const { CustomModuleManager } = require('./custom-module-manager'); + const customMgr = new CustomModuleManager(); + const customSource = await customMgr.findModuleSourceByCode(moduleCode, options); + if (customSource) { + return customSource; + } + return null; } diff --git a/tools/installer/modules/registry-client.js b/tools/installer/modules/registry-client.js new file mode 100644 index 000000000..31965e00c --- /dev/null +++ b/tools/installer/modules/registry-client.js @@ -0,0 +1,66 @@ +const https = require('node:https'); +const yaml = require('yaml'); + +/** + * Shared HTTP client for fetching registry data from GitHub. + * Used by ExternalModuleManager, CommunityModuleManager, and CustomModuleManager. + */ +class RegistryClient { + constructor(options = {}) { + this.timeout = options.timeout || 10_000; + } + + /** + * Fetch a URL and return the response body as a string. + * Follows one redirect (GitHub sometimes 301s). + * @param {string} url - URL to fetch + * @param {number} [timeout] - Timeout in ms (overrides default) + * @returns {Promise} Response body + */ + fetch(url, timeout) { + const timeoutMs = timeout || this.timeout; + return new Promise((resolve, reject) => { + const req = https + .get(url, { timeout: timeoutMs }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + return this.fetch(res.headers.location, timeoutMs).then(resolve, reject); + } + if (res.statusCode !== 200) { + return reject(new Error(`HTTP ${res.statusCode}`)); + } + let data = ''; + res.on('data', (chunk) => (data += chunk)); + res.on('end', () => resolve(data)); + }) + .on('error', reject) + .on('timeout', () => { + req.destroy(); + reject(new Error('Request timed out')); + }); + }); + } + + /** + * Fetch a URL and parse the response as YAML. + * @param {string} url - URL to fetch + * @param {number} [timeout] - Timeout in ms + * @returns {Promise} Parsed YAML content + */ + async fetchYaml(url, timeout) { + const content = await this.fetch(url, timeout); + return yaml.parse(content); + } + + /** + * Fetch a URL and parse the response as JSON. + * @param {string} url - URL to fetch + * @param {number} [timeout] - Timeout in ms + * @returns {Promise} Parsed JSON content + */ + async fetchJson(url, timeout) { + const content = await this.fetch(url, timeout); + return JSON.parse(content); + } +} + +module.exports = { RegistryClient }; diff --git a/tools/installer/ui.js b/tools/installer/ui.js index 2c5c34479..e94d1c6ab 100644 --- a/tools/installer/ui.js +++ b/tools/installer/ui.js @@ -563,22 +563,38 @@ class UI { } /** - * Select all modules (official + community) using grouped multiselect. - * Core is shown as locked but filtered from the result since it's always installed separately. + * Select all modules across three tiers: official, community, and custom URL. * @param {Set} installedModuleIds - Currently installed module IDs * @returns {Array} Selected module codes (excluding core) */ async selectAllModules(installedModuleIds = new Set()) { - // Registry is the single source of truth for the module list + // Phase 1: Official modules + const officialSelected = await this._selectOfficialModules(installedModuleIds); + + // Phase 2: Community modules (category drill-down) + const communitySelected = await this._browseCommunityModules(installedModuleIds); + + // Phase 3: Custom URL modules + const customSelected = await this._addCustomUrlModules(installedModuleIds); + + // Merge all selections + return [...officialSelected, ...communitySelected, ...customSelected]; + } + + /** + * Select official modules using autocompleteMultiselect. + * Extracted from the original selectAllModules - unchanged behavior. + * @param {Set} installedModuleIds - Currently installed module IDs + * @returns {Array} Selected official module codes + */ + async _selectOfficialModules(installedModuleIds = new Set()) { const externalManager = new ExternalModuleManager(); const registryModules = await externalManager.listAvailable(); - // Build flat options list with group hints for autocompleteMultiselect const allOptions = []; const initialValues = []; const lockedValues = ['core']; - // Helper to build module entry with proper sorting and selection const buildModuleEntry = async (mod) => { const isInstalled = installedModuleIds.has(mod.code); const version = await getMarketplaceVersion(mod.code); @@ -591,7 +607,6 @@ class UI { }; }; - // Registry order is display order; core is always locked for (const mod of registryModules) { const entry = await buildModuleEntry(mod); allOptions.push({ label: entry.label, value: entry.value, hint: entry.hint }); @@ -601,7 +616,7 @@ class UI { } const selected = await prompts.autocompleteMultiselect({ - message: 'Select modules to install:', + message: 'Select official modules to install:', options: allOptions, initialValues: initialValues.length > 0 ? initialValues : undefined, lockedValues, @@ -611,18 +626,262 @@ class UI { const result = selected ? [...selected] : []; - // Display selected modules as bulleted list if (result.length > 0) { const moduleLines = result.map((moduleId) => { const opt = allOptions.find((o) => o.value === moduleId); return ` \u2022 ${opt?.label || moduleId}`; }); - await prompts.log.message('Selected modules:\n' + moduleLines.join('\n')); + await prompts.log.message('Selected official modules:\n' + moduleLines.join('\n')); } return result; } + /** + * Browse and select community modules using category drill-down. + * Featured/promoted modules appear at the top. + * @param {Set} installedModuleIds - Currently installed module IDs + * @returns {Array} Selected community module codes + */ + async _browseCommunityModules(installedModuleIds = new Set()) { + const browseCommunity = await prompts.confirm({ + message: 'Would you like to browse community modules?', + default: false, + }); + if (!browseCommunity) return []; + + const { CommunityModuleManager } = require('./modules/community-manager'); + const communityMgr = new CommunityModuleManager(); + + const s = await prompts.spinner(); + s.start('Loading community module catalog...'); + + let categories, featured, allCommunity; + try { + [categories, featured, allCommunity] = await Promise.all([ + communityMgr.getCategoryList(), + communityMgr.listFeatured(), + communityMgr.listAll(), + ]); + s.stop(`Community catalog loaded (${allCommunity.length} modules)`); + } catch (error) { + s.error('Failed to load community catalog'); + await prompts.log.warn(` ${error.message}`); + return []; + } + + if (allCommunity.length === 0) { + await prompts.log.info('No community modules are currently available.'); + return []; + } + + const selectedCodes = new Set(); + let browsing = true; + + while (browsing) { + const categoryChoices = []; + + // Featured section at top + if (featured.length > 0) { + categoryChoices.push({ + value: '__featured__', + label: `\u2605 Featured (${featured.length} module${featured.length === 1 ? '' : 's'})`, + }); + } + + // Categories with module counts + for (const cat of categories) { + categoryChoices.push({ + value: cat.slug, + label: `${cat.name} (${cat.moduleCount} module${cat.moduleCount === 1 ? '' : 's'})`, + }); + } + + // Special actions at bottom + categoryChoices.push( + { value: '__all__', label: '\u25CE View all community modules' }, + { value: '__search__', label: '\u25CE Search by keyword' }, + { value: '__done__', label: '\u2713 Done browsing' }, + ); + + const selectedCount = selectedCodes.size; + const categoryChoice = await prompts.select({ + message: `Browse community modules${selectedCount > 0 ? ` (${selectedCount} selected)` : ''}:`, + choices: categoryChoices, + }); + + if (categoryChoice === '__done__') { + browsing = false; + continue; + } + + let modulesToShow; + switch (categoryChoice) { + case '__featured__': { + modulesToShow = featured; + + break; + } + case '__all__': { + modulesToShow = allCommunity; + + break; + } + case '__search__': { + const query = await prompts.text({ + message: 'Search community modules:', + placeholder: 'e.g., design, testing, game', + }); + if (!query || query.trim() === '') continue; + modulesToShow = await communityMgr.searchByKeyword(query.trim()); + if (modulesToShow.length === 0) { + await prompts.log.warn('No matching modules found.'); + continue; + } + + break; + } + default: { + modulesToShow = await communityMgr.listByCategory(categoryChoice); + } + } + + // Build options for autocompleteMultiselect + const trustBadge = (tier) => { + if (tier === 'bmad-certified') return '\u2713'; + if (tier === 'community-reviewed') return '\u25CB'; + return '\u26A0'; + }; + + const options = modulesToShow.map((mod) => { + const versionStr = mod.version ? ` (v${mod.version})` : ''; + const badge = trustBadge(mod.trustTier); + return { + label: `${mod.displayName}${versionStr} [${badge}]`, + value: mod.code, + hint: mod.description, + }; + }); + + // Pre-check modules that are already selected or installed + const initialValues = modulesToShow.filter((m) => selectedCodes.has(m.code) || installedModuleIds.has(m.code)).map((m) => m.code); + + const selected = await prompts.autocompleteMultiselect({ + message: 'Select community modules:', + options, + initialValues: initialValues.length > 0 ? initialValues : undefined, + required: false, + maxItems: Math.min(options.length, 10), + }); + + // Update accumulated selections: sync with what user selected in this view + const shownCodes = new Set(modulesToShow.map((m) => m.code)); + for (const code of shownCodes) { + if (selected && selected.includes(code)) { + selectedCodes.add(code); + } else { + selectedCodes.delete(code); + } + } + } + + if (selectedCodes.size > 0) { + const communityMgrForDisplay = new (require('./modules/community-manager').CommunityModuleManager)(); + const moduleLines = []; + for (const code of selectedCodes) { + const mod = await communityMgrForDisplay.getModuleByCode(code); + moduleLines.push(` \u2022 ${mod?.displayName || code}`); + } + await prompts.log.message('Selected community modules:\n' + moduleLines.join('\n')); + } + + return [...selectedCodes]; + } + + /** + * Prompt user to install modules from custom GitHub URLs. + * @param {Set} installedModuleIds - Currently installed module IDs + * @returns {Array} Selected custom module objects with metadata + */ + async _addCustomUrlModules(installedModuleIds = new Set()) { + const addCustom = await prompts.confirm({ + message: 'Would you like to install from a custom GitHub URL?', + default: false, + }); + if (!addCustom) return []; + + const { CustomModuleManager } = require('./modules/custom-module-manager'); + const customMgr = new CustomModuleManager(); + const selectedModules = []; + + let addMore = true; + while (addMore) { + const url = await prompts.text({ + message: 'GitHub repository URL:', + placeholder: 'https://github.com/owner/repo', + validate: (input) => { + if (!input || input.trim() === '') return 'URL is required'; + const result = customMgr.validateGitHubUrl(input.trim()); + return result.isValid ? undefined : result.error; + }, + }); + + const s = await prompts.spinner(); + s.start('Fetching module info...'); + + try { + const plugins = await customMgr.discoverModules(url.trim()); + s.stop('Module info loaded'); + + await prompts.log.warn( + 'UNVERIFIED MODULE: This module has not been reviewed by the BMad team.\n' + ' Only install modules from sources you trust.', + ); + + for (const plugin of plugins) { + const versionStr = plugin.version ? ` v${plugin.version}` : ''; + await prompts.log.info(` ${plugin.name}${versionStr}\n ${plugin.description}\n Author: ${plugin.author}`); + } + + const confirmInstall = await prompts.confirm({ + message: `Install ${plugins.length} plugin${plugins.length === 1 ? '' : 's'} from ${url.trim()}?`, + default: false, + }); + + if (confirmInstall) { + // Pre-clone the repo so it's cached for the install pipeline + s.start('Cloning repository...'); + try { + await customMgr.cloneRepo(url.trim()); + s.stop('Repository cloned'); + } catch (cloneError) { + s.error('Failed to clone repository'); + await prompts.log.error(` ${cloneError.message}`); + addMore = await prompts.confirm({ message: 'Try another URL?', default: false }); + continue; + } + + for (const plugin of plugins) { + selectedModules.push(plugin.code); + } + } + } catch (error) { + s.error('Failed to load module info'); + await prompts.log.error(` ${error.message}`); + } + + addMore = await prompts.confirm({ + message: 'Add another custom module?', + default: false, + }); + } + + if (selectedModules.length > 0) { + await prompts.log.message('Selected custom modules:\n' + selectedModules.map((c) => ` \u2022 ${c}`).join('\n')); + } + + return selectedModules; + } + /** * Get default modules for non-interactive mode * @param {Set} installedModuleIds - Already installed module IDs @@ -946,6 +1205,7 @@ class UI { // Group modules by source const builtIn = modules.filter((m) => m.source === 'built-in'); const external = modules.filter((m) => m.source === 'external'); + const community = modules.filter((m) => m.source === 'community'); const custom = modules.filter((m) => m.source === 'custom'); const unknown = modules.filter((m) => m.source === 'unknown'); @@ -966,6 +1226,7 @@ class UI { formatGroup(builtIn, 'Built-in Modules'); formatGroup(external, 'External Modules (Official)'); + formatGroup(community, 'Community Modules'); formatGroup(custom, 'Custom Modules'); formatGroup(unknown, 'Other Modules');