diff --git a/test/test-installation-components.js b/test/test-installation-components.js index f1c1be486..c5d3540b3 100644 --- a/test/test-installation-components.js +++ b/test/test-installation-components.js @@ -1926,6 +1926,112 @@ async function runTests() { console.log(''); + // ============================================================ + // Test Suite 34: RegistryClient GitHub API Cascade + // ============================================================ + console.log(`${colors.yellow}Test Suite 34: RegistryClient GitHub API Cascade${colors.reset}\n`); + + { + const { RegistryClient } = require('../tools/installer/modules/registry-client'); + + // Build a RegistryClient with stubbed fetch paths so we can assert on cascade behavior + // without making real network calls. + function createStubbedClient({ apiResult, rawResult }) { + const client = new RegistryClient(); + const calls = []; + + // Stub _fetchWithHeaders (GitHub API path) + client._fetchWithHeaders = async (url) => { + calls.push(`api:${url}`); + if (apiResult instanceof Error) throw apiResult; + return apiResult; + }; + + // Stub fetch (raw CDN path) — only intercept raw.githubusercontent.com calls + const originalFetch = client.fetch.bind(client); + client.fetch = async (url, timeout) => { + if (url.includes('raw.githubusercontent.com')) { + calls.push(`raw:${url}`); + if (rawResult instanceof Error) throw rawResult; + return rawResult; + } + return originalFetch(url, timeout); + }; + + return { client, calls }; + } + + // --- API success skips raw CDN --- + { + const { client, calls } = createStubbedClient({ apiResult: 'api-content', rawResult: 'raw-content' }); + const result = await client.fetchGitHubFile('owner', 'repo', 'path/file.txt', 'main'); + + assert(result === 'api-content', 'RegistryClient API success returns API content'); + assert(calls.length === 1, 'RegistryClient API success makes exactly one call'); + assert(calls[0].startsWith('api:'), 'RegistryClient API success calls API endpoint'); + } + + // --- API failure falls back to raw CDN --- + { + const { client, calls } = createStubbedClient({ apiResult: new Error('HTTP 403'), rawResult: 'raw-content' }); + const result = await client.fetchGitHubFile('owner', 'repo', 'path/file.txt', 'main'); + + assert(result === 'raw-content', 'RegistryClient API failure returns raw CDN content'); + assert(calls.length === 2, 'RegistryClient API failure makes two calls'); + assert(calls[0].startsWith('api:'), 'RegistryClient first call is to API'); + assert(calls[1].startsWith('raw:'), 'RegistryClient second call is to raw CDN'); + } + + // --- Both endpoints failing throws --- + { + const { client } = createStubbedClient({ apiResult: new Error('HTTP 403'), rawResult: new Error('HTTP 404') }); + let threw = false; + try { + await client.fetchGitHubFile('owner', 'repo', 'path/file.txt', 'main'); + } catch { + threw = true; + } + assert(threw, 'RegistryClient both endpoints failing throws an error'); + } + + // --- API URL construction --- + { + const { client, calls } = createStubbedClient({ apiResult: 'content', rawResult: 'content' }); + await client.fetchGitHubFile('bmad-code-org', 'bmad-plugins-marketplace', 'registry/official.yaml', 'main'); + + const apiCall = calls[0]; + assert( + apiCall.includes('api.github.com/repos/bmad-code-org/bmad-plugins-marketplace/contents/registry/official.yaml'), + 'RegistryClient API URL contains correct path', + ); + assert(apiCall.includes('ref=main'), 'RegistryClient API URL contains ref parameter'); + } + + // --- Raw CDN URL construction --- + { + const { client, calls } = createStubbedClient({ apiResult: new Error('fail'), rawResult: 'content' }); + await client.fetchGitHubFile('bmad-code-org', 'bmad-plugins-marketplace', 'registry/official.yaml', 'main'); + + const rawCall = calls[1]; + assert( + rawCall.includes('raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main/registry/official.yaml'), + 'RegistryClient raw CDN URL contains correct path', + ); + } + + // --- fetchGitHubYaml parses YAML --- + { + const yamlContent = 'modules:\n - name: test\n description: A test module\n'; + const { client } = createStubbedClient({ apiResult: yamlContent, rawResult: yamlContent }); + const result = await client.fetchGitHubYaml('owner', 'repo', 'file.yaml', 'main'); + + assert(Array.isArray(result.modules), 'fetchGitHubYaml parses YAML correctly'); + assert(result.modules[0].name === 'test', 'fetchGitHubYaml preserves YAML values'); + } + } + + console.log(''); + // ============================================================ // Summary // ============================================================ diff --git a/tools/installer/modules/community-manager.js b/tools/installer/modules/community-manager.js index 3e0217688..aff54ca44 100644 --- a/tools/installer/modules/community-manager.js +++ b/tools/installer/modules/community-manager.js @@ -5,9 +5,9 @@ const { execSync } = require('node:child_process'); const prompts = require('../prompts'); const { RegistryClient } = require('./registry-client'); -const MARKETPLACE_BASE = 'https://raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main'; -const COMMUNITY_INDEX_URL = `${MARKETPLACE_BASE}/registry/community-index.yaml`; -const CATEGORIES_URL = `${MARKETPLACE_BASE}/categories.yaml`; +const MARKETPLACE_OWNER = 'bmad-code-org'; +const MARKETPLACE_REPO = 'bmad-plugins-marketplace'; +const MARKETPLACE_REF = 'main'; /** * Manages community modules from the BMad marketplace registry. @@ -33,7 +33,12 @@ class CommunityModuleManager { if (this._cachedIndex) return this._cachedIndex; try { - const config = await this._client.fetchYaml(COMMUNITY_INDEX_URL); + const config = await this._client.fetchGitHubYaml( + MARKETPLACE_OWNER, + MARKETPLACE_REPO, + 'registry/community-index.yaml', + MARKETPLACE_REF, + ); if (config?.modules?.length) { this._cachedIndex = config; return config; @@ -54,7 +59,7 @@ class CommunityModuleManager { if (this._cachedCategories) return this._cachedCategories; try { - const config = await this._client.fetchYaml(CATEGORIES_URL); + const config = await this._client.fetchGitHubYaml(MARKETPLACE_OWNER, MARKETPLACE_REPO, 'categories.yaml', MARKETPLACE_REF); if (config?.categories) { this._cachedCategories = config; return config; diff --git a/tools/installer/modules/external-manager.js b/tools/installer/modules/external-manager.js index 5169ffb50..b91d353af 100644 --- a/tools/installer/modules/external-manager.js +++ b/tools/installer/modules/external-manager.js @@ -6,7 +6,9 @@ const yaml = require('yaml'); const prompts = require('../prompts'); const { RegistryClient } = require('./registry-client'); -const REGISTRY_RAW_URL = 'https://raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main/registry/official.yaml'; +const MARKETPLACE_OWNER = 'bmad-code-org'; +const MARKETPLACE_REPO = 'bmad-plugins-marketplace'; +const MARKETPLACE_REF = 'main'; const FALLBACK_CONFIG_PATH = path.join(__dirname, 'registry-fallback.yaml'); /** @@ -33,8 +35,7 @@ class ExternalModuleManager { // Try remote registry first try { - const content = await this._client.fetch(REGISTRY_RAW_URL); - const config = yaml.parse(content); + const config = await this._client.fetchGitHubYaml(MARKETPLACE_OWNER, MARKETPLACE_REPO, 'registry/official.yaml', MARKETPLACE_REF); if (config?.modules?.length) { this.cachedModules = config; return config; diff --git a/tools/installer/modules/registry-client.js b/tools/installer/modules/registry-client.js index 53d220678..31a38f8d3 100644 --- a/tools/installer/modules/registry-client.js +++ b/tools/installer/modules/registry-client.js @@ -1,6 +1,37 @@ const https = require('node:https'); const yaml = require('yaml'); +/** + * Build a rich Error from a non-2xx response. Includes the URL, the GitHub + * JSON error message (or a truncated body snippet), rate-limit reset time, + * and Retry-After — anything present that would help a user recover. + */ +function buildHttpError(url, res, body) { + const parts = [`HTTP ${res.statusCode} ${url}`]; + + if (body) { + try { + const parsed = JSON.parse(body); + if (parsed.message) parts.push(parsed.message); + if (parsed.documentation_url) parts.push(`(see ${parsed.documentation_url})`); + } catch { + const snippet = body.slice(0, 200).trim(); + if (snippet) parts.push(snippet); + } + } + + const remaining = res.headers['x-ratelimit-remaining']; + const reset = res.headers['x-ratelimit-reset']; + if (remaining === '0' && reset) { + parts.push(`rate limit exhausted; resets at ${new Date(Number(reset) * 1000).toISOString()}`); + } + + const retryAfter = res.headers['retry-after']; + if (retryAfter) parts.push(`retry after ${retryAfter}`); + + return new Error(parts.join(' — ')); +} + /** * Shared HTTP client for fetching registry data from GitHub. * Used by ExternalModuleManager, CommunityModuleManager, and CustomModuleManager. @@ -12,25 +43,31 @@ class RegistryClient { /** * Fetch a URL and return the response body as a string. - * Follows one redirect (GitHub sometimes 301s). + * Follows up to 3 redirects (GitHub sometimes 301s). * @param {string} url - URL to fetch * @param {number} [timeout] - Timeout in ms (overrides default) + * @param {number} [maxRedirects=3] - Maximum redirects to follow * @returns {Promise} Response body */ - fetch(url, timeout) { + fetch(url, timeout, maxRedirects = 3) { const timeoutMs = timeout || this.timeout; return new Promise((resolve, reject) => { const req = https .get(url, { timeout: timeoutMs }, (res) => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { - return this.fetch(res.headers.location, timeoutMs).then(resolve, reject); - } - if (res.statusCode !== 200) { - return reject(new Error(`HTTP ${res.statusCode}`)); + if (maxRedirects <= 0) { + return reject(new Error('Too many redirects')); + } + return this.fetch(res.headers.location, timeoutMs, maxRedirects - 1).then(resolve, reject); } let data = ''; res.on('data', (chunk) => (data += chunk)); - res.on('end', () => resolve(data)); + res.on('end', () => { + if (res.statusCode !== 200) { + return reject(buildHttpError(url, res, data)); + } + resolve(data); + }); }) .on('error', reject) .on('timeout', () => { @@ -50,6 +87,101 @@ class RegistryClient { const content = await this.fetch(url, timeout); return yaml.parse(content); } + + /** + * Fetch a file from a GitHub repo using the Contents API first, + * falling back to raw.githubusercontent.com if the API fails. + * + * The API endpoint (`api.github.com`) is tried first because corporate + * proxies commonly block `raw.githubusercontent.com` while allowing + * `api.github.com` under the "Software Development" category. + * + * @param {string} owner - Repository owner (e.g., 'bmad-code-org') + * @param {string} repo - Repository name (e.g., 'bmad-plugins-marketplace') + * @param {string} filePath - Path within the repo (e.g., 'registry/official.yaml') + * @param {string} ref - Git ref (branch, tag, or SHA; e.g., 'main') + * @param {number} [timeout] - Timeout in ms (overrides default) + * @returns {Promise} Raw file content + */ + async fetchGitHubFile(owner, repo, filePath, ref, timeout) { + const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${filePath}?ref=${ref}`; + const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${filePath}`; + + // Try GitHub Contents API first (with raw content accept header) + try { + return await this._fetchWithHeaders(apiUrl, { Accept: 'application/vnd.github.raw+json' }, timeout); + } catch (apiError) { + // API failed — fall back to raw CDN + try { + return await this.fetch(rawUrl, timeout); + } catch (cdnError) { + throw new AggregateError([apiError, cdnError], `Both GitHub API and raw CDN failed for ${filePath}`); + } + } + } + + /** + * Fetch a file from GitHub and parse as YAML. + * @param {string} owner - Repository owner + * @param {string} repo - Repository name + * @param {string} filePath - Path within the repo + * @param {string} ref - Git ref + * @param {number} [timeout] - Timeout in ms + * @returns {Promise} Parsed YAML content + */ + async fetchGitHubYaml(owner, repo, filePath, ref, timeout) { + const content = await this.fetchGitHubFile(owner, repo, filePath, ref, timeout); + return yaml.parse(content); + } + + /** + * Fetch a URL with custom headers. Used for GitHub API requests. + * Follows up to 3 redirects. + * @param {string} url - URL to fetch + * @param {Object} headers - Request headers + * @param {number} [timeout] - Timeout in ms + * @param {number} [maxRedirects=3] - Maximum redirects to follow + * @returns {Promise} Response body + * @private + */ + _fetchWithHeaders(url, headers, timeout, maxRedirects = 3) { + const timeoutMs = timeout || this.timeout; + const parsed = new URL(url); + const options = { + hostname: parsed.hostname, + path: parsed.pathname + parsed.search, + timeout: timeoutMs, + headers: { + 'User-Agent': 'bmad-installer', + ...headers, + }, + }; + + return new Promise((resolve, reject) => { + const req = https + .get(options, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + if (maxRedirects <= 0) { + return reject(new Error('Too many redirects')); + } + return this._fetchWithHeaders(res.headers.location, headers, timeoutMs, maxRedirects - 1).then(resolve, reject); + } + let data = ''; + res.on('data', (chunk) => (data += chunk)); + res.on('end', () => { + if (res.statusCode !== 200) { + return reject(buildHttpError(url, res, data)); + } + resolve(data); + }); + }) + .on('error', reject) + .on('timeout', () => { + req.destroy(); + reject(new Error('Request timed out')); + }); + }); + } } module.exports = { RegistryClient };