Merge branch 'main' into agent-customization

test/test-installation-components.js

@@ -1926,6 +1926,112 @@ async function runTests() {
 
   console.log('');
 
+  // ============================================================
+  // Test Suite 34: RegistryClient GitHub API Cascade
+  // ============================================================
+  console.log(`${colors.yellow}Test Suite 34: RegistryClient GitHub API Cascade${colors.reset}\n`);
+
+  {
+    const { RegistryClient } = require('../tools/installer/modules/registry-client');
+
+    // Build a RegistryClient with stubbed fetch paths so we can assert on cascade behavior
+    // without making real network calls.
+    function createStubbedClient({ apiResult, rawResult }) {
+      const client = new RegistryClient();
+      const calls = [];
+
+      // Stub _fetchWithHeaders (GitHub API path)
+      client._fetchWithHeaders = async (url) => {
+        calls.push(`api:${url}`);
+        if (apiResult instanceof Error) throw apiResult;
+        return apiResult;
+      };
+
+      // Stub fetch (raw CDN path) — only intercept raw.githubusercontent.com calls
+      const originalFetch = client.fetch.bind(client);
+      client.fetch = async (url, timeout) => {
+        if (url.includes('raw.githubusercontent.com')) {
+          calls.push(`raw:${url}`);
+          if (rawResult instanceof Error) throw rawResult;
+          return rawResult;
+        }
+        return originalFetch(url, timeout);
+      };
+
+      return { client, calls };
+    }
+
+    // --- API success skips raw CDN ---
+    {
+      const { client, calls } = createStubbedClient({ apiResult: 'api-content', rawResult: 'raw-content' });
+      const result = await client.fetchGitHubFile('owner', 'repo', 'path/file.txt', 'main');
+
+      assert(result === 'api-content', 'RegistryClient API success returns API content');
+      assert(calls.length === 1, 'RegistryClient API success makes exactly one call');
+      assert(calls[0].startsWith('api:'), 'RegistryClient API success calls API endpoint');
+    }
+
+    // --- API failure falls back to raw CDN ---
+    {
+      const { client, calls } = createStubbedClient({ apiResult: new Error('HTTP 403'), rawResult: 'raw-content' });
+      const result = await client.fetchGitHubFile('owner', 'repo', 'path/file.txt', 'main');
+
+      assert(result === 'raw-content', 'RegistryClient API failure returns raw CDN content');
+      assert(calls.length === 2, 'RegistryClient API failure makes two calls');
+      assert(calls[0].startsWith('api:'), 'RegistryClient first call is to API');
+      assert(calls[1].startsWith('raw:'), 'RegistryClient second call is to raw CDN');
+    }
+
+    // --- Both endpoints failing throws ---
+    {
+      const { client } = createStubbedClient({ apiResult: new Error('HTTP 403'), rawResult: new Error('HTTP 404') });
+      let threw = false;
+      try {
+        await client.fetchGitHubFile('owner', 'repo', 'path/file.txt', 'main');
+      } catch {
+        threw = true;
+      }
+      assert(threw, 'RegistryClient both endpoints failing throws an error');
+    }
+
+    // --- API URL construction ---
+    {
+      const { client, calls } = createStubbedClient({ apiResult: 'content', rawResult: 'content' });
+      await client.fetchGitHubFile('bmad-code-org', 'bmad-plugins-marketplace', 'registry/official.yaml', 'main');
+
+      const apiCall = calls[0];
+      assert(
+        apiCall.includes('api.github.com/repos/bmad-code-org/bmad-plugins-marketplace/contents/registry/official.yaml'),
+        'RegistryClient API URL contains correct path',
+      );
+      assert(apiCall.includes('ref=main'), 'RegistryClient API URL contains ref parameter');
+    }
+
+    // --- Raw CDN URL construction ---
+    {
+      const { client, calls } = createStubbedClient({ apiResult: new Error('fail'), rawResult: 'content' });
+      await client.fetchGitHubFile('bmad-code-org', 'bmad-plugins-marketplace', 'registry/official.yaml', 'main');
+
+      const rawCall = calls[1];
+      assert(
+        rawCall.includes('raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main/registry/official.yaml'),
+        'RegistryClient raw CDN URL contains correct path',
+      );
+    }
+
+    // --- fetchGitHubYaml parses YAML ---
+    {
+      const yamlContent = 'modules:\n  - name: test\n    description: A test module\n';
+      const { client } = createStubbedClient({ apiResult: yamlContent, rawResult: yamlContent });
+      const result = await client.fetchGitHubYaml('owner', 'repo', 'file.yaml', 'main');
+
+      assert(Array.isArray(result.modules), 'fetchGitHubYaml parses YAML correctly');
+      assert(result.modules[0].name === 'test', 'fetchGitHubYaml preserves YAML values');
+    }
+  }
+
+  console.log('');
+
   // ============================================================
   // Summary
   // ============================================================

tools/installer/modules/community-manager.js

@@ -5,9 +5,9 @@ const { execSync } = require('node:child_process');
 const prompts = require('../prompts');
 const { RegistryClient } = require('./registry-client');
 
-const MARKETPLACE_BASE = 'https://raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main';
-const COMMUNITY_INDEX_URL = `${MARKETPLACE_BASE}/registry/community-index.yaml`;
-const CATEGORIES_URL = `${MARKETPLACE_BASE}/categories.yaml`;
+const MARKETPLACE_OWNER = 'bmad-code-org';
+const MARKETPLACE_REPO = 'bmad-plugins-marketplace';
+const MARKETPLACE_REF = 'main';
 
 /**
  * Manages community modules from the BMad marketplace registry.
@@ -33,7 +33,12 @@ class CommunityModuleManager {
     if (this._cachedIndex) return this._cachedIndex;
 
     try {
-      const config = await this._client.fetchYaml(COMMUNITY_INDEX_URL);
+      const config = await this._client.fetchGitHubYaml(
+        MARKETPLACE_OWNER,
+        MARKETPLACE_REPO,
+        'registry/community-index.yaml',
+        MARKETPLACE_REF,
+      );
       if (config?.modules?.length) {
         this._cachedIndex = config;
         return config;
@@ -54,7 +59,7 @@ class CommunityModuleManager {
     if (this._cachedCategories) return this._cachedCategories;
 
     try {
-      const config = await this._client.fetchYaml(CATEGORIES_URL);
+      const config = await this._client.fetchGitHubYaml(MARKETPLACE_OWNER, MARKETPLACE_REPO, 'categories.yaml', MARKETPLACE_REF);
       if (config?.categories) {
         this._cachedCategories = config;
         return config;

tools/installer/modules/external-manager.js

@@ -6,7 +6,9 @@ const yaml = require('yaml');
 const prompts = require('../prompts');
 const { RegistryClient } = require('./registry-client');
 
-const REGISTRY_RAW_URL = 'https://raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main/registry/official.yaml';
+const MARKETPLACE_OWNER = 'bmad-code-org';
+const MARKETPLACE_REPO = 'bmad-plugins-marketplace';
+const MARKETPLACE_REF = 'main';
 const FALLBACK_CONFIG_PATH = path.join(__dirname, 'registry-fallback.yaml');
 
 /**
@@ -33,8 +35,7 @@ class ExternalModuleManager {
 
     // Try remote registry first
     try {
-      const content = await this._client.fetch(REGISTRY_RAW_URL);
-      const config = yaml.parse(content);
+      const config = await this._client.fetchGitHubYaml(MARKETPLACE_OWNER, MARKETPLACE_REPO, 'registry/official.yaml', MARKETPLACE_REF);
       if (config?.modules?.length) {
         this.cachedModules = config;
         return config;

tools/installer/modules/registry-client.js

@@ -1,6 +1,37 @@
 const https = require('node:https');
 const yaml = require('yaml');
 
+/**
+ * Build a rich Error from a non-2xx response. Includes the URL, the GitHub
+ * JSON error message (or a truncated body snippet), rate-limit reset time,
+ * and Retry-After — anything present that would help a user recover.
+ */
+function buildHttpError(url, res, body) {
+  const parts = [`HTTP ${res.statusCode} ${url}`];
+
+  if (body) {
+    try {
+      const parsed = JSON.parse(body);
+      if (parsed.message) parts.push(parsed.message);
+      if (parsed.documentation_url) parts.push(`(see ${parsed.documentation_url})`);
+    } catch {
+      const snippet = body.slice(0, 200).trim();
+      if (snippet) parts.push(snippet);
+    }
+  }
+
+  const remaining = res.headers['x-ratelimit-remaining'];
+  const reset = res.headers['x-ratelimit-reset'];
+  if (remaining === '0' && reset) {
+    parts.push(`rate limit exhausted; resets at ${new Date(Number(reset) * 1000).toISOString()}`);
+  }
+
+  const retryAfter = res.headers['retry-after'];
+  if (retryAfter) parts.push(`retry after ${retryAfter}`);
+
+  return new Error(parts.join(' — '));
+}
+
 /**
  * Shared HTTP client for fetching registry data from GitHub.
  * Used by ExternalModuleManager, CommunityModuleManager, and CustomModuleManager.
@@ -12,25 +43,31 @@ class RegistryClient {
 
   /**
    * Fetch a URL and return the response body as a string.
-   * Follows one redirect (GitHub sometimes 301s).
+   * Follows up to 3 redirects (GitHub sometimes 301s).
    * @param {string} url - URL to fetch
    * @param {number} [timeout] - Timeout in ms (overrides default)
+   * @param {number} [maxRedirects=3] - Maximum redirects to follow
    * @returns {Promise<string>} Response body
    */
-  fetch(url, timeout) {
+  fetch(url, timeout, maxRedirects = 3) {
     const timeoutMs = timeout || this.timeout;
     return new Promise((resolve, reject) => {
       const req = https
         .get(url, { timeout: timeoutMs }, (res) => {
           if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
-            return this.fetch(res.headers.location, timeoutMs).then(resolve, reject);
-          }
-          if (res.statusCode !== 200) {
-            return reject(new Error(`HTTP ${res.statusCode}`));
+            if (maxRedirects <= 0) {
+              return reject(new Error('Too many redirects'));
+            }
+            return this.fetch(res.headers.location, timeoutMs, maxRedirects - 1).then(resolve, reject);
           }
           let data = '';
           res.on('data', (chunk) => (data += chunk));
-          res.on('end', () => resolve(data));
+          res.on('end', () => {
+            if (res.statusCode !== 200) {
+              return reject(buildHttpError(url, res, data));
+            }
+            resolve(data);
+          });
         })
         .on('error', reject)
         .on('timeout', () => {
@@ -50,6 +87,101 @@ class RegistryClient {
     const content = await this.fetch(url, timeout);
     return yaml.parse(content);
   }
+
+  /**
+   * Fetch a file from a GitHub repo using the Contents API first,
+   * falling back to raw.githubusercontent.com if the API fails.
+   *
+   * The API endpoint (`api.github.com`) is tried first because corporate
+   * proxies commonly block `raw.githubusercontent.com` while allowing
+   * `api.github.com` under the "Software Development" category.
+   *
+   * @param {string} owner - Repository owner (e.g., 'bmad-code-org')
+   * @param {string} repo  - Repository name (e.g., 'bmad-plugins-marketplace')
+   * @param {string} filePath - Path within the repo (e.g., 'registry/official.yaml')
+   * @param {string} ref   - Git ref (branch, tag, or SHA; e.g., 'main')
+   * @param {number} [timeout] - Timeout in ms (overrides default)
+   * @returns {Promise<string>} Raw file content
+   */
+  async fetchGitHubFile(owner, repo, filePath, ref, timeout) {
+    const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${filePath}?ref=${ref}`;
+    const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${filePath}`;
+
+    // Try GitHub Contents API first (with raw content accept header)
+    try {
+      return await this._fetchWithHeaders(apiUrl, { Accept: 'application/vnd.github.raw+json' }, timeout);
+    } catch (apiError) {
+      // API failed — fall back to raw CDN
+      try {
+        return await this.fetch(rawUrl, timeout);
+      } catch (cdnError) {
+        throw new AggregateError([apiError, cdnError], `Both GitHub API and raw CDN failed for ${filePath}`);
+      }
+    }
+  }
+
+  /**
+   * Fetch a file from GitHub and parse as YAML.
+   * @param {string} owner - Repository owner
+   * @param {string} repo  - Repository name
+   * @param {string} filePath - Path within the repo
+   * @param {string} ref   - Git ref
+   * @param {number} [timeout] - Timeout in ms
+   * @returns {Promise<Object>} Parsed YAML content
+   */
+  async fetchGitHubYaml(owner, repo, filePath, ref, timeout) {
+    const content = await this.fetchGitHubFile(owner, repo, filePath, ref, timeout);
+    return yaml.parse(content);
+  }
+
+  /**
+   * Fetch a URL with custom headers. Used for GitHub API requests.
+   * Follows up to 3 redirects.
+   * @param {string} url - URL to fetch
+   * @param {Object} headers - Request headers
+   * @param {number} [timeout] - Timeout in ms
+   * @param {number} [maxRedirects=3] - Maximum redirects to follow
+   * @returns {Promise<string>} Response body
+   * @private
+   */
+  _fetchWithHeaders(url, headers, timeout, maxRedirects = 3) {
+    const timeoutMs = timeout || this.timeout;
+    const parsed = new URL(url);
+    const options = {
+      hostname: parsed.hostname,
+      path: parsed.pathname + parsed.search,
+      timeout: timeoutMs,
+      headers: {
+        'User-Agent': 'bmad-installer',
+        ...headers,
+      },
+    };
+
+    return new Promise((resolve, reject) => {
+      const req = https
+        .get(options, (res) => {
+          if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+            if (maxRedirects <= 0) {
+              return reject(new Error('Too many redirects'));
+            }
+            return this._fetchWithHeaders(res.headers.location, headers, timeoutMs, maxRedirects - 1).then(resolve, reject);
+          }
+          let data = '';
+          res.on('data', (chunk) => (data += chunk));
+          res.on('end', () => {
+            if (res.statusCode !== 200) {
+              return reject(buildHttpError(url, res, data));
+            }
+            resolve(data);
+          });
+        })
+        .on('error', reject)
+        .on('timeout', () => {
+          req.destroy();
+          reject(new Error('Request timed out'));
+        });
+    });
+  }
 }
 
 module.exports = { RegistryClient };