🔄 Fetch with Retry for HTML build (#1793)

Limits outgoing connections to five.

See #1775, #1336

.changeset/late-phones-jam.md

@@ -0,0 +1,5 @@
+---
+"myst-cli": patch
+---
+
+Retry html pages build and limit initial outgoing connections.

packages/myst-cli/src/build/html/index.ts

@@ -9,6 +9,10 @@ import type { StartOptions } from '../site/start.js';
 import { startServer } from '../site/start.js';
 import { getSiteTemplate } from '../site/template.js';
 import { slugToUrl } from 'myst-common';
+import pLimit from 'p-limit';
+import { fetchWithRetry } from '../../utils/fetchWithRetry.js';
+
+const limitConnections = pLimit(5);
 
 export async function currentSiteRoutes(
   session: ISession,
@@ -141,25 +145,27 @@ export async function buildHtml(session: ISession, opts: StartOptions) {
 
   // Fetch all HTML pages and assets by the template
   await Promise.all(
-    routes.map(async (route) => {
-      const resp = await session.fetch(route.url);
-      if (!resp.ok) {
-        session.log.error(`Error fetching ${route.url}`);
-        return;
-      }
-      if (route.binary && resp.body) {
-        await new Promise<void>((resolve) => {
-          const filename = path.join(htmlDir, route.path);
-          if (!fs.existsSync(filename)) fs.mkdirSync(path.dirname(filename), { recursive: true });
-          const fileWriteStream = fs.createWriteStream(filename);
-          resp.body!.pipe(fileWriteStream);
-          fileWriteStream.on('finish', resolve);
-        });
-      } else {
-        const content = await resp.text();
-        writeFileToFolder(path.join(htmlDir, route.path), content);
-      }
-    }),
+    routes.map(async (route) =>
+      limitConnections(async () => {
+        const resp = await fetchWithRetry(session, route.url);
+        if (!resp.ok) {
+          session.log.error(`Error fetching ${route.url}`);
+          return;
+        }
+        if (route.binary && resp.body) {
+          await new Promise<void>((resolve) => {
+            const filename = path.join(htmlDir, route.path);
+            if (!fs.existsSync(filename)) fs.mkdirSync(path.dirname(filename), { recursive: true });
+            const fileWriteStream = fs.createWriteStream(filename);
+            resp.body!.pipe(fileWriteStream);
+            fileWriteStream.on('finish', resolve);
+          });
+        } else {
+          const content = await resp.text();
+          writeFileToFolder(path.join(htmlDir, route.path), content);
+        }
+      }),
+    ),
   );
   appServer.stop();
 

packages/myst-cli/src/utils/fetchWithRetry.ts

@@ -0,0 +1,45 @@
+import type { RequestInfo, RequestInit, Response } from 'node-fetch';
+import type { ISession } from '../session/types.js';
+
+/**
+ * Recursively fetch a URL with retry and exponential backoff.
+ */
+export async function fetchWithRetry(
+  session: Pick<ISession, 'log' | 'fetch'>,
+  /** The URL to fetch. */
+  url: URL | RequestInfo,
+  /** Options to pass to fetch (e.g., headers, method). */
+  options?: RequestInit,
+  /** How many times total to attempt the fetch. */
+  maxRetries = 3,
+  /** The current attempt number. */
+  attempt = 1,
+  /** The current backoff duration in milliseconds. */
+  backoff = 250,
+): Promise<Response> {
+  try {
+    const resp = await session.fetch(url, options);
+    if (resp.ok) {
+      // If it's a 2xx response, we consider it a success and return it
+      return resp;
+    } else {
+      // For non-2xx, we treat it as a failure that triggers a retry
+      session.log.warn(
+        `Fetch of ${url} failed with HTTP status ${resp.status} for URL: ${url} (Attempt #${attempt})`,
+      );
+    }
+  } catch (error) {
+    // This covers network failures and other errors that cause fetch to reject
+    session.log.warn(`Fetch of ${url} threw an error (Attempt #${attempt})`, error);
+  }
+
+  // If we haven't reached the max retries, wait and recurse
+  if (attempt < maxRetries) {
+    session.log.debug(`Waiting ${backoff}ms before retry #${attempt + 1}...`);
+    await new Promise((resolve) => setTimeout(resolve, backoff));
+    return fetchWithRetry(session, url, options, maxRetries, attempt + 1, backoff * 2);
+  }
+
+  // If we made it here, all retries have been exhausted
+  throw new Error(`Failed to fetch ${url} after ${maxRetries} attempts.`);
+}

packages/myst-cli/src/utils/index.ts

@@ -16,6 +16,7 @@ export * from './toc.js';
 export * from './uniqueArray.js';
 export * from './github.js';
 export * from './whiteLabelling.js';
+export * from './fetchWithRetry.js';
 
 export * as ffmpeg from './ffmpeg.js';
 export * as imagemagick from './imagemagick.js';