fix: llms-txt with code snippets (#1835)

fern-api · Nov 19, 2024 · 4db15dd · 4db15dd
1 parent 6e25efa
commit 4db15dd
Show file tree

Hide file tree

Showing 7 changed files with 292 additions and 100 deletions.
diff --git a/packages/ui/docs-bundle/src/pages/api/fern-docs/llms-full.txt.ts b/packages/ui/docs-bundle/src/pages/api/fern-docs/llms-full.txt.ts
@@ -1,13 +1,14 @@
 import { DocsLoader } from "@/server/DocsLoader";
+import { getMarkdownForPath } from "@/server/getMarkdownForPath";
 import { getSectionRoot } from "@/server/getSectionRoot";
 import { getStringParam } from "@/server/getStringParam";
-import { convertToLlmTxtMarkdown } from "@/server/llm-txt-md";
 import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node";
 import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
 import { CONTINUE, SKIP } from "@fern-api/fdr-sdk/traversers";
 import { isNonNullish } from "@fern-api/ui-core-utils";
+import { getFeatureFlags } from "@fern-ui/fern-docs-edge-config";
 import { COOKIE_FERN_TOKEN } from "@fern-ui/fern-docs-utils";
-import { uniqWith } from "es-toolkit/array";
+import { uniqBy } from "es-toolkit/array";
 import { NextApiRequest, NextApiResponse } from "next";
 
 /**
@@ -32,19 +33,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const domain = getDocsDomainNode(req);
     const host = getHostNode(req) ?? domain;
     const fern_token = req.cookies[COOKIE_FERN_TOKEN];
-    const loader = DocsLoader.for(domain, host, fern_token);
+    const featureFlags = await getFeatureFlags(domain);
+    const loader = DocsLoader.for(domain, host, fern_token).withFeatureFlags(featureFlags);
 
     const root = getSectionRoot(await loader.root(), path);
-    const pages = await loader.pages();
 
     if (root == null) {
         return res.status(404).end();
     }
 
-    const pageInfos: {
-        pageId: FernNavigation.PageId;
-        nodeTitle: string;
-    }[] = [];
+    const nodes: FernNavigation.NavigationNodePage[] = [];
 
     // traverse the tree in a depth-first manner to collect all the nodes that have markdown content
     // in the order that they appear in the sidebar
@@ -57,42 +55,24 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
             }
         }
 
-        if (FernNavigation.hasMarkdown(node)) {
-            // if the node is noindexed, don't include it in the list
-            // TODO: include "noindexed" nodes in `llms-full.txt`
-            if (node.noindex) {
-                return SKIP;
-            }
-
-            const pageId = FernNavigation.getPageId(node);
-            if (pageId != null) {
-                pageInfos.push({
-                    pageId,
-                    nodeTitle: node.title,
-                });
-            }
-        }
-
-        if (FernNavigation.isApiLeaf(node)) {
-            // TODO: construct a markdown-compatible page for the API reference
+        if (FernNavigation.isPage(node)) {
+            nodes.push(node);
         }
 
         return CONTINUE;
     });
 
-    const markdowns = uniqWith(pageInfos, (a, b) => a.pageId === b.pageId)
-        .map((pageInfo) => {
-            const page = pages[pageInfo.pageId];
-            if (page == null) {
-                return undefined;
-            }
-            return convertToLlmTxtMarkdown(
-                page.markdown,
-                pageInfo.nodeTitle,
-                pageInfo.pageId.endsWith(".mdx") ? "mdx" : "md",
-            );
-        })
-        .filter(isNonNullish);
+    const markdowns = (
+        await Promise.all(
+            uniqBy(nodes, (a) => FernNavigation.getPageId(a) ?? a.canonicalSlug ?? a.slug).map(async (node) => {
+                const markdown = await getMarkdownForPath(node, loader, featureFlags);
+                if (markdown == null) {
+                    return undefined;
+                }
+                return markdown.content;
+            }),
+        )
+    ).filter(isNonNullish);
 
     if (markdowns.length === 0) {
         return res.status(404).end();

diff --git a/packages/ui/docs-bundle/src/pages/api/fern-docs/llms.txt.ts b/packages/ui/docs-bundle/src/pages/api/fern-docs/llms.txt.ts
@@ -1,12 +1,14 @@
 import { DocsLoader } from "@/server/DocsLoader";
 import { addLeadingSlash } from "@/server/addLeadingSlash";
+import { getMarkdownForPath } from "@/server/getMarkdownForPath";
 import { getSectionRoot } from "@/server/getSectionRoot";
 import { getStringParam } from "@/server/getStringParam";
-import { convertToLlmTxtMarkdown, getLlmTxtMetadata } from "@/server/llm-txt-md";
+import { getLlmTxtMetadata } from "@/server/llm-txt-md";
 import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node";
 import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
 import { CONTINUE, SKIP } from "@fern-api/fdr-sdk/traversers";
 import { isNonNullish, withDefaultProtocol } from "@fern-api/ui-core-utils";
+import { getFeatureFlags } from "@fern-ui/fern-docs-edge-config";
 import { COOKIE_FERN_TOKEN } from "@fern-ui/fern-docs-utils";
 import { NextApiRequest, NextApiResponse } from "next";
 
@@ -44,7 +46,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const domain = getDocsDomainNode(req);
     const host = getHostNode(req) ?? domain;
     const fern_token = req.cookies[COOKIE_FERN_TOKEN];
-    const loader = DocsLoader.for(domain, host, fern_token);
+    const featureFlags = await getFeatureFlags(domain);
+    const loader = DocsLoader.for(domain, host, fern_token).withFeatureFlags(featureFlags);
 
     const root = getSectionRoot(await loader.root(), path);
     const pages = await loader.pages();
@@ -70,16 +73,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     }[] = [];
 
     const landingPage = getLandingPage(root);
-    const landingPageId = landingPage != null ? FernNavigation.getPageId(landingPage) : undefined;
-    const landingPageRawMarkdown = landingPageId != null ? pages[landingPageId]?.markdown : undefined;
-    const landingPageLlmTxtMarkdown =
-        landingPageRawMarkdown != null
-            ? convertToLlmTxtMarkdown(
-                  landingPageRawMarkdown,
-                  landingPage?.title ?? root.title,
-                  landingPageId?.endsWith(".mdx") ? "mdx" : "md",
-              )
-            : undefined;
+    const markdown = landingPage != null ? await getMarkdownForPath(landingPage, loader, featureFlags) : undefined;
 
     // traverse the tree in a depth-first manner to collect all the nodes that have markdown content
     // in the order that they appear in the sidebar
@@ -173,7 +167,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         .map((endpointPageInfo) => {
             return {
                 title: endpointPageInfo.nodeTitle,
-                href: String(new URL(addLeadingSlash(endpointPageInfo.slug), withDefaultProtocol(domain))),
+                href: String(
+                    new URL(
+                        addLeadingSlash(endpointPageInfo.slug) + (endpointPageInfo.endpointId != null ? ".mdx" : ""),
+                        withDefaultProtocol(domain),
+                    ),
+                ),
                 breadcrumb: endpointPageInfo.breadcrumb,
             };
         })
@@ -188,7 +187,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         .send(
             [
                 // if there's a landing page, use the llm-friendly markdown version instead of the ${root.title}
-                landingPageLlmTxtMarkdown ?? `# ${root.title}`,
+                markdown?.content ?? `# ${root.title}`,
                 docs.length > 0 ? `## Docs\n\n${docs.join("\n")}` : undefined,
                 endpoints.length > 0 ? `## API Docs\n\n${endpoints.join("\n")}` : undefined,
             ]
@@ -201,7 +200,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
 function getLandingPage(
     root: FernNavigation.NavigationNodeWithMetadata,
-): FernNavigation.LandingPageNode | FernNavigation.NavigationNodeWithMarkdown | undefined {
+): FernNavigation.LandingPageNode | FernNavigation.NavigationNodePage | undefined {
     if (root.type === "version") {
         return root.landingPage;
     } else if (root.type === "root") {
@@ -213,7 +212,7 @@ function getLandingPage(
         }
     }
 
-    if (FernNavigation.hasMarkdown(root)) {
+    if (FernNavigation.isPage(root)) {
         return root;
     }
 

diff --git a/packages/ui/docs-bundle/src/pages/api/fern-docs/markdown.ts b/packages/ui/docs-bundle/src/pages/api/fern-docs/markdown.ts
@@ -1,9 +1,8 @@
 import { DocsLoader } from "@/server/DocsLoader";
+import { getMarkdownForPath, getPageNodeForPath } from "@/server/getMarkdownForPath";
 import { getStringParam } from "@/server/getStringParam";
-import { convertToLlmTxtMarkdown } from "@/server/llm-txt-md";
-import { removeLeadingSlash } from "@/server/removeLeadingSlash";
 import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node";
-import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
+import { getFeatureFlags } from "@fern-ui/fern-docs-edge-config";
 import { COOKIE_FERN_TOKEN } from "@fern-ui/fern-docs-utils";
 import { NextApiRequest, NextApiResponse } from "next";
 
@@ -29,57 +28,26 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const domain = getDocsDomainNode(req);
     const host = getHostNode(req) ?? domain;
     const fern_token = req.cookies[COOKIE_FERN_TOKEN];
-    const loader = DocsLoader.for(domain, host, fern_token);
+    const featureFlags = await getFeatureFlags(domain);
+    const loader = DocsLoader.for(domain, host, fern_token).withFeatureFlags(featureFlags);
 
-    const root = await loader.root();
-    const pages = await loader.pages();
-
-    const pageInfo = getPageInfo(root, FernNavigation.Slug(removeLeadingSlash(path)));
-
-    // TODO: add support for api reference endpoint pages
-    if (pageInfo == null) {
+    const node = getPageNodeForPath(await loader.root(), path);
+    if (node == null) {
         return res.status(404).end();
     }
 
-    const page = pages[pageInfo.pageId];
-
-    if (!page) {
+    const markdown = await getMarkdownForPath(node, loader, featureFlags);
+    if (markdown == null) {
         return res.status(404).end();
     }
 
     res.status(200)
-        .setHeader("Content-Type", `text/${pageInfo.pageId.endsWith(".mdx") ? "mdx" : "markdown"}`)
+        .setHeader("Content-Type", `text/${markdown.contentType}`)
         // prevent search engines from indexing this page
         .setHeader("X-Robots-Tag", "noindex")
         // cannot guarantee that the content won't change, so we only cache for 60 seconds
         .setHeader("Cache-Control", "s-maxage=60")
-        .send(
-            convertToLlmTxtMarkdown(page.markdown, pageInfo.nodeTitle, pageInfo.pageId.endsWith(".mdx") ? "mdx" : "md"),
-        );
+        .send(markdown.content);
 
     return;
 }
-
-function getPageInfo(
-    root: FernNavigation.RootNode | undefined,
-    slug: FernNavigation.Slug,
-): { pageId: FernNavigation.PageId; nodeTitle: string } | undefined {
-    if (root == null) {
-        return undefined;
-    }
-
-    const foundNode = FernNavigation.utils.findNode(root, slug);
-    if (foundNode == null || foundNode.type !== "found" || !FernNavigation.hasMarkdown(foundNode.node)) {
-        return undefined;
-    }
-
-    const pageId = FernNavigation.getPageId(foundNode.node);
-    if (pageId == null) {
-        return undefined;
-    }
-
-    return {
-        pageId,
-        nodeTitle: foundNode.node.title,
-    };
-}
diff --git a/packages/ui/docs-bundle/src/server/DocsLoader.ts b/packages/ui/docs-bundle/src/server/DocsLoader.ts
@@ -1,14 +1,21 @@
 import type { DocsV1Read, DocsV2Read } from "@fern-api/fdr-sdk";
+import { ApiDefinition, ApiDefinitionV1ToLatest } from "@fern-api/fdr-sdk/api-definition";
 import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
 import type { AuthEdgeConfig } from "@fern-ui/fern-docs-auth";
 import { getAuthEdgeConfig } from "@fern-ui/fern-docs-edge-config";
+import { ApiDefinitionLoader } from "@fern-ui/fern-docs-server";
 import { getAuthState, type AuthState } from "./auth/getAuthState";
 import { loadWithUrl } from "./loadWithUrl";
 import { pruneWithAuthState } from "./withRbac";
 
 interface DocsLoaderFlags {
     isBatchStreamToggleDisabled: boolean;
     isApiScrollingDisabled: boolean;
+
+    // for api definition:
+    useJavaScriptAsTypeScript: boolean;
+    alwaysEnableJavaScriptFetch: boolean;
+    usesApplicationJsonInFormDataValue: boolean;
 }
 
 export class DocsLoader {
@@ -25,6 +32,9 @@ export class DocsLoader {
     private featureFlags: DocsLoaderFlags = {
         isBatchStreamToggleDisabled: false,
         isApiScrollingDisabled: false,
+        useJavaScriptAsTypeScript: false,
+        alwaysEnableJavaScriptFetch: false,
+        usesApplicationJsonInFormDataValue: false,
     };
     public withFeatureFlags(featureFlags: DocsLoaderFlags): DocsLoader {
         this.featureFlags = featureFlags;
@@ -66,6 +76,24 @@ export class DocsLoader {
         return this;
     }
 
+    public async getApiDefinition(key: FernNavigation.ApiDefinitionId): Promise<ApiDefinition | undefined> {
+        const res = await this.loadDocs();
+        if (!res) {
+            return undefined;
+        }
+        const v1 = res.definition.apis[key];
+        if (!v1) {
+            return undefined;
+        }
+        const latest = ApiDefinitionV1ToLatest.from(v1, this.featureFlags).migrate();
+        return ApiDefinitionLoader.create(this.domain, key)
+            .withApiDefinition(latest)
+            .withFlags(this.featureFlags)
+            .withResolveDescriptions(false)
+            .withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN)
+            .load();
+    }
+
     private async loadDocs(): Promise<DocsV2Read.LoadDocsForUrlResponse | undefined> {
         if (!this.#loadForDocsUrlResponse) {
             const response = await loadWithUrl(this.domain);