From 4db15dd148f8a9cfd84c52ea47c296b0dd031c18 Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Mon, 18 Nov 2024 19:40:29 -0500 Subject: [PATCH] fix: llms-txt with code snippets (#1835) --- .../src/pages/api/fern-docs/llms-full.txt.ts | 58 ++--- .../src/pages/api/fern-docs/llms.txt.ts | 31 ++- .../src/pages/api/fern-docs/markdown.ts | 52 +---- .../ui/docs-bundle/src/server/DocsLoader.ts | 28 +++ .../src/server/getMarkdownForPath.ts | 207 ++++++++++++++++++ .../docs-bundle/src/server/headerKeyCase.ts | 10 + .../src/ApiDefinitionLoader.ts | 6 +- 7 files changed, 292 insertions(+), 100 deletions(-) create mode 100644 packages/ui/docs-bundle/src/server/getMarkdownForPath.ts create mode 100644 packages/ui/docs-bundle/src/server/headerKeyCase.ts diff --git a/packages/ui/docs-bundle/src/pages/api/fern-docs/llms-full.txt.ts b/packages/ui/docs-bundle/src/pages/api/fern-docs/llms-full.txt.ts index e4f13f61c8..b6c9df6dfd 100644 --- a/packages/ui/docs-bundle/src/pages/api/fern-docs/llms-full.txt.ts +++ b/packages/ui/docs-bundle/src/pages/api/fern-docs/llms-full.txt.ts @@ -1,13 +1,14 @@ import { DocsLoader } from "@/server/DocsLoader"; +import { getMarkdownForPath } from "@/server/getMarkdownForPath"; import { getSectionRoot } from "@/server/getSectionRoot"; import { getStringParam } from "@/server/getStringParam"; -import { convertToLlmTxtMarkdown } from "@/server/llm-txt-md"; import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { CONTINUE, SKIP } from "@fern-api/fdr-sdk/traversers"; import { isNonNullish } from "@fern-api/ui-core-utils"; +import { getFeatureFlags } from "@fern-ui/fern-docs-edge-config"; import { COOKIE_FERN_TOKEN } from "@fern-ui/fern-docs-utils"; -import { uniqWith } from "es-toolkit/array"; +import { uniqBy } from "es-toolkit/array"; import { NextApiRequest, NextApiResponse } from "next"; /** @@ -32,19 +33,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const domain = getDocsDomainNode(req); const host = getHostNode(req) ?? domain; const fern_token = req.cookies[COOKIE_FERN_TOKEN]; - const loader = DocsLoader.for(domain, host, fern_token); + const featureFlags = await getFeatureFlags(domain); + const loader = DocsLoader.for(domain, host, fern_token).withFeatureFlags(featureFlags); const root = getSectionRoot(await loader.root(), path); - const pages = await loader.pages(); if (root == null) { return res.status(404).end(); } - const pageInfos: { - pageId: FernNavigation.PageId; - nodeTitle: string; - }[] = []; + const nodes: FernNavigation.NavigationNodePage[] = []; // traverse the tree in a depth-first manner to collect all the nodes that have markdown content // in the order that they appear in the sidebar @@ -57,42 +55,24 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) } } - if (FernNavigation.hasMarkdown(node)) { - // if the node is noindexed, don't include it in the list - // TODO: include "noindexed" nodes in `llms-full.txt` - if (node.noindex) { - return SKIP; - } - - const pageId = FernNavigation.getPageId(node); - if (pageId != null) { - pageInfos.push({ - pageId, - nodeTitle: node.title, - }); - } - } - - if (FernNavigation.isApiLeaf(node)) { - // TODO: construct a markdown-compatible page for the API reference + if (FernNavigation.isPage(node)) { + nodes.push(node); } return CONTINUE; }); - const markdowns = uniqWith(pageInfos, (a, b) => a.pageId === b.pageId) - .map((pageInfo) => { - const page = pages[pageInfo.pageId]; - if (page == null) { - return undefined; - } - return convertToLlmTxtMarkdown( - page.markdown, - pageInfo.nodeTitle, - pageInfo.pageId.endsWith(".mdx") ? "mdx" : "md", - ); - }) - .filter(isNonNullish); + const markdowns = ( + await Promise.all( + uniqBy(nodes, (a) => FernNavigation.getPageId(a) ?? a.canonicalSlug ?? a.slug).map(async (node) => { + const markdown = await getMarkdownForPath(node, loader, featureFlags); + if (markdown == null) { + return undefined; + } + return markdown.content; + }), + ) + ).filter(isNonNullish); if (markdowns.length === 0) { return res.status(404).end(); diff --git a/packages/ui/docs-bundle/src/pages/api/fern-docs/llms.txt.ts b/packages/ui/docs-bundle/src/pages/api/fern-docs/llms.txt.ts index cdb5633c42..2f760ba255 100644 --- a/packages/ui/docs-bundle/src/pages/api/fern-docs/llms.txt.ts +++ b/packages/ui/docs-bundle/src/pages/api/fern-docs/llms.txt.ts @@ -1,12 +1,14 @@ import { DocsLoader } from "@/server/DocsLoader"; import { addLeadingSlash } from "@/server/addLeadingSlash"; +import { getMarkdownForPath } from "@/server/getMarkdownForPath"; import { getSectionRoot } from "@/server/getSectionRoot"; import { getStringParam } from "@/server/getStringParam"; -import { convertToLlmTxtMarkdown, getLlmTxtMetadata } from "@/server/llm-txt-md"; +import { getLlmTxtMetadata } from "@/server/llm-txt-md"; import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { CONTINUE, SKIP } from "@fern-api/fdr-sdk/traversers"; import { isNonNullish, withDefaultProtocol } from "@fern-api/ui-core-utils"; +import { getFeatureFlags } from "@fern-ui/fern-docs-edge-config"; import { COOKIE_FERN_TOKEN } from "@fern-ui/fern-docs-utils"; import { NextApiRequest, NextApiResponse } from "next"; @@ -44,7 +46,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const domain = getDocsDomainNode(req); const host = getHostNode(req) ?? domain; const fern_token = req.cookies[COOKIE_FERN_TOKEN]; - const loader = DocsLoader.for(domain, host, fern_token); + const featureFlags = await getFeatureFlags(domain); + const loader = DocsLoader.for(domain, host, fern_token).withFeatureFlags(featureFlags); const root = getSectionRoot(await loader.root(), path); const pages = await loader.pages(); @@ -70,16 +73,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) }[] = []; const landingPage = getLandingPage(root); - const landingPageId = landingPage != null ? FernNavigation.getPageId(landingPage) : undefined; - const landingPageRawMarkdown = landingPageId != null ? pages[landingPageId]?.markdown : undefined; - const landingPageLlmTxtMarkdown = - landingPageRawMarkdown != null - ? convertToLlmTxtMarkdown( - landingPageRawMarkdown, - landingPage?.title ?? root.title, - landingPageId?.endsWith(".mdx") ? "mdx" : "md", - ) - : undefined; + const markdown = landingPage != null ? await getMarkdownForPath(landingPage, loader, featureFlags) : undefined; // traverse the tree in a depth-first manner to collect all the nodes that have markdown content // in the order that they appear in the sidebar @@ -173,7 +167,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) .map((endpointPageInfo) => { return { title: endpointPageInfo.nodeTitle, - href: String(new URL(addLeadingSlash(endpointPageInfo.slug), withDefaultProtocol(domain))), + href: String( + new URL( + addLeadingSlash(endpointPageInfo.slug) + (endpointPageInfo.endpointId != null ? ".mdx" : ""), + withDefaultProtocol(domain), + ), + ), breadcrumb: endpointPageInfo.breadcrumb, }; }) @@ -188,7 +187,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) .send( [ // if there's a landing page, use the llm-friendly markdown version instead of the ${root.title} - landingPageLlmTxtMarkdown ?? `# ${root.title}`, + markdown?.content ?? `# ${root.title}`, docs.length > 0 ? `## Docs\n\n${docs.join("\n")}` : undefined, endpoints.length > 0 ? `## API Docs\n\n${endpoints.join("\n")}` : undefined, ] @@ -201,7 +200,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) function getLandingPage( root: FernNavigation.NavigationNodeWithMetadata, -): FernNavigation.LandingPageNode | FernNavigation.NavigationNodeWithMarkdown | undefined { +): FernNavigation.LandingPageNode | FernNavigation.NavigationNodePage | undefined { if (root.type === "version") { return root.landingPage; } else if (root.type === "root") { @@ -213,7 +212,7 @@ function getLandingPage( } } - if (FernNavigation.hasMarkdown(root)) { + if (FernNavigation.isPage(root)) { return root; } diff --git a/packages/ui/docs-bundle/src/pages/api/fern-docs/markdown.ts b/packages/ui/docs-bundle/src/pages/api/fern-docs/markdown.ts index ef4a1a9ddb..c8cdbe1db8 100644 --- a/packages/ui/docs-bundle/src/pages/api/fern-docs/markdown.ts +++ b/packages/ui/docs-bundle/src/pages/api/fern-docs/markdown.ts @@ -1,9 +1,8 @@ import { DocsLoader } from "@/server/DocsLoader"; +import { getMarkdownForPath, getPageNodeForPath } from "@/server/getMarkdownForPath"; import { getStringParam } from "@/server/getStringParam"; -import { convertToLlmTxtMarkdown } from "@/server/llm-txt-md"; -import { removeLeadingSlash } from "@/server/removeLeadingSlash"; import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node"; -import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; +import { getFeatureFlags } from "@fern-ui/fern-docs-edge-config"; import { COOKIE_FERN_TOKEN } from "@fern-ui/fern-docs-utils"; import { NextApiRequest, NextApiResponse } from "next"; @@ -29,57 +28,26 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const domain = getDocsDomainNode(req); const host = getHostNode(req) ?? domain; const fern_token = req.cookies[COOKIE_FERN_TOKEN]; - const loader = DocsLoader.for(domain, host, fern_token); + const featureFlags = await getFeatureFlags(domain); + const loader = DocsLoader.for(domain, host, fern_token).withFeatureFlags(featureFlags); - const root = await loader.root(); - const pages = await loader.pages(); - - const pageInfo = getPageInfo(root, FernNavigation.Slug(removeLeadingSlash(path))); - - // TODO: add support for api reference endpoint pages - if (pageInfo == null) { + const node = getPageNodeForPath(await loader.root(), path); + if (node == null) { return res.status(404).end(); } - const page = pages[pageInfo.pageId]; - - if (!page) { + const markdown = await getMarkdownForPath(node, loader, featureFlags); + if (markdown == null) { return res.status(404).end(); } res.status(200) - .setHeader("Content-Type", `text/${pageInfo.pageId.endsWith(".mdx") ? "mdx" : "markdown"}`) + .setHeader("Content-Type", `text/${markdown.contentType}`) // prevent search engines from indexing this page .setHeader("X-Robots-Tag", "noindex") // cannot guarantee that the content won't change, so we only cache for 60 seconds .setHeader("Cache-Control", "s-maxage=60") - .send( - convertToLlmTxtMarkdown(page.markdown, pageInfo.nodeTitle, pageInfo.pageId.endsWith(".mdx") ? "mdx" : "md"), - ); + .send(markdown.content); return; } - -function getPageInfo( - root: FernNavigation.RootNode | undefined, - slug: FernNavigation.Slug, -): { pageId: FernNavigation.PageId; nodeTitle: string } | undefined { - if (root == null) { - return undefined; - } - - const foundNode = FernNavigation.utils.findNode(root, slug); - if (foundNode == null || foundNode.type !== "found" || !FernNavigation.hasMarkdown(foundNode.node)) { - return undefined; - } - - const pageId = FernNavigation.getPageId(foundNode.node); - if (pageId == null) { - return undefined; - } - - return { - pageId, - nodeTitle: foundNode.node.title, - }; -} diff --git a/packages/ui/docs-bundle/src/server/DocsLoader.ts b/packages/ui/docs-bundle/src/server/DocsLoader.ts index 0e70dece4a..f23e314dc8 100644 --- a/packages/ui/docs-bundle/src/server/DocsLoader.ts +++ b/packages/ui/docs-bundle/src/server/DocsLoader.ts @@ -1,7 +1,9 @@ import type { DocsV1Read, DocsV2Read } from "@fern-api/fdr-sdk"; +import { ApiDefinition, ApiDefinitionV1ToLatest } from "@fern-api/fdr-sdk/api-definition"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import type { AuthEdgeConfig } from "@fern-ui/fern-docs-auth"; import { getAuthEdgeConfig } from "@fern-ui/fern-docs-edge-config"; +import { ApiDefinitionLoader } from "@fern-ui/fern-docs-server"; import { getAuthState, type AuthState } from "./auth/getAuthState"; import { loadWithUrl } from "./loadWithUrl"; import { pruneWithAuthState } from "./withRbac"; @@ -9,6 +11,11 @@ import { pruneWithAuthState } from "./withRbac"; interface DocsLoaderFlags { isBatchStreamToggleDisabled: boolean; isApiScrollingDisabled: boolean; + + // for api definition: + useJavaScriptAsTypeScript: boolean; + alwaysEnableJavaScriptFetch: boolean; + usesApplicationJsonInFormDataValue: boolean; } export class DocsLoader { @@ -25,6 +32,9 @@ export class DocsLoader { private featureFlags: DocsLoaderFlags = { isBatchStreamToggleDisabled: false, isApiScrollingDisabled: false, + useJavaScriptAsTypeScript: false, + alwaysEnableJavaScriptFetch: false, + usesApplicationJsonInFormDataValue: false, }; public withFeatureFlags(featureFlags: DocsLoaderFlags): DocsLoader { this.featureFlags = featureFlags; @@ -66,6 +76,24 @@ export class DocsLoader { return this; } + public async getApiDefinition(key: FernNavigation.ApiDefinitionId): Promise { + const res = await this.loadDocs(); + if (!res) { + return undefined; + } + const v1 = res.definition.apis[key]; + if (!v1) { + return undefined; + } + const latest = ApiDefinitionV1ToLatest.from(v1, this.featureFlags).migrate(); + return ApiDefinitionLoader.create(this.domain, key) + .withApiDefinition(latest) + .withFlags(this.featureFlags) + .withResolveDescriptions(false) + .withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN) + .load(); + } + private async loadDocs(): Promise { if (!this.#loadForDocsUrlResponse) { const response = await loadWithUrl(this.domain); diff --git a/packages/ui/docs-bundle/src/server/getMarkdownForPath.ts b/packages/ui/docs-bundle/src/server/getMarkdownForPath.ts new file mode 100644 index 0000000000..cac636e5ea --- /dev/null +++ b/packages/ui/docs-bundle/src/server/getMarkdownForPath.ts @@ -0,0 +1,207 @@ +import { ApiDefinition, FernNavigation } from "@fern-api/fdr-sdk"; +import { EndpointDefinition, TypeDefinition, TypeShape } from "@fern-api/fdr-sdk/api-definition"; +import { MarkdownText } from "@fern-api/fdr-sdk/docs"; +import { isNonNullish } from "@fern-api/ui-core-utils"; +import { FeatureFlags } from "@fern-ui/fern-docs-utils"; +import { isString } from "es-toolkit"; +import { DocsLoader } from "./DocsLoader"; +import { pascalCaseHeaderKey } from "./headerKeyCase"; +import { convertToLlmTxtMarkdown } from "./llm-txt-md"; +import { removeLeadingSlash } from "./removeLeadingSlash"; + +export async function getMarkdownForPath( + node: FernNavigation.NavigationNodePage, + loader: DocsLoader, + featureFlags: FeatureFlags, +): Promise<{ content: string; contentType: "markdown" | "mdx" } | undefined> { + loader = loader.withFeatureFlags(featureFlags); + const pages = await loader.pages(); + + if (FernNavigation.isApiLeaf(node)) { + const apiDefinition = await loader.getApiDefinition(node.apiDefinitionId); + if (apiDefinition == null) { + return undefined; + } + + if (node.type === "endpoint") { + const endpoint = apiDefinition.endpoints[node.endpointId]; + if (endpoint == null) { + return undefined; + } + return { + content: endpointDefinitionToMarkdown( + endpoint, + apiDefinition.globalHeaders, + apiDefinition.types, + node.title, + ), + contentType: "mdx", + }; + } + } + + const pageId = FernNavigation.getPageId(node); + if (pageId == null) { + return undefined; + } + + const page = pages[pageId]; + if (!page) { + return undefined; + } + + return { + content: convertToLlmTxtMarkdown(page.markdown, node.title, pageId.endsWith(".mdx") ? "mdx" : "md"), + contentType: pageId.endsWith(".mdx") ? "mdx" : "markdown", + }; +} + +export function getPageNodeForPath( + root: FernNavigation.RootNode | undefined, + path: string, +): FernNavigation.NavigationNodePage | undefined { + if (root == null) { + return undefined; + } + const found = FernNavigation.utils.findNode(root, FernNavigation.Slug(removeLeadingSlash(path))); + if (found.type !== "found" || !FernNavigation.isPage(found.node)) { + return undefined; + } + return found.node; +} + +// function getPageInfo( +// root: FernNavigation.RootNode | undefined, +// slug: FernNavigation.Slug, +// ): +// | { +// nodeTitle: string; +// pageId?: FernNavigation.PageId; +// apiLeaf?: FernNavigation.NavigationNodeApiLeaf; +// } +// | undefined { +// if (root == null) { +// return undefined; +// } + +// const foundNode = FernNavigation.utils.findNode(root, slug); +// if (foundNode == null || foundNode.type !== "found" || !FernNavigation.isPage(foundNode.node)) { +// return undefined; +// } + +// if (FernNavigation.isApiLeaf(foundNode.node)) { +// return { +// nodeTitle: foundNode.node.title, +// apiLeaf: foundNode.node, +// }; +// } + +// const pageId = FernNavigation.getPageId(foundNode.node); +// if (pageId == null) { +// return undefined; +// } + +// return { +// nodeTitle: foundNode.node.title, +// pageId, +// }; +// } + +export function endpointDefinitionToMarkdown( + endpoint: EndpointDefinition, + globalHeaders: ApiDefinition.ObjectProperty[] | undefined, + types: Record, + nodeTitle: string, +): string { + const headers = [...(endpoint.requestHeaders ?? []), ...(globalHeaders ?? [])]; + return [ + `# ${nodeTitle}`, + [ + "```http", + `${endpoint.method} ${endpoint.environments?.find((env) => env.id === endpoint.defaultEnvironment)?.baseUrl ?? endpoint.environments?.[0]?.baseUrl ?? ""}${ApiDefinition.toCurlyBraceEndpointPathLiteral(endpoint.path)}`, + endpoint.request != null ? `Content-Type: ${endpoint.request.contentType}` : undefined, + "```", + ] + .filter(isNonNullish) + .join("\n"), + typeof endpoint.description === "string" ? endpoint.description : undefined, + headers.length ? "## Request Headers" : undefined, + headers + ?.map( + (header) => + `- ${pascalCaseHeaderKey(header.key)}${getShorthand(header.valueShape, types, header.description)}`, + ) + .join("\n"), + endpoint.pathParameters?.length ? "## Path Parameters" : undefined, + endpoint.pathParameters + ?.map( + (param) => + `- ${pascalCaseHeaderKey(param.key)}${getShorthand(param.valueShape, types, param.description)}`, + ) + .join("\n"), + endpoint.queryParameters?.length ? "## Query Parameters" : undefined, + endpoint.queryParameters + ?.map( + (param) => + `- ${pascalCaseHeaderKey(param.key)}${getShorthand(param.valueShape, types, param.description)}`, + ) + .join("\n"), + endpoint.request != null ? "## Request Body" : undefined, + typeof endpoint.request?.description === "string" ? endpoint.request?.description : undefined, + endpoint.request != null ? `\`\`\`json\n${JSON.stringify(endpoint.request.body)}\n\`\`\`` : undefined, + endpoint.responseHeaders?.length ? "## Response Headers" : undefined, + endpoint.responseHeaders + ?.map( + (header) => + `- ${pascalCaseHeaderKey(header.key)}${getShorthand(header.valueShape, types, header.description)}`, + ) + .join("\n"), + endpoint.response != null || endpoint.errors?.length ? "## Response Body" : undefined, + endpoint.response != null || endpoint.errors?.length + ? [ + typeof endpoint.response?.description === "string" + ? `- ${endpoint.response.statusCode}: ${endpoint.response?.description}` + : undefined, + ...(endpoint.errors + ?.filter((error) => typeof error.description === "string") + .map((error) => `- ${error.statusCode}: ${error.description}`) ?? []), + ].join("\n") + : undefined, + + "## Examples", + endpoint.examples + ?.flatMap((example) => + Object.entries(example.snippets ?? {}).flatMap(([language, snippets]) => + snippets.map( + (snippet) => + ({ + language, + snippet, + name: snippet.name ?? example.name, + }) as const, + ), + ), + ) + .map( + ({ language, snippet, name }) => + `\`\`\`${language === "curl" ? "shell" : language}${name != null ? ` ${name}` : ""}\n${snippet.code}\n\`\`\``, + ) + .join("\n\n"), + ] + .filter(isNonNullish) + .join("\n\n"); +} + +function getShorthand( + shape: TypeShape, + types: Record, + shapeDescription: MarkdownText | undefined, +): string | undefined { + const unwrapped = ApiDefinition.unwrapReference(shape, types); + const description = [shapeDescription, ...unwrapped.descriptions].filter(isString)[0]; + if (unwrapped.isOptional) { + return description ? ` (optional): ${description}` : " (optional)"; + } + + return description ? ` (required): ${description}` : " (required)"; +} diff --git a/packages/ui/docs-bundle/src/server/headerKeyCase.ts b/packages/ui/docs-bundle/src/server/headerKeyCase.ts new file mode 100644 index 0000000000..544f86eaee --- /dev/null +++ b/packages/ui/docs-bundle/src/server/headerKeyCase.ts @@ -0,0 +1,10 @@ +import { mapKeys } from "es-toolkit/object"; +import { pascalCase } from "es-toolkit/string"; + +export function pascalCaseHeaderKey(key: string): string { + return key.split("-").map(pascalCase).join("-"); +} + +export function pascalCaseHeaderKeys(headers: Record = {}): Record { + return mapKeys(headers, (_, key) => pascalCaseHeaderKey(key)); +} diff --git a/packages/ui/fern-docs-server/src/ApiDefinitionLoader.ts b/packages/ui/fern-docs-server/src/ApiDefinitionLoader.ts index d4ceee4550..023ff80474 100644 --- a/packages/ui/fern-docs-server/src/ApiDefinitionLoader.ts +++ b/packages/ui/fern-docs-server/src/ApiDefinitionLoader.ts @@ -64,8 +64,8 @@ export class ApiDefinitionLoader { } private flags: FeatureFlags = DEFAULT_FEATURE_FLAGS; - public withFlags = (flags: FeatureFlags): ApiDefinitionLoader => { - this.flags = flags; + public withFlags = (flags: Partial): ApiDefinitionLoader => { + this.flags = { ...this.flags, ...flags }; return this; }; @@ -238,7 +238,7 @@ export class ApiDefinitionLoader { if (code != null) { pushSnippet({ - name: "HTTP Request", + name: undefined, language: targetId, install: undefined, code,