Llms.txt (#724)

<!--

Adds llms.txt compatibility to the docs pages.
-->

---------

Co-authored-by: Konsti Wohlwend <n2d4xc@gmail.com>
Co-authored-by: Madison Kennedy <madison@Madisons-MacBook-Pro.local>
Co-authored-by: BilalG1 <bg2002@gmail.com>

docs/lib/get-llm-text.ts

@@ -0,0 +1,36 @@
+import type { InferPageType } from 'fumadocs-core/source';
+import { remarkInclude } from 'fumadocs-mdx/config';
+import { remark } from 'remark';
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+import { apiSource, source } from './source';
+
+const processor = remark()
+  .use(remarkMdx)
+  // needed for Fumadocs MDX
+  .use(remarkInclude)
+  .use(remarkGfm);
+
+export async function getLLMText(page: InferPageType<typeof source> | InferPageType<typeof apiSource>) {
+  // Remove the Fumadocs generated comment before processing
+  let content = page.data.content;
+  
+  // Remove the specific Fumadocs comment that appears in generated API docs
+  content = content.replace(
+    /\{\s*\/\*\s*This file was generated by Fumadocs\. Do not edit this file directly\. Any changes should be made by running the generation command again\.\s*\*\/\s*\}/g,
+    ''
+  );
+  
+  const processed = await processor.process({
+    path: page.data._file.absolutePath,
+    value: content,
+  });
+
+  return `# ${page.data.title}
+URL: ${page.url}
+Source: ${page.data._file.absolutePath}
+
+${page.data.description || ''}
+
+${processed.value}`;
+} 

docs/next.config.mjs

@@ -26,6 +26,15 @@ const config = {
   },
   async rewrites() {
     return [
+      // Redirect .mdx requests to the llms.mdx route handler
+      {
+        source: '/docs/:path*.mdx',
+        destination: '/llms.mdx/:path*',
+      },
+      {
+        source: '/api/:path*.mdx',
+        destination: '/llms.mdx/:path*',
+      },
       // Serve OpenAPI files from the openapi directory
       {
         source: '/openapi/:path*',

docs/package.json

@@ -41,6 +41,9 @@
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-remove-scroll": "^2.7.0",
+    "remark": "^15.0.1",
+    "remark-gfm": "^4.0.1",
+    "remark-mdx": "^3.1.0",
     "shiki": "^3.4.2",
     "tailwind-merge": "^3.3.0"
   },

docs/src/app/llms.mdx/[[...slug]]/route.ts

@@ -0,0 +1,33 @@
+import { getLLMText } from 'lib/get-llm-text';
+import { apiSource, source } from 'lib/source';
+import { notFound } from 'next/navigation';
+import { type NextRequest, NextResponse } from 'next/server';
+
+export const revalidate = false;
+
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: Promise<{ slug?: string[] }> },
+) {
+  const { slug } = await params;
+
+  // Try to find the page in either source
+  let page = source.getPage(slug);
+
+  // If not found in main docs, try API docs
+  if (!page) {
+    page = apiSource.getPage(slug);
+  }
+
+  if (!page) notFound();
+
+  return new NextResponse(await getLLMText(page));
+}
+
+export function generateStaticParams() {
+  // Generate static params for both main docs and API docs
+  const docsParams = source.generateParams();
+  const apiParams = apiSource.generateParams();
+
+  return [...docsParams, ...apiParams];
+}

docs/src/app/llms.txt/route.ts

@@ -0,0 +1,25 @@
+import { getLLMText } from 'lib/get-llm-text';
+import { apiSource, source } from 'lib/source';
+
+// cached forever
+export const revalidate = false;
+
+export async function GET() {
+  // Get all pages from both main docs and API docs
+  const docsPages = source.getPages();
+  const apiPages = apiSource.getPages();
+
+  // Process all pages
+  const docsPromises = docsPages.map(getLLMText);
+  const apiPromises = apiPages.map(getLLMText);
+
+  const [docsContent, apiContent] = await Promise.all([
+    Promise.all(docsPromises),
+    Promise.all(apiPromises)
+  ]);
+
+  // Combine all content
+  const allContent = [...docsContent, ...apiContent];
+
+  return new Response(allContent.join('\n\n'));
+}

pnpm-lock.yaml

@@ -556,6 +556,15 @@ importers:
       react-remove-scroll:
         specifier: ^2.7.0
         version: 2.7.1(@types/react@18.3.12)(react@18.3.1)
+      remark:
+        specifier: ^15.0.1
+        version: 15.0.1
+      remark-gfm:
+        specifier: ^4.0.1
+        version: 4.0.1
+      remark-mdx:
+        specifier: ^3.1.0
+        version: 3.1.0
       shiki:
         specifier: ^3.4.2
         version: 3.6.0