diff --git a/docs/lib/get-llm-text.ts b/docs/lib/get-llm-text.ts new file mode 100644 index 000000000..e211c85d9 --- /dev/null +++ b/docs/lib/get-llm-text.ts @@ -0,0 +1,36 @@ +import type { InferPageType } from 'fumadocs-core/source'; +import { remarkInclude } from 'fumadocs-mdx/config'; +import { remark } from 'remark'; +import remarkGfm from 'remark-gfm'; +import remarkMdx from 'remark-mdx'; +import { apiSource, source } from './source'; + +const processor = remark() + .use(remarkMdx) + // needed for Fumadocs MDX + .use(remarkInclude) + .use(remarkGfm); + +export async function getLLMText(page: InferPageType | InferPageType) { + // Remove the Fumadocs generated comment before processing + let content = page.data.content; + + // Remove the specific Fumadocs comment that appears in generated API docs + content = content.replace( + /\{\s*\/\*\s*This file was generated by Fumadocs\. Do not edit this file directly\. Any changes should be made by running the generation command again\.\s*\*\/\s*\}/g, + '' + ); + + const processed = await processor.process({ + path: page.data._file.absolutePath, + value: content, + }); + + return `# ${page.data.title} +URL: ${page.url} +Source: ${page.data._file.absolutePath} + +${page.data.description || ''} + +${processed.value}`; +} diff --git a/docs/next.config.mjs b/docs/next.config.mjs index 0e8fba19c..1958dd19d 100644 --- a/docs/next.config.mjs +++ b/docs/next.config.mjs @@ -26,6 +26,15 @@ const config = { }, async rewrites() { return [ + // Redirect .mdx requests to the llms.mdx route handler + { + source: '/docs/:path*.mdx', + destination: '/llms.mdx/:path*', + }, + { + source: '/api/:path*.mdx', + destination: '/llms.mdx/:path*', + }, // Serve OpenAPI files from the openapi directory { source: '/openapi/:path*', diff --git a/docs/package.json b/docs/package.json index 254e629b8..ac2db1fe4 100644 --- a/docs/package.json +++ b/docs/package.json @@ -41,6 +41,9 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-remove-scroll": "^2.7.0", + "remark": "^15.0.1", + "remark-gfm": "^4.0.1", + "remark-mdx": "^3.1.0", "shiki": "^3.4.2", "tailwind-merge": "^3.3.0" }, diff --git a/docs/src/app/llms.mdx/[[...slug]]/route.ts b/docs/src/app/llms.mdx/[[...slug]]/route.ts new file mode 100644 index 000000000..adcb33f17 --- /dev/null +++ b/docs/src/app/llms.mdx/[[...slug]]/route.ts @@ -0,0 +1,33 @@ +import { getLLMText } from 'lib/get-llm-text'; +import { apiSource, source } from 'lib/source'; +import { notFound } from 'next/navigation'; +import { type NextRequest, NextResponse } from 'next/server'; + +export const revalidate = false; + +export async function GET( + _req: NextRequest, + { params }: { params: Promise<{ slug?: string[] }> }, +) { + const { slug } = await params; + + // Try to find the page in either source + let page = source.getPage(slug); + + // If not found in main docs, try API docs + if (!page) { + page = apiSource.getPage(slug); + } + + if (!page) notFound(); + + return new NextResponse(await getLLMText(page)); +} + +export function generateStaticParams() { + // Generate static params for both main docs and API docs + const docsParams = source.generateParams(); + const apiParams = apiSource.generateParams(); + + return [...docsParams, ...apiParams]; +} diff --git a/docs/src/app/llms.txt/route.ts b/docs/src/app/llms.txt/route.ts new file mode 100644 index 000000000..bb86882fb --- /dev/null +++ b/docs/src/app/llms.txt/route.ts @@ -0,0 +1,25 @@ +import { getLLMText } from 'lib/get-llm-text'; +import { apiSource, source } from 'lib/source'; + +// cached forever +export const revalidate = false; + +export async function GET() { + // Get all pages from both main docs and API docs + const docsPages = source.getPages(); + const apiPages = apiSource.getPages(); + + // Process all pages + const docsPromises = docsPages.map(getLLMText); + const apiPromises = apiPages.map(getLLMText); + + const [docsContent, apiContent] = await Promise.all([ + Promise.all(docsPromises), + Promise.all(apiPromises) + ]); + + // Combine all content + const allContent = [...docsContent, ...apiContent]; + + return new Response(allContent.join('\n\n')); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7b42c194c..7fdceae4e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -556,6 +556,15 @@ importers: react-remove-scroll: specifier: ^2.7.0 version: 2.7.1(@types/react@18.3.12)(react@18.3.1) + remark: + specifier: ^15.0.1 + version: 15.0.1 + remark-gfm: + specifier: ^4.0.1 + version: 4.0.1 + remark-mdx: + specifier: ^3.1.0 + version: 3.1.0 shiki: specifier: ^3.4.2 version: 3.6.0