Refactor error handling and URL generation in LLM routes

### Summary of Changes
- Replaced `notFound()` with a `NextResponse` for 404 status in the LLM route to improve error handling.
- Added a try-catch block to handle potential errors when generating LLM text, returning a 500 status on failure.
- Updated URL generation in `llms.txt` to dynamically construct base URLs using the request's origin.

These modifications enhance the robustness and clarity of the LLM routing logic.

docs/src/app/llms.txt/route.ts

@@ -7,8 +7,9 @@ export const revalidate = false;
 export async function GET(request: Request) {
   const docsUrls = new Set<string>();
   const apiUrls = new Set<string>();
-  const docsBaseUrl = new URL('/llms/docs/', request.url).toString();
-  const apiBaseUrl = new URL('/llms/api/', request.url).toString();
+  const origin = new URL(request.url).origin;
+  const docsBaseUrl = `${origin}/llms/docs/`;
+  const apiBaseUrl = `${origin}/llms/api/`;
 
   for (const page of source.getPages()) {
     const relativeUrl = page.url.replace(/^\/docs\/?/, '');

docs/src/app/llms/[[...slug]]/route.ts

@@ -1,4 +1,3 @@
-import { notFound } from 'next/navigation';
 import { NextResponse, type NextRequest } from 'next/server';
 import { getLLMText } from '../../../../lib/get-llm-text';
 import { apiSource, source } from '../../../../lib/source';
@@ -36,14 +35,19 @@ export async function GET(
   const page = resolvePage(slug);
 
   if (!page) {
-    notFound();
+    return new NextResponse(null, { status: 404 });
   }
 
-  return new NextResponse(await getLLMText(page), {
-    headers: {
-      'Content-Type': 'text/plain; charset=utf-8',
-    },
-  });
+  try {
+    return new NextResponse(await getLLMText(page), {
+      headers: {
+        'Content-Type': 'text/plain; charset=utf-8',
+      },
+    });
+  } catch (error) {
+    console.error('Error generating LLM text:', error);
+    return new NextResponse('Error generating content', { status: 500 });
+  }
 }
 
 export function generateStaticParams() {