Replace local ai-agent-detection with @vercel/agent-readability

apps/docs/lib/geistdocs/md-tracking.ts

@@ -1,5 +1,5 @@
 import { siteId } from "@/geistdocs";
-import type { DetectionMethod } from "@/lib/ai-agent-detection";
+import type { DetectionMethod } from "@vercel/agent-readability";
 
 const PLATFORM_URL = "https://geistdocs.com/md-tracking";
 

apps/docs/package.json

@@ -1,4 +1,4 @@
 {
   "name": "docs",
   "version": "0.0.0",
   "private": true,
@@ -15,6 +15,7 @@
     "@orama/tokenizers": "^3.1.16",
     "@streamdown/cjk": "^1.0.1",
     "@streamdown/code": "^1.0.1",
+    "@vercel/agent-readability": "^0.2.1",
     "@vercel/analytics": "^1.6.1",
     "@vercel/speed-insights": "^1.3.1",
     "@vercel/toolbar": "0.1.36",
@@ -60,4 +61,4 @@
     "typescript": "^5.9.3"
   },
   "packageManager": "pnpm@10.19.0"
-}
+}

apps/docs/proxy.ts

@@ -1,3 +1,4 @@
+import { generateNotFoundMarkdown, isAIAgent } from "@vercel/agent-readability";
 import { precompute } from "flags/next";
 import { createI18nMiddleware } from "fumadocs-core/i18n/middleware";
 import { isMarkdownPreferred, rewritePath } from "fumadocs-core/negotiation";
@@ -8,7 +9,6 @@ import {
 } from "next/server";
 import { rootFlags } from "@/flags";
 import { i18n } from "@/lib/geistdocs/i18n";
-import { isAIAgent } from "@/lib/ai-agent-detection";
 import { trackMdRequest } from "@/lib/geistdocs/md-tracking";
 
 const { rewrite: rewriteLLM } = rewritePath(
@@ -65,7 +65,6 @@ const proxy = async (request: NextRequest, context: NextFetchEvent) => {
   }
 
   // AI agent detection — rewrite docs pages to markdown for agents
-  // so they always get structured content without needing .md URLs or Accept headers
   if (
     (pathname === "/docs" || pathname.startsWith("/docs/")) &&
     !pathname.includes("/llms.mdx/")
@@ -88,8 +87,23 @@ const proxy = async (request: NextRequest, context: NextFetchEvent) => {
             detectionMethod: agentResult.method,
           })
         );
-        return NextResponse.rewrite(new URL(result, request.nextUrl));
+        const response = NextResponse.rewrite(new URL(result, request.nextUrl));
+        response.headers.set("Vary", "Accept");
+        return response;
       }
+      // Agent requested a non-existent docs URL — return helpful markdown
+      return new NextResponse(
+        generateNotFoundMarkdown(pathname, {
+          sitemapUrl: "/sitemap.md",
+          indexUrl: "/llms.txt",
+        }),
+        {
+          headers: {
+            "Content-Type": "text/markdown; charset=utf-8",
+            Vary: "Accept",
+          },
+        },
+      );
     }
   }
 
@@ -115,10 +129,9 @@ const proxy = async (request: NextRequest, context: NextFetchEvent) => {
 };
 
 export const config = {
-  // Matcher ignoring `/_next/`, `/api/`, static assets, favicon, sitemap, robots, etc.
   matcher: [
     "/((?!api|_next/static|_next/image|favicon.ico|sitemap.xml|robots.txt|\\.well-known/vercel/flags).*)",
   ],
 };
 
-export default proxy;
+export default proxy;