diff --git a/apps/docs/app/robots.ts b/apps/docs/app/robots.ts index 0a85227d93..951101bf89 100644 --- a/apps/docs/app/robots.ts +++ b/apps/docs/app/robots.ts @@ -13,6 +13,27 @@ import { absoluteUrl } from '@/lib/site'; * not fixed here. * * Static: the content depends on nothing per-request. + * + * ## No `Disallow` lines, deliberately + * + * This file shipped without any and the question was left to the card that owns + * the duplicate-copy problem. The answer is that it stays without any, because + * the fix for that problem is an `X-Robots-Tag: noindex` header on the + * agent-reader endpoints (see the `headers()` block in `next.config.mjs`) and a + * `Disallow` would defeat it: a crawler that is told not to fetch a URL never + * sees the header on it, and a disallowed URL can still be indexed URL-only from + * an inbound link. Allow the crawl, refuse the index. Anything added here later + * must not cover `/docs/**.mdx`, `/llms.mdx/**`, `/llms.txt` or `/llms-full.txt`. + * + * ## Why `llms.txt` is named under `Allow:` + * + * `Allow: /` already permits both aggregate endpoints, so these two lines change + * no crawler's behaviour -- they are declarative, and that is the whole job. The + * robots.txt grammar has exactly one discovery directive, `Sitemap:`, and these + * are not sitemaps; naming the paths in the file agents already fetch first is + * the available way to make them findable on purpose rather than by guessing at + * a convention. It also records, at the point of the crawl rules, that the + * `noindex` on those paths is about indexing and not about access. */ export const dynamic = 'force-static'; export const revalidate = false; @@ -22,7 +43,7 @@ export default function robots(): MetadataRoute.Robots { rules: [ { userAgent: '*', - allow: '/', + allow: ['/', '/llms.txt', '/llms-full.txt'], }, ], sitemap: absoluteUrl('/sitemap.xml'), diff --git a/apps/docs/next.config.mjs b/apps/docs/next.config.mjs index b458568e77..25f153c8e0 100644 --- a/apps/docs/next.config.mjs +++ b/apps/docs/next.config.mjs @@ -62,6 +62,51 @@ const config = { 'lucide-react': './node_modules/lucide-react', }, }, + async headers() { + // The agent-reader surfaces below each serve the full text of documentation + // pages as `text/markdown` / `text/plain`, at URLs a crawler can reach. They + // are a deliberate feature -- this is how AI agents read these docs -- so + // they keep answering 200 with their full body to anyone who asks. The only + // thing added here is a directive telling *search engines* which copy is the + // one worth indexing: the HTML page. + // + // `noindex` rather than `Link: <...>; rel="canonical"`, and the reasons are + // not interchangeable: + // + // 1. Coverage. Only the per-page markdown has an HTML twin to point a + // canonical at. `/llms.txt` is an index of every page and + // `/llms-full.txt` is all 400+ of them concatenated (8 MB today); there + // is no single HTML URL either one is a duplicate *of*, so a canonical + // header cannot say anything honest about them and would leave the two + // largest parallel copies on the site undirected. `noindex` states the + // same intent for all four sources. + // 2. A canonical link is a hint a search engine weighs against other + // signals and may overrule; `noindex` is a directive. What this card + // wants is the strong form -- keep the copy fetchable, keep it out of + // results. + // + // The two are also not additive: pairing `noindex` with a canonical pointing + // elsewhere is contradictory (the target is asked to absorb the signal of a + // page that has asked to be dropped), so exactly one of them belongs here. + // + // ⚠️ This works only while `robots.txt` still allows these paths to be + // crawled -- a `Disallow` would stop the fetch that reveals the header and + // leave the URLs eligible for URL-only indexing instead. `app/robots.ts` + // names them under `Allow:` on purpose; the two files are one mechanism. + // + // Matching is on the *incoming* request path, before rewrites, which is why + // `/docs/:path*.mdx` is spelled here as the client asks for it. Its rewrite + // destination `/llms.mdx/docs/:path*` is a real route and answers directly + // too (measured: `/llms.mdx/docs/data-modeling/objects` -> 200 + // `text/markdown`), so each page actually has two markdown URLs and both are + // listed. + const noindex = ['/docs/:path*.mdx', '/llms.mdx/:path*', '/llms.txt', '/llms-full.txt']; + + return noindex.map((source) => ({ + source, + headers: [{ key: 'X-Robots-Tag', value: 'noindex' }], + })); + }, async redirects() { return toNextRedirects(); },