From 6f8244cef306a0547e0c3e3fe482760b8fcac28f Mon Sep 17 00:00:00 2001 From: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:13:19 +0800 Subject: [PATCH] docs(site): keep the agent-reader copies of every page out of the search index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/docs/.mdx`, its rewrite destination `/llms.mdx/docs/`, `/llms.txt` and `/llms-full.txt` each serve the full text of documentation pages at a crawlable URL with no robots directive of any kind, so every page exists twice (three times, counting the aggregates) as far as a search engine is concerned. Measured on production before this change: `/docs/data-modeling/objects.mdx` → 200 `text/markdown`, 28080 bytes, no `X-Robots-Tag` and no `Link` header. Add `X-Robots-Tag: noindex` to all four from `next.config.mjs`'s `headers()`, which matches the incoming request path and so covers `/docs/**.mdx` before the rewrite rewrites it. The endpoints keep answering 200 with their full body — they are how AI agents read these docs and nothing here gates, redirects or content-negotiates them. `noindex` over `Link: rel="canonical"` because only the per-page markdown has an HTML twin to canonicalise to: `/llms.txt` and the 8 MB `/llms-full.txt` are whole-site aggregates that are a duplicate of no single page, so a canonical header could say nothing honest about the two largest copies on the site. It is also a directive rather than a hint. `robots.txt` keeps allowing all four paths — a `Disallow` would prevent the fetch that reveals the header — and now names `llms.txt` and `llms-full.txt` explicitly so agent crawlers find them deliberately rather than by convention. Co-Authored-By: Claude Opus 5 --- apps/docs/app/robots.ts | 23 +++++++++++++++++++- apps/docs/next.config.mjs | 45 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/apps/docs/app/robots.ts b/apps/docs/app/robots.ts index 0a85227d93..951101bf89 100644 --- a/apps/docs/app/robots.ts +++ b/apps/docs/app/robots.ts @@ -13,6 +13,27 @@ import { absoluteUrl } from '@/lib/site'; * not fixed here. * * Static: the content depends on nothing per-request. + * + * ## No `Disallow` lines, deliberately + * + * This file shipped without any and the question was left to the card that owns + * the duplicate-copy problem. The answer is that it stays without any, because + * the fix for that problem is an `X-Robots-Tag: noindex` header on the + * agent-reader endpoints (see the `headers()` block in `next.config.mjs`) and a + * `Disallow` would defeat it: a crawler that is told not to fetch a URL never + * sees the header on it, and a disallowed URL can still be indexed URL-only from + * an inbound link. Allow the crawl, refuse the index. Anything added here later + * must not cover `/docs/**.mdx`, `/llms.mdx/**`, `/llms.txt` or `/llms-full.txt`. + * + * ## Why `llms.txt` is named under `Allow:` + * + * `Allow: /` already permits both aggregate endpoints, so these two lines change + * no crawler's behaviour -- they are declarative, and that is the whole job. The + * robots.txt grammar has exactly one discovery directive, `Sitemap:`, and these + * are not sitemaps; naming the paths in the file agents already fetch first is + * the available way to make them findable on purpose rather than by guessing at + * a convention. It also records, at the point of the crawl rules, that the + * `noindex` on those paths is about indexing and not about access. */ export const dynamic = 'force-static'; export const revalidate = false; @@ -22,7 +43,7 @@ export default function robots(): MetadataRoute.Robots { rules: [ { userAgent: '*', - allow: '/', + allow: ['/', '/llms.txt', '/llms-full.txt'], }, ], sitemap: absoluteUrl('/sitemap.xml'), diff --git a/apps/docs/next.config.mjs b/apps/docs/next.config.mjs index b458568e77..25f153c8e0 100644 --- a/apps/docs/next.config.mjs +++ b/apps/docs/next.config.mjs @@ -62,6 +62,51 @@ const config = { 'lucide-react': './node_modules/lucide-react', }, }, + async headers() { + // The agent-reader surfaces below each serve the full text of documentation + // pages as `text/markdown` / `text/plain`, at URLs a crawler can reach. They + // are a deliberate feature -- this is how AI agents read these docs -- so + // they keep answering 200 with their full body to anyone who asks. The only + // thing added here is a directive telling *search engines* which copy is the + // one worth indexing: the HTML page. + // + // `noindex` rather than `Link: <...>; rel="canonical"`, and the reasons are + // not interchangeable: + // + // 1. Coverage. Only the per-page markdown has an HTML twin to point a + // canonical at. `/llms.txt` is an index of every page and + // `/llms-full.txt` is all 400+ of them concatenated (8 MB today); there + // is no single HTML URL either one is a duplicate *of*, so a canonical + // header cannot say anything honest about them and would leave the two + // largest parallel copies on the site undirected. `noindex` states the + // same intent for all four sources. + // 2. A canonical link is a hint a search engine weighs against other + // signals and may overrule; `noindex` is a directive. What this card + // wants is the strong form -- keep the copy fetchable, keep it out of + // results. + // + // The two are also not additive: pairing `noindex` with a canonical pointing + // elsewhere is contradictory (the target is asked to absorb the signal of a + // page that has asked to be dropped), so exactly one of them belongs here. + // + // ⚠️ This works only while `robots.txt` still allows these paths to be + // crawled -- a `Disallow` would stop the fetch that reveals the header and + // leave the URLs eligible for URL-only indexing instead. `app/robots.ts` + // names them under `Allow:` on purpose; the two files are one mechanism. + // + // Matching is on the *incoming* request path, before rewrites, which is why + // `/docs/:path*.mdx` is spelled here as the client asks for it. Its rewrite + // destination `/llms.mdx/docs/:path*` is a real route and answers directly + // too (measured: `/llms.mdx/docs/data-modeling/objects` -> 200 + // `text/markdown`), so each page actually has two markdown URLs and both are + // listed. + const noindex = ['/docs/:path*.mdx', '/llms.mdx/:path*', '/llms.txt', '/llms-full.txt']; + + return noindex.map((source) => ({ + source, + headers: [{ key: 'X-Robots-Tag', value: 'noindex' }], + })); + }, async redirects() { return toNextRedirects(); },