feat(docs): serve a real /robots.txt and /sitemap.xml

apps/docs/app/robots.ts

@@ -0,0 +1,30 @@
+import type { MetadataRoute } from 'next';
+
+import { absoluteUrl } from '@/lib/site';
+
+/**
+ * `/robots.txt`.
+ *
+ * Before this file existed the path had no route at all, so `app/[lang]/page.tsx`
+ * matched it as `lang = "robots.txt"` and answered `200 text/html` with the
+ * homepage — a crawler asking for crawl rules got a web page. A literal segment
+ * outranks a dynamic one in the app router, so this file takes the path back; the
+ * `[lang]` catch-all swallowing *other* dotted paths is a separate defect and is
+ * not fixed here.
+ *
+ * Static: the content depends on nothing per-request.
+ */
+export const dynamic = 'force-static';
+export const revalidate = false;
+
+export default function robots(): MetadataRoute.Robots {
+ return {
+ rules: [
+ {
+ userAgent: '*',
+ allow: '/',
+ },
+ ],
+ sitemap: absoluteUrl('/sitemap.xml'),
+ };
+}

apps/docs/app/sitemap.ts

@@ -0,0 +1,149 @@
+import { execFileSync } from 'node:child_process';
+
+import type { MetadataRoute } from 'next';
+
+import { absoluteUrl } from '@/lib/site';
+import { blog, source } from '@/lib/source';
+
+/**
+ * `/sitemap.xml`.
+ *
+ * Same story as `app/robots.ts`: with no route here the path fell through to
+ * `app/[lang]/page.tsx` and answered `200 text/html`, so a sitemap submitted to
+ * Search Console would have failed to parse. Every indexable URL is derived from
+ * `source` / `blog` — never a hand-maintained list, which is guaranteed to rot the
+ * first time a page is added.
+ *
+ * Static: generated once at build, so the `git log` below runs in the build
+ * process and never in a request.
+ */
+export const dynamic = 'force-static';
+export const revalidate = false;
+
+/** Repo-relative roots of the two MDX collections, matching `source.config.ts`. */
+const DOCS_CONTENT_ROOT = 'content/docs';
+const BLOG_CONTENT_ROOT = 'content/blog';
+
+/**
+ * `lastModified` comes from the git committer date of each source `.mdx`, not from
+ * build time. Build time would restamp all 400+ pages on every deploy, which tells
+ * a crawler that the whole site changed whenever anything did — a signal that gets
+ * discounted precisely because it is never false.
+ *
+ * One `git log` pass over both collections covers every file (~1s over 11k commits
+ * locally, measured), rather than one `git log` per page.
+ *
+ * When the date cannot be known — no git directory, or a clone shallow enough that
+ * no commit in the window touched the file — the entry ships **without**
+ * `lastModified`. `lastmod` is optional in the sitemap protocol, and omitting it is
+ * the honest answer; substituting build time would reintroduce the exact lie this
+ * function exists to avoid. Degrading silently is not on the table either: the
+ * build prints a counted warning naming the remedy.
+ */
+let cachedGitDates: Map<string, Date> | undefined;
+
+function loadGitDates(): Map<string, Date> {
+ if (cachedGitDates) return cachedGitDates;
+
+ const dates = new Map<string, Date>();
+ const git = (args: string[], cwd: string) =>
+ execFileSync('git', args, {
+ cwd,
+ encoding: 'utf8',
+ maxBuffer: 64 * 1024 * 1024,
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+
+ try {
+ const repoRoot = git(['rev-parse', '--show-toplevel'], process.cwd()).trim();
+ const shallow = git(['rev-parse', '--is-shallow-repository'], repoRoot).trim() === 'true';
+ if (shallow) {
+ console.warn(
+ '[sitemap] the checkout is a shallow clone; pages whose last commit predates the ' +
+ 'clone depth will ship without <lastmod>. Deepen the clone to restore the dates.',
+ );
+ }
+
+ // `--format` marker cannot collide with a path: no file under content/ starts
+ // with "commit-date:". `diff.relative=false` pins the printed paths to
+ // repo-root-relative regardless of local git config.
+ const log = git(
+ [
+ '-c',
+ 'diff.relative=false',
+ 'log',
+ '--format=commit-date:%cI',
+ '--name-only',
+ '--no-renames',
+ '--',
+ DOCS_CONTENT_ROOT,
+ BLOG_CONTENT_ROOT,
+ ],
+ repoRoot,
+ );
+
+ // `git log` is newest-first, so the first date seen for a path is its latest.
+ let current: Date | undefined;
+ for (const line of log.split('\n')) {
+ if (line.startsWith('commit-date:')) {
+ current = new Date(line.slice('commit-date:'.length));
+ continue;
+ }
+ if (!line || !current || dates.has(line)) continue;
+ dates.set(line, current);
+ }
+ } catch (error) {
+ console.warn(
+ `[sitemap] could not read commit dates from git (${
+ error instanceof Error ? error.message : String(error)
+ }); every entry will ship without <lastmod>.`,
+ );
+ }
+
+ cachedGitDates = dates;
+ return dates;
+}
+
+type SitemapEntry = MetadataRoute.Sitemap[number];
+
+export default function sitemap(): MetadataRoute.Sitemap {
+ const dates = loadGitDates();
+ const undated: string[] = [];
+
+ /**
+ * `sourcePath` is repo-relative; `undefined` for routes with no MDX file behind
+ * them (the homepage, the blog index), which are not counted as missing dates.
+ */
+ const entry = (url: string, sourcePath?: string): SitemapEntry => {
+ const lastModified = sourcePath ? dates.get(sourcePath) : undefined;
+ if (sourcePath && !lastModified) undated.push(sourcePath);
+ return lastModified ? { url: absoluteUrl(url), lastModified } : { url: absoluteUrl(url) };
+ };
+
+ const byUrl = (a: SitemapEntry, b: SitemapEntry) => a.url.localeCompare(b.url);
+
+ // `getPages()` with no argument lists every language. English is the only one
+ // today, and a future locale belongs in the sitemap under its own prefixed URL,
+ // so leaving it unfiltered is the forward-correct spelling.
+ const docs = source
+ .getPages()
+ .map((page) => entry(page.url, `${DOCS_CONTENT_ROOT}/${page.path}`))
+ .sort(byUrl);
+
+ const posts = blog
+ .getPages()
+ .map((page) => entry(page.url, `${BLOG_CONTENT_ROOT}/${page.path}`))
+ .sort(byUrl);
+
+ if (undated.length > 0) {
+ console.warn(
+ `[sitemap] ${undated.length} of ${docs.length + posts.length} content pages have no git ` +
+ `commit date and ship without <lastmod>; first: ${undated.slice(0, 3).join(', ')}`,
+ );
+ }
+
+ // No `priority` or `changeFrequency`: Google ignores both, and inventing values
+ // for 400+ pages would put numbers into a machine-readable surface that nothing
+ // measured.
+ return [entry('/'), ...docs, entry('/blog'), ...posts];
+}

apps/docs/lib/site.ts

@@ -0,0 +1,38 @@
+/**
+ * Canonical identity of the documentation site.
+ *
+ * The origin is a maintainer ruling, not configuration. Every absolute URL this
+ * site emits — sitemap entries, the `Sitemap:` line in `robots.txt`, and (as the
+ * remaining indexability work lands) `metadataBase`, canonical links and JSON-LD
+ * identifiers — must name this host and no other. Hard-coding it twice is how the
+ * two halves drift; hence one constant, imported.
+ *
+ * Deliberately NOT read from an environment variable. A preview deployment that
+ * derived its own origin would emit canonical links and a sitemap pointing at the
+ * preview host — precisely the duplicate-content signal a canonical link exists to
+ * suppress. One host, declared once, here.
+ */
+export const SITE_ORIGIN = 'https://objectstack.ai';
+
+/**
+ * Absolute URL for a **site-relative** path.
+ *
+ * `path` must start with `/`. Anything else throws at build time rather than
+ * quietly emitting a URL on the wrong host: `new URL(path, SITE_ORIGIN)` on its
+ * own would hand an already-absolute `https://elsewhere/...` straight back, and a
+ * sitemap listing another host is discarded wholesale by search engines rather
+ * than reported.
+ *
+ * Next's `metadataBase` wants a `URL` rather than a string — write
+ * `new URL(SITE_ORIGIN)` there, so this file stays the only place the origin is
+ * spelled out.
+ */
+export function absoluteUrl(path: string): string {
+ if (!path.startsWith('/')) {
+ throw new Error(
+ `absoluteUrl() expects a site-relative path starting with "/", received ${JSON.stringify(path)}`,
+ );
+ }
+
+ return new URL(path, SITE_ORIGIN).toString();
+}