add custom Docusaurus plugin generating llms(-full).txt

docusaurus.config.js

@@ -108,6 +108,7 @@ module.exports = async function createConfigAsync() {
  },
  ],
  require.resolve('./src/plugins/webpack-config-plugin'),
+ require.resolve('./src/plugins/llms-txt-plugin'),
  [
  '@docusaurus/plugin-client-redirects',
  {

src/plugins/llms-txt-plugin.js

@@ -0,0 +1,172 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SITE_URL = 'https://docs.etherlink.com';
+
+// Directories under docs/ that are not full pages.
+const SKIP_DIRS = ['conrefs'];
+
+// Rebranding markers used internally: <!--TX-->, <!--TEVM-->, <!--TXN-->,
+// {/* TX */}, {/* TEVM */}, {/* TXN */}, and # tx / # tevm / # txn in titles.
+const MARKER_RE = /<!--T(?:X|EVM|XN)-->|\{\/\* T(?:X|EVM|XN) \*\/\}|\s*#\s*t(?:evm|xn?)\b/gi;
+
+function extractFrontmatter(raw) {
+ const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
+ if (!match) return { title: null, body: raw };
+ const fm = match[1];
+ const m = fm.match(/^title:\s*(.+)$/m);
+ const title = m
+ ? m[1].trim().replace(MARKER_RE, '').trim().replace(/^['"]|['"]$/g, '').trim()
+ : null;
+ return { title, body: raw.slice(match[0].length).trim() };
+}
+
+function stripContent(text) {
+ return text
+ .replace(MARKER_RE, '')
+ .replace(/^import\s+.+from\s+['"].+['"];?\s*$/gm, '') // JSX imports in .mdx
+ .trim();
+}
+
+function fileToUrl(filePath, docsDir) {
+ let rel = path.relative(docsDir, filePath).replace(/\\/g, '/');
+ rel = rel.replace(/\.mdx?$/, '');
+ if (rel === 'index') return '/';
+ if (rel.endsWith('/index')) rel = rel.slice(0, -'/index'.length);
+ return '/' + rel;
+}
+
+// Convert a Docusaurus doc ID to the URL path it renders at.
+function idToUrl(id) {
+ if (id.endsWith('/index')) return '/' + id.slice(0, -'/index'.length);
+ return '/' + id;
+}
+
+// Read every doc file into a map keyed by URL.
+function collectDocsByUrl(docsDir) {
+ const byUrl = {};
+
+ function walk(dir) {
+ const entries = fs.readdirSync(dir, { withFileTypes: true })
+ .sort((a, b) => a.name.localeCompare(b.name));
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ const rel = path.relative(docsDir, full);
+ if (SKIP_DIRS.includes(rel.split(path.sep)[0])) continue;
+ walk(full);
+ } else if (entry.isFile() && /\.mdx?$/.test(entry.name)) {
+ const raw = fs.readFileSync(full, 'utf8');
+ const { title, body } = extractFrontmatter(raw);
+ const url = fileToUrl(full, docsDir);
+ byUrl[url] = { url, title, body: stripContent(body) };
+ }
+ }
+ }
+
+ walk(docsDir);
+ return byUrl;
+}
+
+// Walk the sidebar tree depth-first, yielding {url, section} in sidebar order.
+// `section` is the label of the top-level category the page belongs to.
+function extractSidebarOrder(sidebar) {
+ const result = [];
+
+ function walkItems(items, section) {
+ for (const item of items) {
+ if (typeof item === 'string') {
+ result.push({ url: idToUrl(item), section });
+ } else if (item.type === 'doc') {
+ result.push({ url: idToUrl(item.id), section });
+ } else if (item.type === 'category') {
+ // Category link page comes first, then the category's children.
+ if (item.link && item.link.type === 'doc') {
+ result.push({ url: idToUrl(item.link.id), section });
+ }
+ walkItems(item.items || [], section);
+ }
+ // type: 'link' (external URL) — skip
+ }
+ }
+
+ for (const topItem of sidebar) {
+ if (topItem.type !== 'category') continue;
+ // Top-level categories can also have a link page (e.g. "Tutorials").
+ if (topItem.link && topItem.link.type === 'doc') {
+ result.push({ url: idToUrl(topItem.link.id), section: topItem.label });
+ }
+ walkItems(topItem.items || [], topItem.label);
+ }
+
+ return result;
+}
+
+module.exports = function llmsTxtPlugin(context) {
+ return {
+ name: 'llms-txt-plugin',
+
+ async postBuild({ siteDir, outDir }) {
+ const docsDir = path.join(siteDir, 'docs');
+ const byUrl = collectDocsByUrl(docsDir);
+
+ // Load the sidebar and derive a flat, ordered list of pages.
+ const sidebars = require(path.resolve(siteDir, 'sidebars.js'));
+ const ordered = extractSidebarOrder(sidebars.documentationSidebar);
+
+ // Deduplicate: a page may appear as a category `link` and again in `items`.
+ const seen = new Set();
+ const deduped = ordered.filter(({ url }) => {
+ if (seen.has(url)) return false;
+ seen.add(url);
+ return true;
+ });
+
+ // Group pages by section, preserving the sidebar's section and page order.
+ const sectionLabels = [];
+ const sectionPages = {};
+ for (const { url, section } of deduped) {
+ const doc = byUrl[url];
+ if (!doc) continue; // sidebar entry with no matching file
+ if (!sectionPages[section]) {
+ sectionPages[section] = [];
+ sectionLabels.push(section);
+ }
+ sectionPages[section].push(doc);
+ }
+
+ // llms.txt — index
+ let index = `# Etherlink documentation\n\n`;
+ index += `> Etherlink is an EVM-compatible Layer 2 on Tezos with distributed sequencing and native atomic composability between EVM and Michelson smart contracts.\n\n`;
+ index += `Full content: ${SITE_URL}/llms-full.txt\n\n`;
+
+ for (const section of sectionLabels) {
+ index += `## ${section}\n\n`;
+ for (const { url, title } of sectionPages[section]) {
+ const label = title ?? url.split('/').filter(Boolean).pop() ?? 'index';
+ index += `- [${label}](${SITE_URL}${url})\n`;
+ }
+ index += '\n';
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms.txt'), index);
+
+ // llms-full.txt — all pages concatenated in sidebar order.
+ // Every page body starts with a # heading; inject the URL after it.
+ const allDocs = sectionLabels.flatMap(s => sectionPages[s]);
+ let full = '';
+ for (const { url, body } of allDocs) {
+ const nl = body.indexOf('\n');
+ const heading = nl === -1 ? body : body.slice(0, nl);
+ const rest = nl === -1 ? '' : body.slice(nl).trimStart();
+ full += `\n\n${heading}\n\nURL: ${SITE_URL}${url}\n\n${rest}\n`;
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms-full.txt'), full.trimStart());
+
+ console.log(`[llms-txt-plugin] Wrote llms.txt and llms-full.txt (${allDocs.length} pages).`);
+ },
+ };
+};

docusaurus.config.js

@@ -108,6 +108,7 @@ module.exports = async function createConfigAsync() {
  },
  ],
  require.resolve('./src/plugins/webpack-config-plugin'),
+ require.resolve('./src/plugins/llms-txt-plugin'),
  [
  '@docusaurus/plugin-client-redirects',
  {

src/plugins/llms-txt-plugin.js

@@ -0,0 +1,172 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SITE_URL = 'https://docs.etherlink.com';
+
+// Directories under docs/ that are not full pages.
+const SKIP_DIRS = ['conrefs'];
+
+// Rebranding markers used internally: <!--TX-->, <!--TEVM-->, <!--TXN-->,
+// {/* TX */}, {/* TEVM */}, {/* TXN */}, and # tx / # tevm / # txn in titles.
+const MARKER_RE = /<!--T(?:X|EVM|XN)-->|\{\/\* T(?:X|EVM|XN) \*\/\}|\s*#\s*t(?:evm|xn?)\b/gi;
+
+function extractFrontmatter(raw) {
+ const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
+ if (!match) return { title: null, body: raw };
+ const fm = match[1];
+ const m = fm.match(/^title:\s*(.+)$/m);
+ const title = m
+ ? m[1].trim().replace(MARKER_RE, '').trim().replace(/^['"]|['"]$/g, '').trim()
+ : null;
+ return { title, body: raw.slice(match[0].length).trim() };
+}
+
+function stripContent(text) {
+ return text
+ .replace(MARKER_RE, '')
+ .replace(/^import\s+.+from\s+['"].+['"];?\s*$/gm, '') // JSX imports in .mdx
+ .trim();
+}
+
+function fileToUrl(filePath, docsDir) {
+ let rel = path.relative(docsDir, filePath).replace(/\\/g, '/');
+ rel = rel.replace(/\.mdx?$/, '');
+ if (rel === 'index') return '/';
+ if (rel.endsWith('/index')) rel = rel.slice(0, -'/index'.length);
+ return '/' + rel;
+}
+
+// Convert a Docusaurus doc ID to the URL path it renders at.
+function idToUrl(id) {
+ if (id.endsWith('/index')) return '/' + id.slice(0, -'/index'.length);
+ return '/' + id;
+}
+
+// Read every doc file into a map keyed by URL.
+function collectDocsByUrl(docsDir) {
+ const byUrl = {};
+
+ function walk(dir) {
+ const entries = fs.readdirSync(dir, { withFileTypes: true })
+ .sort((a, b) => a.name.localeCompare(b.name));
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ const rel = path.relative(docsDir, full);
+ if (SKIP_DIRS.includes(rel.split(path.sep)[0])) continue;
+ walk(full);
+ } else if (entry.isFile() && /\.mdx?$/.test(entry.name)) {
+ const raw = fs.readFileSync(full, 'utf8');
+ const { title, body } = extractFrontmatter(raw);
+ const url = fileToUrl(full, docsDir);
+ byUrl[url] = { url, title, body: stripContent(body) };
+ }
+ }
+ }
+
+ walk(docsDir);
+ return byUrl;
+}
+
+// Walk the sidebar tree depth-first, yielding {url, section} in sidebar order.
+// `section` is the label of the top-level category the page belongs to.
+function extractSidebarOrder(sidebar) {
+ const result = [];
+
+ function walkItems(items, section) {
+ for (const item of items) {
+ if (typeof item === 'string') {
+ result.push({ url: idToUrl(item), section });
+ } else if (item.type === 'doc') {
+ result.push({ url: idToUrl(item.id), section });
+ } else if (item.type === 'category') {
+ // Category link page comes first, then the category's children.
+ if (item.link && item.link.type === 'doc') {
+ result.push({ url: idToUrl(item.link.id), section });
+ }
+ walkItems(item.items || [], section);
+ }
+ // type: 'link' (external URL) — skip
+ }
+ }
+
+ for (const topItem of sidebar) {
+ if (topItem.type !== 'category') continue;
+ // Top-level categories can also have a link page (e.g. "Tutorials").
+ if (topItem.link && topItem.link.type === 'doc') {
+ result.push({ url: idToUrl(topItem.link.id), section: topItem.label });
+ }
+ walkItems(topItem.items || [], topItem.label);
+ }
+
+ return result;
+}
+
+module.exports = function llmsTxtPlugin(context) {
+ return {
+ name: 'llms-txt-plugin',
+
+ async postBuild({ siteDir, outDir }) {
+ const docsDir = path.join(siteDir, 'docs');
+ const byUrl = collectDocsByUrl(docsDir);
+
+ // Load the sidebar and derive a flat, ordered list of pages.
+ const sidebars = require(path.resolve(siteDir, 'sidebars.js'));
+ const ordered = extractSidebarOrder(sidebars.documentationSidebar);
+
+ // Deduplicate: a page may appear as a category `link` and again in `items`.
+ const seen = new Set();
+ const deduped = ordered.filter(({ url }) => {
+ if (seen.has(url)) return false;
+ seen.add(url);
+ return true;
+ });
+
+ // Group pages by section, preserving the sidebar's section and page order.
+ const sectionLabels = [];
+ const sectionPages = {};
+ for (const { url, section } of deduped) {
+ const doc = byUrl[url];
+ if (!doc) continue; // sidebar entry with no matching file
+ if (!sectionPages[section]) {
+ sectionPages[section] = [];
+ sectionLabels.push(section);
+ }
+ sectionPages[section].push(doc);
+ }
+
+ // llms.txt — index
+ let index = `# Etherlink documentation\n\n`;
+ index += `> Etherlink is an EVM-compatible Layer 2 on Tezos with distributed sequencing and native atomic composability between EVM and Michelson smart contracts.\n\n`;
+ index += `Full content: ${SITE_URL}/llms-full.txt\n\n`;
+
+ for (const section of sectionLabels) {
+ index += `## ${section}\n\n`;
+ for (const { url, title } of sectionPages[section]) {
+ const label = title ?? url.split('/').filter(Boolean).pop() ?? 'index';
+ index += `- [${label}](${SITE_URL}${url})\n`;
+ }
+ index += '\n';
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms.txt'), index);
+
+ // llms-full.txt — all pages concatenated in sidebar order.
+ // Every page body starts with a # heading; inject the URL after it.
+ const allDocs = sectionLabels.flatMap(s => sectionPages[s]);
+ let full = '';
+ for (const { url, body } of allDocs) {
+ const nl = body.indexOf('\n');
+ const heading = nl === -1 ? body : body.slice(0, nl);
+ const rest = nl === -1 ? '' : body.slice(nl).trimStart();
+ full += `\n\n${heading}\n\nURL: ${SITE_URL}${url}\n\n${rest}\n`;
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms-full.txt'), full.trimStart());
+
+ console.log(`[llms-txt-plugin] Wrote llms.txt and llms-full.txt (${allDocs.length} pages).`);
+ },
+ };
+};

docusaurus.config.js

@@ -108,6 +108,7 @@ module.exports = async function createConfigAsync() {
  },
  ],
  require.resolve('./src/plugins/webpack-config-plugin'),
+ require.resolve('./src/plugins/llms-txt-plugin'),
  [
  '@docusaurus/plugin-client-redirects',
  {

src/plugins/llms-txt-plugin.js

@@ -0,0 +1,172 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SITE_URL = 'https://docs.etherlink.com';
+
+// Directories under docs/ that are not full pages.
+const SKIP_DIRS = ['conrefs'];
+
+// Rebranding markers used internally: <!--TX-->, <!--TEVM-->, <!--TXN-->,
+// {/* TX */}, {/* TEVM */}, {/* TXN */}, and # tx / # tevm / # txn in titles.
+const MARKER_RE = /<!--T(?:X|EVM|XN)-->|\{\/\* T(?:X|EVM|XN) \*\/\}|\s*#\s*t(?:evm|xn?)\b/gi;
+
+function extractFrontmatter(raw) {
+ const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
+ if (!match) return { title: null, body: raw };
+ const fm = match[1];
+ const m = fm.match(/^title:\s*(.+)$/m);
+ const title = m
+ ? m[1].trim().replace(MARKER_RE, '').trim().replace(/^['"]|['"]$/g, '').trim()
+ : null;
+ return { title, body: raw.slice(match[0].length).trim() };
+}
+
+function stripContent(text) {
+ return text
+ .replace(MARKER_RE, '')
+ .replace(/^import\s+.+from\s+['"].+['"];?\s*$/gm, '') // JSX imports in .mdx
+ .trim();
+}
+
+function fileToUrl(filePath, docsDir) {
+ let rel = path.relative(docsDir, filePath).replace(/\\/g, '/');
+ rel = rel.replace(/\.mdx?$/, '');
+ if (rel === 'index') return '/';
+ if (rel.endsWith('/index')) rel = rel.slice(0, -'/index'.length);
+ return '/' + rel;
+}
+
+// Convert a Docusaurus doc ID to the URL path it renders at.
+function idToUrl(id) {
+ if (id.endsWith('/index')) return '/' + id.slice(0, -'/index'.length);
+ return '/' + id;
+}
+
+// Read every doc file into a map keyed by URL.
+function collectDocsByUrl(docsDir) {
+ const byUrl = {};
+
+ function walk(dir) {
+ const entries = fs.readdirSync(dir, { withFileTypes: true })
+ .sort((a, b) => a.name.localeCompare(b.name));
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ const rel = path.relative(docsDir, full);
+ if (SKIP_DIRS.includes(rel.split(path.sep)[0])) continue;
+ walk(full);
+ } else if (entry.isFile() && /\.mdx?$/.test(entry.name)) {
+ const raw = fs.readFileSync(full, 'utf8');
+ const { title, body } = extractFrontmatter(raw);
+ const url = fileToUrl(full, docsDir);
+ byUrl[url] = { url, title, body: stripContent(body) };
+ }
+ }
+ }
+
+ walk(docsDir);
+ return byUrl;
+}
+
+// Walk the sidebar tree depth-first, yielding {url, section} in sidebar order.
+// `section` is the label of the top-level category the page belongs to.
+function extractSidebarOrder(sidebar) {
+ const result = [];
+
+ function walkItems(items, section) {
+ for (const item of items) {
+ if (typeof item === 'string') {
+ result.push({ url: idToUrl(item), section });
+ } else if (item.type === 'doc') {
+ result.push({ url: idToUrl(item.id), section });
+ } else if (item.type === 'category') {
+ // Category link page comes first, then the category's children.
+ if (item.link && item.link.type === 'doc') {
+ result.push({ url: idToUrl(item.link.id), section });
+ }
+ walkItems(item.items || [], section);
+ }
+ // type: 'link' (external URL) — skip
+ }
+ }
+
+ for (const topItem of sidebar) {
+ if (topItem.type !== 'category') continue;
+ // Top-level categories can also have a link page (e.g. "Tutorials").
+ if (topItem.link && topItem.link.type === 'doc') {
+ result.push({ url: idToUrl(topItem.link.id), section: topItem.label });
+ }
+ walkItems(topItem.items || [], topItem.label);
+ }
+
+ return result;
+}
+
+module.exports = function llmsTxtPlugin(context) {
+ return {
+ name: 'llms-txt-plugin',
+
+ async postBuild({ siteDir, outDir }) {
+ const docsDir = path.join(siteDir, 'docs');
+ const byUrl = collectDocsByUrl(docsDir);
+
+ // Load the sidebar and derive a flat, ordered list of pages.
+ const sidebars = require(path.resolve(siteDir, 'sidebars.js'));
+ const ordered = extractSidebarOrder(sidebars.documentationSidebar);
+
+ // Deduplicate: a page may appear as a category `link` and again in `items`.
+ const seen = new Set();
+ const deduped = ordered.filter(({ url }) => {
+ if (seen.has(url)) return false;
+ seen.add(url);
+ return true;
+ });
+
+ // Group pages by section, preserving the sidebar's section and page order.
+ const sectionLabels = [];
+ const sectionPages = {};
+ for (const { url, section } of deduped) {
+ const doc = byUrl[url];
+ if (!doc) continue; // sidebar entry with no matching file
+ if (!sectionPages[section]) {
+ sectionPages[section] = [];
+ sectionLabels.push(section);
+ }
+ sectionPages[section].push(doc);
+ }
+
+ // llms.txt — index
+ let index = `# Etherlink documentation\n\n`;
+ index += `> Etherlink is an EVM-compatible Layer 2 on Tezos with distributed sequencing and native atomic composability between EVM and Michelson smart contracts.\n\n`;
+ index += `Full content: ${SITE_URL}/llms-full.txt\n\n`;
+
+ for (const section of sectionLabels) {
+ index += `## ${section}\n\n`;
+ for (const { url, title } of sectionPages[section]) {
+ const label = title ?? url.split('/').filter(Boolean).pop() ?? 'index';
+ index += `- [${label}](${SITE_URL}${url})\n`;
+ }
+ index += '\n';
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms.txt'), index);
+
+ // llms-full.txt — all pages concatenated in sidebar order.
+ // Every page body starts with a # heading; inject the URL after it.
+ const allDocs = sectionLabels.flatMap(s => sectionPages[s]);
+ let full = '';
+ for (const { url, body } of allDocs) {
+ const nl = body.indexOf('\n');
+ const heading = nl === -1 ? body : body.slice(0, nl);
+ const rest = nl === -1 ? '' : body.slice(nl).trimStart();
+ full += `\n\n${heading}\n\nURL: ${SITE_URL}${url}\n\n${rest}\n`;
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms-full.txt'), full.trimStart());
+
+ console.log(`[llms-txt-plugin] Wrote llms.txt and llms-full.txt (${allDocs.length} pages).`);
+ },
+ };
+};

docusaurus.config.js

@@ -108,6 +108,7 @@ module.exports = async function createConfigAsync() {
  },
  ],
  require.resolve('./src/plugins/webpack-config-plugin'),
+ require.resolve('./src/plugins/llms-txt-plugin'),
  [
  '@docusaurus/plugin-client-redirects',
  {

src/plugins/llms-txt-plugin.js

@@ -0,0 +1,172 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SITE_URL = 'https://docs.etherlink.com';
+
+// Directories under docs/ that are not full pages.
+const SKIP_DIRS = ['conrefs'];
+
+// Rebranding markers used internally: <!--TX-->, <!--TEVM-->, <!--TXN-->,
+// {/* TX */}, {/* TEVM */}, {/* TXN */}, and # tx / # tevm / # txn in titles.
+const MARKER_RE = /<!--T(?:X|EVM|XN)-->|\{\/\* T(?:X|EVM|XN) \*\/\}|\s*#\s*t(?:evm|xn?)\b/gi;
+
+function extractFrontmatter(raw) {
+ const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
+ if (!match) return { title: null, body: raw };
+ const fm = match[1];
+ const m = fm.match(/^title:\s*(.+)$/m);
+ const title = m
+ ? m[1].trim().replace(MARKER_RE, '').trim().replace(/^['"]|['"]$/g, '').trim()
+ : null;
+ return { title, body: raw.slice(match[0].length).trim() };
+}
+
+function stripContent(text) {
+ return text
+ .replace(MARKER_RE, '')
+ .replace(/^import\s+.+from\s+['"].+['"];?\s*$/gm, '') // JSX imports in .mdx
+ .trim();
+}
+
+function fileToUrl(filePath, docsDir) {
+ let rel = path.relative(docsDir, filePath).replace(/\\/g, '/');
+ rel = rel.replace(/\.mdx?$/, '');
+ if (rel === 'index') return '/';
+ if (rel.endsWith('/index')) rel = rel.slice(0, -'/index'.length);
+ return '/' + rel;
+}
+
+// Convert a Docusaurus doc ID to the URL path it renders at.
+function idToUrl(id) {
+ if (id.endsWith('/index')) return '/' + id.slice(0, -'/index'.length);
+ return '/' + id;
+}
+
+// Read every doc file into a map keyed by URL.
+function collectDocsByUrl(docsDir) {
+ const byUrl = {};
+
+ function walk(dir) {
+ const entries = fs.readdirSync(dir, { withFileTypes: true })
+ .sort((a, b) => a.name.localeCompare(b.name));
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ const rel = path.relative(docsDir, full);
+ if (SKIP_DIRS.includes(rel.split(path.sep)[0])) continue;
+ walk(full);
+ } else if (entry.isFile() && /\.mdx?$/.test(entry.name)) {
+ const raw = fs.readFileSync(full, 'utf8');
+ const { title, body } = extractFrontmatter(raw);
+ const url = fileToUrl(full, docsDir);
+ byUrl[url] = { url, title, body: stripContent(body) };
+ }
+ }
+ }
+
+ walk(docsDir);
+ return byUrl;
+}
+
+// Walk the sidebar tree depth-first, yielding {url, section} in sidebar order.
+// `section` is the label of the top-level category the page belongs to.
+function extractSidebarOrder(sidebar) {
+ const result = [];
+
+ function walkItems(items, section) {
+ for (const item of items) {
+ if (typeof item === 'string') {
+ result.push({ url: idToUrl(item), section });
+ } else if (item.type === 'doc') {
+ result.push({ url: idToUrl(item.id), section });
+ } else if (item.type === 'category') {
+ // Category link page comes first, then the category's children.
+ if (item.link && item.link.type === 'doc') {
+ result.push({ url: idToUrl(item.link.id), section });
+ }
+ walkItems(item.items || [], section);
+ }
+ // type: 'link' (external URL) — skip
+ }
+ }
+
+ for (const topItem of sidebar) {
+ if (topItem.type !== 'category') continue;
+ // Top-level categories can also have a link page (e.g. "Tutorials").
+ if (topItem.link && topItem.link.type === 'doc') {
+ result.push({ url: idToUrl(topItem.link.id), section: topItem.label });
+ }
+ walkItems(topItem.items || [], topItem.label);
+ }
+
+ return result;
+}
+
+module.exports = function llmsTxtPlugin(context) {
+ return {
+ name: 'llms-txt-plugin',
+
+ async postBuild({ siteDir, outDir }) {
+ const docsDir = path.join(siteDir, 'docs');
+ const byUrl = collectDocsByUrl(docsDir);
+
+ // Load the sidebar and derive a flat, ordered list of pages.
+ const sidebars = require(path.resolve(siteDir, 'sidebars.js'));
+ const ordered = extractSidebarOrder(sidebars.documentationSidebar);
+
+ // Deduplicate: a page may appear as a category `link` and again in `items`.
+ const seen = new Set();
+ const deduped = ordered.filter(({ url }) => {
+ if (seen.has(url)) return false;
+ seen.add(url);
+ return true;
+ });
+
+ // Group pages by section, preserving the sidebar's section and page order.
+ const sectionLabels = [];
+ const sectionPages = {};
+ for (const { url, section } of deduped) {
+ const doc = byUrl[url];
+ if (!doc) continue; // sidebar entry with no matching file
+ if (!sectionPages[section]) {
+ sectionPages[section] = [];
+ sectionLabels.push(section);
+ }
+ sectionPages[section].push(doc);
+ }
+
+ // llms.txt — index
+ let index = `# Etherlink documentation\n\n`;
+ index += `> Etherlink is an EVM-compatible Layer 2 on Tezos with distributed sequencing and native atomic composability between EVM and Michelson smart contracts.\n\n`;
+ index += `Full content: ${SITE_URL}/llms-full.txt\n\n`;
+
+ for (const section of sectionLabels) {
+ index += `## ${section}\n\n`;
+ for (const { url, title } of sectionPages[section]) {
+ const label = title ?? url.split('/').filter(Boolean).pop() ?? 'index';
+ index += `- [${label}](${SITE_URL}${url})\n`;
+ }
+ index += '\n';
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms.txt'), index);
+
+ // llms-full.txt — all pages concatenated in sidebar order.
+ // Every page body starts with a # heading; inject the URL after it.
+ const allDocs = sectionLabels.flatMap(s => sectionPages[s]);
+ let full = '';
+ for (const { url, body } of allDocs) {
+ const nl = body.indexOf('\n');
+ const heading = nl === -1 ? body : body.slice(0, nl);
+ const rest = nl === -1 ? '' : body.slice(nl).trimStart();
+ full += `\n\n${heading}\n\nURL: ${SITE_URL}${url}\n\n${rest}\n`;
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms-full.txt'), full.trimStart());
+
+ console.log(`[llms-txt-plugin] Wrote llms.txt and llms-full.txt (${allDocs.length} pages).`);
+ },
+ };
+};

docusaurus.config.js

@@ -108,6 +108,7 @@ module.exports = async function createConfigAsync() {
  },
  ],
  require.resolve('./src/plugins/webpack-config-plugin'),
+ require.resolve('./src/plugins/llms-txt-plugin'),
  [
  '@docusaurus/plugin-client-redirects',
  {

src/plugins/llms-txt-plugin.js

@@ -0,0 +1,172 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SITE_URL = 'https://docs.etherlink.com';
+
+// Directories under docs/ that are not full pages.
+const SKIP_DIRS = ['conrefs'];
+
+// Rebranding markers used internally: <!--TX-->, <!--TEVM-->, <!--TXN-->,
+// {/* TX */}, {/* TEVM */}, {/* TXN */}, and # tx / # tevm / # txn in titles.
+const MARKER_RE = /<!--T(?:X|EVM|XN)-->|\{\/\* T(?:X|EVM|XN) \*\/\}|\s*#\s*t(?:evm|xn?)\b/gi;
+
+function extractFrontmatter(raw) {
+ const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
+ if (!match) return { title: null, body: raw };
+ const fm = match[1];
+ const m = fm.match(/^title:\s*(.+)$/m);
+ const title = m
+ ? m[1].trim().replace(MARKER_RE, '').trim().replace(/^['"]|['"]$/g, '').trim()
+ : null;
+ return { title, body: raw.slice(match[0].length).trim() };
+}
+
+function stripContent(text) {
+ return text
+ .replace(MARKER_RE, '')
+ .replace(/^import\s+.+from\s+['"].+['"];?\s*$/gm, '') // JSX imports in .mdx
+ .trim();
+}
+
+function fileToUrl(filePath, docsDir) {
+ let rel = path.relative(docsDir, filePath).replace(/\\/g, '/');
+ rel = rel.replace(/\.mdx?$/, '');
+ if (rel === 'index') return '/';
+ if (rel.endsWith('/index')) rel = rel.slice(0, -'/index'.length);
+ return '/' + rel;
+}
+
+// Convert a Docusaurus doc ID to the URL path it renders at.
+function idToUrl(id) {
+ if (id.endsWith('/index')) return '/' + id.slice(0, -'/index'.length);
+ return '/' + id;
+}
+
+// Read every doc file into a map keyed by URL.
+function collectDocsByUrl(docsDir) {
+ const byUrl = {};
+
+ function walk(dir) {
+ const entries = fs.readdirSync(dir, { withFileTypes: true })
+ .sort((a, b) => a.name.localeCompare(b.name));
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ const rel = path.relative(docsDir, full);
+ if (SKIP_DIRS.includes(rel.split(path.sep)[0])) continue;
+ walk(full);
+ } else if (entry.isFile() && /\.mdx?$/.test(entry.name)) {
+ const raw = fs.readFileSync(full, 'utf8');
+ const { title, body } = extractFrontmatter(raw);
+ const url = fileToUrl(full, docsDir);
+ byUrl[url] = { url, title, body: stripContent(body) };
+ }
+ }
+ }
+
+ walk(docsDir);
+ return byUrl;
+}
+
+// Walk the sidebar tree depth-first, yielding {url, section} in sidebar order.
+// `section` is the label of the top-level category the page belongs to.
+function extractSidebarOrder(sidebar) {
+ const result = [];
+
+ function walkItems(items, section) {
+ for (const item of items) {
+ if (typeof item === 'string') {
+ result.push({ url: idToUrl(item), section });
+ } else if (item.type === 'doc') {
+ result.push({ url: idToUrl(item.id), section });
+ } else if (item.type === 'category') {
+ // Category link page comes first, then the category's children.
+ if (item.link && item.link.type === 'doc') {
+ result.push({ url: idToUrl(item.link.id), section });
+ }
+ walkItems(item.items || [], section);
+ }
+ // type: 'link' (external URL) — skip
+ }
+ }
+
+ for (const topItem of sidebar) {
+ if (topItem.type !== 'category') continue;
+ // Top-level categories can also have a link page (e.g. "Tutorials").
+ if (topItem.link && topItem.link.type === 'doc') {
+ result.push({ url: idToUrl(topItem.link.id), section: topItem.label });
+ }
+ walkItems(topItem.items || [], topItem.label);
+ }
+
+ return result;
+}
+
+module.exports = function llmsTxtPlugin(context) {
+ return {
+ name: 'llms-txt-plugin',
+
+ async postBuild({ siteDir, outDir }) {
+ const docsDir = path.join(siteDir, 'docs');
+ const byUrl = collectDocsByUrl(docsDir);
+
+ // Load the sidebar and derive a flat, ordered list of pages.
+ const sidebars = require(path.resolve(siteDir, 'sidebars.js'));
+ const ordered = extractSidebarOrder(sidebars.documentationSidebar);
+
+ // Deduplicate: a page may appear as a category `link` and again in `items`.
+ const seen = new Set();
+ const deduped = ordered.filter(({ url }) => {
+ if (seen.has(url)) return false;
+ seen.add(url);
+ return true;
+ });
+
+ // Group pages by section, preserving the sidebar's section and page order.
+ const sectionLabels = [];
+ const sectionPages = {};
+ for (const { url, section } of deduped) {
+ const doc = byUrl[url];
+ if (!doc) continue; // sidebar entry with no matching file
+ if (!sectionPages[section]) {
+ sectionPages[section] = [];
+ sectionLabels.push(section);
+ }
+ sectionPages[section].push(doc);
+ }
+
+ // llms.txt — index
+ let index = `# Etherlink documentation\n\n`;
+ index += `> Etherlink is an EVM-compatible Layer 2 on Tezos with distributed sequencing and native atomic composability between EVM and Michelson smart contracts.\n\n`;
+ index += `Full content: ${SITE_URL}/llms-full.txt\n\n`;
+
+ for (const section of sectionLabels) {
+ index += `## ${section}\n\n`;
+ for (const { url, title } of sectionPages[section]) {
+ const label = title ?? url.split('/').filter(Boolean).pop() ?? 'index';
+ index += `- [${label}](${SITE_URL}${url})\n`;
+ }
+ index += '\n';
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms.txt'), index);
+
+ // llms-full.txt — all pages concatenated in sidebar order.
+ // Every page body starts with a # heading; inject the URL after it.
+ const allDocs = sectionLabels.flatMap(s => sectionPages[s]);
+ let full = '';
+ for (const { url, body } of allDocs) {
+ const nl = body.indexOf('\n');
+ const heading = nl === -1 ? body : body.slice(0, nl);
+ const rest = nl === -1 ? '' : body.slice(nl).trimStart();
+ full += `\n\n${heading}\n\nURL: ${SITE_URL}${url}\n\n${rest}\n`;
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms-full.txt'), full.trimStart());
+
+ console.log(`[llms-txt-plugin] Wrote llms.txt and llms-full.txt (${allDocs.length} pages).`);
+ },
+ };
+};

docusaurus.config.js

@@ -108,6 +108,7 @@ module.exports = async function createConfigAsync() {
  },
  ],
  require.resolve('./src/plugins/webpack-config-plugin'),
+ require.resolve('./src/plugins/llms-txt-plugin'),
  [
  '@docusaurus/plugin-client-redirects',
  {

src/plugins/llms-txt-plugin.js

@@ -0,0 +1,172 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SITE_URL = 'https://docs.etherlink.com';
+
+// Directories under docs/ that are not full pages.
+const SKIP_DIRS = ['conrefs'];
+
+// Rebranding markers used internally: <!--TX-->, <!--TEVM-->, <!--TXN-->,
+// {/* TX */}, {/* TEVM */}, {/* TXN */}, and # tx / # tevm / # txn in titles.
+const MARKER_RE = /<!--T(?:X|EVM|XN)-->|\{\/\* T(?:X|EVM|XN) \*\/\}|\s*#\s*t(?:evm|xn?)\b/gi;
+
+function extractFrontmatter(raw) {
+ const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
+ if (!match) return { title: null, body: raw };
+ const fm = match[1];
+ const m = fm.match(/^title:\s*(.+)$/m);
+ const title = m
+ ? m[1].trim().replace(MARKER_RE, '').trim().replace(/^['"]|['"]$/g, '').trim()
+ : null;
+ return { title, body: raw.slice(match[0].length).trim() };
+}
+
+function stripContent(text) {
+ return text
+ .replace(MARKER_RE, '')
+ .replace(/^import\s+.+from\s+['"].+['"];?\s*$/gm, '') // JSX imports in .mdx
+ .trim();
+}
+
+function fileToUrl(filePath, docsDir) {
+ let rel = path.relative(docsDir, filePath).replace(/\\/g, '/');
+ rel = rel.replace(/\.mdx?$/, '');
+ if (rel === 'index') return '/';
+ if (rel.endsWith('/index')) rel = rel.slice(0, -'/index'.length);
+ return '/' + rel;
+}
+
+// Convert a Docusaurus doc ID to the URL path it renders at.
+function idToUrl(id) {
+ if (id.endsWith('/index')) return '/' + id.slice(0, -'/index'.length);
+ return '/' + id;
+}
+
+// Read every doc file into a map keyed by URL.
+function collectDocsByUrl(docsDir) {
+ const byUrl = {};
+
+ function walk(dir) {
+ const entries = fs.readdirSync(dir, { withFileTypes: true })
+ .sort((a, b) => a.name.localeCompare(b.name));
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ const rel = path.relative(docsDir, full);
+ if (SKIP_DIRS.includes(rel.split(path.sep)[0])) continue;
+ walk(full);
+ } else if (entry.isFile() && /\.mdx?$/.test(entry.name)) {
+ const raw = fs.readFileSync(full, 'utf8');
+ const { title, body } = extractFrontmatter(raw);
+ const url = fileToUrl(full, docsDir);
+ byUrl[url] = { url, title, body: stripContent(body) };
+ }
+ }
+ }
+
+ walk(docsDir);
+ return byUrl;
+}
+
+// Walk the sidebar tree depth-first, yielding {url, section} in sidebar order.
+// `section` is the label of the top-level category the page belongs to.
+function extractSidebarOrder(sidebar) {
+ const result = [];
+
+ function walkItems(items, section) {
+ for (const item of items) {
+ if (typeof item === 'string') {
+ result.push({ url: idToUrl(item), section });
+ } else if (item.type === 'doc') {
+ result.push({ url: idToUrl(item.id), section });
+ } else if (item.type === 'category') {
+ // Category link page comes first, then the category's children.
+ if (item.link && item.link.type === 'doc') {
+ result.push({ url: idToUrl(item.link.id), section });
+ }
+ walkItems(item.items || [], section);
+ }
+ // type: 'link' (external URL) — skip
+ }
+ }
+
+ for (const topItem of sidebar) {
+ if (topItem.type !== 'category') continue;
+ // Top-level categories can also have a link page (e.g. "Tutorials").
+ if (topItem.link && topItem.link.type === 'doc') {
+ result.push({ url: idToUrl(topItem.link.id), section: topItem.label });
+ }
+ walkItems(topItem.items || [], topItem.label);
+ }
+
+ return result;
+}
+
+module.exports = function llmsTxtPlugin(context) {
+ return {
+ name: 'llms-txt-plugin',
+
+ async postBuild({ siteDir, outDir }) {
+ const docsDir = path.join(siteDir, 'docs');
+ const byUrl = collectDocsByUrl(docsDir);
+
+ // Load the sidebar and derive a flat, ordered list of pages.
+ const sidebars = require(path.resolve(siteDir, 'sidebars.js'));
+ const ordered = extractSidebarOrder(sidebars.documentationSidebar);
+
+ // Deduplicate: a page may appear as a category `link` and again in `items`.
+ const seen = new Set();
+ const deduped = ordered.filter(({ url }) => {
+ if (seen.has(url)) return false;
+ seen.add(url);
+ return true;
+ });
+
+ // Group pages by section, preserving the sidebar's section and page order.
+ const sectionLabels = [];
+ const sectionPages = {};
+ for (const { url, section } of deduped) {
+ const doc = byUrl[url];
+ if (!doc) continue; // sidebar entry with no matching file
+ if (!sectionPages[section]) {
+ sectionPages[section] = [];
+ sectionLabels.push(section);
+ }
+ sectionPages[section].push(doc);
+ }
+
+ // llms.txt — index
+ let index = `# Etherlink documentation\n\n`;
+ index += `> Etherlink is an EVM-compatible Layer 2 on Tezos with distributed sequencing and native atomic composability between EVM and Michelson smart contracts.\n\n`;
+ index += `Full content: ${SITE_URL}/llms-full.txt\n\n`;
+
+ for (const section of sectionLabels) {
+ index += `## ${section}\n\n`;
+ for (const { url, title } of sectionPages[section]) {
+ const label = title ?? url.split('/').filter(Boolean).pop() ?? 'index';
+ index += `- [${label}](${SITE_URL}${url})\n`;
+ }
+ index += '\n';
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms.txt'), index);
+
+ // llms-full.txt — all pages concatenated in sidebar order.
+ // Every page body starts with a # heading; inject the URL after it.
+ const allDocs = sectionLabels.flatMap(s => sectionPages[s]);
+ let full = '';
+ for (const { url, body } of allDocs) {
+ const nl = body.indexOf('\n');
+ const heading = nl === -1 ? body : body.slice(0, nl);
+ const rest = nl === -1 ? '' : body.slice(nl).trimStart();
+ full += `\n\n${heading}\n\nURL: ${SITE_URL}${url}\n\n${rest}\n`;
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms-full.txt'), full.trimStart());
+
+ console.log(`[llms-txt-plugin] Wrote llms.txt and llms-full.txt (${allDocs.length} pages).`);
+ },
+ };
+};

docusaurus.config.js

@@ -108,6 +108,7 @@ module.exports = async function createConfigAsync() {
  },
  ],
  require.resolve('./src/plugins/webpack-config-plugin'),
+ require.resolve('./src/plugins/llms-txt-plugin'),
  [
  '@docusaurus/plugin-client-redirects',
  {

src/plugins/llms-txt-plugin.js

@@ -0,0 +1,172 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SITE_URL = 'https://docs.etherlink.com';
+
+// Directories under docs/ that are not full pages.
+const SKIP_DIRS = ['conrefs'];
+
+// Rebranding markers used internally: <!--TX-->, <!--TEVM-->, <!--TXN-->,
+// {/* TX */}, {/* TEVM */}, {/* TXN */}, and # tx / # tevm / # txn in titles.
+const MARKER_RE = /<!--T(?:X|EVM|XN)-->|\{\/\* T(?:X|EVM|XN) \*\/\}|\s*#\s*t(?:evm|xn?)\b/gi;
+
+function extractFrontmatter(raw) {
+ const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
+ if (!match) return { title: null, body: raw };
+ const fm = match[1];
+ const m = fm.match(/^title:\s*(.+)$/m);
+ const title = m
+ ? m[1].trim().replace(MARKER_RE, '').trim().replace(/^['"]|['"]$/g, '').trim()
+ : null;
+ return { title, body: raw.slice(match[0].length).trim() };
+}
+
+function stripContent(text) {
+ return text
+ .replace(MARKER_RE, '')
+ .replace(/^import\s+.+from\s+['"].+['"];?\s*$/gm, '') // JSX imports in .mdx
+ .trim();
+}
+
+function fileToUrl(filePath, docsDir) {
+ let rel = path.relative(docsDir, filePath).replace(/\\/g, '/');
+ rel = rel.replace(/\.mdx?$/, '');
+ if (rel === 'index') return '/';
+ if (rel.endsWith('/index')) rel = rel.slice(0, -'/index'.length);
+ return '/' + rel;
+}
+
+// Convert a Docusaurus doc ID to the URL path it renders at.
+function idToUrl(id) {
+ if (id.endsWith('/index')) return '/' + id.slice(0, -'/index'.length);
+ return '/' + id;
+}
+
+// Read every doc file into a map keyed by URL.
+function collectDocsByUrl(docsDir) {
+ const byUrl = {};
+
+ function walk(dir) {
+ const entries = fs.readdirSync(dir, { withFileTypes: true })
+ .sort((a, b) => a.name.localeCompare(b.name));
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ const rel = path.relative(docsDir, full);
+ if (SKIP_DIRS.includes(rel.split(path.sep)[0])) continue;
+ walk(full);
+ } else if (entry.isFile() && /\.mdx?$/.test(entry.name)) {
+ const raw = fs.readFileSync(full, 'utf8');
+ const { title, body } = extractFrontmatter(raw);
+ const url = fileToUrl(full, docsDir);
+ byUrl[url] = { url, title, body: stripContent(body) };
+ }
+ }
+ }
+
+ walk(docsDir);
+ return byUrl;
+}
+
+// Walk the sidebar tree depth-first, yielding {url, section} in sidebar order.
+// `section` is the label of the top-level category the page belongs to.
+function extractSidebarOrder(sidebar) {
+ const result = [];
+
+ function walkItems(items, section) {
+ for (const item of items) {
+ if (typeof item === 'string') {
+ result.push({ url: idToUrl(item), section });
+ } else if (item.type === 'doc') {
+ result.push({ url: idToUrl(item.id), section });
+ } else if (item.type === 'category') {
+ // Category link page comes first, then the category's children.
+ if (item.link && item.link.type === 'doc') {
+ result.push({ url: idToUrl(item.link.id), section });
+ }
+ walkItems(item.items || [], section);
+ }
+ // type: 'link' (external URL) — skip
+ }
+ }
+
+ for (const topItem of sidebar) {
+ if (topItem.type !== 'category') continue;
+ // Top-level categories can also have a link page (e.g. "Tutorials").
+ if (topItem.link && topItem.link.type === 'doc') {
+ result.push({ url: idToUrl(topItem.link.id), section: topItem.label });
+ }
+ walkItems(topItem.items || [], topItem.label);
+ }
+
+ return result;
+}
+
+module.exports = function llmsTxtPlugin(context) {
+ return {
+ name: 'llms-txt-plugin',
+
+ async postBuild({ siteDir, outDir }) {
+ const docsDir = path.join(siteDir, 'docs');
+ const byUrl = collectDocsByUrl(docsDir);
+
+ // Load the sidebar and derive a flat, ordered list of pages.
+ const sidebars = require(path.resolve(siteDir, 'sidebars.js'));
+ const ordered = extractSidebarOrder(sidebars.documentationSidebar);
+
+ // Deduplicate: a page may appear as a category `link` and again in `items`.
+ const seen = new Set();
+ const deduped = ordered.filter(({ url }) => {
+ if (seen.has(url)) return false;
+ seen.add(url);
+ return true;
+ });
+
+ // Group pages by section, preserving the sidebar's section and page order.
+ const sectionLabels = [];
+ const sectionPages = {};
+ for (const { url, section } of deduped) {
+ const doc = byUrl[url];
+ if (!doc) continue; // sidebar entry with no matching file
+ if (!sectionPages[section]) {
+ sectionPages[section] = [];
+ sectionLabels.push(section);
+ }
+ sectionPages[section].push(doc);
+ }
+
+ // llms.txt — index
+ let index = `# Etherlink documentation\n\n`;
+ index += `> Etherlink is an EVM-compatible Layer 2 on Tezos with distributed sequencing and native atomic composability between EVM and Michelson smart contracts.\n\n`;
+ index += `Full content: ${SITE_URL}/llms-full.txt\n\n`;
+
+ for (const section of sectionLabels) {
+ index += `## ${section}\n\n`;
+ for (const { url, title } of sectionPages[section]) {
+ const label = title ?? url.split('/').filter(Boolean).pop() ?? 'index';
+ index += `- [${label}](${SITE_URL}${url})\n`;
+ }
+ index += '\n';
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms.txt'), index);
+
+ // llms-full.txt — all pages concatenated in sidebar order.
+ // Every page body starts with a # heading; inject the URL after it.
+ const allDocs = sectionLabels.flatMap(s => sectionPages[s]);
+ let full = '';
+ for (const { url, body } of allDocs) {
+ const nl = body.indexOf('\n');
+ const heading = nl === -1 ? body : body.slice(0, nl);
+ const rest = nl === -1 ? '' : body.slice(nl).trimStart();
+ full += `\n\n${heading}\n\nURL: ${SITE_URL}${url}\n\n${rest}\n`;
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms-full.txt'), full.trimStart());
+
+ console.log(`[llms-txt-plugin] Wrote llms.txt and llms-full.txt (${allDocs.length} pages).`);
+ },
+ };
+};

docusaurus.config.js

@@ -108,6 +108,7 @@ module.exports = async function createConfigAsync() {
  },
  ],
  require.resolve('./src/plugins/webpack-config-plugin'),
+ require.resolve('./src/plugins/llms-txt-plugin'),
  [
  '@docusaurus/plugin-client-redirects',
  {

src/plugins/llms-txt-plugin.js

@@ -0,0 +1,172 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SITE_URL = 'https://docs.etherlink.com';
+
+// Directories under docs/ that are not full pages.
+const SKIP_DIRS = ['conrefs'];
+
+// Rebranding markers used internally: <!--TX-->, <!--TEVM-->, <!--TXN-->,
+// {/* TX */}, {/* TEVM */}, {/* TXN */}, and # tx / # tevm / # txn in titles.
+const MARKER_RE = /<!--T(?:X|EVM|XN)-->|\{\/\* T(?:X|EVM|XN) \*\/\}|\s*#\s*t(?:evm|xn?)\b/gi;
+
+function extractFrontmatter(raw) {
+ const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
+ if (!match) return { title: null, body: raw };
+ const fm = match[1];
+ const m = fm.match(/^title:\s*(.+)$/m);
+ const title = m
+ ? m[1].trim().replace(MARKER_RE, '').trim().replace(/^['"]|['"]$/g, '').trim()
+ : null;
+ return { title, body: raw.slice(match[0].length).trim() };
+}
+
+function stripContent(text) {
+ return text
+ .replace(MARKER_RE, '')
+ .replace(/^import\s+.+from\s+['"].+['"];?\s*$/gm, '') // JSX imports in .mdx
+ .trim();
+}
+
+function fileToUrl(filePath, docsDir) {
+ let rel = path.relative(docsDir, filePath).replace(/\\/g, '/');
+ rel = rel.replace(/\.mdx?$/, '');
+ if (rel === 'index') return '/';
+ if (rel.endsWith('/index')) rel = rel.slice(0, -'/index'.length);
+ return '/' + rel;
+}
+
+// Convert a Docusaurus doc ID to the URL path it renders at.
+function idToUrl(id) {
+ if (id.endsWith('/index')) return '/' + id.slice(0, -'/index'.length);
+ return '/' + id;
+}
+
+// Read every doc file into a map keyed by URL.
+function collectDocsByUrl(docsDir) {
+ const byUrl = {};
+
+ function walk(dir) {
+ const entries = fs.readdirSync(dir, { withFileTypes: true })
+ .sort((a, b) => a.name.localeCompare(b.name));
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ const rel = path.relative(docsDir, full);
+ if (SKIP_DIRS.includes(rel.split(path.sep)[0])) continue;
+ walk(full);
+ } else if (entry.isFile() && /\.mdx?$/.test(entry.name)) {
+ const raw = fs.readFileSync(full, 'utf8');
+ const { title, body } = extractFrontmatter(raw);
+ const url = fileToUrl(full, docsDir);
+ byUrl[url] = { url, title, body: stripContent(body) };
+ }
+ }
+ }
+
+ walk(docsDir);
+ return byUrl;
+}
+
+// Walk the sidebar tree depth-first, yielding {url, section} in sidebar order.
+// `section` is the label of the top-level category the page belongs to.
+function extractSidebarOrder(sidebar) {
+ const result = [];
+
+ function walkItems(items, section) {
+ for (const item of items) {
+ if (typeof item === 'string') {
+ result.push({ url: idToUrl(item), section });
+ } else if (item.type === 'doc') {
+ result.push({ url: idToUrl(item.id), section });
+ } else if (item.type === 'category') {
+ // Category link page comes first, then the category's children.
+ if (item.link && item.link.type === 'doc') {
+ result.push({ url: idToUrl(item.link.id), section });
+ }
+ walkItems(item.items || [], section);
+ }
+ // type: 'link' (external URL) — skip
+ }
+ }
+
+ for (const topItem of sidebar) {
+ if (topItem.type !== 'category') continue;
+ // Top-level categories can also have a link page (e.g. "Tutorials").
+ if (topItem.link && topItem.link.type === 'doc') {
+ result.push({ url: idToUrl(topItem.link.id), section: topItem.label });
+ }
+ walkItems(topItem.items || [], topItem.label);
+ }
+
+ return result;
+}
+
+module.exports = function llmsTxtPlugin(context) {
+ return {
+ name: 'llms-txt-plugin',
+
+ async postBuild({ siteDir, outDir }) {
+ const docsDir = path.join(siteDir, 'docs');
+ const byUrl = collectDocsByUrl(docsDir);
+
+ // Load the sidebar and derive a flat, ordered list of pages.
+ const sidebars = require(path.resolve(siteDir, 'sidebars.js'));
+ const ordered = extractSidebarOrder(sidebars.documentationSidebar);
+
+ // Deduplicate: a page may appear as a category `link` and again in `items`.
+ const seen = new Set();
+ const deduped = ordered.filter(({ url }) => {
+ if (seen.has(url)) return false;
+ seen.add(url);
+ return true;
+ });
+
+ // Group pages by section, preserving the sidebar's section and page order.
+ const sectionLabels = [];
+ const sectionPages = {};
+ for (const { url, section } of deduped) {
+ const doc = byUrl[url];
+ if (!doc) continue; // sidebar entry with no matching file
+ if (!sectionPages[section]) {
+ sectionPages[section] = [];
+ sectionLabels.push(section);
+ }
+ sectionPages[section].push(doc);
+ }
+
+ // llms.txt — index
+ let index = `# Etherlink documentation\n\n`;
+ index += `> Etherlink is an EVM-compatible Layer 2 on Tezos with distributed sequencing and native atomic composability between EVM and Michelson smart contracts.\n\n`;
+ index += `Full content: ${SITE_URL}/llms-full.txt\n\n`;
+
+ for (const section of sectionLabels) {
+ index += `## ${section}\n\n`;
+ for (const { url, title } of sectionPages[section]) {
+ const label = title ?? url.split('/').filter(Boolean).pop() ?? 'index';
+ index += `- [${label}](${SITE_URL}${url})\n`;
+ }
+ index += '\n';
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms.txt'), index);
+
+ // llms-full.txt — all pages concatenated in sidebar order.
+ // Every page body starts with a # heading; inject the URL after it.
+ const allDocs = sectionLabels.flatMap(s => sectionPages[s]);
+ let full = '';
+ for (const { url, body } of allDocs) {
+ const nl = body.indexOf('\n');
+ const heading = nl === -1 ? body : body.slice(0, nl);
+ const rest = nl === -1 ? '' : body.slice(nl).trimStart();
+ full += `\n\n${heading}\n\nURL: ${SITE_URL}${url}\n\n${rest}\n`;
+ }
+
+ fs.writeFileSync(path.join(outDir, 'llms-full.txt'), full.trimStart());
+
+ console.log(`[llms-txt-plugin] Wrote llms.txt and llms-full.txt (${allDocs.length} pages).`);
+ },
+ };
+};