') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); feat: add getContents() for reading external markdown files into table descriptions by AmmanuelT · Pull Request #2138 · dataform-co/dataform · GitHub
Skip to content

feat: add getContents() for reading external markdown files into table descriptions - #2138

Merged
kolina merged 8 commits into
dataform-co:mainfrom
AmmanuelT:read_description_from_external_file
Apr 21, 2026
Merged

feat: add getContents() for reading external markdown files into table descriptions#2138
kolina merged 8 commits into
dataform-co:mainfrom
AmmanuelT:read_description_from_external_file

Conversation

@AmmanuelT

@AmmanuelTAmmanuelT commented Apr 3, 2026

Copy link
Copy Markdown
Contributor

Closes#1723

Allows users to source table descriptions from external markdown files, keeping .sqlx files smaller and enabling reusable, renderable descriptions across the project.

Usage:

config {
type: "table",
description: getContents('./my_table_description.md')
}
SELECT ...

Changes:

  • core/compilers.ts : .md files now compile to exports.contents = ...``
  • core/session.ts : new getContents(filePath) method; resolves the path relative to the calling file using the call stack, then builds an absolute path for nativeRequire
  • core/main.ts : getContents bound to the global scope
  • cli/vm/compile.ts + testing/run_core.ts: "md" added to sourceExtensions so vm2 applies the compiler to .md files
  • core/main_test.ts + core/compilers_test.ts : tests for happy path and missing file error

Notes:

  • The method is currently named getContents, inspired by the suggestion in the issue thread, but happy to rename before merge if the maintainers have a preference!
  • When getContents references a file that doesn't exist, the error surfaces as a VMError: Cannot find module '/absolute/path/to/file.md' compilation error. A cleaner message could be produced by intercepting in dataformCompile in main.ts also happy to add this if preferred, just wanted to flag the tradeoff before doing extra work.

@AmmanuelT
AmmanuelT requested a review from a team as a code ownerApril 3, 2026 16:06
@AmmanuelT
AmmanuelT requested review from krushangSk17 and removed request for a teamApril 3, 2026 16:06
@kolina

kolina commented Apr 11, 2026

Copy link
Copy Markdown
Contributor

/gcbrun

kolina
kolina requested changes Apr 11, 2026
View reviewed changes

@kolinakolina left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you very much for your contribution!

Overall looks good, I've left some not very big comments to address.

Currently tests are failing due to expired BQ credentials, can you please rebase your PR onto #2139 so I can re-run them?

Comment threadcore/session.ts Outdated
Comment threaddocs/reference/session.md Outdated
Comment threadcore/session.ts
@AmmanuelT

AmmanuelT commented Apr 11, 2026

Copy link
Copy Markdown
ContributorAuthor

Thanks for the comments! Working on the fixes!

@AmmanuelT
AmmanuelTforce-pushed the read_description_from_external_file branch from 8cf17c1 to cf359f2CompareApril 11, 2026 17:54
@AmmanuelT

AmmanuelT commented Apr 11, 2026

Copy link
Copy Markdown
ContributorAuthor

@kolina ready for another check 🫡

kolina
kolina approved these changes Apr 15, 2026
View reviewed changes
Comment threadcore/session.ts Outdated
@kolina

kolina commented Apr 21, 2026

Copy link
Copy Markdown
Contributor

/gcbrun

@kolina
kolina merged commit e7dc2b5 into dataform-co:mainApr 21, 2026
3 checks passed
kolina added a commit that referenced this pull request May 4, 2026
@kolina
Revert "feat: add getContents() for reading external markdown files i…
530f8eb
…nto table descriptions (#2138)" (#2155)
It broke compilation in GCP https://issuetracker.google.com/509294486
@kolina

kolina commented May 4, 2026

Copy link
Copy Markdown
Contributor

@AmmanuelT, unfortunately I had to revert your commit in #2155 because it broke compilation in GCP.

I think the reason for breakage is that we use pure V8 for compilation in GCP and we can't use NodeJs modules like pathhere. Feel free to send a new PR, I think it'll be quite feasible to remove these dependency: you use it for join call here, you can instead use our helper

@AmmanuelT

AmmanuelT commented May 7, 2026

Copy link
Copy Markdown
ContributorAuthor

Sorry about that! I'll get that replaced!

@AmmanuelTAmmanuelT mentioned this pull request May 9, 2026
Merged
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@kolinakolinakolina approved these changes
@krushangSk17krushangSk17Awaiting requested review from krushangSk17krushangSk17 is a code owner automatically assigned from dataform-co/dataform-reviewers

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

Read table description from external files

2 participants

@AmmanuelT@kolina