') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); fix: quote reserved words in `fmt` by prql-bot · Pull Request #6214 · PRQL/prql · GitHub
Skip to content

fix: quote reserved words in fmt - #6214

Merged
max-sixty merged 4 commits into
mainfrom
fix/fmt-quote-all-keywords
Aug 20, 2026
Merged

fix: quote reserved words in fmt#6214
max-sixty merged 4 commits into
mainfrom
fix/fmt-quote-all-keywords

Conversation

@prql-bot

@prql-botprql-bot commented Aug 20, 2026

Copy link
Copy Markdown
Collaborator

fmt emitted source that no longer round-trips for names that collide with reserved words. Two distinct causes, both now fixed: the codegen kept its own hand-written copy of the keyword list (two entries behind the lexer's, missing import and enum), and expression-position identifiers bypassed the quoting logic entirely by going through pr::Ident's Display impl — a fifth copy of the "needs backticks" rule that knows nothing about reserved words.

#6200 fixed this class of bug for names that aren't valid bare idents; these slipped through because the lists and code paths write_ident_part sits behind were already inconsistent. #6210 gave the grammars/ files and the playground a "keep in sync with the lexer" comment for the same reason — this removes the need for one on the codegen side by deleting the duplicate.

Details

Three bugs, in ascending order of nastiness:

  1. Declaration names collide with a keyword — loud failure, output doesn't lex.
$ printf'let `import` = 5\n'| prqlc fmtlet import = 5
$ printf'let import = 5\n'| prqlc fmtError: ╭─[ :1:5 ] 1 │ let import = 5 │ ───┬── │ ╰──── expected something else, but found keyword import───╯

Same for enum. Cause: write_ident_part consulted a local keywords() copy of the list that had drifted.

  1. Reserved literalstrue / false / null are reserved just as firmly (boolean() and null() both end with end_expr()), but lex as TokenKind::Literal rather than TokenKind::Keyword, so they were absent from the keyword list entirely.

  2. Expression position — silent change of meaning, since the output still parses:

$ printf'from t\nselect {`true`, `null`}\n'| prqlc fmtfrom tselect {true, null}

SELECT "true", "null" before the round-trip, SELECT true, NULL after. This one affected every reserved word, including let and the others already in the list — ExprKind::Ident wrote via ident.to_string(), and pr::Ident's Display has its own quoting rule that only checks character validity.

Changes:

  • prqlc/prqlc-parser/src/lexer/mod.rs — extract pub const KEYWORDS: [&str; 10], add the sibling pub const RESERVED_LITERALS: [&str; 3]; keyword() now builds its parser from the former via choice(KEYWORDS.map(just)).
  • prqlc/prqlc/src/codegen/ast.rs — delete the local keywords()OnceLock<HashSet>, read both lexer lists directly. A linear scan of 13 &str beats hashing, so this drops the HashSet import along with the duplication. ExprKind::Ident now goes through a shared write_ident helper instead of Display.
  • web/book/src/reference/syntax/keywords.md — the documented list had drifted the same two entries; adds enum and import. (It already listed true/false/null correctly — the docs were ahead of the code there.)

Tests:test_every_reserved_word_is_quoted iterates KEYWORDS.iter().chain(RESERVED_LITERALS.iter()) and checks both declaration and expression position, so a word added to either lexer list in future is covered without touching the test. test_reserved_literals_lex_as_literals in the parser pins RESERVED_LITERALS to actual lexer behaviour, since it's hand-written next to boolean()/null() rather than derived.

Snapshot changes: two fmt integration snapshots change from tracks. + backtick-star to tracks.*. That's the same Display-vs-write_ident_part divergence in the other direction — Display was over-quoting *, which valid_prql_ident() explicitly allows. Both forms compile to identical SQL, and the new output matches the .prql source.

Verification:cargo test -p prqlc -p prqlc-parser passes; cargo clippy --all-targets and cargo fmt --check are clean. task prqlc:pull-request couldn't run locally — cargo-insta isn't on the sandbox's PATH, which is what #6144 addresses — so CI is the first full-matrix run.

@prql-botprql-bot left a comment

Copy link
Copy Markdown
CollaboratorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fmt still drops backticks on true / false / null. It's the same round-trip break as import/enum, but a worse failure mode — the output parses, as a different query — and test_every_keyword_is_quoted can't catch it, since those three lex as Literals rather than TokenKind::Keyword and so aren't in lexer::KEYWORDS. Details inline, plus a suggestion on the doc comment.

The choice(KEYWORDS.map(just)) rewrite preserves the original alternation order, and cargo test -p prqlc -p prqlc-parser passes locally on the merged tree.

Comment threadprqlc/prqlc/src/codegen/ast.rs Outdated
Comment threadprqlc/prqlc-parser/src/lexer/mod.rs Outdated
`true` / `false` / `null` are reserved by the lexer (`boolean()` and
`null()` both end with `end_expr()`) but produce `TokenKind::Literal`
rather than `TokenKind::Keyword`, so they were absent from
`lexer::KEYWORDS` and `write_ident_part` left them unquoted. Adds a
sibling `lexer::RESERVED_LITERALS` and rejects both lists.
Separately, `ExprKind::Ident` wrote idents via `pr::Ident`'s `Display`
impl, which carries its own copy of the quoting rule and knows nothing
about reserved words — so expression-position names lost their backticks
even for keywords already in `KEYWORDS`. Routes it through
`write_ident_part` like every other codegen path.
@prql-botprql-bot changed the title fix: quote all lexer keywords in fmtfix: quote reserved words in fmtAug 20, 2026

@prql-botprql-bot left a comment

Copy link
Copy Markdown
CollaboratorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cargo doc fails on this branch. The RESERVED_LITERALS doc comment links three private items — boolean, null, end_expr — from a pub const, which rustdoc::private_intra_doc_links denies under the workflow-level RUSTDOCFLAGS: -Dwarnings. Reproduced on the merged tree:

$ RUSTDOCFLAGS="-Dwarnings" cargo doc -p prqlc-parser --no-depserror: public documentation for `RESERVED_LITERALS` links to private item `boolean` --> prqlc/prqlc-parser/src/lexer/mod.rs:206:36 |206 | /// [`TokenKind::Keyword`] — see [`boolean`] and [`null`]. They're reserved just | ^^^^^^^ this item is private...error: could not document `prqlc-parser`

This PR's own CI won't catch it: the 🗒️ Doc step in test-rust.yaml is gated on inputs.nightly == 'true', which rules resolved to false here — so test-rust is green while cargo doc is broken, and it would surface on the nightly run after merge instead. The suggestion inline drops the link brackets; with it, RUSTDOCFLAGS="-Dwarnings" cargo doc -p prqlc-parser --no-deps is clean. (KEYWORDS's own doc comment above is fine — TokenKind::Keyword and RESERVED_LITERALS are both public.)

Nothing else outstanding. cargo test -p prqlc-parser --lib, cargo test -p prqlc --lib -- codegen and cargo test -p prqlc --test integration -- fmt all pass on the merged tree, and the two tracks.* snapshot changes are covered by the idempotence assertion in the fmt harness — pl_to_prql output is re-parsed and re-formatted — so the unquoted wildcard is verified to round-trip rather than just asserted to.

Comment threadprqlc/prqlc-parser/src/lexer/mod.rs Outdated
`boolean`, `null` and `end_expr` are private fns, so linking them from
a `pub const`'s docs trips `rustdoc::private_intra_doc_links`, which is
an error under the workflow's `RUSTDOCFLAGS: -Dwarnings`. Plain code
spans keep the pointer without breaking `cargo doc`.

@prql-botprql-bot left a comment

Copy link
Copy Markdown
CollaboratorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

b932806 applies the suggestion verbatim, and RUSTDOCFLAGS="-Dwarnings" cargo doc -p prqlc-parser --no-deps now exits clean on the merged tree — the private_intra_doc_links error from my previous review is gone. Nothing else outstanding from either review; I can't approve my own PR, so this comment is the close-out.

@max-sixty
max-sixty merged commit 9ebf245 into mainAug 20, 2026
43 checks passed
@max-sixty
max-sixty deleted the fix/fmt-quote-all-keywords branch August 20, 2026 21:52
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants

@prql-bot@max-sixty