Rollup merge of #146340 - fmease:frontmatter-containment, r=fee1-dead,Urgau

Strip frontmatter in fewer places
* Stop stripping frontmatter in `proc_macro::Literal::from_str` (RUST-146132)
* Stop stripping frontmatter in expr-ctxt (but not item-ctxt!) `include`s (RUST-145945)
* Stop stripping shebang (!) in `proc_macro::Literal::from_str`
* Not a breaking change because it did compare spans already to ensure there wasn't extra whitespace or comments (`Literal::from_str("#!\n0")` already yields `Err(_)` thankfully!)
* Stop stripping frontmatter+shebang inside some rustdoc code where it doesn't make any observable difference (see self review comments)
* (Stop stripping frontmatter+shebang inside internal test code)
Fixes#145945.
Fixes#146132.
r? fee1-dead

Author: matthiaskrgr

compiler/rustc_builtin_macros/src/source_util.rs

@@ -1,3 +1,5 @@
+//! The implementation of built-in macros which relate to the file system.
+
 use std::path::{Path,PathBuf};
 use std::rc::Rc;
 use std::sync::Arc;
@@ -11,9 +13,11 @@ use rustc_expand::base::{
 };
 use rustc_expand::module::DirOwnership;
 use rustc_lint_defs::BuiltinLintDiag;
-use rustc_parse::parser::{ForceCollect,Parser};
+use rustc_parse::lexer::StripTokens;
+use rustc_parse::parser::ForceCollect;
 use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal, utf8_error};
 use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
+use rustc_session::parse::ParseSess;
 use rustc_span::source_map::SourceMap;
 use rustc_span::{ByteSymbol,Pos,Span,Symbol};
 use smallvec::SmallVec;
@@ -23,11 +27,7 @@ use crate::util::{
  check_zero_tts, get_single_str_from_tts, get_single_str_spanned_from_tts, parse_expr,
 };
 
-// These macros all relate to the file system; they either return
-// the column/row/filename of the expression, or they include
-// a given file into the current one.
-
-/// line!(): expands to the current line number
+/// Expand `line!()` to the current line number.
 pub(crate)fnexpand_line(
 cx:&mutExtCtxt<'_>,
 sp:Span,
@@ -42,7 +42,7 @@ pub(crate) fn expand_line(
 ExpandResult::Ready(MacEager::expr(cx.expr_u32(topmost, loc.lineasu32)))
 }
 
-/* column!(): expands to the current column number */
+/// Expand `column!()` to the current column number.
 pub(crate)fnexpand_column(
 cx:&mutExtCtxt<'_>,
 sp:Span,
@@ -57,9 +57,7 @@ pub(crate) fn expand_column(
 ExpandResult::Ready(MacEager::expr(cx.expr_u32(topmost, loc.col.to_usize()asu32 + 1)))
 }
 
-/// file!(): expands to the current filename */
-/// The source_file (`loc.file`) contains a bunch more information we could spit
-/// out if we wanted.
+/// Expand `file!()` to the current filename.
 pub(crate)fnexpand_file(
 cx:&mutExtCtxt<'_>,
 sp:Span,
@@ -81,6 +79,7 @@ pub(crate) fn expand_file(
 )))
 }
 
+/// Expand `stringify!($input)`.
 pub(crate)fnexpand_stringify(
 cx:&mutExtCtxt<'_>,
 sp:Span,
@@ -91,6 +90,7 @@ pub(crate) fn expand_stringify(
 ExpandResult::Ready(MacEager::expr(cx.expr_str(sp,Symbol::intern(&s))))
 }
 
+/// Expand `module_path!()` to (a textual representation of) the current module path.
 pub(crate)fnexpand_mod(
 cx:&mutExtCtxt<'_>,
 sp:Span,
@@ -104,9 +104,9 @@ pub(crate) fn expand_mod(
 ExpandResult::Ready(MacEager::expr(cx.expr_str(sp,Symbol::intern(&string))))
 }
 
-/// include! : parse the given file as an expr
-/// This is generally a bad idea because it's going to behave
-/// unhygienically.
+/// Expand `include!($input)`.
+///
+/// This works in item and expression position. Notably, it doesn't work in pattern position.
 pub(crate)fnexpand_include<'cx>(
 cx:&'cxmutExtCtxt<'_>,
 sp:Span,
@@ -116,64 +116,76 @@ pub(crate) fn expand_include<'cx>(
 letExpandResult::Ready(mac) = get_single_str_from_tts(cx, sp, tts,"include!")else{
 returnExpandResult::Retry(());
 };
-letfile = match mac {
-Ok(file) => file,
+letpath = match mac {
+Ok(path) => path,
 Err(guar) => returnExpandResult::Ready(DummyResult::any(sp, guar)),
 };
 // The file will be added to the code map by the parser
-letfile = matchresolve_path(&cx.sess,file.as_str(), sp){
-Ok(f) => f,
+letpath = matchresolve_path(&cx.sess,path.as_str(), sp){
+Ok(path) => path,
 Err(err) => {
 let guar = err.emit();
 returnExpandResult::Ready(DummyResult::any(sp, guar));
 }
 };
-let p = unwrap_or_emit_fatal(new_parser_from_file(cx.psess(),&file,Some(sp)));
 
 // If in the included file we have e.g., `mod bar;`,
-// then the path of `bar.rs` should be relative to the directory of `file`.
+// then the path of `bar.rs` should be relative to the directory of `path`.
 // See https://github.com/rust-lang/rust/pull/69838/files#r395217057 for a discussion.
 // `MacroExpander::fully_expand_fragment` later restores, so "stack discipline" is maintained.
-let dir_path = file.parent().unwrap_or(&file).to_owned();
+let dir_path = path.parent().unwrap_or(&path).to_owned();
  cx.current_expansion.module = Rc::new(cx.current_expansion.module.with_dir_path(dir_path));
  cx.current_expansion.dir_ownership = DirOwnership::Owned{relative:None};
 
 structExpandInclude<'a>{
-p:Parser<'a>,
+psess:&'aParseSess,
+path:PathBuf,
 node_id: ast::NodeId,
+span:Span,
 }
 impl<'a>MacResultforExpandInclude<'a>{
-fnmake_expr(mutself:Box<ExpandInclude<'a>>) -> Option<Box<ast::Expr>>{
-let expr = parse_expr(&mutself.p).ok()?;
-ifself.p.token != token::Eof{
-self.p.psess.buffer_lint(
+fnmake_expr(self:Box<ExpandInclude<'a>>) -> Option<Box<ast::Expr>>{
+letmut p = unwrap_or_emit_fatal(new_parser_from_file(
+self.psess,
+&self.path,
+// Don't strip frontmatter for backward compatibility, `---` may be the start of a
+// manifold negation. FIXME: Ideally, we wouldn't strip shebangs here either.
+StripTokens::Shebang,
+Some(self.span),
+));
+let expr = parse_expr(&mut p).ok()?;
+if p.token != token::Eof{
+ p.psess.buffer_lint(
 INCOMPLETE_INCLUDE,
-self.p.token.span,
+ p.token.span,
 self.node_id,
 BuiltinLintDiag::IncompleteInclude,
 );
 }
 Some(expr)
 }
 
-fnmake_items(mutself:Box<ExpandInclude<'a>>) -> Option<SmallVec<[Box<ast::Item>;1]>>{
+fnmake_items(self:Box<ExpandInclude<'a>>) -> Option<SmallVec<[Box<ast::Item>;1]>>{
+letmut p = unwrap_or_emit_fatal(new_parser_from_file(
+self.psess,
+&self.path,
+StripTokens::ShebangAndFrontmatter,
+Some(self.span),
+));
 letmut ret = SmallVec::new();
 loop{
-matchself.p.parse_item(ForceCollect::No){
+match p.parse_item(ForceCollect::No){
 Err(err) => {
  err.emit();
 break;
 }
 Ok(Some(item)) => ret.push(item),
 Ok(None) => {
-ifself.p.token != token::Eof{
-self.p
-.dcx()
-.create_err(errors::ExpectedItem{
-span:self.p.token.span,
-token:&pprust::token_to_string(&self.p.token),
-})
-.emit();
+if p.token != token::Eof{
+ p.dcx().emit_err(errors::ExpectedItem{
+span: p.token.span,
+token:&pprust::token_to_string(&p.token),
+});
 }
 
 break;
@@ -184,10 +196,17 @@ pub(crate) fn expand_include<'cx>(
 }
 }
 
-ExpandResult::Ready(Box::new(ExpandInclude{ p,node_id: cx.current_expansion.lint_node_id}))
+ExpandResult::Ready(Box::new(ExpandInclude{
+psess: cx.psess(),
+ path,
+node_id: cx.current_expansion.lint_node_id,
+span: sp,
+}))
 }
 
-/// `include_str!`: read the given file, insert it as a literal string expr
+/// Expand `include_str!($input)` to the content of the UTF-8-encoded file given by path `$input` as a string literal.
+///
+/// This works in expression, pattern and statement position.
 pub(crate)fnexpand_include_str(
 cx:&mutExtCtxt<'_>,
 sp:Span,
@@ -206,6 +225,7 @@ pub(crate) fn expand_include_str(
 Ok((bytes, bsp)) => match std::str::from_utf8(&bytes){
 Ok(src) => {
 let interned_src = Symbol::intern(src);
+// MacEager converts the expr into a pat if need be.
 MacEager::expr(cx.expr_str(cx.with_def_site_ctxt(bsp), interned_src))
 }
 Err(utf8err) => {
@@ -218,6 +238,9 @@ pub(crate) fn expand_include_str(
 })
 }
 
+/// Expand `include_bytes!($input)` to the content of the file given by path `$input`.
+///
+/// This works in expression, pattern and statement position.
 pub(crate)fnexpand_include_bytes(
 cx:&mutExtCtxt<'_>,
 sp:Span,
@@ -237,6 +260,7 @@ pub(crate) fn expand_include_bytes(
 // Don't care about getting the span for the raw bytes,
 // because the console can't really show them anyway.
 let expr = cx.expr(sp, ast::ExprKind::IncludedBytes(ByteSymbol::intern(&bytes)));
+// MacEager converts the expr into a pat if need be.
 MacEager::expr(expr)
 }
 Err(dummy) => dummy,

compiler/rustc_driver_impl/src/lib.rs

@@ -51,6 +51,7 @@ use rustc_lint::unerased_lint_store;
 use rustc_metadata::creader::MetadataLoader;
 use rustc_metadata::locator;
 use rustc_middle::ty::TyCtxt;
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_session::config::{
 CG_OPTIONS,CrateType,ErrorOutputType,Input,OptionDesc,OutFileName,OutputType,Sysroot,
@@ -1288,10 +1289,15 @@ fn warn_on_confusing_output_filename_flag(
 
 fn parse_crate_attrs<'a>(sess:&'a Session) -> PResult<'a, ast::AttrVec> {
 letmut parser = unwrap_or_emit_fatal(match&sess.io.input {
-Input::File(file) => new_parser_from_file(&sess.psess, file,None),
-Input::Str{ name, input } => {
- new_parser_from_source_str(&sess.psess, name.clone(), input.clone())
+Input::File(file) => {
+ new_parser_from_file(&sess.psess, file,StripTokens::ShebangAndFrontmatter,None)
 }
+Input::Str{ name, input } => new_parser_from_source_str(
+&sess.psess,
+ name.clone(),
+ input.clone(),
+StripTokens::ShebangAndFrontmatter,
+),
 });
  parser.parse_inner_attributes()
 }

compiler/rustc_expand/src/module.rs

@@ -4,6 +4,7 @@ use std::path::{self, Path, PathBuf};
 use rustc_ast::{AttrVec,Attribute,Inline,Item,ModSpans};
 use rustc_attr_parsing::validate_attr;
 use rustc_errors::{Diag,ErrorGuaranteed};
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::{exp, new_parser_from_file, unwrap_or_emit_fatal};
 use rustc_session::Session;
 use rustc_session::parse::ParseSess;
@@ -67,8 +68,12 @@ pub(crate) fn parse_external_mod(
 }
 
 // Actually parse the external file as a module.
-letmut parser =
-unwrap_or_emit_fatal(new_parser_from_file(&sess.psess,&mp.file_path,Some(span)));
+letmut parser = unwrap_or_emit_fatal(new_parser_from_file(
+&sess.psess,
+&mp.file_path,
+StripTokens::ShebangAndFrontmatter,
+Some(span),
+));
 let(inner_attrs, items, inner_span) =
  parser.parse_mod(exp!(Eof)).map_err(|err| ModError::ParserError(err))?;
  attrs.extend(inner_attrs);

compiler/rustc_expand/src/proc_macro_server.rs

@@ -8,7 +8,7 @@ use rustc_ast::util::literal::escape_byte_str_symbol;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Diag,ErrorGuaranteed,MultiSpan,PResult};
-use rustc_parse::lexer::nfc_normalize;
+use rustc_parse::lexer::{StripTokens,nfc_normalize};
 use rustc_parse::parser::Parser;
 use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
 use rustc_proc_macro::bridge::{
@@ -485,8 +485,13 @@ impl server::FreeFunctions for Rustc<'_, '_> {
 
 fnliteral_from_str(&mutself,s:&str) -> Result<Literal<Self::Span,Self::Symbol>,()>{
 let name = FileName::proc_macro_source_code(s);
-letmut parser =
-unwrap_or_emit_fatal(new_parser_from_source_str(self.psess(), name, s.to_owned()));
+
+letmut parser = unwrap_or_emit_fatal(new_parser_from_source_str(
+self.psess(),
+ name,
+ s.to_owned(),
+StripTokens::Nothing,
+));
 
 let first_span = parser.token.span.data();
 let minus_present = parser.eat(exp!(Minus));

compiler/rustc_interface/src/interface.rs

@@ -13,7 +13,8 @@ use rustc_lint::LintStore;
 use rustc_middle::ty;
 use rustc_middle::ty::CurrentGcx;
 use rustc_middle::util::Providers;
-use rustc_parse::new_parser_from_simple_source_str;
+use rustc_parse::lexer::StripTokens;
+use rustc_parse::new_parser_from_source_str;
 use rustc_parse::parser::attr::AllowLeadingUnsafe;
 use rustc_query_impl::QueryCtxt;
 use rustc_query_system::query::print_query_stack;
@@ -68,7 +69,8 @@ pub(crate) fn parse_cfg(dcx: DiagCtxtHandle<'_>, cfgs: Vec<String>) -> Cfg {
 };
 }
 
-matchnew_parser_from_simple_source_str(&psess, filename, s.to_string()){
+matchnew_parser_from_source_str(&psess, filename, s.to_string(),StripTokens::Nothing)
+{
 Ok(mut parser) => match parser.parse_meta_item(AllowLeadingUnsafe::No){
 Ok(meta_item)if parser.token == token::Eof => {
 if meta_item.path.segments.len() != 1{
@@ -166,13 +168,15 @@ pub(crate) fn parse_check_cfg(dcx: DiagCtxtHandle<'_>, specs: Vec<String>) -> Ch
 error!("expected `cfg(name, values(\"value1\", \"value2\", ... \"valueN\"))`")
 };
 
-letmut parser = matchnew_parser_from_simple_source_str(&psess, filename, s.to_string()){
-Ok(parser) => parser,
-Err(errs) => {
- errs.into_iter().for_each(|err| err.cancel());
-expected_error();
-}
-};
+letmut parser =
+matchnew_parser_from_source_str(&psess, filename, s.to_string(),StripTokens::Nothing)
+{
+Ok(parser) => parser,
+Err(errs) => {
+ errs.into_iter().for_each(|err| err.cancel());
+expected_error();
+}
+};
 
 let meta_item = match parser.parse_meta_item(AllowLeadingUnsafe::No){
 Ok(meta_item)if parser.token == token::Eof => meta_item,

compiler/rustc_interface/src/passes.rs

@@ -28,6 +28,7 @@ use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepsType;
 use rustc_middle::ty::{self,CurrentGcx,GlobalCtxt,RegisteredTools,TyCtxt};
 use rustc_middle::util::Providers;
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_passes::{abi_test, input_stats, layout_test};
 use rustc_resolve::{Resolver,ResolverOutputs};
@@ -52,10 +53,18 @@ pub fn parse<'a>(sess: &'a Session) -> ast::Crate {
 letmut krate = sess
 .time("parse_crate", || {
 letmut parser = unwrap_or_emit_fatal(match&sess.io.input{
-Input::File(file) => new_parser_from_file(&sess.psess, file,None),
-Input::Str{ input, name } => {
-new_parser_from_source_str(&sess.psess, name.clone(), input.clone())
-}
+Input::File(file) => new_parser_from_file(
+&sess.psess,
+ file,
+StripTokens::ShebangAndFrontmatter,
+None,
+),
+Input::Str{ input, name } => new_parser_from_source_str(
+&sess.psess,
+ name.clone(),
+ input.clone(),
+StripTokens::ShebangAndFrontmatter,
+),
 });
  parser.parse_crate_mod()
 })

compiler/rustc_parse/src/lexer/mod.rs

@@ -45,7 +45,7 @@ pub(crate) struct UnmatchedDelim {
 }
 
 /// Which tokens should be stripped before lexing the tokens.
-pub(crate)enumStripTokens{
+pubenumStripTokens{
 /// Strip both shebang and frontmatter.
 ShebangAndFrontmatter,
 /// Strip the shebang but not frontmatter.