Rollup merge of #134693 - SpriteOvO:proc-macro-use-to-tokens-in-quote, r=tgross35

proc_macro: Use `ToTokens` trait in `quote` macro
Tracking issues: #130977, #54722
This PR changed `proc_macro::quote!` to use `ToTokens` trait instead of `TokenStream::from`, and migrated test cases from `quote` crate.
r? `@dtolnay`
CC `@tgross35`

Author: jhpratt

library/proc_macro/src/lib.rs

@@ -419,7 +419,7 @@ pub mod token_stream {
 /// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
 /// To quote `$` itself, use `$$`.
 #[unstable(feature = "proc_macro_quote", issue = "54722")]
-#[allow_internal_unstable(proc_macro_def_site, proc_macro_internals)]
+#[allow_internal_unstable(proc_macro_def_site, proc_macro_internals, proc_macro_totokens)]
 #[rustc_builtin_macro]
 pub macro quote($($t:tt)*){
 /* compiler built-in */

library/proc_macro/src/quote.rs

@@ -4,12 +4,14 @@
 //! This quasiquoter uses macros 2.0 hygiene to reliably access
 //! items from `proc_macro`, to build a `proc_macro::TokenStream`.
 
-usecrate::{Delimiter,Group,Ident,Literal,Punct,Spacing,Span,TokenStream,TokenTree};
+usecrate::{
+Delimiter,Group,Ident,Literal,Punct,Spacing,Span,ToTokens,TokenStream,TokenTree,
+};
 
-macro_rules!quote_tt{
-(($($t:tt)*)) => {Group::new(Delimiter::Parenthesis,quote!($($t)*))};
-([$($t:tt)*]) => {Group::new(Delimiter::Bracket,quote!($($t)*))};
-({$($t:tt)*}) => {Group::new(Delimiter::Brace,quote!($($t)*))};
+macro_rules!minimal_quote_tt{
+(($($t:tt)*)) => {Group::new(Delimiter::Parenthesis,minimal_quote!($($t)*))};
+([$($t:tt)*]) => {Group::new(Delimiter::Bracket,minimal_quote!($($t)*))};
+({$($t:tt)*}) => {Group::new(Delimiter::Brace,minimal_quote!($($t)*))};
 (,) => {Punct::new(',',Spacing::Alone)};
 (.) => {Punct::new('.',Spacing::Alone)};
 (;) => {Punct::new(';',Spacing::Alone)};
@@ -21,21 +23,20 @@ macro_rules! quote_tt {
 ($i:ident) => {Ident::new(stringify!($i),Span::def_site())};
 }
 
-macro_rules!quote_ts{
+macro_rules!minimal_quote_ts{
 ((@ $($t:tt)*)) => { $($t)*};
 (::) => {
-[
-TokenTree::from(Punct::new(':',Spacing::Joint)),
-TokenTree::from(Punct::new(':',Spacing::Alone)),
-].iter()
-.cloned()
-.map(|mut x| {
- x.set_span(Span::def_site());
- x
-})
-.collect::<TokenStream>()
+{
+letmut c = (
+TokenTree::from(Punct::new(':',Spacing::Joint)),
+TokenTree::from(Punct::new(':',Spacing::Alone))
+);
+ c.0.set_span(Span::def_site());
+ c.1.set_span(Span::def_site());
+[c.0, c.1].into_iter().collect::<TokenStream>()
+}
 };
-($t:tt) => {TokenTree::from(quote_tt!($t))};
+($t:tt) => {TokenTree::from(minimal_quote_tt!($t))};
 }
 
 /// Simpler version of the real `quote!` macro, implemented solely
@@ -46,12 +47,14 @@ macro_rules! quote_ts {
 ///
 /// Note: supported tokens are a subset of the real `quote!`, but
 /// unquoting is different: instead of `$x`, this uses `(@ expr)`.
-macro_rules! quote {
-() => {TokenStream::new()};
+macro_rules! minimal_quote {
 ($($t:tt)*) => {
-[
- $(TokenStream::from(quote_ts!($t)),)*
-].iter().cloned().collect::<TokenStream>()
+{
+ #[allow(unused_mut)]// In case the expansion is empty
+letmut ts = TokenStream::new();
+ $(ToTokens::to_tokens(&minimal_quote_ts!($t),&mut ts);)*
+ ts
+}
 };
 }
 
@@ -62,52 +65,66 @@ macro_rules! quote {
 #[unstable(feature = "proc_macro_quote", issue = "54722")]
 pubfnquote(stream:TokenStream) -> TokenStream{
 if stream.is_empty(){
-returnquote!(crate::TokenStream::new());
+returnminimal_quote!(crate::TokenStream::new());
 }
-let proc_macro_crate = quote!(crate);
+let proc_macro_crate = minimal_quote!(crate);
 letmut after_dollar = false;
-let tokens = stream
-.into_iter()
-.filter_map(|tree| {
-if after_dollar {
- after_dollar = false;
-match tree {
-TokenTree::Ident(_) => {
-returnSome(quote!(Into::<crate::TokenStream>::into(
-Clone::clone(&(@ tree))),));
-}
-TokenTree::Punct(ref tt)if tt.as_char() == '$' => {}
- _ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
-}
-}elseifletTokenTree::Punct(ref tt) = tree {
-if tt.as_char() == '$'{
- after_dollar = true;
-returnNone;
+
+letmut tokens = crate::TokenStream::new();
+for tree in stream {
+if after_dollar {
+ after_dollar = false;
+match tree {
+TokenTree::Ident(_) => {
+minimal_quote!(crate::ToTokens::to_tokens(&(@ tree),&mut ts);)
+.to_tokens(&mut tokens);
+continue;
 }
+TokenTree::Punct(ref tt)if tt.as_char() == '$' => {}
+ _ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
 }
+}elseifletTokenTree::Punct(ref tt) = tree {
+if tt.as_char() == '$'{
+ after_dollar = true;
+continue;
+}
+}
 
-Some(quote!(crate::TokenStream::from((@ match tree {
-TokenTree::Punct(tt) => quote!(crate::TokenTree::Punct(crate::Punct::new(
+match tree {
+TokenTree::Punct(tt) => {
+minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(
 (@ TokenTree::from(Literal::character(tt.as_char()))),
 (@ match tt.spacing(){
-Spacing::Alone => quote!(crate::Spacing::Alone),
-Spacing::Joint => quote!(crate::Spacing::Joint),
+Spacing::Alone => minimal_quote!(crate::Spacing::Alone),
+Spacing::Joint => minimal_quote!(crate::Spacing::Joint),
 }),
-))),
-TokenTree::Group(tt) => quote!(crate::TokenTree::Group(crate::Group::new(
+)),&mut ts);)
+}
+TokenTree::Group(tt) => {
+minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Group(crate::Group::new(
 (@ match tt.delimiter(){
-Delimiter::Parenthesis => quote!(crate::Delimiter::Parenthesis),
-Delimiter::Brace => quote!(crate::Delimiter::Brace),
-Delimiter::Bracket => quote!(crate::Delimiter::Bracket),
-Delimiter::None => quote!(crate::Delimiter::None),
+Delimiter::Parenthesis => minimal_quote!(crate::Delimiter::Parenthesis),
+Delimiter::Brace => minimal_quote!(crate::Delimiter::Brace),
+Delimiter::Bracket => minimal_quote!(crate::Delimiter::Bracket),
+Delimiter::None => minimal_quote!(crate::Delimiter::None),
 }),
 (@ quote(tt.stream())),
-))),
-TokenTree::Ident(tt) => quote!(crate::TokenTree::Ident(crate::Ident::new(
-(@ TokenTree::from(Literal::string(&tt.to_string()))),
+)),&mut ts);)
+}
+TokenTree::Ident(tt) => {
+let literal = tt.to_string();
+let(literal, ctor) = ifletSome(stripped) = literal.strip_prefix("r#"){
+(stripped,minimal_quote!(crate::Ident::new_raw))
+}else{
+(literal.as_str(),minimal_quote!(crate::Ident::new))
+};
+minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Ident((@ ctor)(
+(@ TokenTree::from(Literal::string(literal))),
 (@ quote_span(proc_macro_crate.clone(), tt.span())),
-))),
-TokenTree::Literal(tt) => quote!(crate::TokenTree::Literal({
+)),&mut ts);)
+}
+TokenTree::Literal(tt) => {
+minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
 letmut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
 .parse::<crate::TokenStream>()
 .unwrap()
@@ -120,22 +137,28 @@ pub fn quote(stream: TokenStream) -> TokenStream {
 } else {
  unreachable!()
 }
-}))
-})),))
-})
-.collect::<TokenStream>();
-
+}),&mut ts);)
+}
+}
+.to_tokens(&mut tokens);
+}
 if after_dollar {
 panic!("unexpected trailing `$` in `quote!`");
 }
 
-quote!([(@ tokens)].iter().cloned().collect::<crate::TokenStream>())
+minimal_quote!{
+{
+letmut ts = crate::TokenStream::new();
+(@ tokens)
+ ts
+}
+}
 }
 
 /// Quote a `Span` into a `TokenStream`.
 /// This is needed to implement a custom quoter.
 #[unstable(feature = "proc_macro_quote", issue = "54722")]
 pubfnquote_span(proc_macro_crate:TokenStream,span:Span) -> TokenStream{
 let id = span.save_span();
-quote!((@ proc_macro_crate )::Span::recover_proc_macro_span((@ TokenTree::from(Literal::usize_unsuffixed(id)))))
+minimal_quote!((@ proc_macro_crate )::Span::recover_proc_macro_span((@ TokenTree::from(Literal::usize_unsuffixed(id)))))
 }