internal: Filter out opaque tokens in some of IDE feature macro descensions

Author: Veykril

src/tools/rust-analyzer/crates/hir-expand/src/db.rs

@@ -16,7 +16,10 @@ use crate::{
  cfg_process,
  declarative::DeclarativeMacroExpander,
  fixup::{self,SyntaxFixupUndoInfo},
- hygiene::{span_with_call_site_ctxt, span_with_def_site_ctxt, span_with_mixed_site_ctxt},
+ hygiene::{
+ span_with_call_site_ctxt, span_with_def_site_ctxt, span_with_mixed_site_ctxt,
+SyntaxContextExtas _,
+},
  proc_macro::ProcMacros,
  span_map::{RealSpanMap,SpanMap,SpanMapRef},
  tt,AstId,BuiltinAttrExpander,BuiltinDeriveExpander,BuiltinFnLikeExpander,
@@ -300,14 +303,16 @@ pub fn expand_speculative(
 token_tree_to_syntax_node(&speculative_expansion.value, expand_to, loc.def.edition);
 
 let syntax_node = node.syntax_node();
-let token = rev_tmap
+let(token, _) = rev_tmap
 .ranges_with_span(span_map.span_for_range(token_to_map.text_range()))
-.filter_map(|range| syntax_node.covering_element(range).into_token())
-.min_by_key(|t| {
-// prefer tokens of the same kind and text
+.filter_map(|(range, ctx)| syntax_node.covering_element(range).into_token().zip(Some(ctx)))
+.min_by_key(|(t, ctx)| {
+// prefer tokens of the same kind and text, as well as non opaque marked ones
 // Note the inversion of the score here, as we want to prefer the first token in case
 // of all tokens having the same score
-(t.kind() != token_to_map.kind())asu8 + 2*((t.text() != token_to_map.text())asu8)
+ ctx.is_opaque(db)asu8
+ + 2*(t.kind() != token_to_map.kind())asu8
+ + 4*((t.text() != token_to_map.text())asu8)
 })?;
 Some((node.syntax_node(), token))
 }

src/tools/rust-analyzer/crates/hir-expand/src/hygiene.rs

@@ -151,6 +151,7 @@ pub trait SyntaxContextExt {
 fnremove_mark(&mutself,db:&dynExpandDatabase) -> (Option<MacroCallId>,Transparency);
 fnouter_mark(self,db:&dynExpandDatabase) -> (Option<MacroCallId>,Transparency);
 fnmarks(self,db:&dynExpandDatabase) -> Vec<(MacroCallId,Transparency)>;
+fnis_opaque(self,db:&dynExpandDatabase) -> bool;
 }
 
 implSyntaxContextExtforSyntaxContextId{
@@ -177,6 +178,9 @@ impl SyntaxContextExt for SyntaxContextId {
  marks.reverse();
  marks
 }
+fnis_opaque(self,db:&dynExpandDatabase) -> bool{
+ !self.is_root() && db.lookup_intern_syntax_context(self).outer_transparency.is_opaque()
+}
 }
 
 // FIXME: Make this a SyntaxContextExt method once we have RPIT

src/tools/rust-analyzer/crates/hir-expand/src/lib.rs

@@ -25,6 +25,7 @@ mod prettify_macro_expansion_;
 
 use attrs::collect_attrs;
 use rustc_hash::FxHashMap;
+use stdx::TupleExt;
 use triomphe::Arc;
 
 use std::hash::Hash;
@@ -772,14 +773,15 @@ impl ExpansionInfo {
 /// Maps the passed in file range down into a macro expansion if it is the input to a macro call.
 ///
 /// Note this does a linear search through the entire backing vector of the spanmap.
+// FIXME: Consider adding a reverse map to ExpansionInfo to get rid of the linear search which
+// potentially results in quadratic look ups (notably this might improve semantic highlighting perf)
 pubfnmap_range_down_exact(
 &self,
 span:Span,
-) -> Option<InMacroFile<implIterator<Item = SyntaxToken> + '_>>{
-let tokens = self
-.exp_map
-.ranges_with_span_exact(span)
-.flat_map(move |range| self.expanded.value.covering_element(range).into_token());
+) -> Option<InMacroFile<implIterator<Item = (SyntaxToken,SyntaxContextId)> + '_>>{
+let tokens = self.exp_map.ranges_with_span_exact(span).flat_map(move |(range, ctx)| {
+self.expanded.value.covering_element(range).into_token().zip(Some(ctx))
+});
 
 Some(InMacroFile::new(self.expanded.file_id, tokens))
 }
@@ -791,11 +793,10 @@ impl ExpansionInfo {
 pubfnmap_range_down(
 &self,
 span:Span,
-) -> Option<InMacroFile<implIterator<Item = SyntaxToken> + '_>>{
-let tokens = self
-.exp_map
-.ranges_with_span(span)
-.flat_map(move |range| self.expanded.value.covering_element(range).into_token());
+) -> Option<InMacroFile<implIterator<Item = (SyntaxToken,SyntaxContextId)> + '_>>{
+let tokens = self.exp_map.ranges_with_span(span).flat_map(move |(range, ctx)| {
+self.expanded.value.covering_element(range).into_token().zip(Some(ctx))
+});
 
 Some(InMacroFile::new(self.expanded.file_id, tokens))
 }
@@ -845,7 +846,8 @@ impl ExpansionInfo {
 self.arg.file_id,
  arg_map
 .ranges_with_span_exact(span)
-.filter(|range| range.intersect(arg_range).is_some())
+.filter(|(range, _)| range.intersect(arg_range).is_some())
+.map(TupleExt::head)
 .collect(),
 )
 }

src/tools/rust-analyzer/crates/hir/src/semantics.rs

@@ -24,6 +24,7 @@ use hir_expand::{
  builtin::{BuiltinFnLikeExpander,EagerExpander},
  db::ExpandDatabase,
  files::InRealFile,
+ hygiene::SyntaxContextExtas _,
  inert_attr_macro::find_builtin_attr_idx,
  name::AsName,
 FileRange,InMacroFile,MacroCallId,MacroFileId,MacroFileIdExt,
@@ -32,7 +33,7 @@ use intern::Symbol;
 use itertools::Itertools;
 use rustc_hash::{FxHashMap,FxHashSet};
 use smallvec::{smallvec,SmallVec};
-use span::{EditionedFileId,FileId,HirFileIdRepr};
+use span::{EditionedFileId,FileId,HirFileIdRepr,SyntaxContextId};
 use stdx::TupleExt;
 use syntax::{
  algo::skip_trivia_token,
@@ -608,7 +609,7 @@ impl<'db> SemanticsImpl<'db> {
 let quote = string.open_quote_text_range()?;
 
 let token = self.wrap_token_infile(string.syntax().clone()).into_real_file().ok()?;
-self.descend_into_macros_breakable(token, |token| {
+self.descend_into_macros_breakable(token, |token, _| {
 (|| {
 let token = token.value;
 let string = ast::String::cast(token)?;
@@ -655,7 +656,7 @@ impl<'db> SemanticsImpl<'db> {
 let original_string = ast::String::cast(original_token.clone())?;
 let original_token = self.wrap_token_infile(original_token).into_real_file().ok()?;
 let quote = original_string.open_quote_text_range()?;
-self.descend_into_macros_breakable(original_token, |token| {
+self.descend_into_macros_breakable(original_token, |token, _| {
 (|| {
 let token = token.value;
 self.resolve_offset_in_format_args(
@@ -718,7 +719,7 @@ impl<'db> SemanticsImpl<'db> {
 // node is just the token, so descend the token
 self.descend_into_macros_impl(
 InRealFile::new(file_id, first),
-&mut |InFile{ value, .. }| {
+&mut |InFile{ value, .. }, _ctx| {
 ifletSome(node) = value
 .parent_ancestors()
 .take_while(|it| it.text_range() == value.text_range())
@@ -732,15 +733,15 @@ impl<'db> SemanticsImpl<'db> {
 }else{
 // Descend first and last token, then zip them to look for the node they belong to
 letmut scratch:SmallVec<[_;1]> = smallvec![];
-self.descend_into_macros_impl(InRealFile::new(file_id, first),&mut |token| {
+self.descend_into_macros_impl(InRealFile::new(file_id, first),&mut |token, _ctx| {
  scratch.push(token);
 CONTINUE_NO_BREAKS
 });
 
 letmut scratch = scratch.into_iter();
 self.descend_into_macros_impl(
 InRealFile::new(file_id, last),
-&mut |InFile{value: last,file_id: last_fid }| {
+&mut |InFile{value: last,file_id: last_fid }, _ctx| {
 ifletSome(InFile{value: first,file_id: first_fid }) = scratch.next(){
 if first_fid == last_fid {
 ifletSome(p) = first.parent(){
@@ -763,7 +764,9 @@ impl<'db> SemanticsImpl<'db> {
  res
 }
 
-fnis_inside_macro_call(token:&SyntaxToken) -> bool{
+// FIXME: This isn't quite right wrt to inner attributes
+/// Does a syntactic traversal to check whether this token might be inside a macro call
+pubfnmight_be_inside_macro_call(&self,token:&SyntaxToken) -> bool{
  token.parent_ancestors().any(|ancestor| {
 if ast::MacroCall::can_cast(ancestor.kind()){
 returntrue;
@@ -781,25 +784,14 @@ impl<'db> SemanticsImpl<'db> {
 })
 }
 
-pubfndescend_into_macros_exact_if_in_macro(
-&self,
-token:SyntaxToken,
-) -> SmallVec<[SyntaxToken;1]>{
-ifSelf::is_inside_macro_call(&token){
-self.descend_into_macros_exact(token)
-}else{
-smallvec![token]
-}
-}
-
 pubfndescend_into_macros_cb(
 &self,
 token:SyntaxToken,
-mutcb:implFnMut(InFile<SyntaxToken>),
+mutcb:implFnMut(InFile<SyntaxToken>,SyntaxContextId),
 ){
 ifletOk(token) = self.wrap_token_infile(token).into_real_file(){
-self.descend_into_macros_impl(token,&mut |t| {
-cb(t);
+self.descend_into_macros_impl(token,&mut |t, ctx| {
+cb(t, ctx);
 CONTINUE_NO_BREAKS
 });
 }
@@ -808,7 +800,7 @@ impl<'db> SemanticsImpl<'db> {
 pubfndescend_into_macros(&self,token:SyntaxToken) -> SmallVec<[SyntaxToken;1]>{
 letmut res = smallvec![];
 ifletOk(token) = self.wrap_token_infile(token.clone()).into_real_file(){
-self.descend_into_macros_impl(token,&mut |t| {
+self.descend_into_macros_impl(token,&mut |t, _ctx| {
  res.push(t.value);
 CONTINUE_NO_BREAKS
 });
@@ -819,10 +811,27 @@ impl<'db> SemanticsImpl<'db> {
  res
 }
 
+pubfndescend_into_macros_no_opaque(&self,token:SyntaxToken) -> SmallVec<[SyntaxToken;1]>{
+letmut res = smallvec![];
+ifletOk(token) = self.wrap_token_infile(token.clone()).into_real_file(){
+self.descend_into_macros_impl(token,&mut |t, ctx| {
+if !ctx.is_opaque(self.db.upcast()){
+// Don't descend into opaque contexts
+ res.push(t.value);
+}
+CONTINUE_NO_BREAKS
+});
+}
+if res.is_empty(){
+ res.push(token);
+}
+ res
+}
+
 pubfndescend_into_macros_breakable<T>(
 &self,
 token:InRealFile<SyntaxToken>,
-mutcb:implFnMut(InFile<SyntaxToken>) -> ControlFlow<T>,
+mutcb:implFnMut(InFile<SyntaxToken>,SyntaxContextId) -> ControlFlow<T>,
 ) -> Option<T>{
 self.descend_into_macros_impl(token.clone(),&mut cb)
 }
@@ -834,10 +843,12 @@ impl<'db> SemanticsImpl<'db> {
 let text = token.text();
 let kind = token.kind();
 
-self.descend_into_macros_cb(token.clone(), |InFile{ value,file_id: _ }| {
+self.descend_into_macros_cb(token.clone(), |InFile{ value,file_id: _ }, ctx| {
 let mapped_kind = value.kind();
 let any_ident_match = || kind.is_any_identifier() && value.kind().is_any_identifier();
-let matches = (kind == mapped_kind || any_ident_match()) && text == value.text();
+let matches = (kind == mapped_kind || any_ident_match())
+ && text == value.text()
+ && !ctx.is_opaque(self.db.upcast());
 if matches {
  r.push(value);
 }
@@ -854,17 +865,21 @@ impl<'db> SemanticsImpl<'db> {
 let text = token.text();
 let kind = token.kind();
 ifletOk(token) = self.wrap_token_infile(token.clone()).into_real_file(){
-self.descend_into_macros_breakable(token.clone(), |InFile{ value,file_id: _ }| {
-let mapped_kind = value.kind();
-let any_ident_match =
- || kind.is_any_identifier() && value.kind().is_any_identifier();
-let matches = (kind == mapped_kind || any_ident_match()) && text == value.text();
-if matches {
-ControlFlow::Break(value)
-}else{
-ControlFlow::Continue(())
-}
-})
+self.descend_into_macros_breakable(
+ token.clone(),
+ |InFile{ value,file_id: _ }, _ctx| {
+let mapped_kind = value.kind();
+let any_ident_match =
+ || kind.is_any_identifier() && value.kind().is_any_identifier();
+let matches =
+(kind == mapped_kind || any_ident_match()) && text == value.text();
+if matches {
+ControlFlow::Break(value)
+}else{
+ControlFlow::Continue(())
+}
+},
+)
 }else{
 None
 }
@@ -874,7 +889,7 @@ impl<'db> SemanticsImpl<'db> {
 fndescend_into_macros_impl<T>(
 &self,
 InRealFile{value: token, file_id }:InRealFile<SyntaxToken>,
-f:&mutdynFnMut(InFile<SyntaxToken>) -> ControlFlow<T>,
+f:&mutdynFnMut(InFile<SyntaxToken>,SyntaxContextId) -> ControlFlow<T>,
 ) -> Option<T>{
 let _p = tracing::info_span!("descend_into_macros_impl").entered();
 let(sa, span, file_id) = token
@@ -898,7 +913,8 @@ impl<'db> SemanticsImpl<'db> {
 // These are tracked to know which macro calls we still have to look into
 // the tokens themselves aren't that interesting as the span that is being used to map
 // things down never changes.
-letmut stack:Vec<(_,SmallVec<[_;2]>)> = vec![(file_id, smallvec![token])];
+letmut stack:Vec<(_,SmallVec<[_;2]>)> =
+vec![(file_id, smallvec![(token,SyntaxContextId::ROOT)])];
 
 // Process the expansion of a call, pushing all tokens with our span in the expansion back onto our stack
 let process_expansion_for_token = |stack:&mutVec<_>, macro_file| {
@@ -921,11 +937,11 @@ impl<'db> SemanticsImpl<'db> {
 // Filters out all tokens that contain the given range (usually the macro call), any such
 // token is redundant as the corresponding macro call has already been processed
 let filter_duplicates = |tokens:&mutSmallVec<_>,range:TextRange| {
- tokens.retain(|t:&mutSyntaxToken| !range.contains_range(t.text_range()))
+ tokens.retain(|(t, _):&mut(SyntaxToken,_)| !range.contains_range(t.text_range()))
 };
 
 whileletSome((expansion,refmut tokens)) = stack.pop(){
-whileletSome(token) = tokens.pop(){
+whileletSome((token, ctx)) = tokens.pop(){
 let was_not_remapped = (|| {
 // First expand into attribute invocations
 let containing_attribute_macro_call = self.with_ctx(|ctx| {
@@ -1036,7 +1052,7 @@ impl<'db> SemanticsImpl<'db> {
 let text_range = attr.syntax().text_range();
 // remove any other token in this macro input, all their mappings are the
 // same as this
- tokens.retain(|t| {
+ tokens.retain(|(t, _)| {
  !text_range.contains_range(t.text_range())
 });
 returnprocess_expansion_for_token(
@@ -1093,7 +1109,7 @@ impl<'db> SemanticsImpl<'db> {
 .is_none();
 
 if was_not_remapped {
-ifletControlFlow::Break(b) = f(InFile::new(expansion, token)){
+ifletControlFlow::Break(b) = f(InFile::new(expansion, token), ctx){
 returnSome(b);
 }
 }

src/tools/rust-analyzer/crates/ide-db/src/search.rs

@@ -529,9 +529,13 @@ impl<'a> FindUsages<'a> {
 })
 .into_iter()
 .flat_map(move |token| {
- sema.descend_into_macros_exact_if_in_macro(token)
-.into_iter()
-.filter_map(|it| it.parent())
+if sema.might_be_inside_macro_call(&token){
+ sema.descend_into_macros_exact(token)
+}else{
+ <_>::from([token])
+}
+.into_iter()
+.filter_map(|it| it.parent())
 })
 }
 

src/tools/rust-analyzer/crates/ide/src/goto_declaration.rs

@@ -29,7 +29,7 @@ pub(crate) fn goto_declaration(
 .find(|it| matches!(it.kind(),IDENT | T![self] | T![super] | T![crate] | T![Self]))?;
 let range = original_token.text_range();
 let info:Vec<NavigationTarget> = sema
-.descend_into_macros(original_token)
+.descend_into_macros_no_opaque(original_token)
 .iter()
 .filter_map(|token| {
 let parent = token.parent()?;

src/tools/rust-analyzer/crates/ide/src/goto_definition.rs

@@ -83,7 +83,7 @@ pub(crate) fn goto_definition(
 }
 
 let navs = sema
-.descend_into_macros(original_token.clone())
+.descend_into_macros_no_opaque(original_token.clone())
 .into_iter()
 .filter_map(|token| {
 let parent = token.parent()?;

src/tools/rust-analyzer/crates/ide/src/goto_type_definition.rs

@@ -69,7 +69,7 @@ pub(crate) fn goto_type_definition(
 }
 
 let range = token.text_range();
- sema.descend_into_macros(token)
+ sema.descend_into_macros_no_opaque(token)
 .into_iter()
 .filter_map(|token| {
 let ty = sema

src/tools/rust-analyzer/crates/ide/src/syntax_highlighting.rs

@@ -409,7 +409,8 @@ fn traverse(
 letmut r = 0;
  sema.descend_into_macros_breakable(
 InRealFile::new(file_id, token.clone()),
- |tok| {
+ |tok, _ctx| {
+// FIXME: Consider checking ctx transparency for being opaque?
 let tok = tok.value;
 let tok_kind = tok.kind();