Rollup of 5 pull requests

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -174,11 +174,6 @@ impl fmt::Debug for DepNode {
 /// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual
 /// jump table instead of large matches.
 pub struct DepKindVTable<'tcx> {
- /// Anonymous queries cannot be replayed from one compiler invocation to the next.
- /// When their result is needed, it is recomputed. They are useful for fine-grained
- /// dependency tracking, and caching within one compiler invocation.
- pub is_anon: bool,
-
  /// Eval-always queries do not track their dependencies, and are always recomputed, even if
  /// their inputs have not changed since the last compiler invocation. The result is still
  /// cached within one compiler invocation.

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::QueryInvocationId;
 use rustc_data_structures::sharded::{self, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU64, Lock};
+use rustc_data_structures::sync::{AtomicU64, Lock, is_dyn_thread_safe};
 use rustc_data_structures::unord::UnordMap;
 use rustc_data_structures::{assert_matches, outline};
 use rustc_errors::DiagInner;
@@ -1302,7 +1302,9 @@ impl CurrentDepGraph {
  prev_graph: &SerializedDepGraph,
  prev_index: SerializedDepNodeIndex,
  ) {
- if let Some(ref nodes_in_current_session) = self.nodes_in_current_session {
+ if !is_dyn_thread_safe()
+ && let Some(ref nodes_in_current_session) = self.nodes_in_current_session
+ {
  debug_assert!(
  !nodes_in_current_session
  .lock()

compiler/rustc_middle/src/queries.rs

@@ -150,10 +150,10 @@ use crate::{dep_graph, mir, thir};
 // `Providers` that the driver creates (using several `rustc_*` crates).
 //
 // The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
+// `ty::query::from_cycle_error::FromCycleError`, which produces an appropriate
+// placeholder (error) value if the query resulted in a query cycle.
 // Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
+// as they will raise a fatal error on query cycles instead.
 rustc_queries! {
  /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`.
  /// The key is:

compiler/rustc_middle/src/query/plumbing.rs

@@ -62,18 +62,6 @@ pub enum CycleErrorHandling {
  Stash,
 }
 
-pub type WillCacheOnDiskForKeyFn<'tcx, Key> = fn(tcx: TyCtxt<'tcx>, key: &Key) -> bool;
-
-pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn(
- tcx: TyCtxt<'tcx>,
- key: &Key,
- prev_index: SerializedDepNodeIndex,
- index: DepNodeIndex,
-) -> Option<Value>;
-
-pub type IsLoadableFromDiskFn<'tcx, Key> =
- fn(tcx: TyCtxt<'tcx>, key: &Key, index: SerializedDepNodeIndex) -> bool;
-
 pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
 #[derive(Clone, Debug)]
@@ -128,7 +116,7 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  pub cycle_error_handling: CycleErrorHandling,
  pub state: QueryState<'tcx, C::Key>,
  pub cache: C,
- pub will_cache_on_disk_for_key_fn: Option<WillCacheOnDiskForKeyFn<'tcx, C::Key>>,
+ pub will_cache_on_disk_for_key_fn: Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool>,
 
  /// Function pointer that calls `tcx.$query(key)` for this query and
  /// discards the returned value.
@@ -144,11 +132,19 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  /// This should be the only code that calls the provider function.
  pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
 
- pub try_load_from_disk_fn: Option<TryLoadFromDiskFn<'tcx, C::Key, C::Value>>,
- pub is_loadable_from_disk_fn: Option<IsLoadableFromDiskFn<'tcx, C::Key>>,
+ pub try_load_from_disk_fn: Option<
+ fn(
+ tcx: TyCtxt<'tcx>,
+ key: &C::Key,
+ prev_index: SerializedDepNodeIndex,
+ index: DepNodeIndex,
+ ) -> Option<C::Value>,
+ >,
+ pub is_loadable_from_disk_fn:
+ Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool>,
  pub hash_result: HashResult<C::Value>,
  pub value_from_cycle_error:
- fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
+ fn(tcx: TyCtxt<'tcx>, cycle_error: CycleError, guar: ErrorGuaranteed) -> C::Value,
  pub format_value: fn(&C::Value) -> String,
 
  /// Formats a human-readable description of this query and its key, as
@@ -213,7 +209,7 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> {
  pub fn value_from_cycle_error(
  &self,
  tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
+ cycle_error: CycleError,
  guar: ErrorGuaranteed,
  ) -> C::Value {
  (self.value_from_cycle_error)(tcx, cycle_error, guar)
@@ -647,18 +643,6 @@ macro_rules! define_callbacks {
  };
 }
 
-// Each of these queries corresponds to a function pointer field in the
-// `Providers` struct for requesting a value of that type, and a method
-// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
-// which memoizes and does dep-graph tracking, wrapping around the actual
-// `Providers` that the driver creates (using several `rustc_*` crates).
-//
-// The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
-// Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
-
 mod sealed {
  use rustc_hir::def_id::{LocalModDefId, ModDefId};
 

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -13,7 +13,6 @@ mod non_query {
  // We use this for most things when incr. comp. is turned off.
  pub(crate) fn Null<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -26,7 +25,6 @@ mod non_query {
  // We use this for the forever-red node.
  pub(crate) fn Red<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -38,7 +36,6 @@ mod non_query {
 
  pub(crate) fn SideEffect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|tcx, _, prev_index| {
@@ -51,7 +48,6 @@ mod non_query {
 
  pub(crate) fn AnonZeroDeps<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
@@ -61,7 +57,6 @@ mod non_query {
 
  pub(crate) fn TraitSelect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -71,7 +66,6 @@ mod non_query {
 
  pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -81,7 +75,6 @@ mod non_query {
 
  pub(crate) fn CompileMonoItem<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -91,7 +84,6 @@ mod non_query {
 
  pub(crate) fn Metadata<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -117,7 +109,6 @@ where
 
  if is_anon || !key_fingerprint_style.reconstructible() {
  return DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: None,
@@ -126,7 +117,6 @@ where
  }
 
  DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: Some(|tcx, dep_node, _| {

compiler/rustc_query_impl/src/execution.rs

@@ -4,7 +4,7 @@ use std::mem;
 use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::{outline, sharded, sync};
-use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_errors::{FatalError, StashKey};
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
 use rustc_middle::query::plumbing::QueryVTable;
 use rustc_middle::query::{
@@ -108,24 +108,14 @@ fn mk_cycle<'tcx, C: QueryCache>(
  cycle_error: CycleError,
 ) -> C::Value {
  let error = report_cycle(tcx.sess, &cycle_error);
- handle_cycle_error(query, tcx, &cycle_error, error)
-}
-
-fn handle_cycle_error<'tcx, C: QueryCache>(
- query: &'tcx QueryVTable<'tcx, C>,
- tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
- error: Diag<'_>,
-) -> C::Value {
  match query.cycle_error_handling {
  CycleErrorHandling::Error => {
  let guar = error.emit();
  query.value_from_cycle_error(tcx, cycle_error, guar)
  }
  CycleErrorHandling::Fatal => {
- error.emit();
- tcx.dcx().abort_if_errors();
- unreachable!()
+ let guar = error.emit();
+ guar.raise_fatal();
  }
  CycleErrorHandling::DelayBug => {
  let guar = error.delay_as_bug();
@@ -322,15 +312,15 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
 
  // Only call `wait_for_query` if we're using a Rayon thread pool
  // as it will attempt to mark the worker thread as blocked.
- return wait_for_query(query, tcx, span, key, latch, current_job_id);
- }
-
- let id = job.id;
- drop(state_lock);
+ wait_for_query(query, tcx, span, key, latch, current_job_id)
+ } else {
+ let id = job.id;
+ drop(state_lock);
 
- // If we are single-threaded we know that we have cycle error,
- // so we just return the error.
- cycle_error(query, tcx, id, span)
+ // If we are single-threaded we know that we have cycle error,
+ // so we just return the error.
+ cycle_error(query, tcx, id, span)
+ }
  }
  ActiveKeyStatus::Poisoned => FatalError.raise(),
  }
@@ -412,27 +402,21 @@ fn execute_job_non_incr<'tcx, C: QueryCache>(
 ) -> (C::Value, DepNodeIndex) {
  debug_assert!(!tcx.dep_graph.is_fully_enabled());
 
- // Fingerprint the key, just to assert that it doesn't
- // have anything we don't consider hashable
- if cfg!(debug_assertions) {
- let _ = key.to_fingerprint(tcx);
- }
-
  let prof_timer = tcx.prof.query_provider();
  // Call the query provider.
  let result =
  start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key));
  let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
  prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
- // Similarly, fingerprint the result to assert that
-// it doesn't have anything not considered hashable.
-if cfg!(debug_assertions)
-&& let Some(hash_result) = query.hash_result
- {
-tcx.with_stable_hashing_context(|mut hcx| {
-hash_result(&mut hcx, &result);
- });
+ // Sanity: Fingerprint the key and the result to assert they don't contain anything unhashable.
+if cfg!(debug_assertions) {
+ let _ = key.to_fingerprint(tcx);
+if let Some(hash_result) = query.hash_result {
+ tcx.with_stable_hashing_context(|mut hcx| {
+ hash_result(&mut hcx, &result);
+});
+ }
  }
 
  (result, dep_node_index)

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -174,11 +174,6 @@ impl fmt::Debug for DepNode {
 /// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual
 /// jump table instead of large matches.
 pub struct DepKindVTable<'tcx> {
- /// Anonymous queries cannot be replayed from one compiler invocation to the next.
- /// When their result is needed, it is recomputed. They are useful for fine-grained
- /// dependency tracking, and caching within one compiler invocation.
- pub is_anon: bool,
-
  /// Eval-always queries do not track their dependencies, and are always recomputed, even if
  /// their inputs have not changed since the last compiler invocation. The result is still
  /// cached within one compiler invocation.

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::QueryInvocationId;
 use rustc_data_structures::sharded::{self, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU64, Lock};
+use rustc_data_structures::sync::{AtomicU64, Lock, is_dyn_thread_safe};
 use rustc_data_structures::unord::UnordMap;
 use rustc_data_structures::{assert_matches, outline};
 use rustc_errors::DiagInner;
@@ -1302,7 +1302,9 @@ impl CurrentDepGraph {
  prev_graph: &SerializedDepGraph,
  prev_index: SerializedDepNodeIndex,
  ) {
- if let Some(ref nodes_in_current_session) = self.nodes_in_current_session {
+ if !is_dyn_thread_safe()
+ && let Some(ref nodes_in_current_session) = self.nodes_in_current_session
+ {
  debug_assert!(
  !nodes_in_current_session
  .lock()

compiler/rustc_middle/src/queries.rs

@@ -150,10 +150,10 @@ use crate::{dep_graph, mir, thir};
 // `Providers` that the driver creates (using several `rustc_*` crates).
 //
 // The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
+// `ty::query::from_cycle_error::FromCycleError`, which produces an appropriate
+// placeholder (error) value if the query resulted in a query cycle.
 // Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
+// as they will raise a fatal error on query cycles instead.
 rustc_queries! {
  /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`.
  /// The key is:

compiler/rustc_middle/src/query/plumbing.rs

@@ -62,18 +62,6 @@ pub enum CycleErrorHandling {
  Stash,
 }
 
-pub type WillCacheOnDiskForKeyFn<'tcx, Key> = fn(tcx: TyCtxt<'tcx>, key: &Key) -> bool;
-
-pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn(
- tcx: TyCtxt<'tcx>,
- key: &Key,
- prev_index: SerializedDepNodeIndex,
- index: DepNodeIndex,
-) -> Option<Value>;
-
-pub type IsLoadableFromDiskFn<'tcx, Key> =
- fn(tcx: TyCtxt<'tcx>, key: &Key, index: SerializedDepNodeIndex) -> bool;
-
 pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
 #[derive(Clone, Debug)]
@@ -128,7 +116,7 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  pub cycle_error_handling: CycleErrorHandling,
  pub state: QueryState<'tcx, C::Key>,
  pub cache: C,
- pub will_cache_on_disk_for_key_fn: Option<WillCacheOnDiskForKeyFn<'tcx, C::Key>>,
+ pub will_cache_on_disk_for_key_fn: Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool>,
 
  /// Function pointer that calls `tcx.$query(key)` for this query and
  /// discards the returned value.
@@ -144,11 +132,19 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  /// This should be the only code that calls the provider function.
  pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
 
- pub try_load_from_disk_fn: Option<TryLoadFromDiskFn<'tcx, C::Key, C::Value>>,
- pub is_loadable_from_disk_fn: Option<IsLoadableFromDiskFn<'tcx, C::Key>>,
+ pub try_load_from_disk_fn: Option<
+ fn(
+ tcx: TyCtxt<'tcx>,
+ key: &C::Key,
+ prev_index: SerializedDepNodeIndex,
+ index: DepNodeIndex,
+ ) -> Option<C::Value>,
+ >,
+ pub is_loadable_from_disk_fn:
+ Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool>,
  pub hash_result: HashResult<C::Value>,
  pub value_from_cycle_error:
- fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
+ fn(tcx: TyCtxt<'tcx>, cycle_error: CycleError, guar: ErrorGuaranteed) -> C::Value,
  pub format_value: fn(&C::Value) -> String,
 
  /// Formats a human-readable description of this query and its key, as
@@ -213,7 +209,7 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> {
  pub fn value_from_cycle_error(
  &self,
  tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
+ cycle_error: CycleError,
  guar: ErrorGuaranteed,
  ) -> C::Value {
  (self.value_from_cycle_error)(tcx, cycle_error, guar)
@@ -647,18 +643,6 @@ macro_rules! define_callbacks {
  };
 }
 
-// Each of these queries corresponds to a function pointer field in the
-// `Providers` struct for requesting a value of that type, and a method
-// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
-// which memoizes and does dep-graph tracking, wrapping around the actual
-// `Providers` that the driver creates (using several `rustc_*` crates).
-//
-// The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
-// Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
-
 mod sealed {
  use rustc_hir::def_id::{LocalModDefId, ModDefId};
 

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -13,7 +13,6 @@ mod non_query {
  // We use this for most things when incr. comp. is turned off.
  pub(crate) fn Null<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -26,7 +25,6 @@ mod non_query {
  // We use this for the forever-red node.
  pub(crate) fn Red<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -38,7 +36,6 @@ mod non_query {
 
  pub(crate) fn SideEffect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|tcx, _, prev_index| {
@@ -51,7 +48,6 @@ mod non_query {
 
  pub(crate) fn AnonZeroDeps<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
@@ -61,7 +57,6 @@ mod non_query {
 
  pub(crate) fn TraitSelect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -71,7 +66,6 @@ mod non_query {
 
  pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -81,7 +75,6 @@ mod non_query {
 
  pub(crate) fn CompileMonoItem<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -91,7 +84,6 @@ mod non_query {
 
  pub(crate) fn Metadata<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -117,7 +109,6 @@ where
 
  if is_anon || !key_fingerprint_style.reconstructible() {
  return DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: None,
@@ -126,7 +117,6 @@ where
  }
 
  DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: Some(|tcx, dep_node, _| {

compiler/rustc_query_impl/src/execution.rs

@@ -4,7 +4,7 @@ use std::mem;
 use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::{outline, sharded, sync};
-use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_errors::{FatalError, StashKey};
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
 use rustc_middle::query::plumbing::QueryVTable;
 use rustc_middle::query::{
@@ -108,24 +108,14 @@ fn mk_cycle<'tcx, C: QueryCache>(
  cycle_error: CycleError,
 ) -> C::Value {
  let error = report_cycle(tcx.sess, &cycle_error);
- handle_cycle_error(query, tcx, &cycle_error, error)
-}
-
-fn handle_cycle_error<'tcx, C: QueryCache>(
- query: &'tcx QueryVTable<'tcx, C>,
- tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
- error: Diag<'_>,
-) -> C::Value {
  match query.cycle_error_handling {
  CycleErrorHandling::Error => {
  let guar = error.emit();
  query.value_from_cycle_error(tcx, cycle_error, guar)
  }
  CycleErrorHandling::Fatal => {
- error.emit();
- tcx.dcx().abort_if_errors();
- unreachable!()
+ let guar = error.emit();
+ guar.raise_fatal();
  }
  CycleErrorHandling::DelayBug => {
  let guar = error.delay_as_bug();
@@ -322,15 +312,15 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
 
  // Only call `wait_for_query` if we're using a Rayon thread pool
  // as it will attempt to mark the worker thread as blocked.
- return wait_for_query(query, tcx, span, key, latch, current_job_id);
- }
-
- let id = job.id;
- drop(state_lock);
+ wait_for_query(query, tcx, span, key, latch, current_job_id)
+ } else {
+ let id = job.id;
+ drop(state_lock);
 
- // If we are single-threaded we know that we have cycle error,
- // so we just return the error.
- cycle_error(query, tcx, id, span)
+ // If we are single-threaded we know that we have cycle error,
+ // so we just return the error.
+ cycle_error(query, tcx, id, span)
+ }
  }
  ActiveKeyStatus::Poisoned => FatalError.raise(),
  }
@@ -412,27 +402,21 @@ fn execute_job_non_incr<'tcx, C: QueryCache>(
 ) -> (C::Value, DepNodeIndex) {
  debug_assert!(!tcx.dep_graph.is_fully_enabled());
 
- // Fingerprint the key, just to assert that it doesn't
- // have anything we don't consider hashable
- if cfg!(debug_assertions) {
- let _ = key.to_fingerprint(tcx);
- }
-
  let prof_timer = tcx.prof.query_provider();
  // Call the query provider.
  let result =
  start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key));
  let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
  prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
- // Similarly, fingerprint the result to assert that
-// it doesn't have anything not considered hashable.
-if cfg!(debug_assertions)
-&& let Some(hash_result) = query.hash_result
- {
-tcx.with_stable_hashing_context(|mut hcx| {
-hash_result(&mut hcx, &result);
- });
+ // Sanity: Fingerprint the key and the result to assert they don't contain anything unhashable.
+if cfg!(debug_assertions) {
+ let _ = key.to_fingerprint(tcx);
+if let Some(hash_result) = query.hash_result {
+ tcx.with_stable_hashing_context(|mut hcx| {
+ hash_result(&mut hcx, &result);
+});
+ }
  }
 
  (result, dep_node_index)

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -174,11 +174,6 @@ impl fmt::Debug for DepNode {
 /// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual
 /// jump table instead of large matches.
 pub struct DepKindVTable<'tcx> {
- /// Anonymous queries cannot be replayed from one compiler invocation to the next.
- /// When their result is needed, it is recomputed. They are useful for fine-grained
- /// dependency tracking, and caching within one compiler invocation.
- pub is_anon: bool,
-
  /// Eval-always queries do not track their dependencies, and are always recomputed, even if
  /// their inputs have not changed since the last compiler invocation. The result is still
  /// cached within one compiler invocation.

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::QueryInvocationId;
 use rustc_data_structures::sharded::{self, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU64, Lock};
+use rustc_data_structures::sync::{AtomicU64, Lock, is_dyn_thread_safe};
 use rustc_data_structures::unord::UnordMap;
 use rustc_data_structures::{assert_matches, outline};
 use rustc_errors::DiagInner;
@@ -1302,7 +1302,9 @@ impl CurrentDepGraph {
  prev_graph: &SerializedDepGraph,
  prev_index: SerializedDepNodeIndex,
  ) {
- if let Some(ref nodes_in_current_session) = self.nodes_in_current_session {
+ if !is_dyn_thread_safe()
+ && let Some(ref nodes_in_current_session) = self.nodes_in_current_session
+ {
  debug_assert!(
  !nodes_in_current_session
  .lock()

compiler/rustc_middle/src/queries.rs

@@ -150,10 +150,10 @@ use crate::{dep_graph, mir, thir};
 // `Providers` that the driver creates (using several `rustc_*` crates).
 //
 // The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
+// `ty::query::from_cycle_error::FromCycleError`, which produces an appropriate
+// placeholder (error) value if the query resulted in a query cycle.
 // Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
+// as they will raise a fatal error on query cycles instead.
 rustc_queries! {
  /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`.
  /// The key is:

compiler/rustc_middle/src/query/plumbing.rs

@@ -62,18 +62,6 @@ pub enum CycleErrorHandling {
  Stash,
 }
 
-pub type WillCacheOnDiskForKeyFn<'tcx, Key> = fn(tcx: TyCtxt<'tcx>, key: &Key) -> bool;
-
-pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn(
- tcx: TyCtxt<'tcx>,
- key: &Key,
- prev_index: SerializedDepNodeIndex,
- index: DepNodeIndex,
-) -> Option<Value>;
-
-pub type IsLoadableFromDiskFn<'tcx, Key> =
- fn(tcx: TyCtxt<'tcx>, key: &Key, index: SerializedDepNodeIndex) -> bool;
-
 pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
 #[derive(Clone, Debug)]
@@ -128,7 +116,7 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  pub cycle_error_handling: CycleErrorHandling,
  pub state: QueryState<'tcx, C::Key>,
  pub cache: C,
- pub will_cache_on_disk_for_key_fn: Option<WillCacheOnDiskForKeyFn<'tcx, C::Key>>,
+ pub will_cache_on_disk_for_key_fn: Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool>,
 
  /// Function pointer that calls `tcx.$query(key)` for this query and
  /// discards the returned value.
@@ -144,11 +132,19 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  /// This should be the only code that calls the provider function.
  pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
 
- pub try_load_from_disk_fn: Option<TryLoadFromDiskFn<'tcx, C::Key, C::Value>>,
- pub is_loadable_from_disk_fn: Option<IsLoadableFromDiskFn<'tcx, C::Key>>,
+ pub try_load_from_disk_fn: Option<
+ fn(
+ tcx: TyCtxt<'tcx>,
+ key: &C::Key,
+ prev_index: SerializedDepNodeIndex,
+ index: DepNodeIndex,
+ ) -> Option<C::Value>,
+ >,
+ pub is_loadable_from_disk_fn:
+ Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool>,
  pub hash_result: HashResult<C::Value>,
  pub value_from_cycle_error:
- fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
+ fn(tcx: TyCtxt<'tcx>, cycle_error: CycleError, guar: ErrorGuaranteed) -> C::Value,
  pub format_value: fn(&C::Value) -> String,
 
  /// Formats a human-readable description of this query and its key, as
@@ -213,7 +209,7 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> {
  pub fn value_from_cycle_error(
  &self,
  tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
+ cycle_error: CycleError,
  guar: ErrorGuaranteed,
  ) -> C::Value {
  (self.value_from_cycle_error)(tcx, cycle_error, guar)
@@ -647,18 +643,6 @@ macro_rules! define_callbacks {
  };
 }
 
-// Each of these queries corresponds to a function pointer field in the
-// `Providers` struct for requesting a value of that type, and a method
-// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
-// which memoizes and does dep-graph tracking, wrapping around the actual
-// `Providers` that the driver creates (using several `rustc_*` crates).
-//
-// The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
-// Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
-
 mod sealed {
  use rustc_hir::def_id::{LocalModDefId, ModDefId};
 

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -13,7 +13,6 @@ mod non_query {
  // We use this for most things when incr. comp. is turned off.
  pub(crate) fn Null<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -26,7 +25,6 @@ mod non_query {
  // We use this for the forever-red node.
  pub(crate) fn Red<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -38,7 +36,6 @@ mod non_query {
 
  pub(crate) fn SideEffect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|tcx, _, prev_index| {
@@ -51,7 +48,6 @@ mod non_query {
 
  pub(crate) fn AnonZeroDeps<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
@@ -61,7 +57,6 @@ mod non_query {
 
  pub(crate) fn TraitSelect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -71,7 +66,6 @@ mod non_query {
 
  pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -81,7 +75,6 @@ mod non_query {
 
  pub(crate) fn CompileMonoItem<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -91,7 +84,6 @@ mod non_query {
 
  pub(crate) fn Metadata<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -117,7 +109,6 @@ where
 
  if is_anon || !key_fingerprint_style.reconstructible() {
  return DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: None,
@@ -126,7 +117,6 @@ where
  }
 
  DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: Some(|tcx, dep_node, _| {

compiler/rustc_query_impl/src/execution.rs

@@ -4,7 +4,7 @@ use std::mem;
 use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::{outline, sharded, sync};
-use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_errors::{FatalError, StashKey};
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
 use rustc_middle::query::plumbing::QueryVTable;
 use rustc_middle::query::{
@@ -108,24 +108,14 @@ fn mk_cycle<'tcx, C: QueryCache>(
  cycle_error: CycleError,
 ) -> C::Value {
  let error = report_cycle(tcx.sess, &cycle_error);
- handle_cycle_error(query, tcx, &cycle_error, error)
-}
-
-fn handle_cycle_error<'tcx, C: QueryCache>(
- query: &'tcx QueryVTable<'tcx, C>,
- tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
- error: Diag<'_>,
-) -> C::Value {
  match query.cycle_error_handling {
  CycleErrorHandling::Error => {
  let guar = error.emit();
  query.value_from_cycle_error(tcx, cycle_error, guar)
  }
  CycleErrorHandling::Fatal => {
- error.emit();
- tcx.dcx().abort_if_errors();
- unreachable!()
+ let guar = error.emit();
+ guar.raise_fatal();
  }
  CycleErrorHandling::DelayBug => {
  let guar = error.delay_as_bug();
@@ -322,15 +312,15 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
 
  // Only call `wait_for_query` if we're using a Rayon thread pool
  // as it will attempt to mark the worker thread as blocked.
- return wait_for_query(query, tcx, span, key, latch, current_job_id);
- }
-
- let id = job.id;
- drop(state_lock);
+ wait_for_query(query, tcx, span, key, latch, current_job_id)
+ } else {
+ let id = job.id;
+ drop(state_lock);
 
- // If we are single-threaded we know that we have cycle error,
- // so we just return the error.
- cycle_error(query, tcx, id, span)
+ // If we are single-threaded we know that we have cycle error,
+ // so we just return the error.
+ cycle_error(query, tcx, id, span)
+ }
  }
  ActiveKeyStatus::Poisoned => FatalError.raise(),
  }
@@ -412,27 +402,21 @@ fn execute_job_non_incr<'tcx, C: QueryCache>(
 ) -> (C::Value, DepNodeIndex) {
  debug_assert!(!tcx.dep_graph.is_fully_enabled());
 
- // Fingerprint the key, just to assert that it doesn't
- // have anything we don't consider hashable
- if cfg!(debug_assertions) {
- let _ = key.to_fingerprint(tcx);
- }
-
  let prof_timer = tcx.prof.query_provider();
  // Call the query provider.
  let result =
  start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key));
  let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
  prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
- // Similarly, fingerprint the result to assert that
-// it doesn't have anything not considered hashable.
-if cfg!(debug_assertions)
-&& let Some(hash_result) = query.hash_result
- {
-tcx.with_stable_hashing_context(|mut hcx| {
-hash_result(&mut hcx, &result);
- });
+ // Sanity: Fingerprint the key and the result to assert they don't contain anything unhashable.
+if cfg!(debug_assertions) {
+ let _ = key.to_fingerprint(tcx);
+if let Some(hash_result) = query.hash_result {
+ tcx.with_stable_hashing_context(|mut hcx| {
+ hash_result(&mut hcx, &result);
+});
+ }
  }
 
  (result, dep_node_index)

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -174,11 +174,6 @@ impl fmt::Debug for DepNode {
 /// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual
 /// jump table instead of large matches.
 pub struct DepKindVTable<'tcx> {
- /// Anonymous queries cannot be replayed from one compiler invocation to the next.
- /// When their result is needed, it is recomputed. They are useful for fine-grained
- /// dependency tracking, and caching within one compiler invocation.
- pub is_anon: bool,
-
  /// Eval-always queries do not track their dependencies, and are always recomputed, even if
  /// their inputs have not changed since the last compiler invocation. The result is still
  /// cached within one compiler invocation.

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::QueryInvocationId;
 use rustc_data_structures::sharded::{self, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU64, Lock};
+use rustc_data_structures::sync::{AtomicU64, Lock, is_dyn_thread_safe};
 use rustc_data_structures::unord::UnordMap;
 use rustc_data_structures::{assert_matches, outline};
 use rustc_errors::DiagInner;
@@ -1302,7 +1302,9 @@ impl CurrentDepGraph {
  prev_graph: &SerializedDepGraph,
  prev_index: SerializedDepNodeIndex,
  ) {
- if let Some(ref nodes_in_current_session) = self.nodes_in_current_session {
+ if !is_dyn_thread_safe()
+ && let Some(ref nodes_in_current_session) = self.nodes_in_current_session
+ {
  debug_assert!(
  !nodes_in_current_session
  .lock()

compiler/rustc_middle/src/queries.rs

@@ -150,10 +150,10 @@ use crate::{dep_graph, mir, thir};
 // `Providers` that the driver creates (using several `rustc_*` crates).
 //
 // The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
+// `ty::query::from_cycle_error::FromCycleError`, which produces an appropriate
+// placeholder (error) value if the query resulted in a query cycle.
 // Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
+// as they will raise a fatal error on query cycles instead.
 rustc_queries! {
  /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`.
  /// The key is:

compiler/rustc_middle/src/query/plumbing.rs

@@ -62,18 +62,6 @@ pub enum CycleErrorHandling {
  Stash,
 }
 
-pub type WillCacheOnDiskForKeyFn<'tcx, Key> = fn(tcx: TyCtxt<'tcx>, key: &Key) -> bool;
-
-pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn(
- tcx: TyCtxt<'tcx>,
- key: &Key,
- prev_index: SerializedDepNodeIndex,
- index: DepNodeIndex,
-) -> Option<Value>;
-
-pub type IsLoadableFromDiskFn<'tcx, Key> =
- fn(tcx: TyCtxt<'tcx>, key: &Key, index: SerializedDepNodeIndex) -> bool;
-
 pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
 #[derive(Clone, Debug)]
@@ -128,7 +116,7 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  pub cycle_error_handling: CycleErrorHandling,
  pub state: QueryState<'tcx, C::Key>,
  pub cache: C,
- pub will_cache_on_disk_for_key_fn: Option<WillCacheOnDiskForKeyFn<'tcx, C::Key>>,
+ pub will_cache_on_disk_for_key_fn: Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool>,
 
  /// Function pointer that calls `tcx.$query(key)` for this query and
  /// discards the returned value.
@@ -144,11 +132,19 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  /// This should be the only code that calls the provider function.
  pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
 
- pub try_load_from_disk_fn: Option<TryLoadFromDiskFn<'tcx, C::Key, C::Value>>,
- pub is_loadable_from_disk_fn: Option<IsLoadableFromDiskFn<'tcx, C::Key>>,
+ pub try_load_from_disk_fn: Option<
+ fn(
+ tcx: TyCtxt<'tcx>,
+ key: &C::Key,
+ prev_index: SerializedDepNodeIndex,
+ index: DepNodeIndex,
+ ) -> Option<C::Value>,
+ >,
+ pub is_loadable_from_disk_fn:
+ Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool>,
  pub hash_result: HashResult<C::Value>,
  pub value_from_cycle_error:
- fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
+ fn(tcx: TyCtxt<'tcx>, cycle_error: CycleError, guar: ErrorGuaranteed) -> C::Value,
  pub format_value: fn(&C::Value) -> String,
 
  /// Formats a human-readable description of this query and its key, as
@@ -213,7 +209,7 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> {
  pub fn value_from_cycle_error(
  &self,
  tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
+ cycle_error: CycleError,
  guar: ErrorGuaranteed,
  ) -> C::Value {
  (self.value_from_cycle_error)(tcx, cycle_error, guar)
@@ -647,18 +643,6 @@ macro_rules! define_callbacks {
  };
 }
 
-// Each of these queries corresponds to a function pointer field in the
-// `Providers` struct for requesting a value of that type, and a method
-// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
-// which memoizes and does dep-graph tracking, wrapping around the actual
-// `Providers` that the driver creates (using several `rustc_*` crates).
-//
-// The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
-// Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
-
 mod sealed {
  use rustc_hir::def_id::{LocalModDefId, ModDefId};
 

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -13,7 +13,6 @@ mod non_query {
  // We use this for most things when incr. comp. is turned off.
  pub(crate) fn Null<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -26,7 +25,6 @@ mod non_query {
  // We use this for the forever-red node.
  pub(crate) fn Red<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -38,7 +36,6 @@ mod non_query {
 
  pub(crate) fn SideEffect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|tcx, _, prev_index| {
@@ -51,7 +48,6 @@ mod non_query {
 
  pub(crate) fn AnonZeroDeps<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
@@ -61,7 +57,6 @@ mod non_query {
 
  pub(crate) fn TraitSelect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -71,7 +66,6 @@ mod non_query {
 
  pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -81,7 +75,6 @@ mod non_query {
 
  pub(crate) fn CompileMonoItem<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -91,7 +84,6 @@ mod non_query {
 
  pub(crate) fn Metadata<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -117,7 +109,6 @@ where
 
  if is_anon || !key_fingerprint_style.reconstructible() {
  return DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: None,
@@ -126,7 +117,6 @@ where
  }
 
  DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: Some(|tcx, dep_node, _| {

compiler/rustc_query_impl/src/execution.rs

@@ -4,7 +4,7 @@ use std::mem;
 use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::{outline, sharded, sync};
-use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_errors::{FatalError, StashKey};
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
 use rustc_middle::query::plumbing::QueryVTable;
 use rustc_middle::query::{
@@ -108,24 +108,14 @@ fn mk_cycle<'tcx, C: QueryCache>(
  cycle_error: CycleError,
 ) -> C::Value {
  let error = report_cycle(tcx.sess, &cycle_error);
- handle_cycle_error(query, tcx, &cycle_error, error)
-}
-
-fn handle_cycle_error<'tcx, C: QueryCache>(
- query: &'tcx QueryVTable<'tcx, C>,
- tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
- error: Diag<'_>,
-) -> C::Value {
  match query.cycle_error_handling {
  CycleErrorHandling::Error => {
  let guar = error.emit();
  query.value_from_cycle_error(tcx, cycle_error, guar)
  }
  CycleErrorHandling::Fatal => {
- error.emit();
- tcx.dcx().abort_if_errors();
- unreachable!()
+ let guar = error.emit();
+ guar.raise_fatal();
  }
  CycleErrorHandling::DelayBug => {
  let guar = error.delay_as_bug();
@@ -322,15 +312,15 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
 
  // Only call `wait_for_query` if we're using a Rayon thread pool
  // as it will attempt to mark the worker thread as blocked.
- return wait_for_query(query, tcx, span, key, latch, current_job_id);
- }
-
- let id = job.id;
- drop(state_lock);
+ wait_for_query(query, tcx, span, key, latch, current_job_id)
+ } else {
+ let id = job.id;
+ drop(state_lock);
 
- // If we are single-threaded we know that we have cycle error,
- // so we just return the error.
- cycle_error(query, tcx, id, span)
+ // If we are single-threaded we know that we have cycle error,
+ // so we just return the error.
+ cycle_error(query, tcx, id, span)
+ }
  }
  ActiveKeyStatus::Poisoned => FatalError.raise(),
  }
@@ -412,27 +402,21 @@ fn execute_job_non_incr<'tcx, C: QueryCache>(
 ) -> (C::Value, DepNodeIndex) {
  debug_assert!(!tcx.dep_graph.is_fully_enabled());
 
- // Fingerprint the key, just to assert that it doesn't
- // have anything we don't consider hashable
- if cfg!(debug_assertions) {
- let _ = key.to_fingerprint(tcx);
- }
-
  let prof_timer = tcx.prof.query_provider();
  // Call the query provider.
  let result =
  start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key));
  let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
  prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
- // Similarly, fingerprint the result to assert that
-// it doesn't have anything not considered hashable.
-if cfg!(debug_assertions)
-&& let Some(hash_result) = query.hash_result
- {
-tcx.with_stable_hashing_context(|mut hcx| {
-hash_result(&mut hcx, &result);
- });
+ // Sanity: Fingerprint the key and the result to assert they don't contain anything unhashable.
+if cfg!(debug_assertions) {
+ let _ = key.to_fingerprint(tcx);
+if let Some(hash_result) = query.hash_result {
+ tcx.with_stable_hashing_context(|mut hcx| {
+ hash_result(&mut hcx, &result);
+});
+ }
  }
 
  (result, dep_node_index)

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -174,11 +174,6 @@ impl fmt::Debug for DepNode {
 /// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual
 /// jump table instead of large matches.
 pub struct DepKindVTable<'tcx> {
- /// Anonymous queries cannot be replayed from one compiler invocation to the next.
- /// When their result is needed, it is recomputed. They are useful for fine-grained
- /// dependency tracking, and caching within one compiler invocation.
- pub is_anon: bool,
-
  /// Eval-always queries do not track their dependencies, and are always recomputed, even if
  /// their inputs have not changed since the last compiler invocation. The result is still
  /// cached within one compiler invocation.

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::QueryInvocationId;
 use rustc_data_structures::sharded::{self, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU64, Lock};
+use rustc_data_structures::sync::{AtomicU64, Lock, is_dyn_thread_safe};
 use rustc_data_structures::unord::UnordMap;
 use rustc_data_structures::{assert_matches, outline};
 use rustc_errors::DiagInner;
@@ -1302,7 +1302,9 @@ impl CurrentDepGraph {
  prev_graph: &SerializedDepGraph,
  prev_index: SerializedDepNodeIndex,
  ) {
- if let Some(ref nodes_in_current_session) = self.nodes_in_current_session {
+ if !is_dyn_thread_safe()
+ && let Some(ref nodes_in_current_session) = self.nodes_in_current_session
+ {
  debug_assert!(
  !nodes_in_current_session
  .lock()

compiler/rustc_middle/src/queries.rs

@@ -150,10 +150,10 @@ use crate::{dep_graph, mir, thir};
 // `Providers` that the driver creates (using several `rustc_*` crates).
 //
 // The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
+// `ty::query::from_cycle_error::FromCycleError`, which produces an appropriate
+// placeholder (error) value if the query resulted in a query cycle.
 // Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
+// as they will raise a fatal error on query cycles instead.
 rustc_queries! {
  /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`.
  /// The key is:

compiler/rustc_middle/src/query/plumbing.rs

@@ -62,18 +62,6 @@ pub enum CycleErrorHandling {
  Stash,
 }
 
-pub type WillCacheOnDiskForKeyFn<'tcx, Key> = fn(tcx: TyCtxt<'tcx>, key: &Key) -> bool;
-
-pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn(
- tcx: TyCtxt<'tcx>,
- key: &Key,
- prev_index: SerializedDepNodeIndex,
- index: DepNodeIndex,
-) -> Option<Value>;
-
-pub type IsLoadableFromDiskFn<'tcx, Key> =
- fn(tcx: TyCtxt<'tcx>, key: &Key, index: SerializedDepNodeIndex) -> bool;
-
 pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
 #[derive(Clone, Debug)]
@@ -128,7 +116,7 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  pub cycle_error_handling: CycleErrorHandling,
  pub state: QueryState<'tcx, C::Key>,
  pub cache: C,
- pub will_cache_on_disk_for_key_fn: Option<WillCacheOnDiskForKeyFn<'tcx, C::Key>>,
+ pub will_cache_on_disk_for_key_fn: Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool>,
 
  /// Function pointer that calls `tcx.$query(key)` for this query and
  /// discards the returned value.
@@ -144,11 +132,19 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  /// This should be the only code that calls the provider function.
  pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
 
- pub try_load_from_disk_fn: Option<TryLoadFromDiskFn<'tcx, C::Key, C::Value>>,
- pub is_loadable_from_disk_fn: Option<IsLoadableFromDiskFn<'tcx, C::Key>>,
+ pub try_load_from_disk_fn: Option<
+ fn(
+ tcx: TyCtxt<'tcx>,
+ key: &C::Key,
+ prev_index: SerializedDepNodeIndex,
+ index: DepNodeIndex,
+ ) -> Option<C::Value>,
+ >,
+ pub is_loadable_from_disk_fn:
+ Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool>,
  pub hash_result: HashResult<C::Value>,
  pub value_from_cycle_error:
- fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
+ fn(tcx: TyCtxt<'tcx>, cycle_error: CycleError, guar: ErrorGuaranteed) -> C::Value,
  pub format_value: fn(&C::Value) -> String,
 
  /// Formats a human-readable description of this query and its key, as
@@ -213,7 +209,7 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> {
  pub fn value_from_cycle_error(
  &self,
  tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
+ cycle_error: CycleError,
  guar: ErrorGuaranteed,
  ) -> C::Value {
  (self.value_from_cycle_error)(tcx, cycle_error, guar)
@@ -647,18 +643,6 @@ macro_rules! define_callbacks {
  };
 }
 
-// Each of these queries corresponds to a function pointer field in the
-// `Providers` struct for requesting a value of that type, and a method
-// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
-// which memoizes and does dep-graph tracking, wrapping around the actual
-// `Providers` that the driver creates (using several `rustc_*` crates).
-//
-// The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
-// Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
-
 mod sealed {
  use rustc_hir::def_id::{LocalModDefId, ModDefId};
 

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -13,7 +13,6 @@ mod non_query {
  // We use this for most things when incr. comp. is turned off.
  pub(crate) fn Null<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -26,7 +25,6 @@ mod non_query {
  // We use this for the forever-red node.
  pub(crate) fn Red<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -38,7 +36,6 @@ mod non_query {
 
  pub(crate) fn SideEffect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|tcx, _, prev_index| {
@@ -51,7 +48,6 @@ mod non_query {
 
  pub(crate) fn AnonZeroDeps<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
@@ -61,7 +57,6 @@ mod non_query {
 
  pub(crate) fn TraitSelect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -71,7 +66,6 @@ mod non_query {
 
  pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -81,7 +75,6 @@ mod non_query {
 
  pub(crate) fn CompileMonoItem<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -91,7 +84,6 @@ mod non_query {
 
  pub(crate) fn Metadata<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -117,7 +109,6 @@ where
 
  if is_anon || !key_fingerprint_style.reconstructible() {
  return DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: None,
@@ -126,7 +117,6 @@ where
  }
 
  DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: Some(|tcx, dep_node, _| {

compiler/rustc_query_impl/src/execution.rs

@@ -4,7 +4,7 @@ use std::mem;
 use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::{outline, sharded, sync};
-use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_errors::{FatalError, StashKey};
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
 use rustc_middle::query::plumbing::QueryVTable;
 use rustc_middle::query::{
@@ -108,24 +108,14 @@ fn mk_cycle<'tcx, C: QueryCache>(
  cycle_error: CycleError,
 ) -> C::Value {
  let error = report_cycle(tcx.sess, &cycle_error);
- handle_cycle_error(query, tcx, &cycle_error, error)
-}
-
-fn handle_cycle_error<'tcx, C: QueryCache>(
- query: &'tcx QueryVTable<'tcx, C>,
- tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
- error: Diag<'_>,
-) -> C::Value {
  match query.cycle_error_handling {
  CycleErrorHandling::Error => {
  let guar = error.emit();
  query.value_from_cycle_error(tcx, cycle_error, guar)
  }
  CycleErrorHandling::Fatal => {
- error.emit();
- tcx.dcx().abort_if_errors();
- unreachable!()
+ let guar = error.emit();
+ guar.raise_fatal();
  }
  CycleErrorHandling::DelayBug => {
  let guar = error.delay_as_bug();
@@ -322,15 +312,15 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
 
  // Only call `wait_for_query` if we're using a Rayon thread pool
  // as it will attempt to mark the worker thread as blocked.
- return wait_for_query(query, tcx, span, key, latch, current_job_id);
- }
-
- let id = job.id;
- drop(state_lock);
+ wait_for_query(query, tcx, span, key, latch, current_job_id)
+ } else {
+ let id = job.id;
+ drop(state_lock);
 
- // If we are single-threaded we know that we have cycle error,
- // so we just return the error.
- cycle_error(query, tcx, id, span)
+ // If we are single-threaded we know that we have cycle error,
+ // so we just return the error.
+ cycle_error(query, tcx, id, span)
+ }
  }
  ActiveKeyStatus::Poisoned => FatalError.raise(),
  }
@@ -412,27 +402,21 @@ fn execute_job_non_incr<'tcx, C: QueryCache>(
 ) -> (C::Value, DepNodeIndex) {
  debug_assert!(!tcx.dep_graph.is_fully_enabled());
 
- // Fingerprint the key, just to assert that it doesn't
- // have anything we don't consider hashable
- if cfg!(debug_assertions) {
- let _ = key.to_fingerprint(tcx);
- }
-
  let prof_timer = tcx.prof.query_provider();
  // Call the query provider.
  let result =
  start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key));
  let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
  prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
- // Similarly, fingerprint the result to assert that
-// it doesn't have anything not considered hashable.
-if cfg!(debug_assertions)
-&& let Some(hash_result) = query.hash_result
- {
-tcx.with_stable_hashing_context(|mut hcx| {
-hash_result(&mut hcx, &result);
- });
+ // Sanity: Fingerprint the key and the result to assert they don't contain anything unhashable.
+if cfg!(debug_assertions) {
+ let _ = key.to_fingerprint(tcx);
+if let Some(hash_result) = query.hash_result {
+ tcx.with_stable_hashing_context(|mut hcx| {
+ hash_result(&mut hcx, &result);
+});
+ }
  }
 
  (result, dep_node_index)

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -174,11 +174,6 @@ impl fmt::Debug for DepNode {
 /// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual
 /// jump table instead of large matches.
 pub struct DepKindVTable<'tcx> {
- /// Anonymous queries cannot be replayed from one compiler invocation to the next.
- /// When their result is needed, it is recomputed. They are useful for fine-grained
- /// dependency tracking, and caching within one compiler invocation.
- pub is_anon: bool,
-
  /// Eval-always queries do not track their dependencies, and are always recomputed, even if
  /// their inputs have not changed since the last compiler invocation. The result is still
  /// cached within one compiler invocation.

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::QueryInvocationId;
 use rustc_data_structures::sharded::{self, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU64, Lock};
+use rustc_data_structures::sync::{AtomicU64, Lock, is_dyn_thread_safe};
 use rustc_data_structures::unord::UnordMap;
 use rustc_data_structures::{assert_matches, outline};
 use rustc_errors::DiagInner;
@@ -1302,7 +1302,9 @@ impl CurrentDepGraph {
  prev_graph: &SerializedDepGraph,
  prev_index: SerializedDepNodeIndex,
  ) {
- if let Some(ref nodes_in_current_session) = self.nodes_in_current_session {
+ if !is_dyn_thread_safe()
+ && let Some(ref nodes_in_current_session) = self.nodes_in_current_session
+ {
  debug_assert!(
  !nodes_in_current_session
  .lock()

compiler/rustc_middle/src/queries.rs

@@ -150,10 +150,10 @@ use crate::{dep_graph, mir, thir};
 // `Providers` that the driver creates (using several `rustc_*` crates).
 //
 // The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
+// `ty::query::from_cycle_error::FromCycleError`, which produces an appropriate
+// placeholder (error) value if the query resulted in a query cycle.
 // Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
+// as they will raise a fatal error on query cycles instead.
 rustc_queries! {
  /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`.
  /// The key is:

compiler/rustc_middle/src/query/plumbing.rs

@@ -62,18 +62,6 @@ pub enum CycleErrorHandling {
  Stash,
 }
 
-pub type WillCacheOnDiskForKeyFn<'tcx, Key> = fn(tcx: TyCtxt<'tcx>, key: &Key) -> bool;
-
-pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn(
- tcx: TyCtxt<'tcx>,
- key: &Key,
- prev_index: SerializedDepNodeIndex,
- index: DepNodeIndex,
-) -> Option<Value>;
-
-pub type IsLoadableFromDiskFn<'tcx, Key> =
- fn(tcx: TyCtxt<'tcx>, key: &Key, index: SerializedDepNodeIndex) -> bool;
-
 pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
 #[derive(Clone, Debug)]
@@ -128,7 +116,7 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  pub cycle_error_handling: CycleErrorHandling,
  pub state: QueryState<'tcx, C::Key>,
  pub cache: C,
- pub will_cache_on_disk_for_key_fn: Option<WillCacheOnDiskForKeyFn<'tcx, C::Key>>,
+ pub will_cache_on_disk_for_key_fn: Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool>,
 
  /// Function pointer that calls `tcx.$query(key)` for this query and
  /// discards the returned value.
@@ -144,11 +132,19 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  /// This should be the only code that calls the provider function.
  pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
 
- pub try_load_from_disk_fn: Option<TryLoadFromDiskFn<'tcx, C::Key, C::Value>>,
- pub is_loadable_from_disk_fn: Option<IsLoadableFromDiskFn<'tcx, C::Key>>,
+ pub try_load_from_disk_fn: Option<
+ fn(
+ tcx: TyCtxt<'tcx>,
+ key: &C::Key,
+ prev_index: SerializedDepNodeIndex,
+ index: DepNodeIndex,
+ ) -> Option<C::Value>,
+ >,
+ pub is_loadable_from_disk_fn:
+ Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool>,
  pub hash_result: HashResult<C::Value>,
  pub value_from_cycle_error:
- fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
+ fn(tcx: TyCtxt<'tcx>, cycle_error: CycleError, guar: ErrorGuaranteed) -> C::Value,
  pub format_value: fn(&C::Value) -> String,
 
  /// Formats a human-readable description of this query and its key, as
@@ -213,7 +209,7 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> {
  pub fn value_from_cycle_error(
  &self,
  tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
+ cycle_error: CycleError,
  guar: ErrorGuaranteed,
  ) -> C::Value {
  (self.value_from_cycle_error)(tcx, cycle_error, guar)
@@ -647,18 +643,6 @@ macro_rules! define_callbacks {
  };
 }
 
-// Each of these queries corresponds to a function pointer field in the
-// `Providers` struct for requesting a value of that type, and a method
-// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
-// which memoizes and does dep-graph tracking, wrapping around the actual
-// `Providers` that the driver creates (using several `rustc_*` crates).
-//
-// The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
-// Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
-
 mod sealed {
  use rustc_hir::def_id::{LocalModDefId, ModDefId};
 

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -13,7 +13,6 @@ mod non_query {
  // We use this for most things when incr. comp. is turned off.
  pub(crate) fn Null<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -26,7 +25,6 @@ mod non_query {
  // We use this for the forever-red node.
  pub(crate) fn Red<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -38,7 +36,6 @@ mod non_query {
 
  pub(crate) fn SideEffect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|tcx, _, prev_index| {
@@ -51,7 +48,6 @@ mod non_query {
 
  pub(crate) fn AnonZeroDeps<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
@@ -61,7 +57,6 @@ mod non_query {
 
  pub(crate) fn TraitSelect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -71,7 +66,6 @@ mod non_query {
 
  pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -81,7 +75,6 @@ mod non_query {
 
  pub(crate) fn CompileMonoItem<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -91,7 +84,6 @@ mod non_query {
 
  pub(crate) fn Metadata<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -117,7 +109,6 @@ where
 
  if is_anon || !key_fingerprint_style.reconstructible() {
  return DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: None,
@@ -126,7 +117,6 @@ where
  }
 
  DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: Some(|tcx, dep_node, _| {

compiler/rustc_query_impl/src/execution.rs

@@ -4,7 +4,7 @@ use std::mem;
 use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::{outline, sharded, sync};
-use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_errors::{FatalError, StashKey};
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
 use rustc_middle::query::plumbing::QueryVTable;
 use rustc_middle::query::{
@@ -108,24 +108,14 @@ fn mk_cycle<'tcx, C: QueryCache>(
  cycle_error: CycleError,
 ) -> C::Value {
  let error = report_cycle(tcx.sess, &cycle_error);
- handle_cycle_error(query, tcx, &cycle_error, error)
-}
-
-fn handle_cycle_error<'tcx, C: QueryCache>(
- query: &'tcx QueryVTable<'tcx, C>,
- tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
- error: Diag<'_>,
-) -> C::Value {
  match query.cycle_error_handling {
  CycleErrorHandling::Error => {
  let guar = error.emit();
  query.value_from_cycle_error(tcx, cycle_error, guar)
  }
  CycleErrorHandling::Fatal => {
- error.emit();
- tcx.dcx().abort_if_errors();
- unreachable!()
+ let guar = error.emit();
+ guar.raise_fatal();
  }
  CycleErrorHandling::DelayBug => {
  let guar = error.delay_as_bug();
@@ -322,15 +312,15 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
 
  // Only call `wait_for_query` if we're using a Rayon thread pool
  // as it will attempt to mark the worker thread as blocked.
- return wait_for_query(query, tcx, span, key, latch, current_job_id);
- }
-
- let id = job.id;
- drop(state_lock);
+ wait_for_query(query, tcx, span, key, latch, current_job_id)
+ } else {
+ let id = job.id;
+ drop(state_lock);
 
- // If we are single-threaded we know that we have cycle error,
- // so we just return the error.
- cycle_error(query, tcx, id, span)
+ // If we are single-threaded we know that we have cycle error,
+ // so we just return the error.
+ cycle_error(query, tcx, id, span)
+ }
  }
  ActiveKeyStatus::Poisoned => FatalError.raise(),
  }
@@ -412,27 +402,21 @@ fn execute_job_non_incr<'tcx, C: QueryCache>(
 ) -> (C::Value, DepNodeIndex) {
  debug_assert!(!tcx.dep_graph.is_fully_enabled());
 
- // Fingerprint the key, just to assert that it doesn't
- // have anything we don't consider hashable
- if cfg!(debug_assertions) {
- let _ = key.to_fingerprint(tcx);
- }
-
  let prof_timer = tcx.prof.query_provider();
  // Call the query provider.
  let result =
  start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key));
  let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
  prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
- // Similarly, fingerprint the result to assert that
-// it doesn't have anything not considered hashable.
-if cfg!(debug_assertions)
-&& let Some(hash_result) = query.hash_result
- {
-tcx.with_stable_hashing_context(|mut hcx| {
-hash_result(&mut hcx, &result);
- });
+ // Sanity: Fingerprint the key and the result to assert they don't contain anything unhashable.
+if cfg!(debug_assertions) {
+ let _ = key.to_fingerprint(tcx);
+if let Some(hash_result) = query.hash_result {
+ tcx.with_stable_hashing_context(|mut hcx| {
+ hash_result(&mut hcx, &result);
+});
+ }
  }
 
  (result, dep_node_index)

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -174,11 +174,6 @@ impl fmt::Debug for DepNode {
 /// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual
 /// jump table instead of large matches.
 pub struct DepKindVTable<'tcx> {
- /// Anonymous queries cannot be replayed from one compiler invocation to the next.
- /// When their result is needed, it is recomputed. They are useful for fine-grained
- /// dependency tracking, and caching within one compiler invocation.
- pub is_anon: bool,
-
  /// Eval-always queries do not track their dependencies, and are always recomputed, even if
  /// their inputs have not changed since the last compiler invocation. The result is still
  /// cached within one compiler invocation.

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::QueryInvocationId;
 use rustc_data_structures::sharded::{self, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU64, Lock};
+use rustc_data_structures::sync::{AtomicU64, Lock, is_dyn_thread_safe};
 use rustc_data_structures::unord::UnordMap;
 use rustc_data_structures::{assert_matches, outline};
 use rustc_errors::DiagInner;
@@ -1302,7 +1302,9 @@ impl CurrentDepGraph {
  prev_graph: &SerializedDepGraph,
  prev_index: SerializedDepNodeIndex,
  ) {
- if let Some(ref nodes_in_current_session) = self.nodes_in_current_session {
+ if !is_dyn_thread_safe()
+ && let Some(ref nodes_in_current_session) = self.nodes_in_current_session
+ {
  debug_assert!(
  !nodes_in_current_session
  .lock()

compiler/rustc_middle/src/queries.rs

@@ -150,10 +150,10 @@ use crate::{dep_graph, mir, thir};
 // `Providers` that the driver creates (using several `rustc_*` crates).
 //
 // The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
+// `ty::query::from_cycle_error::FromCycleError`, which produces an appropriate
+// placeholder (error) value if the query resulted in a query cycle.
 // Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
+// as they will raise a fatal error on query cycles instead.
 rustc_queries! {
  /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`.
  /// The key is:

compiler/rustc_middle/src/query/plumbing.rs

@@ -62,18 +62,6 @@ pub enum CycleErrorHandling {
  Stash,
 }
 
-pub type WillCacheOnDiskForKeyFn<'tcx, Key> = fn(tcx: TyCtxt<'tcx>, key: &Key) -> bool;
-
-pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn(
- tcx: TyCtxt<'tcx>,
- key: &Key,
- prev_index: SerializedDepNodeIndex,
- index: DepNodeIndex,
-) -> Option<Value>;
-
-pub type IsLoadableFromDiskFn<'tcx, Key> =
- fn(tcx: TyCtxt<'tcx>, key: &Key, index: SerializedDepNodeIndex) -> bool;
-
 pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
 #[derive(Clone, Debug)]
@@ -128,7 +116,7 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  pub cycle_error_handling: CycleErrorHandling,
  pub state: QueryState<'tcx, C::Key>,
  pub cache: C,
- pub will_cache_on_disk_for_key_fn: Option<WillCacheOnDiskForKeyFn<'tcx, C::Key>>,
+ pub will_cache_on_disk_for_key_fn: Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool>,
 
  /// Function pointer that calls `tcx.$query(key)` for this query and
  /// discards the returned value.
@@ -144,11 +132,19 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  /// This should be the only code that calls the provider function.
  pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
 
- pub try_load_from_disk_fn: Option<TryLoadFromDiskFn<'tcx, C::Key, C::Value>>,
- pub is_loadable_from_disk_fn: Option<IsLoadableFromDiskFn<'tcx, C::Key>>,
+ pub try_load_from_disk_fn: Option<
+ fn(
+ tcx: TyCtxt<'tcx>,
+ key: &C::Key,
+ prev_index: SerializedDepNodeIndex,
+ index: DepNodeIndex,
+ ) -> Option<C::Value>,
+ >,
+ pub is_loadable_from_disk_fn:
+ Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool>,
  pub hash_result: HashResult<C::Value>,
  pub value_from_cycle_error:
- fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
+ fn(tcx: TyCtxt<'tcx>, cycle_error: CycleError, guar: ErrorGuaranteed) -> C::Value,
  pub format_value: fn(&C::Value) -> String,
 
  /// Formats a human-readable description of this query and its key, as
@@ -213,7 +209,7 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> {
  pub fn value_from_cycle_error(
  &self,
  tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
+ cycle_error: CycleError,
  guar: ErrorGuaranteed,
  ) -> C::Value {
  (self.value_from_cycle_error)(tcx, cycle_error, guar)
@@ -647,18 +643,6 @@ macro_rules! define_callbacks {
  };
 }
 
-// Each of these queries corresponds to a function pointer field in the
-// `Providers` struct for requesting a value of that type, and a method
-// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
-// which memoizes and does dep-graph tracking, wrapping around the actual
-// `Providers` that the driver creates (using several `rustc_*` crates).
-//
-// The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
-// Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
-
 mod sealed {
  use rustc_hir::def_id::{LocalModDefId, ModDefId};
 

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -13,7 +13,6 @@ mod non_query {
  // We use this for most things when incr. comp. is turned off.
  pub(crate) fn Null<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -26,7 +25,6 @@ mod non_query {
  // We use this for the forever-red node.
  pub(crate) fn Red<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -38,7 +36,6 @@ mod non_query {
 
  pub(crate) fn SideEffect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|tcx, _, prev_index| {
@@ -51,7 +48,6 @@ mod non_query {
 
  pub(crate) fn AnonZeroDeps<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
@@ -61,7 +57,6 @@ mod non_query {
 
  pub(crate) fn TraitSelect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -71,7 +66,6 @@ mod non_query {
 
  pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -81,7 +75,6 @@ mod non_query {
 
  pub(crate) fn CompileMonoItem<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -91,7 +84,6 @@ mod non_query {
 
  pub(crate) fn Metadata<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -117,7 +109,6 @@ where
 
  if is_anon || !key_fingerprint_style.reconstructible() {
  return DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: None,
@@ -126,7 +117,6 @@ where
  }
 
  DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: Some(|tcx, dep_node, _| {

compiler/rustc_query_impl/src/execution.rs

@@ -4,7 +4,7 @@ use std::mem;
 use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::{outline, sharded, sync};
-use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_errors::{FatalError, StashKey};
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
 use rustc_middle::query::plumbing::QueryVTable;
 use rustc_middle::query::{
@@ -108,24 +108,14 @@ fn mk_cycle<'tcx, C: QueryCache>(
  cycle_error: CycleError,
 ) -> C::Value {
  let error = report_cycle(tcx.sess, &cycle_error);
- handle_cycle_error(query, tcx, &cycle_error, error)
-}
-
-fn handle_cycle_error<'tcx, C: QueryCache>(
- query: &'tcx QueryVTable<'tcx, C>,
- tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
- error: Diag<'_>,
-) -> C::Value {
  match query.cycle_error_handling {
  CycleErrorHandling::Error => {
  let guar = error.emit();
  query.value_from_cycle_error(tcx, cycle_error, guar)
  }
  CycleErrorHandling::Fatal => {
- error.emit();
- tcx.dcx().abort_if_errors();
- unreachable!()
+ let guar = error.emit();
+ guar.raise_fatal();
  }
  CycleErrorHandling::DelayBug => {
  let guar = error.delay_as_bug();
@@ -322,15 +312,15 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
 
  // Only call `wait_for_query` if we're using a Rayon thread pool
  // as it will attempt to mark the worker thread as blocked.
- return wait_for_query(query, tcx, span, key, latch, current_job_id);
- }
-
- let id = job.id;
- drop(state_lock);
+ wait_for_query(query, tcx, span, key, latch, current_job_id)
+ } else {
+ let id = job.id;
+ drop(state_lock);
 
- // If we are single-threaded we know that we have cycle error,
- // so we just return the error.
- cycle_error(query, tcx, id, span)
+ // If we are single-threaded we know that we have cycle error,
+ // so we just return the error.
+ cycle_error(query, tcx, id, span)
+ }
  }
  ActiveKeyStatus::Poisoned => FatalError.raise(),
  }
@@ -412,27 +402,21 @@ fn execute_job_non_incr<'tcx, C: QueryCache>(
 ) -> (C::Value, DepNodeIndex) {
  debug_assert!(!tcx.dep_graph.is_fully_enabled());
 
- // Fingerprint the key, just to assert that it doesn't
- // have anything we don't consider hashable
- if cfg!(debug_assertions) {
- let _ = key.to_fingerprint(tcx);
- }
-
  let prof_timer = tcx.prof.query_provider();
  // Call the query provider.
  let result =
  start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key));
  let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
  prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
- // Similarly, fingerprint the result to assert that
-// it doesn't have anything not considered hashable.
-if cfg!(debug_assertions)
-&& let Some(hash_result) = query.hash_result
- {
-tcx.with_stable_hashing_context(|mut hcx| {
-hash_result(&mut hcx, &result);
- });
+ // Sanity: Fingerprint the key and the result to assert they don't contain anything unhashable.
+if cfg!(debug_assertions) {
+ let _ = key.to_fingerprint(tcx);
+if let Some(hash_result) = query.hash_result {
+ tcx.with_stable_hashing_context(|mut hcx| {
+ hash_result(&mut hcx, &result);
+});
+ }
  }
 
  (result, dep_node_index)

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -174,11 +174,6 @@ impl fmt::Debug for DepNode {
 /// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual
 /// jump table instead of large matches.
 pub struct DepKindVTable<'tcx> {
- /// Anonymous queries cannot be replayed from one compiler invocation to the next.
- /// When their result is needed, it is recomputed. They are useful for fine-grained
- /// dependency tracking, and caching within one compiler invocation.
- pub is_anon: bool,
-
  /// Eval-always queries do not track their dependencies, and are always recomputed, even if
  /// their inputs have not changed since the last compiler invocation. The result is still
  /// cached within one compiler invocation.

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::QueryInvocationId;
 use rustc_data_structures::sharded::{self, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU64, Lock};
+use rustc_data_structures::sync::{AtomicU64, Lock, is_dyn_thread_safe};
 use rustc_data_structures::unord::UnordMap;
 use rustc_data_structures::{assert_matches, outline};
 use rustc_errors::DiagInner;
@@ -1302,7 +1302,9 @@ impl CurrentDepGraph {
  prev_graph: &SerializedDepGraph,
  prev_index: SerializedDepNodeIndex,
  ) {
- if let Some(ref nodes_in_current_session) = self.nodes_in_current_session {
+ if !is_dyn_thread_safe()
+ && let Some(ref nodes_in_current_session) = self.nodes_in_current_session
+ {
  debug_assert!(
  !nodes_in_current_session
  .lock()

compiler/rustc_middle/src/queries.rs

@@ -150,10 +150,10 @@ use crate::{dep_graph, mir, thir};
 // `Providers` that the driver creates (using several `rustc_*` crates).
 //
 // The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
+// `ty::query::from_cycle_error::FromCycleError`, which produces an appropriate
+// placeholder (error) value if the query resulted in a query cycle.
 // Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
+// as they will raise a fatal error on query cycles instead.
 rustc_queries! {
  /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`.
  /// The key is:

compiler/rustc_middle/src/query/plumbing.rs

@@ -62,18 +62,6 @@ pub enum CycleErrorHandling {
  Stash,
 }
 
-pub type WillCacheOnDiskForKeyFn<'tcx, Key> = fn(tcx: TyCtxt<'tcx>, key: &Key) -> bool;
-
-pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn(
- tcx: TyCtxt<'tcx>,
- key: &Key,
- prev_index: SerializedDepNodeIndex,
- index: DepNodeIndex,
-) -> Option<Value>;
-
-pub type IsLoadableFromDiskFn<'tcx, Key> =
- fn(tcx: TyCtxt<'tcx>, key: &Key, index: SerializedDepNodeIndex) -> bool;
-
 pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
 #[derive(Clone, Debug)]
@@ -128,7 +116,7 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  pub cycle_error_handling: CycleErrorHandling,
  pub state: QueryState<'tcx, C::Key>,
  pub cache: C,
- pub will_cache_on_disk_for_key_fn: Option<WillCacheOnDiskForKeyFn<'tcx, C::Key>>,
+ pub will_cache_on_disk_for_key_fn: Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool>,
 
  /// Function pointer that calls `tcx.$query(key)` for this query and
  /// discards the returned value.
@@ -144,11 +132,19 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
  /// This should be the only code that calls the provider function.
  pub invoke_provider_fn: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
 
- pub try_load_from_disk_fn: Option<TryLoadFromDiskFn<'tcx, C::Key, C::Value>>,
- pub is_loadable_from_disk_fn: Option<IsLoadableFromDiskFn<'tcx, C::Key>>,
+ pub try_load_from_disk_fn: Option<
+ fn(
+ tcx: TyCtxt<'tcx>,
+ key: &C::Key,
+ prev_index: SerializedDepNodeIndex,
+ index: DepNodeIndex,
+ ) -> Option<C::Value>,
+ >,
+ pub is_loadable_from_disk_fn:
+ Option<fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool>,
  pub hash_result: HashResult<C::Value>,
  pub value_from_cycle_error:
- fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
+ fn(tcx: TyCtxt<'tcx>, cycle_error: CycleError, guar: ErrorGuaranteed) -> C::Value,
  pub format_value: fn(&C::Value) -> String,
 
  /// Formats a human-readable description of this query and its key, as
@@ -213,7 +209,7 @@ impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> {
  pub fn value_from_cycle_error(
  &self,
  tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
+ cycle_error: CycleError,
  guar: ErrorGuaranteed,
  ) -> C::Value {
  (self.value_from_cycle_error)(tcx, cycle_error, guar)
@@ -647,18 +643,6 @@ macro_rules! define_callbacks {
  };
 }
 
-// Each of these queries corresponds to a function pointer field in the
-// `Providers` struct for requesting a value of that type, and a method
-// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
-// which memoizes and does dep-graph tracking, wrapping around the actual
-// `Providers` that the driver creates (using several `rustc_*` crates).
-//
-// The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
-// Queries marked with `cycle_fatal` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
-
 mod sealed {
  use rustc_hir::def_id::{LocalModDefId, ModDefId};
 

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -13,7 +13,6 @@ mod non_query {
  // We use this for most things when incr. comp. is turned off.
  pub(crate) fn Null<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -26,7 +25,6 @@ mod non_query {
  // We use this for the forever-red node.
  pub(crate) fn Red<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|_, dep_node, _| {
@@ -38,7 +36,6 @@ mod non_query {
 
  pub(crate) fn SideEffect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: Some(|tcx, _, prev_index| {
@@ -51,7 +48,6 @@ mod non_query {
 
  pub(crate) fn AnonZeroDeps<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
@@ -61,7 +57,6 @@ mod non_query {
 
  pub(crate) fn TraitSelect<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: true,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -71,7 +66,6 @@ mod non_query {
 
  pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -81,7 +75,6 @@ mod non_query {
 
  pub(crate) fn CompileMonoItem<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Opaque,
  force_from_dep_node: None,
@@ -91,7 +84,6 @@ mod non_query {
 
  pub(crate) fn Metadata<'tcx>() -> DepKindVTable<'tcx> {
  DepKindVTable {
- is_anon: false,
  is_eval_always: false,
  key_fingerprint_style: KeyFingerprintStyle::Unit,
  force_from_dep_node: None,
@@ -117,7 +109,6 @@ where
 
  if is_anon || !key_fingerprint_style.reconstructible() {
  return DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: None,
@@ -126,7 +117,6 @@ where
  }
 
  DepKindVTable {
- is_anon,
  is_eval_always,
  key_fingerprint_style,
  force_from_dep_node: Some(|tcx, dep_node, _| {

compiler/rustc_query_impl/src/execution.rs

@@ -4,7 +4,7 @@ use std::mem;
 use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::{outline, sharded, sync};
-use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_errors::{FatalError, StashKey};
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
 use rustc_middle::query::plumbing::QueryVTable;
 use rustc_middle::query::{
@@ -108,24 +108,14 @@ fn mk_cycle<'tcx, C: QueryCache>(
  cycle_error: CycleError,
 ) -> C::Value {
  let error = report_cycle(tcx.sess, &cycle_error);
- handle_cycle_error(query, tcx, &cycle_error, error)
-}
-
-fn handle_cycle_error<'tcx, C: QueryCache>(
- query: &'tcx QueryVTable<'tcx, C>,
- tcx: TyCtxt<'tcx>,
- cycle_error: &CycleError,
- error: Diag<'_>,
-) -> C::Value {
  match query.cycle_error_handling {
  CycleErrorHandling::Error => {
  let guar = error.emit();
  query.value_from_cycle_error(tcx, cycle_error, guar)
  }
  CycleErrorHandling::Fatal => {
- error.emit();
- tcx.dcx().abort_if_errors();
- unreachable!()
+ let guar = error.emit();
+ guar.raise_fatal();
  }
  CycleErrorHandling::DelayBug => {
  let guar = error.delay_as_bug();
@@ -322,15 +312,15 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
 
  // Only call `wait_for_query` if we're using a Rayon thread pool
  // as it will attempt to mark the worker thread as blocked.
- return wait_for_query(query, tcx, span, key, latch, current_job_id);
- }
-
- let id = job.id;
- drop(state_lock);
+ wait_for_query(query, tcx, span, key, latch, current_job_id)
+ } else {
+ let id = job.id;
+ drop(state_lock);
 
- // If we are single-threaded we know that we have cycle error,
- // so we just return the error.
- cycle_error(query, tcx, id, span)
+ // If we are single-threaded we know that we have cycle error,
+ // so we just return the error.
+ cycle_error(query, tcx, id, span)
+ }
  }
  ActiveKeyStatus::Poisoned => FatalError.raise(),
  }
@@ -412,27 +402,21 @@ fn execute_job_non_incr<'tcx, C: QueryCache>(
 ) -> (C::Value, DepNodeIndex) {
  debug_assert!(!tcx.dep_graph.is_fully_enabled());
 
- // Fingerprint the key, just to assert that it doesn't
- // have anything we don't consider hashable
- if cfg!(debug_assertions) {
- let _ = key.to_fingerprint(tcx);
- }
-
  let prof_timer = tcx.prof.query_provider();
  // Call the query provider.
  let result =
  start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key));
  let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
  prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
- // Similarly, fingerprint the result to assert that
-// it doesn't have anything not considered hashable.
-if cfg!(debug_assertions)
-&& let Some(hash_result) = query.hash_result
- {
-tcx.with_stable_hashing_context(|mut hcx| {
-hash_result(&mut hcx, &result);
- });
+ // Sanity: Fingerprint the key and the result to assert they don't contain anything unhashable.
+if cfg!(debug_assertions) {
+ let _ = key.to_fingerprint(tcx);
+if let Some(hash_result) = query.hash_result {
+ tcx.with_stable_hashing_context(|mut hcx| {
+ hash_result(&mut hcx, &result);
+});
+ }
  }
 
  (result, dep_node_index)