Auto merge of #146096 - adwinwhite:handle_normalization_overflow_in_mono1, r=saethlin

Fix normalization overflow ICEs in monomorphization
Fixes#92004Fixes#92470Fixes#95134Fixes#105275Fixes#105937Fixes#117696-2
Fixes#118590Fixes#122823Fixes#131342Fixes#139659
## Analysis:
The causes of these issues are similar. They contain generic recursive functions that can be instantiated with different args infinitely at monomorphization stage.
Ideally this should be caught by the [`check_recursion_limit`](https://github.com/rust-lang/rust/blob/c0bb3b98bb7aac24a37635e5d36d961e0b14f435/compiler/rustc_monomorphize/src/collector.rs#L468) function. The reality is that normalization can reach recursion limit earlier than monomorphization's check because they calculate depths in different ways.
Since normalization is called everywhere, ICEs appear in different locations.
## Fix:
If we abort on overflow with `TypingMode::PostAnalysis` in the trait solver, it would also catch these errors.
The main challenge is providing good diagnostics for them. So it's quite natural to put the check right before these normalization happening.
I first tried to check the whole MIR body's normalization and `references_error`. (As elaborate_drop handles normalization failure by [returning `ty::Error`](https://github.com/rust-lang/rust/blob/c0bb3b98bb7aac24a37635e5d36d961e0b14f435/compiler/rustc_mir_transform/src/elaborate_drop.rs#L514-L519).)
It turns out that checking all `Local`s seems sufficient.
These types are gonna be normalized anyway. So with cache, these checks shouldn't be expensive.
This fixes these ICEs for both the next and old solver, though I'm not sure the change I made to the old solver is proper. Its overflow handling looks convoluted thus I didn't try to fix it more "upstream".

Author: bors

compiler/rustc_middle/src/mir/mono.rs

@@ -47,6 +47,9 @@ pub enum InstantiationMode {
 LocalCopy,
 }
 
+#[derive(PartialEq,Eq,Clone,Copy,Debug,Hash,HashStable,TyEncodable,TyDecodable)]
+pubstructNormalizationErrorInMono;
+
 #[derive(PartialEq,Eq,Clone,Copy,Debug,Hash,HashStable,TyEncodable,TyDecodable)]
 pubenumMonoItem<'tcx>{
 Fn(Instance<'tcx>),

compiler/rustc_middle/src/query/erase.rs

@@ -3,8 +3,10 @@ use std::intrinsics::transmute_unchecked;
 use std::mem::MaybeUninit;
 
 use rustc_span::ErrorGuaranteed;
+use rustc_span::source_map::Spanned;
 
 usecrate::mir::interpret::EvalToValTreeResult;
+usecrate::mir::mono::{MonoItem,NormalizationErrorInMono};
 usecrate::query::CyclePlaceholder;
 usecrate::traits::solve;
 usecrate::ty::adjustment::CoerceUnsizedInfo;
@@ -171,6 +173,17 @@ impl EraseType for Result<ty::EarlyBinder<'_, Ty<'_>>, CyclePlaceholder> {
 typeResult = [u8;size_of::<Result<ty::EarlyBinder<'static,Ty<'_>>,CyclePlaceholder>>()];
 }
 
+implEraseType
+forResult<(&'_[Spanned<MonoItem<'_>>],&'_[Spanned<MonoItem<'_>>]),NormalizationErrorInMono>
+{
+typeResult = [u8;size_of::<
+Result<
+(&'static[Spanned<MonoItem<'static>>],&'static[Spanned<MonoItem<'static>>]),
+NormalizationErrorInMono,
+>,
+>()];
+}
+
 impl<T>EraseTypeforOption<&'_T>{
 typeResult = [u8;size_of::<Option<&'static()>>()];
 }

compiler/rustc_middle/src/query/mod.rs

@@ -116,7 +116,9 @@ use crate::mir::interpret::{
 EvalStaticInitializerRawResult,EvalToAllocationRawResult,EvalToConstValueResult,
 EvalToValTreeResult,GlobalId,LitToConstInput,
 };
-usecrate::mir::mono::{CodegenUnit,CollectionMode,MonoItem,MonoItemPartitions};
+usecrate::mir::mono::{
+CodegenUnit,CollectionMode,MonoItem,MonoItemPartitions,NormalizationErrorInMono,
+};
 usecrate::query::erase::{Erase, erase, restore};
 usecrate::query::plumbing::{CyclePlaceholder,DynamicQuery};
 usecrate::traits::query::{
@@ -2702,7 +2704,7 @@ rustc_queries! {
  desc {"functions to skip for move-size check"}
 }
 
- query items_of_instance(key:(ty::Instance<'tcx>,CollectionMode)) -> (&'tcx [Spanned<MonoItem<'tcx>>],&'tcx [Spanned<MonoItem<'tcx>>]){
+ query items_of_instance(key:(ty::Instance<'tcx>,CollectionMode)) -> Result<(&'tcx [Spanned<MonoItem<'tcx>>],&'tcx [Spanned<MonoItem<'tcx>>]),NormalizationErrorInMono>{
  desc {"collecting items used by `{}`", key.0}
  cache_on_disk_if {true}
 }

compiler/rustc_monomorphize/src/collector.rs

@@ -208,6 +208,7 @@
 mod autodiff;
 
 use std::cell::OnceCell;
+use std::ops::ControlFlow;
 
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::{MTLock, par_for_each_in};
@@ -220,15 +221,17 @@ use rustc_hir::lang_items::LangItem;
 use rustc_hir::limit::Limit;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{AllocId,ErrorHandled,GlobalAlloc,Scalar};
-use rustc_middle::mir::mono::{CollectionMode,InstantiationMode,MonoItem};
+use rustc_middle::mir::mono::{
+CollectionMode,InstantiationMode,MonoItem,NormalizationErrorInMono,
+};
 use rustc_middle::mir::visit::VisitorasMirVisitor;
-use rustc_middle::mir::{self,Location,MentionedItem, traversal};
+use rustc_middle::mir::{self,Body,Location,MentionedItem, traversal};
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::adjustment::{CustomCoerceUnsized,PointerCoercion};
 use rustc_middle::ty::layout::ValidityRequirement;
 use rustc_middle::ty::{
 self,GenericArgs,GenericParamDefKind,Instance,InstanceKind,Ty,TyCtxt,TypeFoldable,
-TypeVisitableExt,VtblEntry,
+TypeVisitable,TypeVisitableExt,TypeVisitor,VtblEntry,
 };
 use rustc_middle::util::Providers;
 use rustc_middle::{bug, span_bug};
@@ -474,7 +477,22 @@ fn collect_items_rec<'tcx>(
 ));
 
  rustc_data_structures::stack::ensure_sufficient_stack(|| {
-let(used, mentioned) = tcx.items_of_instance((instance, mode));
+letOk((used, mentioned)) = tcx.items_of_instance((instance, mode))else{
+// Normalization errors here are usually due to trait solving overflow.
+// FIXME: I assume that there are few type errors at post-analysis stage, but not
+// entirely sure.
+// We have to emit the error outside of `items_of_instance` to access the
+// span of the `starting_item`.
+let def_id = instance.def_id();
+let def_span = tcx.def_span(def_id);
+let def_path_str = tcx.def_path_str(def_id);
+ tcx.dcx().emit_fatal(RecursionLimit{
+span: starting_item.span,
+ instance,
+ def_span,
+ def_path_str,
+});
+};
  used_items.extend(used.into_iter().copied());
  mentioned_items.extend(mentioned.into_iter().copied());
 });
@@ -603,6 +621,35 @@ fn collect_items_rec<'tcx>(
 }
 }
 
+// Check whether we can normalize every type in the instantiated MIR body.
+fncheck_normalization_error<'tcx>(
+tcx:TyCtxt<'tcx>,
+instance:Instance<'tcx>,
+body:&Body<'tcx>,
+) -> Result<(),NormalizationErrorInMono>{
+structNormalizationChecker<'tcx>{
+tcx:TyCtxt<'tcx>,
+instance:Instance<'tcx>,
+}
+impl<'tcx>TypeVisitor<TyCtxt<'tcx>>forNormalizationChecker<'tcx>{
+typeResult = ControlFlow<()>;
+
+fnvisit_ty(&mutself,t:Ty<'tcx>) -> Self::Result{
+matchself.instance.try_instantiate_mir_and_normalize_erasing_regions(
+self.tcx,
+ ty::TypingEnv::fully_monomorphized(),
+ ty::EarlyBinder::bind(t),
+){
+Ok(_) => ControlFlow::Continue(()),
+Err(_) => ControlFlow::Break(()),
+}
+}
+}
+
+letmut checker = NormalizationChecker{ tcx, instance };
+if body.visit_with(&mut checker).is_break(){Err(NormalizationErrorInMono)}else{Ok(())}
+}
+
 fncheck_recursion_limit<'tcx>(
 tcx:TyCtxt<'tcx>,
 instance:Instance<'tcx>,
@@ -1259,11 +1306,15 @@ fn collect_items_of_instance<'tcx>(
 tcx:TyCtxt<'tcx>,
 instance:Instance<'tcx>,
 mode:CollectionMode,
-) -> (MonoItems<'tcx>,MonoItems<'tcx>){
+) -> Result<(MonoItems<'tcx>,MonoItems<'tcx>),NormalizationErrorInMono>{
 // This item is getting monomorphized, do mono-time checks.
+let body = tcx.instance_mir(instance.def);
+// Plenty of code paths later assume that everything can be normalized. So we have to check
+// normalization first.
+// We choose to emit the error outside to provide helpful diagnostics.
+check_normalization_error(tcx, instance, body)?;
  tcx.ensure_ok().check_mono_item(instance);
 
-let body = tcx.instance_mir(instance.def);
 // Naively, in "used" collection mode, all functions get added to *both* `used_items` and
 // `mentioned_items`. Mentioned items processing will then notice that they have already been
 // visited, but at that point each mentioned item has been monomorphized, added to the
@@ -1313,19 +1364,22 @@ fn collect_items_of_instance<'tcx>(
 }
 }
 
-(used_items, mentioned_items)
+Ok((used_items, mentioned_items))
 }
 
 fnitems_of_instance<'tcx>(
 tcx:TyCtxt<'tcx>,
 (instance, mode):(Instance<'tcx>,CollectionMode),
-) -> (&'tcx[Spanned<MonoItem<'tcx>>],&'tcx[Spanned<MonoItem<'tcx>>]){
-let(used_items, mentioned_items) = collect_items_of_instance(tcx, instance, mode);
+) -> Result<
+(&'tcx[Spanned<MonoItem<'tcx>>],&'tcx[Spanned<MonoItem<'tcx>>]),
+NormalizationErrorInMono,
+>{
+let(used_items, mentioned_items) = collect_items_of_instance(tcx, instance, mode)?;
 
 let used_items = tcx.arena.alloc_from_iter(used_items);
 let mentioned_items = tcx.arena.alloc_from_iter(mentioned_items);
 
-(used_items, mentioned_items)
+Ok((used_items, mentioned_items))
 }
 
 /// `item` must be already monomorphized.

compiler/rustc_traits/src/normalize_erasing_regions.rs

@@ -42,7 +42,18 @@ fn try_normalize_after_erasing_regions<'tcx, T: TypeFoldable<TyCtxt<'tcx>> + Par
 // us a test case.
 debug_assert_eq!(normalized_value, resolved_value);
 let erased = infcx.tcx.erase_and_anonymize_regions(resolved_value);
-debug_assert!(!erased.has_infer(),"{erased:?}");
+if infcx.next_trait_solver(){
+debug_assert!(!erased.has_infer(),"{erased:?}");
+}else{
+// The old solver returns an ty var with the failed obligation in case of
+// selection error. And when the obligation is re-tried, the error should be
+// reported. However in case of overflow error, the obligation may be fulfilled
+// due to the original depth being dropped.
+// In conclusion, overflow results in an unconstrained ty var.
+if erased.has_infer(){
+returnErr(NoSolution);
+}
+}
 Ok(erased)
 }
 Err(NoSolution) => Err(NoSolution),

tests/crashes/105275.rs …ation-overflow/recursion-issue-105275.rstests/crashes/105275.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-105275.rs tests/crashes/105275.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-105275.rs

@@ -1,9 +1,10 @@
-//@ known-bug: #105275
+//@ build-fail
 //@ compile-flags: -Copt-level=0
 
 pubfnencode_num<Writer:ExampleWriter>(n:u32,mutwriter:Writer) -> Result<(),Writer::Error>{
 if n > 15{
 encode_num(n / 16,&mut writer)?;
+//~^ ERROR: reached the recursion limit while instantiating
 }
 Ok(())
 }

tests/ui/codegen/normalization-overflow/recursion-issue-105275.stderr

@@ -0,0 +1,14 @@
+error: reached the recursion limit while instantiating `encode_num::<&mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut Error>`
+ --> $DIR/recursion-issue-105275.rs:6:9
+ |
+LL | encode_num(n / 16, &mut writer)?;
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ |
+note: `encode_num` defined here
+ --> $DIR/recursion-issue-105275.rs:4:1
+ |
+LL | pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+

tests/crashes/105937.rs …ation-overflow/recursion-issue-105937.rstests/crashes/105937.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-105937.rs tests/crashes/105937.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-105937.rs

@@ -1,9 +1,10 @@
-//@ known-bug: #105937
+//@ build-fail
 //@ compile-flags: -Copt-level=0
 
 pubfnencode_num<Writer:ExampleWriter>(n:u32,mutwriter:Writer) -> Result<(),Writer::Error>{
 if n > 15{
 encode_num(n / 16,&mut writer)?;
+//~^ ERROR: reached the recursion limit while instantiating
 }
 Ok(())
 }

tests/ui/codegen/normalization-overflow/recursion-issue-105937.stderr

@@ -0,0 +1,14 @@
+error: reached the recursion limit while instantiating `encode_num::<&mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut Error>`
+ --> $DIR/recursion-issue-105937.rs:6:9
+ |
+LL | encode_num(n / 16, &mut writer)?;
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ |
+note: `encode_num` defined here
+ --> $DIR/recursion-issue-105937.rs:4:1
+ |
+LL | pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+

tests/crashes/117696-1.rs …ion-overflow/recursion-issue-117696-1.rstests/crashes/117696-1.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.rs tests/crashes/117696-1.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.rs

@@ -1,4 +1,4 @@
-//@ known-bug: #117696
+//@ build-fail
 fnmain(){
 letmut it = (Empty);
 rec(&mut it);
@@ -23,6 +23,7 @@ where
 {
 if() == (){
 T::count(it);
+//~^ ERROR: reached the recursion limit while instantiating
 }else{
 rec(identity(&mut it))
 }