Rollup merge of #101339 - the8472:ci-randomize-debug, r=Mark-Simulacrum

enable -Zrandomize-layout in debug CI builds
This builds rustc/libs/tools with `-Zrandomize-layout` on *-debug CI runners.
Only a handful of tests and asserts break with that enabled, which is promising. One test was fixable, the rest is dealt with by disabling them through new cargo features or compiletest directives.
The config.toml flag `rust.randomize-layout` defaults to false, so it has to be explicitly enabled for now.

Author: matthiaskrgr

Cargo.lock

@@ -3569,6 +3569,7 @@ dependencies = [
 "rustc_hir_pretty",
 "rustc_hir_typeck",
 "rustc_incremental",
+"rustc_index",
 "rustc_infer",
 "rustc_interface",
 "rustc_lint",

compiler/rustc/Cargo.toml

@@ -30,5 +30,6 @@ features = ['unprefixed_malloc_on_supported_platforms']
 jemalloc = ['dep:jemalloc-sys']
 llvm = ['rustc_driver_impl/llvm']
 max_level_info = ['rustc_driver_impl/max_level_info']
+rustc_randomized_layouts = ['rustc_driver_impl/rustc_randomized_layouts']
 rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler']
 # tidy-alphabetical-end

compiler/rustc_abi/src/layout.rs

@@ -968,8 +968,8 @@ fn univariant<
 letmut align = if pack.is_some(){ dl.i8_align}else{ dl.aggregate_align};
 letmut max_repr_align = repr.align;
 letmut inverse_memory_index:IndexVec<u32,FieldIdx> = fields.indices().collect();
-letoptimize = !repr.inhibit_struct_field_reordering();
-ifoptimize && fields.len() > 1{
+letoptimize_field_order = !repr.inhibit_struct_field_reordering();
+ifoptimize_field_order && fields.len() > 1{
 let end = ifletStructKind::MaybeUnsized = kind { fields.len() - 1}else{ fields.len()};
 let optimizing = &mut inverse_memory_index.raw[..end];
 let fields_excluding_tail = &fields.raw[..end];
@@ -1176,7 +1176,7 @@ fn univariant<
 // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
 // Field 5 would be the first element, so memory_index is i:
 // Note: if we didn't optimize, it's already right.
-let memory_index = ifoptimize{
+let memory_index = ifoptimize_field_order{
  inverse_memory_index.invert_bijective_mapping()
 }else{
 debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
@@ -1189,6 +1189,9 @@ fn univariant<
 }
 letmut layout_of_single_non_zst_field = None;
 letmut abi = Abi::Aggregate{ sized };
+
+let optimize_abi = !repr.inhibit_newtype_abi_optimization();
+
 // Try to make this a Scalar/ScalarPair.
 if sized && size.bytes() > 0{
 // We skip *all* ZST here and later check if we are good in terms of alignment.
@@ -1205,7 +1208,7 @@ fn univariant<
 match field.abi{
 // For plain scalars, or vectors of them, we can't unpack
 // newtypes for `#[repr(C)]`, as that affects C ABIs.
-Abi::Scalar(_) | Abi::Vector{ .. }ifoptimize => {
+Abi::Scalar(_) | Abi::Vector{ .. }ifoptimize_abi => {
  abi = field.abi;
 }
 // But scalar pairs are Rust-specific and get

compiler/rustc_abi/src/lib.rs

@@ -43,14 +43,17 @@ bitflags! {
 constIS_SIMD = 1 << 1;
 constIS_TRANSPARENT = 1 << 2;
 // Internal only for now. If true, don't reorder fields.
+// On its own it does not prevent ABI optimizations.
 constIS_LINEAR = 1 << 3;
-// If true, the type's layout can be randomized using
-// the seed stored in `ReprOptions.field_shuffle_seed`
+// If true, the type's crate has opted into layout randomization.
+// Other flags can still inhibit reordering and thus randomization.
+// The seed stored in `ReprOptions.field_shuffle_seed`.
 constRANDOMIZE_LAYOUT = 1 << 4;
 // Any of these flags being set prevent field reordering optimisation.
-constIS_UNOPTIMISABLE = ReprFlags::IS_C.bits()
+constFIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits()
  | ReprFlags::IS_SIMD.bits()
  | ReprFlags::IS_LINEAR.bits();
+constABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
 }
 }
 
@@ -139,10 +142,14 @@ impl ReprOptions {
 self.c() || self.int.is_some()
 }
 
+pubfninhibit_newtype_abi_optimization(&self) -> bool{
+self.flags.intersects(ReprFlags::ABI_UNOPTIMIZABLE)
+}
+
 /// Returns `true` if this `#[repr()]` guarantees a fixed field order,
 /// e.g. `repr(C)` or `repr(<int>)`.
 pubfninhibit_struct_field_reordering(&self) -> bool{
-self.flags.intersects(ReprFlags::IS_UNOPTIMISABLE) || self.int.is_some()
+self.flags.intersects(ReprFlags::FIELD_ORDER_UNOPTIMIZABLE) || self.int.is_some()
 }
 
 /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`

compiler/rustc_driver_impl/Cargo.toml

@@ -23,6 +23,7 @@ rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_hir_typeck = { path = "../rustc_hir_typeck" }
 rustc_incremental = { path = "../rustc_incremental" }
+rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_interface = { path = "../rustc_interface" }
 rustc_lint = { path = "../rustc_lint" }
@@ -72,6 +73,10 @@ ctrlc = "3.4.4"
 # tidy-alphabetical-start
 llvm = ['rustc_interface/llvm']
 max_level_info = ['rustc_log/max_level_info']
+rustc_randomized_layouts = [
+'rustc_index/rustc_randomized_layouts',
+'rustc_middle/rustc_randomized_layouts'
+]
 rustc_use_parallel_compiler = [
 'rustc_data_structures/rustc_use_parallel_compiler',
 'rustc_interface/rustc_use_parallel_compiler',

compiler/rustc_index/Cargo.toml

@@ -20,4 +20,5 @@ nightly = [
 "dep:rustc_macros",
 "rustc_index_macros/nightly",
 ]
+rustc_randomized_layouts = []
 # tidy-alphabetical-end

compiler/rustc_index/src/lib.rs

@@ -33,8 +33,19 @@ pub use vec::IndexVec;
 ///
 /// </div>
 #[macro_export]
+#[cfg(not(feature = "rustc_randomized_layouts"))]
 macro_rules! static_assert_size {
 ($ty:ty, $size:expr) => {
 const _:[(); $size] = [();::std::mem::size_of::<$ty>()];
 };
 }
+
+#[macro_export]
+#[cfg(feature = "rustc_randomized_layouts")]
+macro_rules! static_assert_size {
+($ty:ty, $size:expr) => {
+// no effect other than using the statements.
+// struct sizes are not deterministic under randomized layouts
+const _:(usize,usize) = ($size,::std::mem::size_of::<$ty>());
+};
+}

compiler/rustc_middle/Cargo.toml

@@ -40,5 +40,6 @@ tracing = "0.1"
 
 [features]
 # tidy-alphabetical-start
+rustc_randomized_layouts = []
 rustc_use_parallel_compiler = ["dep:rustc-rayon-core"]
 # tidy-alphabetical-end

compiler/rustc_middle/src/query/plumbing.rs

@@ -337,6 +337,7 @@ macro_rules! define_callbacks {
 // Ensure that values grow no larger than 64 bytes by accident.
 // Increase this limit if necessary, but do try to keep the size low if possible
  #[cfg(target_pointer_width = "64")]
+ #[cfg(not(feature = "rustc_randomized_layouts"))]
 const _:() = {
 if mem::size_of::<Value<'static>>() > 64{
  panic!("{}", concat!(

compiler/rustc_middle/src/ty/mod.rs

@@ -35,6 +35,7 @@ use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_errors::{Diag,ErrorGuaranteed,StashKey};
 use rustc_hir::def::{CtorKind,CtorOf,DefKind,DocLinkResMap,LifetimeRes,Res};
 use rustc_hir::def_id::{CrateNum,DefId,DefIdMap,LocalDefId,LocalDefIdMap};
+use rustc_hir::LangItem;
 use rustc_index::IndexVec;
 use rustc_macros::{
  extension,Decodable,Encodable,HashStable,TyDecodable,TyEncodable,TypeFoldable,
@@ -1570,8 +1571,15 @@ impl<'tcx> TyCtxt<'tcx> {
  flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
 }
 
+// box is special, on the one hand the compiler assumes an ordered layout, with the pointer
+// always at offset zero. On the other hand we want scalar abi optimizations.
+let is_box = self.is_lang_item(did.to_def_id(),LangItem::OwnedBox);
+
 // This is here instead of layout because the choice must make it into metadata.
-if !self.consider_optimizing(|| format!("Reorder fields of {:?}",self.def_path_str(did))){
+if is_box
+ || !self
+.consider_optimizing(|| format!("Reorder fields of {:?}",self.def_path_str(did)))
+{
  flags.insert(ReprFlags::IS_LINEAR);
 }