inhibit layout randomization for Box

Author: the8472

compiler/rustc_abi/src/layout.rs

@@ -968,8 +968,8 @@ fn univariant<
 letmut align = if pack.is_some(){ dl.i8_align}else{ dl.aggregate_align};
 letmut max_repr_align = repr.align;
 letmut inverse_memory_index:IndexVec<u32,FieldIdx> = fields.indices().collect();
-letoptimize = !repr.inhibit_struct_field_reordering();
-ifoptimize && fields.len() > 1{
+letoptimize_field_order = !repr.inhibit_struct_field_reordering();
+ifoptimize_field_order && fields.len() > 1{
 let end = ifletStructKind::MaybeUnsized = kind { fields.len() - 1}else{ fields.len()};
 let optimizing = &mut inverse_memory_index.raw[..end];
 let fields_excluding_tail = &fields.raw[..end];
@@ -1176,7 +1176,7 @@ fn univariant<
 // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
 // Field 5 would be the first element, so memory_index is i:
 // Note: if we didn't optimize, it's already right.
-let memory_index = ifoptimize{
+let memory_index = ifoptimize_field_order{
  inverse_memory_index.invert_bijective_mapping()
 }else{
 debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
@@ -1189,6 +1189,9 @@ fn univariant<
 }
 letmut layout_of_single_non_zst_field = None;
 letmut abi = Abi::Aggregate{ sized };
+
+let optimize_abi = !repr.inhibit_newtype_abi_optimization();
+
 // Try to make this a Scalar/ScalarPair.
 if sized && size.bytes() > 0{
 // We skip *all* ZST here and later check if we are good in terms of alignment.
@@ -1205,7 +1208,7 @@ fn univariant<
 match field.abi{
 // For plain scalars, or vectors of them, we can't unpack
 // newtypes for `#[repr(C)]`, as that affects C ABIs.
-Abi::Scalar(_) | Abi::Vector{ .. }ifoptimize => {
+Abi::Scalar(_) | Abi::Vector{ .. }ifoptimize_abi => {
  abi = field.abi;
 }
 // But scalar pairs are Rust-specific and get

compiler/rustc_abi/src/lib.rs

@@ -43,14 +43,17 @@ bitflags! {
 constIS_SIMD = 1 << 1;
 constIS_TRANSPARENT = 1 << 2;
 // Internal only for now. If true, don't reorder fields.
+// On its own it does not prevent ABI optimizations.
 constIS_LINEAR = 1 << 3;
-// If true, the type's layout can be randomized using
-// the seed stored in `ReprOptions.field_shuffle_seed`
+// If true, the type's crate has opted into layout randomization.
+// Other flags can still inhibit reordering and thus randomization.
+// The seed stored in `ReprOptions.field_shuffle_seed`.
 constRANDOMIZE_LAYOUT = 1 << 4;
 // Any of these flags being set prevent field reordering optimisation.
-constIS_UNOPTIMISABLE = ReprFlags::IS_C.bits()
+constFIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits()
  | ReprFlags::IS_SIMD.bits()
  | ReprFlags::IS_LINEAR.bits();
+constABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
 }
 }
 
@@ -139,10 +142,14 @@ impl ReprOptions {
 self.c() || self.int.is_some()
 }
 
+pubfninhibit_newtype_abi_optimization(&self) -> bool{
+self.flags.intersects(ReprFlags::ABI_UNOPTIMIZABLE)
+}
+
 /// Returns `true` if this `#[repr()]` guarantees a fixed field order,
 /// e.g. `repr(C)` or `repr(<int>)`.
 pubfninhibit_struct_field_reordering(&self) -> bool{
-self.flags.intersects(ReprFlags::IS_UNOPTIMISABLE) || self.int.is_some()
+self.flags.intersects(ReprFlags::FIELD_ORDER_UNOPTIMIZABLE) || self.int.is_some()
 }
 
 /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`

compiler/rustc_middle/src/ty/mod.rs

@@ -35,6 +35,7 @@ use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_errors::{Diag,ErrorGuaranteed,StashKey};
 use rustc_hir::def::{CtorKind,CtorOf,DefKind,DocLinkResMap,LifetimeRes,Res};
 use rustc_hir::def_id::{CrateNum,DefId,DefIdMap,LocalDefId,LocalDefIdMap};
+use rustc_hir::LangItem;
 use rustc_index::IndexVec;
 use rustc_macros::{
  extension,Decodable,Encodable,HashStable,TyDecodable,TyEncodable,TypeFoldable,
@@ -1570,8 +1571,15 @@ impl<'tcx> TyCtxt<'tcx> {
  flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
 }
 
+// box is special, on the one hand the compiler assumes an ordered layout, with the pointer
+// always at offset zero. On the other hand we want scalar abi optimizations.
+let is_box = self.is_lang_item(did.to_def_id(),LangItem::OwnedBox);
+
 // This is here instead of layout because the choice must make it into metadata.
-if !self.consider_optimizing(|| format!("Reorder fields of {:?}",self.def_path_str(did))){
+if is_box
+ || !self
+.consider_optimizing(|| format!("Reorder fields of {:?}",self.def_path_str(did)))
+{
  flags.insert(ReprFlags::IS_LINEAR);
 }