Auto merge of #147803 - jsgf:move-copy-codegen, r=madsmtm,saethlin

Add -Zannotate-moves for profiler visibility of move/copy operations (codegen)
**Note:** this is an alternative implementation of #147206; rather than being a MIR transform, it adds the annotations closer to codegen. It's functionally the same but the implementation is lower impact and it could be more correct.
---
This implements a new unstable compiler flag `-Zannotate-moves` that makes move and copy operations visible in profilers by creating synthetic debug information. This is achieved with zero runtime cost by manipulating debug info scopes to make moves/copies appear as calls to `compiler_move<T, SIZE>` and `compiler_copy<T, SIZE>` marker functions in profiling tools.
This allows developers to identify expensive move/copy operations in their code using standard profiling tools, without requiring specialized tooling or runtime instrumentation.
The implementation works at codegen time. When processing MIR operands (`Operand::Move` and `Operand::Copy`), the codegen creates an `OperandRef` with an optional `move_annotation` field containing an `Instance` of the appropriate profiling marker function. When storing the operand, `store_with_annotation()` wraps the store operation in a synthetic debug scope that makes it appear inlined from the marker.
Two marker functions (`compiler_move` and `compiler_copy`) are defined in `library/core/src/profiling.rs`. These are never actually called - they exist solely as debug info anchors.
Operations are only annotated if:
- We're generating debug info and the feature is enabled.
- Meets the size threshold (default: 65 bytes, configurable via `-Zannotate-moves=SIZE`), and is non-zero
- Has a memory representation
This has a very small size impact on object file size. With the default limit it's well under 0.1%, and even with a very small limit of 8 bytes it's still ~1.5%. This could be enabled by default.

Author: bors

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super)constUNKNOWN_LINE_NUMBER:u32 = 0;
 pub(super)constUNKNOWN_COLUMN_NUMBER:u32 = 0;
 
-impl<'a,'gcc,'tcx>DebugInfoBuilderMethodsforBuilder<'a,'gcc,'tcx>{
+impl<'a,'gcc,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'a,'gcc,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(

compiler/rustc_codegen_llvm/src/abi.rs

@@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
 );
  bx.lifetime_end(llscratch, scratch_size);
 }
-_ => {
+PassMode::Pair(..) | PassMode::Direct{ .. } => {
 OperandRef::from_immediate_or_packed_pair(bx, val,self.layout).val.store(bx, dst);
 }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -751,7 +751,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
 }
 }
 
-impl<'ll>DebugInfoBuilderMethodsforBuilder<'_,'ll,'_>{
+impl<'ll,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'_,'ll,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
  llvm::set_value_name(value, name.as_bytes());
 }
 }
+
+/// Annotate move/copy operations with debug info for profiling.
+///
+/// This creates a temporary debug scope that makes the move/copy appear as an inlined call to
+/// `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`. The provided closure is executed
+/// with this temporary debug location active.
+///
+/// The `instance` parameter should be the monomorphized instance of the `compiler_move` or
+/// `compiler_copy` function with the actual type and size.
+fnwith_move_annotation<R>(
+&mutself,
+instance: ty::Instance<'tcx>,
+f:implFnOnce(&mutSelf) -> R,
+) -> R{
+// Save the current debug location
+let saved_loc = self.get_dbg_loc();
+
+// Create a DIScope for the compiler_move/compiler_copy function
+// We use the function's FnAbi for debug info generation
+let fn_abi = self
+.cx()
+.tcx
+.fn_abi_of_instance(
+self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+)
+.unwrap();
+
+let di_scope = self.cx().dbg_scope_fn(instance, fn_abi,None);
+
+// Create an inlined debug location:
+// - scope: the compiler_move/compiler_copy function
+// - inlined_at: the current location (where the move/copy actually occurs)
+// - span: use the function's definition span
+let fn_span = self.cx().tcx.def_span(instance.def_id());
+let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+// Set the temporary debug location
+self.set_dbg_loc(inlined_loc);
+
+// Execute the closure (which will generate the memcpy)
+let result = f(self);
+
+// Restore the original debug location
+ifletSome(loc) = saved_loc {
+self.set_dbg_loc(loc);
+}else{
+self.clear_dbg_loc();
+}
+
+ result
+}
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 let op = matchself.locals[mir::RETURN_PLACE]{
 LocalRef::Operand(op) => op,
 LocalRef::PendingOperand => bug!("use of return before def"),
-LocalRef::Place(cg_place) => {
-OperandRef{val:Ref(cg_place.val),layout: cg_place.layout}
-}
+LocalRef::Place(cg_place) => OperandRef{
+val:Ref(cg_place.val),
+layout: cg_place.layout,
+move_annotation:None,
+},
 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 };
 let llslot = match op.val{
@@ -1155,7 +1157,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  | (&mir::Operand::Constant(_),Ref(PlaceValue{llextra:None, .. })) => {
 let tmp = PlaceRef::alloca(bx, op.layout);
  bx.lifetime_start(tmp.val.llval, tmp.layout.size);
- op.val.store(bx, tmp);
+ op.store_with_annotation(bx, tmp);
  op.val = Ref(tmp.val);
  lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
 }
@@ -1563,13 +1565,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 };
 let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
  bx.lifetime_start(scratch.llval, arg.layout.size);
- op.val.store(bx, scratch.with_type(arg.layout));
+ op.store_with_annotation(bx, scratch.with_type(arg.layout));
  lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
 (scratch.llval, scratch.align,true)
 }
 PassMode::Cast{ .. } => {
 let scratch = PlaceRef::alloca(bx, arg.layout);
- op.val.store(bx, scratch);
+ op.store_with_annotation(bx, scratch);
 (scratch.val.llval, scratch.val.align,true)
 }
  _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi,false),

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 returnlocal(OperandRef{
 val:OperandValue::Pair(a, b),
 layout: arg.layout,
+move_annotation:None,
 });
 }
  _ => {}
@@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  fx.caller_location = Some(OperandRef{
 val:OperandValue::Immediate(bx.get_param(llarg_idx)),
 layout: arg.layout,
+move_annotation:None,
 });
 }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -5,12 +5,13 @@ use rustc_abi as abi;
 use rustc_abi::{
 Align,BackendRepr,FIRST_VARIANT,FieldIdx,Primitive,Size,TagEncoding,VariantIdx,Variants,
 };
+use rustc_hir::LangItem;
 use rustc_middle::mir::interpret::{Pointer,Scalar, alloc_range};
 use rustc_middle::mir::{self,ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf,TyAndLayout};
+use rustc_middle::ty::{self,Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves,DebugInfo,OptLevel};
 use tracing::{debug, instrument};
 
 usesuper::place::{PlaceRef,PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
 /// The layout of value, based on its Rust type.
 publayout:TyAndLayout<'tcx>,
+
+/// Annotation for profiler visibility of move/copy operations.
+/// When set, the store operation should appear as an inlined call to this function.
+pubmove_annotation:Option<ty::Instance<'tcx>>,
 }
 
 impl<V:CodegenObject> fmt::DebugforOperandRef<'_,V>{
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a,'tcx,V:CodegenObject>OperandRef<'tcx,V>{
 pubfnzero_sized(layout:TyAndLayout<'tcx>) -> OperandRef<'tcx,V>{
 assert!(layout.is_zst());
-OperandRef{val:OperandValue::ZeroSized, layout }
+OperandRef{val:OperandValue::ZeroSized, layout,move_annotation:None}
 }
 
 pub(crate)fnfrom_const<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }
 };
 
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 fnfrom_const_alloc<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 let size = s.size(bx);
 assert_eq!(size, layout.size,"abi::Scalar size does not match layout size");
 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-OperandRef{val:OperandValue::Immediate(val), layout }
+OperandRef{val:OperandValue::Immediate(val), layout,move_annotation:None}
 }
 BackendRepr::ScalarPair(
  a @ abi::Scalar::Initialized{ .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
  b,
  bx.scalar_pair_element_backend_type(layout,1,true),
 );
-OperandRef{val:OperandValue::Pair(a_val, b_val), layout }
+OperandRef{val:OperandValue::Pair(a_val, b_val), layout,move_annotation:None}
 }
  _ if layout.is_zst() => OperandRef::zero_sized(layout),
  _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 self.val.deref(layout.align.abi).with_type(layout)
 }
 
+/// Store this operand into a place, applying move/copy annotation if present.
+///
+/// This is the preferred method for storing operands, as it automatically
+/// applies profiler annotations for tracked move/copy operations.
+pubfnstore_with_annotation<Bx:BuilderMethods<'a,'tcx,Value = V>>(
+self,
+bx:&mutBx,
+dest:PlaceRef<'tcx,V>,
+){
+ifletSome(instance) = self.move_annotation{
+ bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+}else{
+self.val.store(bx, dest)
+}
+}
+
 /// If this operand is a `Pair`, we return an aggregate with the two values.
 /// For other cases, see `immediate`.
 pubfnimmediate_or_packed_pair<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }else{
 OperandValue::Immediate(llval)
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 pub(crate)fnextract_field<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 })
 };
 
-OperandRef{ val,layout: field }
+OperandRef{ val,layout: field,move_annotation:None}
 }
 
 /// Obtain the actual discriminant of a value.
@@ -828,10 +849,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
 }
 },
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 }
 
+/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache
+/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only
+/// annotate copies larger than this.
+constMOVE_ANNOTATION_DEFAULT_LIMIT:u64 = 65;
+
 impl<'a,'tcx,V:CodegenObject>OperandValue<V>{
 /// Returns an `OperandValue` that's generally UB to use in any way.
 ///
@@ -961,7 +987,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  abi::Variants::Single{ index: vidx },
 );
 let layout = o.layout.for_variant(bx.cx(), vidx);
- o = OperandRef{val: o.val, layout}
+ o = OperandRef{layout, ..o}
 }
  _ => returnNone,
 }
@@ -1014,7 +1040,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
 match*operand {
  mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-self.codegen_consume(bx, place.as_ref())
+let kind = match operand {
+ mir::Operand::Move(_) => LangItem::CompilerMove,
+ mir::Operand::Copy(_) => LangItem::CompilerCopy,
+ _ => unreachable!(),
+};
+
+// Check if we should annotate this move/copy for profiling
+let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind);
+
+OperandRef{ move_annotation, ..self.codegen_consume(bx, place.as_ref())}
 }
 
  mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 returnOperandRef{
 val:OperandValue::Immediate(llval),
 layout: bx.layout_of(ty),
+move_annotation:None,
 };
 }
 }
 self.eval_mir_constant_to_operand(bx, constant)
 }
 }
 }
+
+/// Creates an `Instance` for annotating a move/copy operation at codegen time.
+///
+/// Returns `Some(instance)` if the operation should be annotated with debug info, `None`
+/// otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>` or
+/// `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+///
+/// There are a number of conditions that must be met for an annotation to be created, but aside
+/// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is
+/// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also
+/// that the underlying representation of the type is in memory.
+fnmove_copy_annotation_instance(
+&self,
+bx:&Bx,
+place: mir::PlaceRef<'tcx>,
+kind:LangItem,
+) -> Option<ty::Instance<'tcx>>{
+let tcx = bx.tcx();
+let sess = tcx.sess;
+
+// Skip if we're not generating debuginfo
+if sess.opts.debuginfo == DebugInfo::None{
+returnNone;
+}
+
+// Check if annotation is enabled and get size limit (otherwise skip)
+let size_limit = match sess.opts.unstable_opts.annotate_moves{
+AnnotateMoves::Disabled => returnNone,
+AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+AnnotateMoves::Enabled(Some(limit)) => limit,
+};
+
+let ty = self.monomorphized_place_ty(place);
+let layout = bx.cx().layout_of(ty);
+let ty_size = layout.size.bytes();
+
+// Only annotate if type has a memory representation and exceeds size limit (and has a
+// non-zero size)
+if layout.is_zst()
+ || ty_size < size_limit
+ || !matches!(layout.backend_repr,BackendRepr::Memory{ .. })
+{
+returnNone;
+}
+
+// Look up the DefId for compiler_move or compiler_copy lang item
+let def_id = tcx.lang_items().get(kind)?;
+
+// Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+let size_const = ty::Const::from_target_usize(tcx, ty_size);
+let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+// Create the Instance
+let typing_env = self.mir.typing_env(tcx);
+let instance = ty::Instance::expect_resolve(
+ tcx,
+ typing_env,
+ def_id,
+ generic_args,
+ rustc_span::DUMMY_SP,// span only used for error messages
+);
+
+Some(instance)
+}
 }

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super)constUNKNOWN_LINE_NUMBER:u32 = 0;
 pub(super)constUNKNOWN_COLUMN_NUMBER:u32 = 0;
 
-impl<'a,'gcc,'tcx>DebugInfoBuilderMethodsforBuilder<'a,'gcc,'tcx>{
+impl<'a,'gcc,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'a,'gcc,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(

compiler/rustc_codegen_llvm/src/abi.rs

@@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
 );
  bx.lifetime_end(llscratch, scratch_size);
 }
-_ => {
+PassMode::Pair(..) | PassMode::Direct{ .. } => {
 OperandRef::from_immediate_or_packed_pair(bx, val,self.layout).val.store(bx, dst);
 }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -751,7 +751,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
 }
 }
 
-impl<'ll>DebugInfoBuilderMethodsforBuilder<'_,'ll,'_>{
+impl<'ll,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'_,'ll,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
  llvm::set_value_name(value, name.as_bytes());
 }
 }
+
+/// Annotate move/copy operations with debug info for profiling.
+///
+/// This creates a temporary debug scope that makes the move/copy appear as an inlined call to
+/// `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`. The provided closure is executed
+/// with this temporary debug location active.
+///
+/// The `instance` parameter should be the monomorphized instance of the `compiler_move` or
+/// `compiler_copy` function with the actual type and size.
+fnwith_move_annotation<R>(
+&mutself,
+instance: ty::Instance<'tcx>,
+f:implFnOnce(&mutSelf) -> R,
+) -> R{
+// Save the current debug location
+let saved_loc = self.get_dbg_loc();
+
+// Create a DIScope for the compiler_move/compiler_copy function
+// We use the function's FnAbi for debug info generation
+let fn_abi = self
+.cx()
+.tcx
+.fn_abi_of_instance(
+self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+)
+.unwrap();
+
+let di_scope = self.cx().dbg_scope_fn(instance, fn_abi,None);
+
+// Create an inlined debug location:
+// - scope: the compiler_move/compiler_copy function
+// - inlined_at: the current location (where the move/copy actually occurs)
+// - span: use the function's definition span
+let fn_span = self.cx().tcx.def_span(instance.def_id());
+let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+// Set the temporary debug location
+self.set_dbg_loc(inlined_loc);
+
+// Execute the closure (which will generate the memcpy)
+let result = f(self);
+
+// Restore the original debug location
+ifletSome(loc) = saved_loc {
+self.set_dbg_loc(loc);
+}else{
+self.clear_dbg_loc();
+}
+
+ result
+}
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 let op = matchself.locals[mir::RETURN_PLACE]{
 LocalRef::Operand(op) => op,
 LocalRef::PendingOperand => bug!("use of return before def"),
-LocalRef::Place(cg_place) => {
-OperandRef{val:Ref(cg_place.val),layout: cg_place.layout}
-}
+LocalRef::Place(cg_place) => OperandRef{
+val:Ref(cg_place.val),
+layout: cg_place.layout,
+move_annotation:None,
+},
 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 };
 let llslot = match op.val{
@@ -1155,7 +1157,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  | (&mir::Operand::Constant(_),Ref(PlaceValue{llextra:None, .. })) => {
 let tmp = PlaceRef::alloca(bx, op.layout);
  bx.lifetime_start(tmp.val.llval, tmp.layout.size);
- op.val.store(bx, tmp);
+ op.store_with_annotation(bx, tmp);
  op.val = Ref(tmp.val);
  lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
 }
@@ -1563,13 +1565,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 };
 let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
  bx.lifetime_start(scratch.llval, arg.layout.size);
- op.val.store(bx, scratch.with_type(arg.layout));
+ op.store_with_annotation(bx, scratch.with_type(arg.layout));
  lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
 (scratch.llval, scratch.align,true)
 }
 PassMode::Cast{ .. } => {
 let scratch = PlaceRef::alloca(bx, arg.layout);
- op.val.store(bx, scratch);
+ op.store_with_annotation(bx, scratch);
 (scratch.val.llval, scratch.val.align,true)
 }
  _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi,false),

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 returnlocal(OperandRef{
 val:OperandValue::Pair(a, b),
 layout: arg.layout,
+move_annotation:None,
 });
 }
  _ => {}
@@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  fx.caller_location = Some(OperandRef{
 val:OperandValue::Immediate(bx.get_param(llarg_idx)),
 layout: arg.layout,
+move_annotation:None,
 });
 }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -5,12 +5,13 @@ use rustc_abi as abi;
 use rustc_abi::{
 Align,BackendRepr,FIRST_VARIANT,FieldIdx,Primitive,Size,TagEncoding,VariantIdx,Variants,
 };
+use rustc_hir::LangItem;
 use rustc_middle::mir::interpret::{Pointer,Scalar, alloc_range};
 use rustc_middle::mir::{self,ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf,TyAndLayout};
+use rustc_middle::ty::{self,Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves,DebugInfo,OptLevel};
 use tracing::{debug, instrument};
 
 usesuper::place::{PlaceRef,PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
 /// The layout of value, based on its Rust type.
 publayout:TyAndLayout<'tcx>,
+
+/// Annotation for profiler visibility of move/copy operations.
+/// When set, the store operation should appear as an inlined call to this function.
+pubmove_annotation:Option<ty::Instance<'tcx>>,
 }
 
 impl<V:CodegenObject> fmt::DebugforOperandRef<'_,V>{
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a,'tcx,V:CodegenObject>OperandRef<'tcx,V>{
 pubfnzero_sized(layout:TyAndLayout<'tcx>) -> OperandRef<'tcx,V>{
 assert!(layout.is_zst());
-OperandRef{val:OperandValue::ZeroSized, layout }
+OperandRef{val:OperandValue::ZeroSized, layout,move_annotation:None}
 }
 
 pub(crate)fnfrom_const<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }
 };
 
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 fnfrom_const_alloc<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 let size = s.size(bx);
 assert_eq!(size, layout.size,"abi::Scalar size does not match layout size");
 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-OperandRef{val:OperandValue::Immediate(val), layout }
+OperandRef{val:OperandValue::Immediate(val), layout,move_annotation:None}
 }
 BackendRepr::ScalarPair(
  a @ abi::Scalar::Initialized{ .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
  b,
  bx.scalar_pair_element_backend_type(layout,1,true),
 );
-OperandRef{val:OperandValue::Pair(a_val, b_val), layout }
+OperandRef{val:OperandValue::Pair(a_val, b_val), layout,move_annotation:None}
 }
  _ if layout.is_zst() => OperandRef::zero_sized(layout),
  _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 self.val.deref(layout.align.abi).with_type(layout)
 }
 
+/// Store this operand into a place, applying move/copy annotation if present.
+///
+/// This is the preferred method for storing operands, as it automatically
+/// applies profiler annotations for tracked move/copy operations.
+pubfnstore_with_annotation<Bx:BuilderMethods<'a,'tcx,Value = V>>(
+self,
+bx:&mutBx,
+dest:PlaceRef<'tcx,V>,
+){
+ifletSome(instance) = self.move_annotation{
+ bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+}else{
+self.val.store(bx, dest)
+}
+}
+
 /// If this operand is a `Pair`, we return an aggregate with the two values.
 /// For other cases, see `immediate`.
 pubfnimmediate_or_packed_pair<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }else{
 OperandValue::Immediate(llval)
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 pub(crate)fnextract_field<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 })
 };
 
-OperandRef{ val,layout: field }
+OperandRef{ val,layout: field,move_annotation:None}
 }
 
 /// Obtain the actual discriminant of a value.
@@ -828,10 +849,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
 }
 },
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 }
 
+/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache
+/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only
+/// annotate copies larger than this.
+constMOVE_ANNOTATION_DEFAULT_LIMIT:u64 = 65;
+
 impl<'a,'tcx,V:CodegenObject>OperandValue<V>{
 /// Returns an `OperandValue` that's generally UB to use in any way.
 ///
@@ -961,7 +987,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  abi::Variants::Single{ index: vidx },
 );
 let layout = o.layout.for_variant(bx.cx(), vidx);
- o = OperandRef{val: o.val, layout}
+ o = OperandRef{layout, ..o}
 }
  _ => returnNone,
 }
@@ -1014,7 +1040,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
 match*operand {
  mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-self.codegen_consume(bx, place.as_ref())
+let kind = match operand {
+ mir::Operand::Move(_) => LangItem::CompilerMove,
+ mir::Operand::Copy(_) => LangItem::CompilerCopy,
+ _ => unreachable!(),
+};
+
+// Check if we should annotate this move/copy for profiling
+let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind);
+
+OperandRef{ move_annotation, ..self.codegen_consume(bx, place.as_ref())}
 }
 
  mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 returnOperandRef{
 val:OperandValue::Immediate(llval),
 layout: bx.layout_of(ty),
+move_annotation:None,
 };
 }
 }
 self.eval_mir_constant_to_operand(bx, constant)
 }
 }
 }
+
+/// Creates an `Instance` for annotating a move/copy operation at codegen time.
+///
+/// Returns `Some(instance)` if the operation should be annotated with debug info, `None`
+/// otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>` or
+/// `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+///
+/// There are a number of conditions that must be met for an annotation to be created, but aside
+/// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is
+/// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also
+/// that the underlying representation of the type is in memory.
+fnmove_copy_annotation_instance(
+&self,
+bx:&Bx,
+place: mir::PlaceRef<'tcx>,
+kind:LangItem,
+) -> Option<ty::Instance<'tcx>>{
+let tcx = bx.tcx();
+let sess = tcx.sess;
+
+// Skip if we're not generating debuginfo
+if sess.opts.debuginfo == DebugInfo::None{
+returnNone;
+}
+
+// Check if annotation is enabled and get size limit (otherwise skip)
+let size_limit = match sess.opts.unstable_opts.annotate_moves{
+AnnotateMoves::Disabled => returnNone,
+AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+AnnotateMoves::Enabled(Some(limit)) => limit,
+};
+
+let ty = self.monomorphized_place_ty(place);
+let layout = bx.cx().layout_of(ty);
+let ty_size = layout.size.bytes();
+
+// Only annotate if type has a memory representation and exceeds size limit (and has a
+// non-zero size)
+if layout.is_zst()
+ || ty_size < size_limit
+ || !matches!(layout.backend_repr,BackendRepr::Memory{ .. })
+{
+returnNone;
+}
+
+// Look up the DefId for compiler_move or compiler_copy lang item
+let def_id = tcx.lang_items().get(kind)?;
+
+// Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+let size_const = ty::Const::from_target_usize(tcx, ty_size);
+let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+// Create the Instance
+let typing_env = self.mir.typing_env(tcx);
+let instance = ty::Instance::expect_resolve(
+ tcx,
+ typing_env,
+ def_id,
+ generic_args,
+ rustc_span::DUMMY_SP,// span only used for error messages
+);
+
+Some(instance)
+}
 }

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super)constUNKNOWN_LINE_NUMBER:u32 = 0;
 pub(super)constUNKNOWN_COLUMN_NUMBER:u32 = 0;
 
-impl<'a,'gcc,'tcx>DebugInfoBuilderMethodsforBuilder<'a,'gcc,'tcx>{
+impl<'a,'gcc,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'a,'gcc,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(

compiler/rustc_codegen_llvm/src/abi.rs

@@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
 );
  bx.lifetime_end(llscratch, scratch_size);
 }
-_ => {
+PassMode::Pair(..) | PassMode::Direct{ .. } => {
 OperandRef::from_immediate_or_packed_pair(bx, val,self.layout).val.store(bx, dst);
 }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -751,7 +751,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
 }
 }
 
-impl<'ll>DebugInfoBuilderMethodsforBuilder<'_,'ll,'_>{
+impl<'ll,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'_,'ll,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
  llvm::set_value_name(value, name.as_bytes());
 }
 }
+
+/// Annotate move/copy operations with debug info for profiling.
+///
+/// This creates a temporary debug scope that makes the move/copy appear as an inlined call to
+/// `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`. The provided closure is executed
+/// with this temporary debug location active.
+///
+/// The `instance` parameter should be the monomorphized instance of the `compiler_move` or
+/// `compiler_copy` function with the actual type and size.
+fnwith_move_annotation<R>(
+&mutself,
+instance: ty::Instance<'tcx>,
+f:implFnOnce(&mutSelf) -> R,
+) -> R{
+// Save the current debug location
+let saved_loc = self.get_dbg_loc();
+
+// Create a DIScope for the compiler_move/compiler_copy function
+// We use the function's FnAbi for debug info generation
+let fn_abi = self
+.cx()
+.tcx
+.fn_abi_of_instance(
+self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+)
+.unwrap();
+
+let di_scope = self.cx().dbg_scope_fn(instance, fn_abi,None);
+
+// Create an inlined debug location:
+// - scope: the compiler_move/compiler_copy function
+// - inlined_at: the current location (where the move/copy actually occurs)
+// - span: use the function's definition span
+let fn_span = self.cx().tcx.def_span(instance.def_id());
+let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+// Set the temporary debug location
+self.set_dbg_loc(inlined_loc);
+
+// Execute the closure (which will generate the memcpy)
+let result = f(self);
+
+// Restore the original debug location
+ifletSome(loc) = saved_loc {
+self.set_dbg_loc(loc);
+}else{
+self.clear_dbg_loc();
+}
+
+ result
+}
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 let op = matchself.locals[mir::RETURN_PLACE]{
 LocalRef::Operand(op) => op,
 LocalRef::PendingOperand => bug!("use of return before def"),
-LocalRef::Place(cg_place) => {
-OperandRef{val:Ref(cg_place.val),layout: cg_place.layout}
-}
+LocalRef::Place(cg_place) => OperandRef{
+val:Ref(cg_place.val),
+layout: cg_place.layout,
+move_annotation:None,
+},
 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 };
 let llslot = match op.val{
@@ -1155,7 +1157,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  | (&mir::Operand::Constant(_),Ref(PlaceValue{llextra:None, .. })) => {
 let tmp = PlaceRef::alloca(bx, op.layout);
  bx.lifetime_start(tmp.val.llval, tmp.layout.size);
- op.val.store(bx, tmp);
+ op.store_with_annotation(bx, tmp);
  op.val = Ref(tmp.val);
  lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
 }
@@ -1563,13 +1565,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 };
 let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
  bx.lifetime_start(scratch.llval, arg.layout.size);
- op.val.store(bx, scratch.with_type(arg.layout));
+ op.store_with_annotation(bx, scratch.with_type(arg.layout));
  lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
 (scratch.llval, scratch.align,true)
 }
 PassMode::Cast{ .. } => {
 let scratch = PlaceRef::alloca(bx, arg.layout);
- op.val.store(bx, scratch);
+ op.store_with_annotation(bx, scratch);
 (scratch.val.llval, scratch.val.align,true)
 }
  _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi,false),

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 returnlocal(OperandRef{
 val:OperandValue::Pair(a, b),
 layout: arg.layout,
+move_annotation:None,
 });
 }
  _ => {}
@@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  fx.caller_location = Some(OperandRef{
 val:OperandValue::Immediate(bx.get_param(llarg_idx)),
 layout: arg.layout,
+move_annotation:None,
 });
 }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -5,12 +5,13 @@ use rustc_abi as abi;
 use rustc_abi::{
 Align,BackendRepr,FIRST_VARIANT,FieldIdx,Primitive,Size,TagEncoding,VariantIdx,Variants,
 };
+use rustc_hir::LangItem;
 use rustc_middle::mir::interpret::{Pointer,Scalar, alloc_range};
 use rustc_middle::mir::{self,ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf,TyAndLayout};
+use rustc_middle::ty::{self,Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves,DebugInfo,OptLevel};
 use tracing::{debug, instrument};
 
 usesuper::place::{PlaceRef,PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
 /// The layout of value, based on its Rust type.
 publayout:TyAndLayout<'tcx>,
+
+/// Annotation for profiler visibility of move/copy operations.
+/// When set, the store operation should appear as an inlined call to this function.
+pubmove_annotation:Option<ty::Instance<'tcx>>,
 }
 
 impl<V:CodegenObject> fmt::DebugforOperandRef<'_,V>{
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a,'tcx,V:CodegenObject>OperandRef<'tcx,V>{
 pubfnzero_sized(layout:TyAndLayout<'tcx>) -> OperandRef<'tcx,V>{
 assert!(layout.is_zst());
-OperandRef{val:OperandValue::ZeroSized, layout }
+OperandRef{val:OperandValue::ZeroSized, layout,move_annotation:None}
 }
 
 pub(crate)fnfrom_const<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }
 };
 
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 fnfrom_const_alloc<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 let size = s.size(bx);
 assert_eq!(size, layout.size,"abi::Scalar size does not match layout size");
 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-OperandRef{val:OperandValue::Immediate(val), layout }
+OperandRef{val:OperandValue::Immediate(val), layout,move_annotation:None}
 }
 BackendRepr::ScalarPair(
  a @ abi::Scalar::Initialized{ .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
  b,
  bx.scalar_pair_element_backend_type(layout,1,true),
 );
-OperandRef{val:OperandValue::Pair(a_val, b_val), layout }
+OperandRef{val:OperandValue::Pair(a_val, b_val), layout,move_annotation:None}
 }
  _ if layout.is_zst() => OperandRef::zero_sized(layout),
  _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 self.val.deref(layout.align.abi).with_type(layout)
 }
 
+/// Store this operand into a place, applying move/copy annotation if present.
+///
+/// This is the preferred method for storing operands, as it automatically
+/// applies profiler annotations for tracked move/copy operations.
+pubfnstore_with_annotation<Bx:BuilderMethods<'a,'tcx,Value = V>>(
+self,
+bx:&mutBx,
+dest:PlaceRef<'tcx,V>,
+){
+ifletSome(instance) = self.move_annotation{
+ bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+}else{
+self.val.store(bx, dest)
+}
+}
+
 /// If this operand is a `Pair`, we return an aggregate with the two values.
 /// For other cases, see `immediate`.
 pubfnimmediate_or_packed_pair<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }else{
 OperandValue::Immediate(llval)
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 pub(crate)fnextract_field<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 })
 };
 
-OperandRef{ val,layout: field }
+OperandRef{ val,layout: field,move_annotation:None}
 }
 
 /// Obtain the actual discriminant of a value.
@@ -828,10 +849,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
 }
 },
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 }
 
+/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache
+/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only
+/// annotate copies larger than this.
+constMOVE_ANNOTATION_DEFAULT_LIMIT:u64 = 65;
+
 impl<'a,'tcx,V:CodegenObject>OperandValue<V>{
 /// Returns an `OperandValue` that's generally UB to use in any way.
 ///
@@ -961,7 +987,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  abi::Variants::Single{ index: vidx },
 );
 let layout = o.layout.for_variant(bx.cx(), vidx);
- o = OperandRef{val: o.val, layout}
+ o = OperandRef{layout, ..o}
 }
  _ => returnNone,
 }
@@ -1014,7 +1040,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
 match*operand {
  mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-self.codegen_consume(bx, place.as_ref())
+let kind = match operand {
+ mir::Operand::Move(_) => LangItem::CompilerMove,
+ mir::Operand::Copy(_) => LangItem::CompilerCopy,
+ _ => unreachable!(),
+};
+
+// Check if we should annotate this move/copy for profiling
+let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind);
+
+OperandRef{ move_annotation, ..self.codegen_consume(bx, place.as_ref())}
 }
 
  mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 returnOperandRef{
 val:OperandValue::Immediate(llval),
 layout: bx.layout_of(ty),
+move_annotation:None,
 };
 }
 }
 self.eval_mir_constant_to_operand(bx, constant)
 }
 }
 }
+
+/// Creates an `Instance` for annotating a move/copy operation at codegen time.
+///
+/// Returns `Some(instance)` if the operation should be annotated with debug info, `None`
+/// otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>` or
+/// `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+///
+/// There are a number of conditions that must be met for an annotation to be created, but aside
+/// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is
+/// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also
+/// that the underlying representation of the type is in memory.
+fnmove_copy_annotation_instance(
+&self,
+bx:&Bx,
+place: mir::PlaceRef<'tcx>,
+kind:LangItem,
+) -> Option<ty::Instance<'tcx>>{
+let tcx = bx.tcx();
+let sess = tcx.sess;
+
+// Skip if we're not generating debuginfo
+if sess.opts.debuginfo == DebugInfo::None{
+returnNone;
+}
+
+// Check if annotation is enabled and get size limit (otherwise skip)
+let size_limit = match sess.opts.unstable_opts.annotate_moves{
+AnnotateMoves::Disabled => returnNone,
+AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+AnnotateMoves::Enabled(Some(limit)) => limit,
+};
+
+let ty = self.monomorphized_place_ty(place);
+let layout = bx.cx().layout_of(ty);
+let ty_size = layout.size.bytes();
+
+// Only annotate if type has a memory representation and exceeds size limit (and has a
+// non-zero size)
+if layout.is_zst()
+ || ty_size < size_limit
+ || !matches!(layout.backend_repr,BackendRepr::Memory{ .. })
+{
+returnNone;
+}
+
+// Look up the DefId for compiler_move or compiler_copy lang item
+let def_id = tcx.lang_items().get(kind)?;
+
+// Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+let size_const = ty::Const::from_target_usize(tcx, ty_size);
+let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+// Create the Instance
+let typing_env = self.mir.typing_env(tcx);
+let instance = ty::Instance::expect_resolve(
+ tcx,
+ typing_env,
+ def_id,
+ generic_args,
+ rustc_span::DUMMY_SP,// span only used for error messages
+);
+
+Some(instance)
+}
 }

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super)constUNKNOWN_LINE_NUMBER:u32 = 0;
 pub(super)constUNKNOWN_COLUMN_NUMBER:u32 = 0;
 
-impl<'a,'gcc,'tcx>DebugInfoBuilderMethodsforBuilder<'a,'gcc,'tcx>{
+impl<'a,'gcc,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'a,'gcc,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(

compiler/rustc_codegen_llvm/src/abi.rs

@@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
 );
  bx.lifetime_end(llscratch, scratch_size);
 }
-_ => {
+PassMode::Pair(..) | PassMode::Direct{ .. } => {
 OperandRef::from_immediate_or_packed_pair(bx, val,self.layout).val.store(bx, dst);
 }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -751,7 +751,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
 }
 }
 
-impl<'ll>DebugInfoBuilderMethodsforBuilder<'_,'ll,'_>{
+impl<'ll,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'_,'ll,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
  llvm::set_value_name(value, name.as_bytes());
 }
 }
+
+/// Annotate move/copy operations with debug info for profiling.
+///
+/// This creates a temporary debug scope that makes the move/copy appear as an inlined call to
+/// `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`. The provided closure is executed
+/// with this temporary debug location active.
+///
+/// The `instance` parameter should be the monomorphized instance of the `compiler_move` or
+/// `compiler_copy` function with the actual type and size.
+fnwith_move_annotation<R>(
+&mutself,
+instance: ty::Instance<'tcx>,
+f:implFnOnce(&mutSelf) -> R,
+) -> R{
+// Save the current debug location
+let saved_loc = self.get_dbg_loc();
+
+// Create a DIScope for the compiler_move/compiler_copy function
+// We use the function's FnAbi for debug info generation
+let fn_abi = self
+.cx()
+.tcx
+.fn_abi_of_instance(
+self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+)
+.unwrap();
+
+let di_scope = self.cx().dbg_scope_fn(instance, fn_abi,None);
+
+// Create an inlined debug location:
+// - scope: the compiler_move/compiler_copy function
+// - inlined_at: the current location (where the move/copy actually occurs)
+// - span: use the function's definition span
+let fn_span = self.cx().tcx.def_span(instance.def_id());
+let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+// Set the temporary debug location
+self.set_dbg_loc(inlined_loc);
+
+// Execute the closure (which will generate the memcpy)
+let result = f(self);
+
+// Restore the original debug location
+ifletSome(loc) = saved_loc {
+self.set_dbg_loc(loc);
+}else{
+self.clear_dbg_loc();
+}
+
+ result
+}
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 let op = matchself.locals[mir::RETURN_PLACE]{
 LocalRef::Operand(op) => op,
 LocalRef::PendingOperand => bug!("use of return before def"),
-LocalRef::Place(cg_place) => {
-OperandRef{val:Ref(cg_place.val),layout: cg_place.layout}
-}
+LocalRef::Place(cg_place) => OperandRef{
+val:Ref(cg_place.val),
+layout: cg_place.layout,
+move_annotation:None,
+},
 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 };
 let llslot = match op.val{
@@ -1155,7 +1157,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  | (&mir::Operand::Constant(_),Ref(PlaceValue{llextra:None, .. })) => {
 let tmp = PlaceRef::alloca(bx, op.layout);
  bx.lifetime_start(tmp.val.llval, tmp.layout.size);
- op.val.store(bx, tmp);
+ op.store_with_annotation(bx, tmp);
  op.val = Ref(tmp.val);
  lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
 }
@@ -1563,13 +1565,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 };
 let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
  bx.lifetime_start(scratch.llval, arg.layout.size);
- op.val.store(bx, scratch.with_type(arg.layout));
+ op.store_with_annotation(bx, scratch.with_type(arg.layout));
  lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
 (scratch.llval, scratch.align,true)
 }
 PassMode::Cast{ .. } => {
 let scratch = PlaceRef::alloca(bx, arg.layout);
- op.val.store(bx, scratch);
+ op.store_with_annotation(bx, scratch);
 (scratch.val.llval, scratch.val.align,true)
 }
  _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi,false),

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 returnlocal(OperandRef{
 val:OperandValue::Pair(a, b),
 layout: arg.layout,
+move_annotation:None,
 });
 }
  _ => {}
@@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  fx.caller_location = Some(OperandRef{
 val:OperandValue::Immediate(bx.get_param(llarg_idx)),
 layout: arg.layout,
+move_annotation:None,
 });
 }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -5,12 +5,13 @@ use rustc_abi as abi;
 use rustc_abi::{
 Align,BackendRepr,FIRST_VARIANT,FieldIdx,Primitive,Size,TagEncoding,VariantIdx,Variants,
 };
+use rustc_hir::LangItem;
 use rustc_middle::mir::interpret::{Pointer,Scalar, alloc_range};
 use rustc_middle::mir::{self,ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf,TyAndLayout};
+use rustc_middle::ty::{self,Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves,DebugInfo,OptLevel};
 use tracing::{debug, instrument};
 
 usesuper::place::{PlaceRef,PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
 /// The layout of value, based on its Rust type.
 publayout:TyAndLayout<'tcx>,
+
+/// Annotation for profiler visibility of move/copy operations.
+/// When set, the store operation should appear as an inlined call to this function.
+pubmove_annotation:Option<ty::Instance<'tcx>>,
 }
 
 impl<V:CodegenObject> fmt::DebugforOperandRef<'_,V>{
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a,'tcx,V:CodegenObject>OperandRef<'tcx,V>{
 pubfnzero_sized(layout:TyAndLayout<'tcx>) -> OperandRef<'tcx,V>{
 assert!(layout.is_zst());
-OperandRef{val:OperandValue::ZeroSized, layout }
+OperandRef{val:OperandValue::ZeroSized, layout,move_annotation:None}
 }
 
 pub(crate)fnfrom_const<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }
 };
 
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 fnfrom_const_alloc<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 let size = s.size(bx);
 assert_eq!(size, layout.size,"abi::Scalar size does not match layout size");
 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-OperandRef{val:OperandValue::Immediate(val), layout }
+OperandRef{val:OperandValue::Immediate(val), layout,move_annotation:None}
 }
 BackendRepr::ScalarPair(
  a @ abi::Scalar::Initialized{ .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
  b,
  bx.scalar_pair_element_backend_type(layout,1,true),
 );
-OperandRef{val:OperandValue::Pair(a_val, b_val), layout }
+OperandRef{val:OperandValue::Pair(a_val, b_val), layout,move_annotation:None}
 }
  _ if layout.is_zst() => OperandRef::zero_sized(layout),
  _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 self.val.deref(layout.align.abi).with_type(layout)
 }
 
+/// Store this operand into a place, applying move/copy annotation if present.
+///
+/// This is the preferred method for storing operands, as it automatically
+/// applies profiler annotations for tracked move/copy operations.
+pubfnstore_with_annotation<Bx:BuilderMethods<'a,'tcx,Value = V>>(
+self,
+bx:&mutBx,
+dest:PlaceRef<'tcx,V>,
+){
+ifletSome(instance) = self.move_annotation{
+ bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+}else{
+self.val.store(bx, dest)
+}
+}
+
 /// If this operand is a `Pair`, we return an aggregate with the two values.
 /// For other cases, see `immediate`.
 pubfnimmediate_or_packed_pair<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }else{
 OperandValue::Immediate(llval)
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 pub(crate)fnextract_field<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 })
 };
 
-OperandRef{ val,layout: field }
+OperandRef{ val,layout: field,move_annotation:None}
 }
 
 /// Obtain the actual discriminant of a value.
@@ -828,10 +849,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
 }
 },
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 }
 
+/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache
+/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only
+/// annotate copies larger than this.
+constMOVE_ANNOTATION_DEFAULT_LIMIT:u64 = 65;
+
 impl<'a,'tcx,V:CodegenObject>OperandValue<V>{
 /// Returns an `OperandValue` that's generally UB to use in any way.
 ///
@@ -961,7 +987,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  abi::Variants::Single{ index: vidx },
 );
 let layout = o.layout.for_variant(bx.cx(), vidx);
- o = OperandRef{val: o.val, layout}
+ o = OperandRef{layout, ..o}
 }
  _ => returnNone,
 }
@@ -1014,7 +1040,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
 match*operand {
  mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-self.codegen_consume(bx, place.as_ref())
+let kind = match operand {
+ mir::Operand::Move(_) => LangItem::CompilerMove,
+ mir::Operand::Copy(_) => LangItem::CompilerCopy,
+ _ => unreachable!(),
+};
+
+// Check if we should annotate this move/copy for profiling
+let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind);
+
+OperandRef{ move_annotation, ..self.codegen_consume(bx, place.as_ref())}
 }
 
  mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 returnOperandRef{
 val:OperandValue::Immediate(llval),
 layout: bx.layout_of(ty),
+move_annotation:None,
 };
 }
 }
 self.eval_mir_constant_to_operand(bx, constant)
 }
 }
 }
+
+/// Creates an `Instance` for annotating a move/copy operation at codegen time.
+///
+/// Returns `Some(instance)` if the operation should be annotated with debug info, `None`
+/// otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>` or
+/// `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+///
+/// There are a number of conditions that must be met for an annotation to be created, but aside
+/// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is
+/// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also
+/// that the underlying representation of the type is in memory.
+fnmove_copy_annotation_instance(
+&self,
+bx:&Bx,
+place: mir::PlaceRef<'tcx>,
+kind:LangItem,
+) -> Option<ty::Instance<'tcx>>{
+let tcx = bx.tcx();
+let sess = tcx.sess;
+
+// Skip if we're not generating debuginfo
+if sess.opts.debuginfo == DebugInfo::None{
+returnNone;
+}
+
+// Check if annotation is enabled and get size limit (otherwise skip)
+let size_limit = match sess.opts.unstable_opts.annotate_moves{
+AnnotateMoves::Disabled => returnNone,
+AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+AnnotateMoves::Enabled(Some(limit)) => limit,
+};
+
+let ty = self.monomorphized_place_ty(place);
+let layout = bx.cx().layout_of(ty);
+let ty_size = layout.size.bytes();
+
+// Only annotate if type has a memory representation and exceeds size limit (and has a
+// non-zero size)
+if layout.is_zst()
+ || ty_size < size_limit
+ || !matches!(layout.backend_repr,BackendRepr::Memory{ .. })
+{
+returnNone;
+}
+
+// Look up the DefId for compiler_move or compiler_copy lang item
+let def_id = tcx.lang_items().get(kind)?;
+
+// Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+let size_const = ty::Const::from_target_usize(tcx, ty_size);
+let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+// Create the Instance
+let typing_env = self.mir.typing_env(tcx);
+let instance = ty::Instance::expect_resolve(
+ tcx,
+ typing_env,
+ def_id,
+ generic_args,
+ rustc_span::DUMMY_SP,// span only used for error messages
+);
+
+Some(instance)
+}
 }

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super)constUNKNOWN_LINE_NUMBER:u32 = 0;
 pub(super)constUNKNOWN_COLUMN_NUMBER:u32 = 0;
 
-impl<'a,'gcc,'tcx>DebugInfoBuilderMethodsforBuilder<'a,'gcc,'tcx>{
+impl<'a,'gcc,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'a,'gcc,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(

compiler/rustc_codegen_llvm/src/abi.rs

@@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
 );
  bx.lifetime_end(llscratch, scratch_size);
 }
-_ => {
+PassMode::Pair(..) | PassMode::Direct{ .. } => {
 OperandRef::from_immediate_or_packed_pair(bx, val,self.layout).val.store(bx, dst);
 }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -751,7 +751,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
 }
 }
 
-impl<'ll>DebugInfoBuilderMethodsforBuilder<'_,'ll,'_>{
+impl<'ll,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'_,'ll,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
  llvm::set_value_name(value, name.as_bytes());
 }
 }
+
+/// Annotate move/copy operations with debug info for profiling.
+///
+/// This creates a temporary debug scope that makes the move/copy appear as an inlined call to
+/// `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`. The provided closure is executed
+/// with this temporary debug location active.
+///
+/// The `instance` parameter should be the monomorphized instance of the `compiler_move` or
+/// `compiler_copy` function with the actual type and size.
+fnwith_move_annotation<R>(
+&mutself,
+instance: ty::Instance<'tcx>,
+f:implFnOnce(&mutSelf) -> R,
+) -> R{
+// Save the current debug location
+let saved_loc = self.get_dbg_loc();
+
+// Create a DIScope for the compiler_move/compiler_copy function
+// We use the function's FnAbi for debug info generation
+let fn_abi = self
+.cx()
+.tcx
+.fn_abi_of_instance(
+self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+)
+.unwrap();
+
+let di_scope = self.cx().dbg_scope_fn(instance, fn_abi,None);
+
+// Create an inlined debug location:
+// - scope: the compiler_move/compiler_copy function
+// - inlined_at: the current location (where the move/copy actually occurs)
+// - span: use the function's definition span
+let fn_span = self.cx().tcx.def_span(instance.def_id());
+let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+// Set the temporary debug location
+self.set_dbg_loc(inlined_loc);
+
+// Execute the closure (which will generate the memcpy)
+let result = f(self);
+
+// Restore the original debug location
+ifletSome(loc) = saved_loc {
+self.set_dbg_loc(loc);
+}else{
+self.clear_dbg_loc();
+}
+
+ result
+}
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 let op = matchself.locals[mir::RETURN_PLACE]{
 LocalRef::Operand(op) => op,
 LocalRef::PendingOperand => bug!("use of return before def"),
-LocalRef::Place(cg_place) => {
-OperandRef{val:Ref(cg_place.val),layout: cg_place.layout}
-}
+LocalRef::Place(cg_place) => OperandRef{
+val:Ref(cg_place.val),
+layout: cg_place.layout,
+move_annotation:None,
+},
 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 };
 let llslot = match op.val{
@@ -1155,7 +1157,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  | (&mir::Operand::Constant(_),Ref(PlaceValue{llextra:None, .. })) => {
 let tmp = PlaceRef::alloca(bx, op.layout);
  bx.lifetime_start(tmp.val.llval, tmp.layout.size);
- op.val.store(bx, tmp);
+ op.store_with_annotation(bx, tmp);
  op.val = Ref(tmp.val);
  lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
 }
@@ -1563,13 +1565,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 };
 let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
  bx.lifetime_start(scratch.llval, arg.layout.size);
- op.val.store(bx, scratch.with_type(arg.layout));
+ op.store_with_annotation(bx, scratch.with_type(arg.layout));
  lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
 (scratch.llval, scratch.align,true)
 }
 PassMode::Cast{ .. } => {
 let scratch = PlaceRef::alloca(bx, arg.layout);
- op.val.store(bx, scratch);
+ op.store_with_annotation(bx, scratch);
 (scratch.val.llval, scratch.val.align,true)
 }
  _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi,false),

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 returnlocal(OperandRef{
 val:OperandValue::Pair(a, b),
 layout: arg.layout,
+move_annotation:None,
 });
 }
  _ => {}
@@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  fx.caller_location = Some(OperandRef{
 val:OperandValue::Immediate(bx.get_param(llarg_idx)),
 layout: arg.layout,
+move_annotation:None,
 });
 }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -5,12 +5,13 @@ use rustc_abi as abi;
 use rustc_abi::{
 Align,BackendRepr,FIRST_VARIANT,FieldIdx,Primitive,Size,TagEncoding,VariantIdx,Variants,
 };
+use rustc_hir::LangItem;
 use rustc_middle::mir::interpret::{Pointer,Scalar, alloc_range};
 use rustc_middle::mir::{self,ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf,TyAndLayout};
+use rustc_middle::ty::{self,Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves,DebugInfo,OptLevel};
 use tracing::{debug, instrument};
 
 usesuper::place::{PlaceRef,PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
 /// The layout of value, based on its Rust type.
 publayout:TyAndLayout<'tcx>,
+
+/// Annotation for profiler visibility of move/copy operations.
+/// When set, the store operation should appear as an inlined call to this function.
+pubmove_annotation:Option<ty::Instance<'tcx>>,
 }
 
 impl<V:CodegenObject> fmt::DebugforOperandRef<'_,V>{
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a,'tcx,V:CodegenObject>OperandRef<'tcx,V>{
 pubfnzero_sized(layout:TyAndLayout<'tcx>) -> OperandRef<'tcx,V>{
 assert!(layout.is_zst());
-OperandRef{val:OperandValue::ZeroSized, layout }
+OperandRef{val:OperandValue::ZeroSized, layout,move_annotation:None}
 }
 
 pub(crate)fnfrom_const<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }
 };
 
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 fnfrom_const_alloc<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 let size = s.size(bx);
 assert_eq!(size, layout.size,"abi::Scalar size does not match layout size");
 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-OperandRef{val:OperandValue::Immediate(val), layout }
+OperandRef{val:OperandValue::Immediate(val), layout,move_annotation:None}
 }
 BackendRepr::ScalarPair(
  a @ abi::Scalar::Initialized{ .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
  b,
  bx.scalar_pair_element_backend_type(layout,1,true),
 );
-OperandRef{val:OperandValue::Pair(a_val, b_val), layout }
+OperandRef{val:OperandValue::Pair(a_val, b_val), layout,move_annotation:None}
 }
  _ if layout.is_zst() => OperandRef::zero_sized(layout),
  _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 self.val.deref(layout.align.abi).with_type(layout)
 }
 
+/// Store this operand into a place, applying move/copy annotation if present.
+///
+/// This is the preferred method for storing operands, as it automatically
+/// applies profiler annotations for tracked move/copy operations.
+pubfnstore_with_annotation<Bx:BuilderMethods<'a,'tcx,Value = V>>(
+self,
+bx:&mutBx,
+dest:PlaceRef<'tcx,V>,
+){
+ifletSome(instance) = self.move_annotation{
+ bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+}else{
+self.val.store(bx, dest)
+}
+}
+
 /// If this operand is a `Pair`, we return an aggregate with the two values.
 /// For other cases, see `immediate`.
 pubfnimmediate_or_packed_pair<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }else{
 OperandValue::Immediate(llval)
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 pub(crate)fnextract_field<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 })
 };
 
-OperandRef{ val,layout: field }
+OperandRef{ val,layout: field,move_annotation:None}
 }
 
 /// Obtain the actual discriminant of a value.
@@ -828,10 +849,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
 }
 },
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 }
 
+/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache
+/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only
+/// annotate copies larger than this.
+constMOVE_ANNOTATION_DEFAULT_LIMIT:u64 = 65;
+
 impl<'a,'tcx,V:CodegenObject>OperandValue<V>{
 /// Returns an `OperandValue` that's generally UB to use in any way.
 ///
@@ -961,7 +987,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  abi::Variants::Single{ index: vidx },
 );
 let layout = o.layout.for_variant(bx.cx(), vidx);
- o = OperandRef{val: o.val, layout}
+ o = OperandRef{layout, ..o}
 }
  _ => returnNone,
 }
@@ -1014,7 +1040,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
 match*operand {
  mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-self.codegen_consume(bx, place.as_ref())
+let kind = match operand {
+ mir::Operand::Move(_) => LangItem::CompilerMove,
+ mir::Operand::Copy(_) => LangItem::CompilerCopy,
+ _ => unreachable!(),
+};
+
+// Check if we should annotate this move/copy for profiling
+let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind);
+
+OperandRef{ move_annotation, ..self.codegen_consume(bx, place.as_ref())}
 }
 
  mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 returnOperandRef{
 val:OperandValue::Immediate(llval),
 layout: bx.layout_of(ty),
+move_annotation:None,
 };
 }
 }
 self.eval_mir_constant_to_operand(bx, constant)
 }
 }
 }
+
+/// Creates an `Instance` for annotating a move/copy operation at codegen time.
+///
+/// Returns `Some(instance)` if the operation should be annotated with debug info, `None`
+/// otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>` or
+/// `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+///
+/// There are a number of conditions that must be met for an annotation to be created, but aside
+/// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is
+/// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also
+/// that the underlying representation of the type is in memory.
+fnmove_copy_annotation_instance(
+&self,
+bx:&Bx,
+place: mir::PlaceRef<'tcx>,
+kind:LangItem,
+) -> Option<ty::Instance<'tcx>>{
+let tcx = bx.tcx();
+let sess = tcx.sess;
+
+// Skip if we're not generating debuginfo
+if sess.opts.debuginfo == DebugInfo::None{
+returnNone;
+}
+
+// Check if annotation is enabled and get size limit (otherwise skip)
+let size_limit = match sess.opts.unstable_opts.annotate_moves{
+AnnotateMoves::Disabled => returnNone,
+AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+AnnotateMoves::Enabled(Some(limit)) => limit,
+};
+
+let ty = self.monomorphized_place_ty(place);
+let layout = bx.cx().layout_of(ty);
+let ty_size = layout.size.bytes();
+
+// Only annotate if type has a memory representation and exceeds size limit (and has a
+// non-zero size)
+if layout.is_zst()
+ || ty_size < size_limit
+ || !matches!(layout.backend_repr,BackendRepr::Memory{ .. })
+{
+returnNone;
+}
+
+// Look up the DefId for compiler_move or compiler_copy lang item
+let def_id = tcx.lang_items().get(kind)?;
+
+// Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+let size_const = ty::Const::from_target_usize(tcx, ty_size);
+let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+// Create the Instance
+let typing_env = self.mir.typing_env(tcx);
+let instance = ty::Instance::expect_resolve(
+ tcx,
+ typing_env,
+ def_id,
+ generic_args,
+ rustc_span::DUMMY_SP,// span only used for error messages
+);
+
+Some(instance)
+}
 }

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super)constUNKNOWN_LINE_NUMBER:u32 = 0;
 pub(super)constUNKNOWN_COLUMN_NUMBER:u32 = 0;
 
-impl<'a,'gcc,'tcx>DebugInfoBuilderMethodsforBuilder<'a,'gcc,'tcx>{
+impl<'a,'gcc,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'a,'gcc,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(

compiler/rustc_codegen_llvm/src/abi.rs

@@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
 );
  bx.lifetime_end(llscratch, scratch_size);
 }
-_ => {
+PassMode::Pair(..) | PassMode::Direct{ .. } => {
 OperandRef::from_immediate_or_packed_pair(bx, val,self.layout).val.store(bx, dst);
 }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -751,7 +751,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
 }
 }
 
-impl<'ll>DebugInfoBuilderMethodsforBuilder<'_,'ll,'_>{
+impl<'ll,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'_,'ll,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
  llvm::set_value_name(value, name.as_bytes());
 }
 }
+
+/// Annotate move/copy operations with debug info for profiling.
+///
+/// This creates a temporary debug scope that makes the move/copy appear as an inlined call to
+/// `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`. The provided closure is executed
+/// with this temporary debug location active.
+///
+/// The `instance` parameter should be the monomorphized instance of the `compiler_move` or
+/// `compiler_copy` function with the actual type and size.
+fnwith_move_annotation<R>(
+&mutself,
+instance: ty::Instance<'tcx>,
+f:implFnOnce(&mutSelf) -> R,
+) -> R{
+// Save the current debug location
+let saved_loc = self.get_dbg_loc();
+
+// Create a DIScope for the compiler_move/compiler_copy function
+// We use the function's FnAbi for debug info generation
+let fn_abi = self
+.cx()
+.tcx
+.fn_abi_of_instance(
+self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+)
+.unwrap();
+
+let di_scope = self.cx().dbg_scope_fn(instance, fn_abi,None);
+
+// Create an inlined debug location:
+// - scope: the compiler_move/compiler_copy function
+// - inlined_at: the current location (where the move/copy actually occurs)
+// - span: use the function's definition span
+let fn_span = self.cx().tcx.def_span(instance.def_id());
+let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+// Set the temporary debug location
+self.set_dbg_loc(inlined_loc);
+
+// Execute the closure (which will generate the memcpy)
+let result = f(self);
+
+// Restore the original debug location
+ifletSome(loc) = saved_loc {
+self.set_dbg_loc(loc);
+}else{
+self.clear_dbg_loc();
+}
+
+ result
+}
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 let op = matchself.locals[mir::RETURN_PLACE]{
 LocalRef::Operand(op) => op,
 LocalRef::PendingOperand => bug!("use of return before def"),
-LocalRef::Place(cg_place) => {
-OperandRef{val:Ref(cg_place.val),layout: cg_place.layout}
-}
+LocalRef::Place(cg_place) => OperandRef{
+val:Ref(cg_place.val),
+layout: cg_place.layout,
+move_annotation:None,
+},
 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 };
 let llslot = match op.val{
@@ -1155,7 +1157,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  | (&mir::Operand::Constant(_),Ref(PlaceValue{llextra:None, .. })) => {
 let tmp = PlaceRef::alloca(bx, op.layout);
  bx.lifetime_start(tmp.val.llval, tmp.layout.size);
- op.val.store(bx, tmp);
+ op.store_with_annotation(bx, tmp);
  op.val = Ref(tmp.val);
  lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
 }
@@ -1563,13 +1565,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 };
 let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
  bx.lifetime_start(scratch.llval, arg.layout.size);
- op.val.store(bx, scratch.with_type(arg.layout));
+ op.store_with_annotation(bx, scratch.with_type(arg.layout));
  lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
 (scratch.llval, scratch.align,true)
 }
 PassMode::Cast{ .. } => {
 let scratch = PlaceRef::alloca(bx, arg.layout);
- op.val.store(bx, scratch);
+ op.store_with_annotation(bx, scratch);
 (scratch.val.llval, scratch.val.align,true)
 }
  _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi,false),

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 returnlocal(OperandRef{
 val:OperandValue::Pair(a, b),
 layout: arg.layout,
+move_annotation:None,
 });
 }
  _ => {}
@@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  fx.caller_location = Some(OperandRef{
 val:OperandValue::Immediate(bx.get_param(llarg_idx)),
 layout: arg.layout,
+move_annotation:None,
 });
 }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -5,12 +5,13 @@ use rustc_abi as abi;
 use rustc_abi::{
 Align,BackendRepr,FIRST_VARIANT,FieldIdx,Primitive,Size,TagEncoding,VariantIdx,Variants,
 };
+use rustc_hir::LangItem;
 use rustc_middle::mir::interpret::{Pointer,Scalar, alloc_range};
 use rustc_middle::mir::{self,ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf,TyAndLayout};
+use rustc_middle::ty::{self,Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves,DebugInfo,OptLevel};
 use tracing::{debug, instrument};
 
 usesuper::place::{PlaceRef,PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
 /// The layout of value, based on its Rust type.
 publayout:TyAndLayout<'tcx>,
+
+/// Annotation for profiler visibility of move/copy operations.
+/// When set, the store operation should appear as an inlined call to this function.
+pubmove_annotation:Option<ty::Instance<'tcx>>,
 }
 
 impl<V:CodegenObject> fmt::DebugforOperandRef<'_,V>{
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a,'tcx,V:CodegenObject>OperandRef<'tcx,V>{
 pubfnzero_sized(layout:TyAndLayout<'tcx>) -> OperandRef<'tcx,V>{
 assert!(layout.is_zst());
-OperandRef{val:OperandValue::ZeroSized, layout }
+OperandRef{val:OperandValue::ZeroSized, layout,move_annotation:None}
 }
 
 pub(crate)fnfrom_const<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }
 };
 
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 fnfrom_const_alloc<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 let size = s.size(bx);
 assert_eq!(size, layout.size,"abi::Scalar size does not match layout size");
 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-OperandRef{val:OperandValue::Immediate(val), layout }
+OperandRef{val:OperandValue::Immediate(val), layout,move_annotation:None}
 }
 BackendRepr::ScalarPair(
  a @ abi::Scalar::Initialized{ .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
  b,
  bx.scalar_pair_element_backend_type(layout,1,true),
 );
-OperandRef{val:OperandValue::Pair(a_val, b_val), layout }
+OperandRef{val:OperandValue::Pair(a_val, b_val), layout,move_annotation:None}
 }
  _ if layout.is_zst() => OperandRef::zero_sized(layout),
  _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 self.val.deref(layout.align.abi).with_type(layout)
 }
 
+/// Store this operand into a place, applying move/copy annotation if present.
+///
+/// This is the preferred method for storing operands, as it automatically
+/// applies profiler annotations for tracked move/copy operations.
+pubfnstore_with_annotation<Bx:BuilderMethods<'a,'tcx,Value = V>>(
+self,
+bx:&mutBx,
+dest:PlaceRef<'tcx,V>,
+){
+ifletSome(instance) = self.move_annotation{
+ bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+}else{
+self.val.store(bx, dest)
+}
+}
+
 /// If this operand is a `Pair`, we return an aggregate with the two values.
 /// For other cases, see `immediate`.
 pubfnimmediate_or_packed_pair<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }else{
 OperandValue::Immediate(llval)
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 pub(crate)fnextract_field<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 })
 };
 
-OperandRef{ val,layout: field }
+OperandRef{ val,layout: field,move_annotation:None}
 }
 
 /// Obtain the actual discriminant of a value.
@@ -828,10 +849,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
 }
 },
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 }
 
+/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache
+/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only
+/// annotate copies larger than this.
+constMOVE_ANNOTATION_DEFAULT_LIMIT:u64 = 65;
+
 impl<'a,'tcx,V:CodegenObject>OperandValue<V>{
 /// Returns an `OperandValue` that's generally UB to use in any way.
 ///
@@ -961,7 +987,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  abi::Variants::Single{ index: vidx },
 );
 let layout = o.layout.for_variant(bx.cx(), vidx);
- o = OperandRef{val: o.val, layout}
+ o = OperandRef{layout, ..o}
 }
  _ => returnNone,
 }
@@ -1014,7 +1040,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
 match*operand {
  mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-self.codegen_consume(bx, place.as_ref())
+let kind = match operand {
+ mir::Operand::Move(_) => LangItem::CompilerMove,
+ mir::Operand::Copy(_) => LangItem::CompilerCopy,
+ _ => unreachable!(),
+};
+
+// Check if we should annotate this move/copy for profiling
+let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind);
+
+OperandRef{ move_annotation, ..self.codegen_consume(bx, place.as_ref())}
 }
 
  mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 returnOperandRef{
 val:OperandValue::Immediate(llval),
 layout: bx.layout_of(ty),
+move_annotation:None,
 };
 }
 }
 self.eval_mir_constant_to_operand(bx, constant)
 }
 }
 }
+
+/// Creates an `Instance` for annotating a move/copy operation at codegen time.
+///
+/// Returns `Some(instance)` if the operation should be annotated with debug info, `None`
+/// otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>` or
+/// `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+///
+/// There are a number of conditions that must be met for an annotation to be created, but aside
+/// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is
+/// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also
+/// that the underlying representation of the type is in memory.
+fnmove_copy_annotation_instance(
+&self,
+bx:&Bx,
+place: mir::PlaceRef<'tcx>,
+kind:LangItem,
+) -> Option<ty::Instance<'tcx>>{
+let tcx = bx.tcx();
+let sess = tcx.sess;
+
+// Skip if we're not generating debuginfo
+if sess.opts.debuginfo == DebugInfo::None{
+returnNone;
+}
+
+// Check if annotation is enabled and get size limit (otherwise skip)
+let size_limit = match sess.opts.unstable_opts.annotate_moves{
+AnnotateMoves::Disabled => returnNone,
+AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+AnnotateMoves::Enabled(Some(limit)) => limit,
+};
+
+let ty = self.monomorphized_place_ty(place);
+let layout = bx.cx().layout_of(ty);
+let ty_size = layout.size.bytes();
+
+// Only annotate if type has a memory representation and exceeds size limit (and has a
+// non-zero size)
+if layout.is_zst()
+ || ty_size < size_limit
+ || !matches!(layout.backend_repr,BackendRepr::Memory{ .. })
+{
+returnNone;
+}
+
+// Look up the DefId for compiler_move or compiler_copy lang item
+let def_id = tcx.lang_items().get(kind)?;
+
+// Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+let size_const = ty::Const::from_target_usize(tcx, ty_size);
+let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+// Create the Instance
+let typing_env = self.mir.typing_env(tcx);
+let instance = ty::Instance::expect_resolve(
+ tcx,
+ typing_env,
+ def_id,
+ generic_args,
+ rustc_span::DUMMY_SP,// span only used for error messages
+);
+
+Some(instance)
+}
 }

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super)constUNKNOWN_LINE_NUMBER:u32 = 0;
 pub(super)constUNKNOWN_COLUMN_NUMBER:u32 = 0;
 
-impl<'a,'gcc,'tcx>DebugInfoBuilderMethodsforBuilder<'a,'gcc,'tcx>{
+impl<'a,'gcc,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'a,'gcc,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(

compiler/rustc_codegen_llvm/src/abi.rs

@@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
 );
  bx.lifetime_end(llscratch, scratch_size);
 }
-_ => {
+PassMode::Pair(..) | PassMode::Direct{ .. } => {
 OperandRef::from_immediate_or_packed_pair(bx, val,self.layout).val.store(bx, dst);
 }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -751,7 +751,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
 }
 }
 
-impl<'ll>DebugInfoBuilderMethodsforBuilder<'_,'ll,'_>{
+impl<'ll,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'_,'ll,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
  llvm::set_value_name(value, name.as_bytes());
 }
 }
+
+/// Annotate move/copy operations with debug info for profiling.
+///
+/// This creates a temporary debug scope that makes the move/copy appear as an inlined call to
+/// `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`. The provided closure is executed
+/// with this temporary debug location active.
+///
+/// The `instance` parameter should be the monomorphized instance of the `compiler_move` or
+/// `compiler_copy` function with the actual type and size.
+fnwith_move_annotation<R>(
+&mutself,
+instance: ty::Instance<'tcx>,
+f:implFnOnce(&mutSelf) -> R,
+) -> R{
+// Save the current debug location
+let saved_loc = self.get_dbg_loc();
+
+// Create a DIScope for the compiler_move/compiler_copy function
+// We use the function's FnAbi for debug info generation
+let fn_abi = self
+.cx()
+.tcx
+.fn_abi_of_instance(
+self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+)
+.unwrap();
+
+let di_scope = self.cx().dbg_scope_fn(instance, fn_abi,None);
+
+// Create an inlined debug location:
+// - scope: the compiler_move/compiler_copy function
+// - inlined_at: the current location (where the move/copy actually occurs)
+// - span: use the function's definition span
+let fn_span = self.cx().tcx.def_span(instance.def_id());
+let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+// Set the temporary debug location
+self.set_dbg_loc(inlined_loc);
+
+// Execute the closure (which will generate the memcpy)
+let result = f(self);
+
+// Restore the original debug location
+ifletSome(loc) = saved_loc {
+self.set_dbg_loc(loc);
+}else{
+self.clear_dbg_loc();
+}
+
+ result
+}
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 let op = matchself.locals[mir::RETURN_PLACE]{
 LocalRef::Operand(op) => op,
 LocalRef::PendingOperand => bug!("use of return before def"),
-LocalRef::Place(cg_place) => {
-OperandRef{val:Ref(cg_place.val),layout: cg_place.layout}
-}
+LocalRef::Place(cg_place) => OperandRef{
+val:Ref(cg_place.val),
+layout: cg_place.layout,
+move_annotation:None,
+},
 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 };
 let llslot = match op.val{
@@ -1155,7 +1157,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  | (&mir::Operand::Constant(_),Ref(PlaceValue{llextra:None, .. })) => {
 let tmp = PlaceRef::alloca(bx, op.layout);
  bx.lifetime_start(tmp.val.llval, tmp.layout.size);
- op.val.store(bx, tmp);
+ op.store_with_annotation(bx, tmp);
  op.val = Ref(tmp.val);
  lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
 }
@@ -1563,13 +1565,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 };
 let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
  bx.lifetime_start(scratch.llval, arg.layout.size);
- op.val.store(bx, scratch.with_type(arg.layout));
+ op.store_with_annotation(bx, scratch.with_type(arg.layout));
  lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
 (scratch.llval, scratch.align,true)
 }
 PassMode::Cast{ .. } => {
 let scratch = PlaceRef::alloca(bx, arg.layout);
- op.val.store(bx, scratch);
+ op.store_with_annotation(bx, scratch);
 (scratch.val.llval, scratch.val.align,true)
 }
  _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi,false),

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 returnlocal(OperandRef{
 val:OperandValue::Pair(a, b),
 layout: arg.layout,
+move_annotation:None,
 });
 }
  _ => {}
@@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  fx.caller_location = Some(OperandRef{
 val:OperandValue::Immediate(bx.get_param(llarg_idx)),
 layout: arg.layout,
+move_annotation:None,
 });
 }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -5,12 +5,13 @@ use rustc_abi as abi;
 use rustc_abi::{
 Align,BackendRepr,FIRST_VARIANT,FieldIdx,Primitive,Size,TagEncoding,VariantIdx,Variants,
 };
+use rustc_hir::LangItem;
 use rustc_middle::mir::interpret::{Pointer,Scalar, alloc_range};
 use rustc_middle::mir::{self,ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf,TyAndLayout};
+use rustc_middle::ty::{self,Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves,DebugInfo,OptLevel};
 use tracing::{debug, instrument};
 
 usesuper::place::{PlaceRef,PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
 /// The layout of value, based on its Rust type.
 publayout:TyAndLayout<'tcx>,
+
+/// Annotation for profiler visibility of move/copy operations.
+/// When set, the store operation should appear as an inlined call to this function.
+pubmove_annotation:Option<ty::Instance<'tcx>>,
 }
 
 impl<V:CodegenObject> fmt::DebugforOperandRef<'_,V>{
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a,'tcx,V:CodegenObject>OperandRef<'tcx,V>{
 pubfnzero_sized(layout:TyAndLayout<'tcx>) -> OperandRef<'tcx,V>{
 assert!(layout.is_zst());
-OperandRef{val:OperandValue::ZeroSized, layout }
+OperandRef{val:OperandValue::ZeroSized, layout,move_annotation:None}
 }
 
 pub(crate)fnfrom_const<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }
 };
 
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 fnfrom_const_alloc<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 let size = s.size(bx);
 assert_eq!(size, layout.size,"abi::Scalar size does not match layout size");
 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-OperandRef{val:OperandValue::Immediate(val), layout }
+OperandRef{val:OperandValue::Immediate(val), layout,move_annotation:None}
 }
 BackendRepr::ScalarPair(
  a @ abi::Scalar::Initialized{ .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
  b,
  bx.scalar_pair_element_backend_type(layout,1,true),
 );
-OperandRef{val:OperandValue::Pair(a_val, b_val), layout }
+OperandRef{val:OperandValue::Pair(a_val, b_val), layout,move_annotation:None}
 }
  _ if layout.is_zst() => OperandRef::zero_sized(layout),
  _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 self.val.deref(layout.align.abi).with_type(layout)
 }
 
+/// Store this operand into a place, applying move/copy annotation if present.
+///
+/// This is the preferred method for storing operands, as it automatically
+/// applies profiler annotations for tracked move/copy operations.
+pubfnstore_with_annotation<Bx:BuilderMethods<'a,'tcx,Value = V>>(
+self,
+bx:&mutBx,
+dest:PlaceRef<'tcx,V>,
+){
+ifletSome(instance) = self.move_annotation{
+ bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+}else{
+self.val.store(bx, dest)
+}
+}
+
 /// If this operand is a `Pair`, we return an aggregate with the two values.
 /// For other cases, see `immediate`.
 pubfnimmediate_or_packed_pair<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }else{
 OperandValue::Immediate(llval)
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 pub(crate)fnextract_field<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 })
 };
 
-OperandRef{ val,layout: field }
+OperandRef{ val,layout: field,move_annotation:None}
 }
 
 /// Obtain the actual discriminant of a value.
@@ -828,10 +849,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
 }
 },
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 }
 
+/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache
+/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only
+/// annotate copies larger than this.
+constMOVE_ANNOTATION_DEFAULT_LIMIT:u64 = 65;
+
 impl<'a,'tcx,V:CodegenObject>OperandValue<V>{
 /// Returns an `OperandValue` that's generally UB to use in any way.
 ///
@@ -961,7 +987,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  abi::Variants::Single{ index: vidx },
 );
 let layout = o.layout.for_variant(bx.cx(), vidx);
- o = OperandRef{val: o.val, layout}
+ o = OperandRef{layout, ..o}
 }
  _ => returnNone,
 }
@@ -1014,7 +1040,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
 match*operand {
  mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-self.codegen_consume(bx, place.as_ref())
+let kind = match operand {
+ mir::Operand::Move(_) => LangItem::CompilerMove,
+ mir::Operand::Copy(_) => LangItem::CompilerCopy,
+ _ => unreachable!(),
+};
+
+// Check if we should annotate this move/copy for profiling
+let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind);
+
+OperandRef{ move_annotation, ..self.codegen_consume(bx, place.as_ref())}
 }
 
  mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 returnOperandRef{
 val:OperandValue::Immediate(llval),
 layout: bx.layout_of(ty),
+move_annotation:None,
 };
 }
 }
 self.eval_mir_constant_to_operand(bx, constant)
 }
 }
 }
+
+/// Creates an `Instance` for annotating a move/copy operation at codegen time.
+///
+/// Returns `Some(instance)` if the operation should be annotated with debug info, `None`
+/// otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>` or
+/// `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+///
+/// There are a number of conditions that must be met for an annotation to be created, but aside
+/// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is
+/// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also
+/// that the underlying representation of the type is in memory.
+fnmove_copy_annotation_instance(
+&self,
+bx:&Bx,
+place: mir::PlaceRef<'tcx>,
+kind:LangItem,
+) -> Option<ty::Instance<'tcx>>{
+let tcx = bx.tcx();
+let sess = tcx.sess;
+
+// Skip if we're not generating debuginfo
+if sess.opts.debuginfo == DebugInfo::None{
+returnNone;
+}
+
+// Check if annotation is enabled and get size limit (otherwise skip)
+let size_limit = match sess.opts.unstable_opts.annotate_moves{
+AnnotateMoves::Disabled => returnNone,
+AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+AnnotateMoves::Enabled(Some(limit)) => limit,
+};
+
+let ty = self.monomorphized_place_ty(place);
+let layout = bx.cx().layout_of(ty);
+let ty_size = layout.size.bytes();
+
+// Only annotate if type has a memory representation and exceeds size limit (and has a
+// non-zero size)
+if layout.is_zst()
+ || ty_size < size_limit
+ || !matches!(layout.backend_repr,BackendRepr::Memory{ .. })
+{
+returnNone;
+}
+
+// Look up the DefId for compiler_move or compiler_copy lang item
+let def_id = tcx.lang_items().get(kind)?;
+
+// Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+let size_const = ty::Const::from_target_usize(tcx, ty_size);
+let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+// Create the Instance
+let typing_env = self.mir.typing_env(tcx);
+let instance = ty::Instance::expect_resolve(
+ tcx,
+ typing_env,
+ def_id,
+ generic_args,
+ rustc_span::DUMMY_SP,// span only used for error messages
+);
+
+Some(instance)
+}
 }

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super)constUNKNOWN_LINE_NUMBER:u32 = 0;
 pub(super)constUNKNOWN_COLUMN_NUMBER:u32 = 0;
 
-impl<'a,'gcc,'tcx>DebugInfoBuilderMethodsforBuilder<'a,'gcc,'tcx>{
+impl<'a,'gcc,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'a,'gcc,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(

compiler/rustc_codegen_llvm/src/abi.rs

@@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
 );
  bx.lifetime_end(llscratch, scratch_size);
 }
-_ => {
+PassMode::Pair(..) | PassMode::Direct{ .. } => {
 OperandRef::from_immediate_or_packed_pair(bx, val,self.layout).val.store(bx, dst);
 }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -751,7 +751,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 OperandValue::Ref(place.val)
 };
 
-OperandRef{ val,layout: place.layout}
+OperandRef{ val,layout: place.layout,move_annotation:None}
 }
 
 fnwrite_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
 }
 }
 
-impl<'ll>DebugInfoBuilderMethodsforBuilder<'_,'ll,'_>{
+impl<'ll,'tcx>DebugInfoBuilderMethods<'tcx>forBuilder<'_,'ll,'tcx>{
 // FIXME(eddyb) find a common convention for all of the debuginfo-related
 // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
 fndbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
  llvm::set_value_name(value, name.as_bytes());
 }
 }
+
+/// Annotate move/copy operations with debug info for profiling.
+///
+/// This creates a temporary debug scope that makes the move/copy appear as an inlined call to
+/// `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`. The provided closure is executed
+/// with this temporary debug location active.
+///
+/// The `instance` parameter should be the monomorphized instance of the `compiler_move` or
+/// `compiler_copy` function with the actual type and size.
+fnwith_move_annotation<R>(
+&mutself,
+instance: ty::Instance<'tcx>,
+f:implFnOnce(&mutSelf) -> R,
+) -> R{
+// Save the current debug location
+let saved_loc = self.get_dbg_loc();
+
+// Create a DIScope for the compiler_move/compiler_copy function
+// We use the function's FnAbi for debug info generation
+let fn_abi = self
+.cx()
+.tcx
+.fn_abi_of_instance(
+self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+)
+.unwrap();
+
+let di_scope = self.cx().dbg_scope_fn(instance, fn_abi,None);
+
+// Create an inlined debug location:
+// - scope: the compiler_move/compiler_copy function
+// - inlined_at: the current location (where the move/copy actually occurs)
+// - span: use the function's definition span
+let fn_span = self.cx().tcx.def_span(instance.def_id());
+let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+// Set the temporary debug location
+self.set_dbg_loc(inlined_loc);
+
+// Execute the closure (which will generate the memcpy)
+let result = f(self);
+
+// Restore the original debug location
+ifletSome(loc) = saved_loc {
+self.set_dbg_loc(loc);
+}else{
+self.clear_dbg_loc();
+}
+
+ result
+}
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 let op = matchself.locals[mir::RETURN_PLACE]{
 LocalRef::Operand(op) => op,
 LocalRef::PendingOperand => bug!("use of return before def"),
-LocalRef::Place(cg_place) => {
-OperandRef{val:Ref(cg_place.val),layout: cg_place.layout}
-}
+LocalRef::Place(cg_place) => OperandRef{
+val:Ref(cg_place.val),
+layout: cg_place.layout,
+move_annotation:None,
+},
 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
 };
 let llslot = match op.val{
@@ -1155,7 +1157,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  | (&mir::Operand::Constant(_),Ref(PlaceValue{llextra:None, .. })) => {
 let tmp = PlaceRef::alloca(bx, op.layout);
  bx.lifetime_start(tmp.val.llval, tmp.layout.size);
- op.val.store(bx, tmp);
+ op.store_with_annotation(bx, tmp);
  op.val = Ref(tmp.val);
  lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
 }
@@ -1563,13 +1565,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 };
 let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
  bx.lifetime_start(scratch.llval, arg.layout.size);
- op.val.store(bx, scratch.with_type(arg.layout));
+ op.store_with_annotation(bx, scratch.with_type(arg.layout));
  lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
 (scratch.llval, scratch.align,true)
 }
 PassMode::Cast{ .. } => {
 let scratch = PlaceRef::alloca(bx, arg.layout);
- op.val.store(bx, scratch);
+ op.store_with_annotation(bx, scratch);
 (scratch.val.llval, scratch.val.align,true)
 }
  _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi,false),

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 returnlocal(OperandRef{
 val:OperandValue::Pair(a, b),
 layout: arg.layout,
+move_annotation:None,
 });
 }
  _ => {}
@@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  fx.caller_location = Some(OperandRef{
 val:OperandValue::Immediate(bx.get_param(llarg_idx)),
 layout: arg.layout,
+move_annotation:None,
 });
 }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -5,12 +5,13 @@ use rustc_abi as abi;
 use rustc_abi::{
 Align,BackendRepr,FIRST_VARIANT,FieldIdx,Primitive,Size,TagEncoding,VariantIdx,Variants,
 };
+use rustc_hir::LangItem;
 use rustc_middle::mir::interpret::{Pointer,Scalar, alloc_range};
 use rustc_middle::mir::{self,ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf,TyAndLayout};
+use rustc_middle::ty::{self,Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves,DebugInfo,OptLevel};
 use tracing::{debug, instrument};
 
 usesuper::place::{PlaceRef,PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
 /// The layout of value, based on its Rust type.
 publayout:TyAndLayout<'tcx>,
+
+/// Annotation for profiler visibility of move/copy operations.
+/// When set, the store operation should appear as an inlined call to this function.
+pubmove_annotation:Option<ty::Instance<'tcx>>,
 }
 
 impl<V:CodegenObject> fmt::DebugforOperandRef<'_,V>{
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a,'tcx,V:CodegenObject>OperandRef<'tcx,V>{
 pubfnzero_sized(layout:TyAndLayout<'tcx>) -> OperandRef<'tcx,V>{
 assert!(layout.is_zst());
-OperandRef{val:OperandValue::ZeroSized, layout }
+OperandRef{val:OperandValue::ZeroSized, layout,move_annotation:None}
 }
 
 pub(crate)fnfrom_const<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }
 };
 
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 fnfrom_const_alloc<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 let size = s.size(bx);
 assert_eq!(size, layout.size,"abi::Scalar size does not match layout size");
 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-OperandRef{val:OperandValue::Immediate(val), layout }
+OperandRef{val:OperandValue::Immediate(val), layout,move_annotation:None}
 }
 BackendRepr::ScalarPair(
  a @ abi::Scalar::Initialized{ .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
  b,
  bx.scalar_pair_element_backend_type(layout,1,true),
 );
-OperandRef{val:OperandValue::Pair(a_val, b_val), layout }
+OperandRef{val:OperandValue::Pair(a_val, b_val), layout,move_annotation:None}
 }
  _ if layout.is_zst() => OperandRef::zero_sized(layout),
  _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 self.val.deref(layout.align.abi).with_type(layout)
 }
 
+/// Store this operand into a place, applying move/copy annotation if present.
+///
+/// This is the preferred method for storing operands, as it automatically
+/// applies profiler annotations for tracked move/copy operations.
+pubfnstore_with_annotation<Bx:BuilderMethods<'a,'tcx,Value = V>>(
+self,
+bx:&mutBx,
+dest:PlaceRef<'tcx,V>,
+){
+ifletSome(instance) = self.move_annotation{
+ bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+}else{
+self.val.store(bx, dest)
+}
+}
+
 /// If this operand is a `Pair`, we return an aggregate with the two values.
 /// For other cases, see `immediate`.
 pubfnimmediate_or_packed_pair<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }else{
 OperandValue::Immediate(llval)
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 
 pub(crate)fnextract_field<Bx:BuilderMethods<'a,'tcx,Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 })
 };
 
-OperandRef{ val,layout: field }
+OperandRef{ val,layout: field,move_annotation:None}
 }
 
 /// Obtain the actual discriminant of a value.
@@ -828,10 +849,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
 }
 },
 };
-OperandRef{ val, layout }
+OperandRef{ val, layout,move_annotation:None}
 }
 }
 
+/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache
+/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only
+/// annotate copies larger than this.
+constMOVE_ANNOTATION_DEFAULT_LIMIT:u64 = 65;
+
 impl<'a,'tcx,V:CodegenObject>OperandValue<V>{
 /// Returns an `OperandValue` that's generally UB to use in any way.
 ///
@@ -961,7 +987,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  abi::Variants::Single{ index: vidx },
 );
 let layout = o.layout.for_variant(bx.cx(), vidx);
- o = OperandRef{val: o.val, layout}
+ o = OperandRef{layout, ..o}
 }
  _ => returnNone,
 }
@@ -1014,7 +1040,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
 match*operand {
  mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-self.codegen_consume(bx, place.as_ref())
+let kind = match operand {
+ mir::Operand::Move(_) => LangItem::CompilerMove,
+ mir::Operand::Copy(_) => LangItem::CompilerCopy,
+ _ => unreachable!(),
+};
+
+// Check if we should annotate this move/copy for profiling
+let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind);
+
+OperandRef{ move_annotation, ..self.codegen_consume(bx, place.as_ref())}
 }
 
  mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 returnOperandRef{
 val:OperandValue::Immediate(llval),
 layout: bx.layout_of(ty),
+move_annotation:None,
 };
 }
 }
 self.eval_mir_constant_to_operand(bx, constant)
 }
 }
 }
+
+/// Creates an `Instance` for annotating a move/copy operation at codegen time.
+///
+/// Returns `Some(instance)` if the operation should be annotated with debug info, `None`
+/// otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>` or
+/// `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+///
+/// There are a number of conditions that must be met for an annotation to be created, but aside
+/// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is
+/// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also
+/// that the underlying representation of the type is in memory.
+fnmove_copy_annotation_instance(
+&self,
+bx:&Bx,
+place: mir::PlaceRef<'tcx>,
+kind:LangItem,
+) -> Option<ty::Instance<'tcx>>{
+let tcx = bx.tcx();
+let sess = tcx.sess;
+
+// Skip if we're not generating debuginfo
+if sess.opts.debuginfo == DebugInfo::None{
+returnNone;
+}
+
+// Check if annotation is enabled and get size limit (otherwise skip)
+let size_limit = match sess.opts.unstable_opts.annotate_moves{
+AnnotateMoves::Disabled => returnNone,
+AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+AnnotateMoves::Enabled(Some(limit)) => limit,
+};
+
+let ty = self.monomorphized_place_ty(place);
+let layout = bx.cx().layout_of(ty);
+let ty_size = layout.size.bytes();
+
+// Only annotate if type has a memory representation and exceeds size limit (and has a
+// non-zero size)
+if layout.is_zst()
+ || ty_size < size_limit
+ || !matches!(layout.backend_repr,BackendRepr::Memory{ .. })
+{
+returnNone;
+}
+
+// Look up the DefId for compiler_move or compiler_copy lang item
+let def_id = tcx.lang_items().get(kind)?;
+
+// Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+let size_const = ty::Const::from_target_usize(tcx, ty_size);
+let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+// Create the Instance
+let typing_env = self.mir.typing_env(tcx);
+let instance = ty::Instance::expect_resolve(
+ tcx,
+ typing_env,
+ def_id,
+ generic_args,
+ rustc_span::DUMMY_SP,// span only used for error messages
+);
+
+Some(instance)
+}
 }