Don't require extern "llvm-intrinsic" return structs to be #[repr(packed)]

Author: beetrees

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -22,7 +22,7 @@ use rustc_codegen_ssa::{MemFlags, RetagInfo};
 use rustc_data_structures::fx::FxHashSet;
 #[cfg(feature = "master")]
 use rustc_middle::ty::layout::FnAbiOf;
-use rustc_middle::ty::layout::LayoutOf;
+use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_session::config::OptLevel;
 use rustc_span::{Span, Symbol, bug, span_bug, sym};
@@ -556,6 +556,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[OperandRef<'tcx, Self::Value>],
+        return_slot: ReturnSlot<Self::Value>,
         is_cleanup: bool,
     ) -> Self::Value {
         let func = if let Some(&func) = self.intrinsic_instances.borrow().get(&instance) {
@@ -630,7 +631,33 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             self.apply_attrs_to_cleanup_callsite(llret);
         }
 
-        llret
+        let fn_sig = match *instance.ty(self.tcx, self.typing_env()).kind() {
+            ty::FnDef(def_id, args) => self.tcx.instantiate_bound_regions_with_erased(
+                self.tcx
+                    .fn_sig(def_id)
+                    .instantiate(self.tcx, args.no_bound_vars().unwrap())
+                    .skip_norm_wip(),
+            ),
+            _ => unreachable!(),
+        };
+        let ret_layout = self.layout_of(fn_sig.output());
+
+        if !ret_layout.is_zst() && ret_layout.is_aggregate() {
+            let fields =
+                rustc_codegen_ssa::mir::intrinsic_return_struct_non_zst_fields(self.cx, ret_layout)
+                    .into_iter()
+                    .enumerate()
+                    .map(|(i, (_, offset))| (offset, self.extract_value(llret, i as u64)))
+                    .collect();
+            rustc_codegen_ssa::mir::store_intrinsic_return_struct(
+                self,
+                ret_layout,
+                return_slot,
+                fields,
+            )
+        } else {
+            llret
+        }
     }
 
     fn abort_immediate(&mut self) {

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -895,6 +895,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[OperandRef<'tcx, Self::Value>],
+        return_slot: ReturnSlot<Self::Value>,
         _is_cleanup: bool,
     ) -> Self::Value {
         let tcx = self.tcx();
@@ -909,10 +910,15 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         assert!(!fn_sig.c_variadic());
 
         let ret_layout = self.layout_of(fn_sig.output());
-        let llreturn_ty = if ret_layout.is_zst() {
-            self.type_void()
+        let (llreturn_ty, return_struct_fields) = if ret_layout.is_zst() {
+            (self.type_void(), vec![])
+        } else if ret_layout.is_aggregate() {
+            let fields =
+                rustc_codegen_ssa::mir::intrinsic_return_struct_non_zst_fields(self.cx, ret_layout);
+            let llvm_fields = fields.iter().copied().map(|(t, _)| t).collect::<Vec<_>>();
+            (self.type_struct(&llvm_fields, false), fields)
         } else {
-            ret_layout.immediate_llvm_type(self)
+            (ret_layout.immediate_llvm_type(self), vec![])
         };
 
         let mut llargument_tys = Vec::with_capacity(fn_sig.inputs().len());
@@ -996,14 +1002,33 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             )
         };
 
-        let src_ty = self.val_ty(llret);
-        let dest_ty = llreturn_ty;
-        assert!(
-            can_autocast(self, dest_ty, src_ty),
-            "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{fn_ptr:?}`"
-        );
+        if return_struct_fields.is_empty() {
+            let src_ty = self.val_ty(llret);
+            let dest_ty = llreturn_ty;
+            assert!(
+                can_autocast(self, dest_ty, src_ty),
+                "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{fn_ptr:?}`"
+            );
 
-        autocast(self, llret, src_ty, dest_ty)
+            autocast(self, llret, src_ty, dest_ty)
+        } else {
+            let fields = return_struct_fields.into_iter().enumerate().map(|(i, (dest_ty, offset))| {
+                let value = self.extract_value(llret, i as u64);
+                let src_ty = self.val_ty(value);
+                assert!(
+                    can_autocast(self, dest_ty, src_ty),
+                    "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{fn_ptr:?}`"
+                );
+                let value = autocast(self, value, src_ty, dest_ty);
+                (offset, value)
+            }).collect();
+            rustc_codegen_ssa::mir::store_intrinsic_return_struct(
+                self,
+                ret_layout,
+                return_slot,
+                fields,
+            )
+        }
     }
 
     fn abort_immediate(&mut self) {
@@ -1089,23 +1114,6 @@ fn can_autocast<'ll>(cx: &CodegenCx<'ll, '_>, rust_ty: &'ll Type, llvm_ty: &'ll
     }
 
     match cx.type_kind(llvm_ty) {
-        // Some LLVM intrinsics return **non-packed** structs, but they can't be mimicked from Rust
-        // due to auto field-alignment in non-packed structs (packed structs are represented in LLVM
-        // as, well, packed structs, so they won't match with those either)
-        TypeKind::Struct if cx.type_kind(rust_ty) == TypeKind::Struct => {
-            let rust_element_tys = cx.struct_element_types(rust_ty);
-            let llvm_element_tys = cx.struct_element_types(llvm_ty);
-
-            if rust_element_tys.len() != llvm_element_tys.len() {
-                return false;
-            }
-
-            iter::zip(rust_element_tys, llvm_element_tys).all(
-                |(rust_element_ty, llvm_element_ty)| {
-                    can_autocast(cx, rust_element_ty, llvm_element_ty)
-                },
-            )
-        }
         TypeKind::Vector => {
             let llvm_element_ty = cx.element_type(llvm_ty);
             let element_count = cx.vector_length(llvm_ty) as u64;
@@ -1152,19 +1160,6 @@ fn autocast<'ll>(
         return val;
     }
     match (bx.type_kind(src_ty), bx.type_kind(dest_ty)) {
-        // re-pack structs
-        (TypeKind::Struct, TypeKind::Struct) => {
-            let mut ret = bx.const_poison(dest_ty);
-            for (idx, (src_element_ty, dest_element_ty)) in
-                iter::zip(bx.struct_element_types(src_ty), bx.struct_element_types(dest_ty))
-                    .enumerate()
-            {
-                let elt = bx.extract_value(val, idx as u64);
-                let casted_elt = autocast(bx, elt, src_element_ty, dest_element_ty);
-                ret = bx.insert_value(ret, casted_elt, idx as u64);
-            }
-            ret
-        }
         // cast from the i1xN vector type to the primitive type
         (TypeKind::Vector, TypeKind::Integer) if bx.element_type(src_ty) == bx.type_i1() => {
             let vector_length = bx.vector_length(src_ty) as u64;
@@ -1258,9 +1253,17 @@ fn intrinsic_fn<'ll, 'tcx>(
         let llvm_argument_tys = bx.func_params_types(llvm_fn_ty);
         let llvm_is_variadic = bx.func_is_variadic(llvm_fn_ty);
 
+        let (rust_return_tys, llvm_return_tys) = if bx.type_kind(rust_return_ty) == TypeKind::Struct
+            && bx.type_kind(llvm_return_ty) == TypeKind::Struct
+        {
+            (bx.struct_element_types(rust_return_ty), bx.struct_element_types(llvm_return_ty))
+        } else {
+            (vec![rust_return_ty], vec![llvm_return_ty])
+        };
         let is_correct_signature = !llvm_is_variadic
             && rust_argument_tys.len() == llvm_argument_tys.len()
-            && iter::once((rust_return_ty, llvm_return_ty))
+            && rust_return_tys.len() == llvm_return_tys.len()
+            && iter::zip(rust_return_tys, llvm_return_tys)
                 .chain(iter::zip(rust_argument_tys, llvm_argument_tys))
                 .all(|(rust_ty, llvm_ty)| can_autocast(bx, rust_ty, llvm_ty));
 

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1,5 +1,5 @@
-use std::cmp;
 use std::ops::Range;
+use std::{assert_matches, cmp};
 
 use rustc_abi::{
     Align, ArmCall, BackendRepr, CanonAbi, ExternAbi, FieldsShape, HasDataLayout, Reg, Size,
@@ -1164,38 +1164,36 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             let result_layout =
                 self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref()));
 
-            let return_dest = if result_layout.is_zst() {
-                ReturnDest::Nothing
-            } else if let Some(index) = destination.as_local() {
-                match self.locals[index] {
-                    LocalRef::Place(dest) => ReturnDest::Store(dest),
-                    LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
-                    LocalRef::PendingOperand => {
-                        // Handle temporary places, specifically `Operand` ones, as
-                        // they don't have `alloca`s.
-                        ReturnDest::DirectOperand(index)
-                    }
-                    LocalRef::Operand(_) => bug!("place local already assigned to"),
+            let return_mode = if result_layout.is_zst() {
+                PassMode::Ignore
+            } else if let BackendRepr::ScalarPair { .. } = result_layout.backend_repr {
+                PassMode::Pair(ArgAttributes::new(), ArgAttributes::new())
+            } else if result_layout.is_aggregate() {
+                PassMode::Indirect {
+                    attrs: ArgAttributes::new(),
+                    address_space: None,
+                    mode: IndirectMode::Pointer,
                 }
             } else {
-                ReturnDest::Store(self.codegen_place(bx, destination.as_ref()))
+                PassMode::Direct(ArgAttributes::new())
             };
+            let return_abi = ArgAbi { layout: result_layout, mode: return_mode };
+            let (return_dest, return_slot) = self.make_return_dest(bx, destination, &return_abi);
 
             let args =
                 args.into_iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect::<Vec<_>>();
 
             self.set_debug_loc(bx, source_info);
 
-            let llret =
-                bx.codegen_llvm_intrinsic_call(instance, &args, self.mir[helper.bb].is_cleanup);
+            let llret = bx.codegen_llvm_intrinsic_call(
+                instance,
+                &args,
+                return_slot,
+                self.mir[helper.bb].is_cleanup,
+            );
 
             if let Some(target) = target {
-                self.store_return(
-                    bx,
-                    return_dest,
-                    &ArgAbi { layout: result_layout, mode: PassMode::Direct(ArgAttributes::new()) },
-                    llret,
-                );
+                self.store_return(bx, return_dest, &return_abi, llret);
                 return helper.funclet_br(self, bx, target, mergeable_succ, &[]);
             } else {
                 bx.unreachable();
@@ -2501,7 +2499,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         }
     }
 
-    // Stores the return value of a function call into it's final location.
+    /// Stores the return value of a function call into it's final location.
     fn store_return(
         &mut self,
         bx: &mut Bx,
@@ -2603,3 +2601,64 @@ pub fn store_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         bx.store(value, ptr, align);
     };
 }
+
+pub fn intrinsic_return_struct_non_zst_fields<'tcx, Cx: CodegenMethods<'tcx>>(
+    cx: &Cx,
+    ret_layout: TyAndLayout<'tcx>,
+) -> Vec<(Cx::Type, Size)> {
+    let ret_layout = ret_layout.peel_transparent_wrappers_from_non_1zst(cx);
+    match ret_layout.ty.kind() {
+        ty::Adt(adt_def, _) if adt_def.is_struct() => {}
+        ty::Tuple(_) => {}
+        _ => bug!("invalid intrinsic return type: {ret_layout:?}"),
+    }
+    let fields = (0..ret_layout.fields.count())
+        .filter_map(|i| {
+            let field = ret_layout.field(cx, i);
+            assert!(
+                field.is_zst() || !field.is_aggregate(),
+                "invalid intrinsic return field: {field:?}"
+            );
+            (!field.is_zst())
+                .then(|| (cx.immediate_backend_type(field), ret_layout.fields.offset(i)))
+        })
+        .collect::<Vec<_>>();
+    assert!(fields.len() >= 2, "invalid intrinsic return type: {ret_layout:?}");
+    fields
+}
+
+pub fn store_intrinsic_return_struct<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+    bx: &mut Bx,
+    ret_layout: TyAndLayout<'tcx>,
+    return_slot: ReturnSlot<Bx::Value>,
+    fields: Vec<(Size, Bx::Value)>,
+) -> Bx::Value {
+    match ret_layout.backend_repr {
+        BackendRepr::ScalarPair { a: _, b: _, b_offset } => {
+            assert_matches!(return_slot, ReturnSlot::Direct);
+            let a_index = if fields[0].0 == b_offset { 1 } else { 0 };
+            let b_index = if fields[1].0 == b_offset { 1 } else { 0 };
+            assert_ne!(a_index, b_index);
+            let mut pair = bx.cx().const_poison(bx.immediate_backend_type(ret_layout));
+            pair = bx.insert_value(pair, fields[a_index].1, 0);
+            bx.insert_value(pair, fields[b_index].1, 1)
+        }
+        BackendRepr::Memory { sized: true } => {
+            let ReturnSlot::Indirect(dest) = return_slot else {
+                bug!("BackendRepr::Memory return with no return slot")
+            };
+            let dest_align = ret_layout.align.abi;
+            for (offset, value) in fields.into_iter() {
+                let pointer = if offset.bytes() != 0 {
+                    bx.inbounds_ptradd(dest, bx.const_usize(offset.bytes()))
+                } else {
+                    dest
+                };
+                bx.store(value, pointer, dest_align.restrict_for_offset(offset));
+            }
+            // Return value is ignored as return_slot is indirect
+            dest
+        }
+        _ => bug!("unexpected aggregate return layout: {ret_layout:?}"),
+    }
+}

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -28,7 +28,9 @@ mod retag;
 mod rvalue;
 mod statement;
 
-pub use self::block::store_cast;
+pub use self::block::{
+    intrinsic_return_struct_non_zst_fields, store_cast, store_intrinsic_return_struct,
+};
 use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo};
 use self::operand::{OperandRef, OperandValue};
 use self::place::PlaceRef;

compiler/rustc_codegen_ssa/src/traits/intrinsic.rs

@@ -6,6 +6,7 @@ use crate::RetagInfo;
 use crate::mir::IntrinsicResult;
 use crate::mir::operand::OperandRef;
 use crate::mir::place::PlaceValue;
+use crate::traits::ReturnSlot;
 
 pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes {
     /// Higher-level interface to emitting calls to intrinsics
@@ -35,6 +36,7 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes {
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[OperandRef<'tcx, Self::Value>],
+        return_slot: ReturnSlot<Self::Value>,
         is_cleanup: bool,
     ) -> Self::Value;