Auto merge of #80594 - bjorn3:abi_refactor3, r=petrochenkov

Various ABI refactorings

This includes changes to the rust abi and various refactorings that will hopefully make it easier to use the abi handling infrastructure of rustc in cg_clif. There are several refactorings that I haven't done. I am opening this draft PR to check that I haven't broken any non x86_64 architectures.

r? `@ghost`

Author: bors

Cargo.lock

@@ -3551,7 +3551,6 @@ version = "0.0.0"
 dependencies = [
  "rustc_ast",
  "rustc_span",
- "rustc_target",
  "tracing",
 ]
 

compiler/rustc_ast_pretty/Cargo.toml

@@ -11,4 +11,3 @@ doctest = false
 tracing = "0.1"
 rustc_span = { path = "../rustc_span" }
 rustc_ast = { path = "../rustc_ast" }
-rustc_target = { path = "../rustc_target" }

compiler/rustc_middle/src/ty/layout.rs

@@ -2514,7 +2514,7 @@ where
         extra_args: &[Ty<'tcx>],
         caller_location: Option<Ty<'tcx>>,
         codegen_fn_attr_flags: CodegenFnAttrFlags,
-        mk_arg_type: impl Fn(Ty<'tcx>, Option<usize>) -> ArgAbi<'tcx, Ty<'tcx>>,
+        make_self_ptr_thin: bool,
     ) -> Self;
     fn adjust_for_abi(&mut self, cx: &C, abi: SpecAbi);
 }
@@ -2574,9 +2574,7 @@ where
         // Assume that fn pointers may always unwind
         let codegen_fn_attr_flags = CodegenFnAttrFlags::UNWIND;
 
-        call::FnAbi::new_internal(cx, sig, extra_args, None, codegen_fn_attr_flags, |ty, _| {
-            ArgAbi::new(cx.layout_of(ty))
-        })
+        call::FnAbi::new_internal(cx, sig, extra_args, None, codegen_fn_attr_flags, false)
     }
 
     fn of_instance(cx: &C, instance: ty::Instance<'tcx>, extra_args: &[Ty<'tcx>]) -> Self {
@@ -2590,55 +2588,14 @@ where
 
         let attrs = cx.tcx().codegen_fn_attrs(instance.def_id()).flags;
 
-        call::FnAbi::new_internal(cx, sig, extra_args, caller_location, attrs, |ty, arg_idx| {
-            let mut layout = cx.layout_of(ty);
-            // Don't pass the vtable, it's not an argument of the virtual fn.
-            // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait`
-            // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen
-            if let (ty::InstanceDef::Virtual(..), Some(0)) = (&instance.def, arg_idx) {
-                let fat_pointer_ty = if layout.is_unsized() {
-                    // unsized `self` is passed as a pointer to `self`
-                    // FIXME (mikeyhew) change this to use &own if it is ever added to the language
-                    cx.tcx().mk_mut_ptr(layout.ty)
-                } else {
-                    match layout.abi {
-                        Abi::ScalarPair(..) => (),
-                        _ => bug!("receiver type has unsupported layout: {:?}", layout),
-                    }
-
-                    // In the case of Rc<Self>, we need to explicitly pass a *mut RcBox<Self>
-                    // with a Scalar (not ScalarPair) ABI. This is a hack that is understood
-                    // elsewhere in the compiler as a method on a `dyn Trait`.
-                    // To get the type `*mut RcBox<Self>`, we just keep unwrapping newtypes until we
-                    // get a built-in pointer type
-                    let mut fat_pointer_layout = layout;
-                    'descend_newtypes: while !fat_pointer_layout.ty.is_unsafe_ptr()
-                        && !fat_pointer_layout.ty.is_region_ptr()
-                    {
-                        for i in 0..fat_pointer_layout.fields.count() {
-                            let field_layout = fat_pointer_layout.field(cx, i);
-
-                            if !field_layout.is_zst() {
-                                fat_pointer_layout = field_layout;
-                                continue 'descend_newtypes;
-                            }
-                        }
-
-                        bug!("receiver has no non-zero-sized fields {:?}", fat_pointer_layout);
-                    }
-
-                    fat_pointer_layout.ty
-                };
-
-                // we now have a type like `*mut RcBox<dyn Trait>`
-                // change its layout to that of `*mut ()`, a thin pointer, but keep the same type
-                // this is understood as a special case elsewhere in the compiler
-                let unit_pointer_ty = cx.tcx().mk_mut_ptr(cx.tcx().mk_unit());
-                layout = cx.layout_of(unit_pointer_ty);
-                layout.ty = fat_pointer_ty;
-            }
-            ArgAbi::new(layout)
-        })
+        call::FnAbi::new_internal(
+            cx,
+            sig,
+            extra_args,
+            caller_location,
+            attrs,
+            matches!(instance.def, ty::InstanceDef::Virtual(..)),
+        )
     }
 
     fn new_internal(
@@ -2647,7 +2604,7 @@ where
         extra_args: &[Ty<'tcx>],
         caller_location: Option<Ty<'tcx>>,
         codegen_fn_attr_flags: CodegenFnAttrFlags,
-        mk_arg_type: impl Fn(Ty<'tcx>, Option<usize>) -> ArgAbi<'tcx, Ty<'tcx>>,
+        force_thin_self_ptr: bool,
     ) -> Self {
         debug!("FnAbi::new_internal({:?}, {:?})", sig, extra_args);
 
@@ -2778,7 +2735,23 @@ where
 
         let arg_of = |ty: Ty<'tcx>, arg_idx: Option<usize>| {
             let is_return = arg_idx.is_none();
-            let mut arg = mk_arg_type(ty, arg_idx);
+
+            let layout = cx.layout_of(ty);
+            let layout = if force_thin_self_ptr && arg_idx == Some(0) {
+                // Don't pass the vtable, it's not an argument of the virtual fn.
+                // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait`
+                // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen
+                make_thin_self_ptr(cx, layout)
+            } else {
+                layout
+            };
+
+            let mut arg = ArgAbi::new(cx, layout, |layout, scalar, offset| {
+                let mut attrs = ArgAttributes::new();
+                adjust_for_rust_scalar(&mut attrs, scalar, *layout, offset, is_return);
+                attrs
+            });
+
             if arg.layout.is_zst() {
                 // For some forsaken reason, x86_64-pc-windows-gnu
                 // doesn't ignore zero-sized struct arguments.
@@ -2794,30 +2767,6 @@ where
                 }
             }
 
-            // FIXME(eddyb) other ABIs don't have logic for scalar pairs.
-            if !is_return && rust_abi {
-                if let Abi::ScalarPair(ref a, ref b) = arg.layout.abi {
-                    let mut a_attrs = ArgAttributes::new();
-                    let mut b_attrs = ArgAttributes::new();
-                    adjust_for_rust_scalar(&mut a_attrs, a, arg.layout, Size::ZERO, false);
-                    adjust_for_rust_scalar(
-                        &mut b_attrs,
-                        b,
-                        arg.layout,
-                        a.value.size(cx).align_to(b.value.align(cx).abi),
-                        false,
-                    );
-                    arg.mode = PassMode::Pair(a_attrs, b_attrs);
-                    return arg;
-                }
-            }
-
-            if let Abi::Scalar(ref scalar) = arg.layout.abi {
-                if let PassMode::Direct(ref mut attrs) = arg.mode {
-                    adjust_for_rust_scalar(attrs, scalar, arg.layout, Size::ZERO, is_return);
-                }
-            }
-
             arg
         };
 
@@ -2894,9 +2843,10 @@ where
                 let max_by_val_size = Pointer.size(cx) * 2;
                 let size = arg.layout.size;
 
-                if arg.layout.is_unsized() || size > max_by_val_size {
-                    arg.make_indirect();
-                } else {
+                let is_indirect_not_on_stack =
+                    matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
+                assert!(is_indirect_not_on_stack, "{:?}", arg);
+                if !arg.layout.is_unsized() && size <= max_by_val_size {
                     // We want to pass small aggregates as immediates, but using
                     // a LLVM aggregate type for this leads to bad optimizations,
                     // so we pick an appropriately sized integer type instead.
@@ -2915,3 +2865,52 @@ where
         }
     }
 }
+
+fn make_thin_self_ptr<'tcx, C>(cx: &C, mut layout: TyAndLayout<'tcx>) -> TyAndLayout<'tcx>
+where
+    C: LayoutOf<Ty = Ty<'tcx>, TyAndLayout = TyAndLayout<'tcx>>
+        + HasTyCtxt<'tcx>
+        + HasParamEnv<'tcx>,
+{
+    let fat_pointer_ty = if layout.is_unsized() {
+        // unsized `self` is passed as a pointer to `self`
+        // FIXME (mikeyhew) change this to use &own if it is ever added to the language
+        cx.tcx().mk_mut_ptr(layout.ty)
+    } else {
+        match layout.abi {
+            Abi::ScalarPair(..) => (),
+            _ => bug!("receiver type has unsupported layout: {:?}", layout),
+        }
+
+        // In the case of Rc<Self>, we need to explicitly pass a *mut RcBox<Self>
+        // with a Scalar (not ScalarPair) ABI. This is a hack that is understood
+        // elsewhere in the compiler as a method on a `dyn Trait`.
+        // To get the type `*mut RcBox<Self>`, we just keep unwrapping newtypes until we
+        // get a built-in pointer type
+        let mut fat_pointer_layout = layout;
+        'descend_newtypes: while !fat_pointer_layout.ty.is_unsafe_ptr()
+            && !fat_pointer_layout.ty.is_region_ptr()
+        {
+            for i in 0..fat_pointer_layout.fields.count() {
+                let field_layout = fat_pointer_layout.field(cx, i);
+
+                if !field_layout.is_zst() {
+                    fat_pointer_layout = field_layout;
+                    continue 'descend_newtypes;
+                }
+            }
+
+            bug!("receiver has no non-zero-sized fields {:?}", fat_pointer_layout);
+        }
+
+        fat_pointer_layout.ty
+    };
+
+    // we now have a type like `*mut RcBox<dyn Trait>`
+    // change its layout to that of `*mut ()`, a thin pointer, but keep the same type
+    // this is understood as a special case elsewhere in the compiler
+    let unit_pointer_ty = cx.tcx().mk_mut_ptr(cx.tcx().mk_unit());
+    layout = cx.layout_of(unit_pointer_ty);
+    layout.ty = fat_pointer_ty;
+    layout
+}

compiler/rustc_target/src/abi/call/mod.rs

@@ -27,10 +27,16 @@ mod x86_win64;
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub enum PassMode {
     /// Ignore the argument.
+    ///
+    /// The argument is either uninhabited or a ZST.
     Ignore,
     /// Pass the argument directly.
+    ///
+    /// The argument has a layout abi of `Scalar` or `Vector`.
     Direct(ArgAttributes),
     /// Pass a pair's elements directly in two arguments.
+    ///
+    /// The argument has a layout abi of `ScalarPair`.
     Pair(ArgAttributes, ArgAttributes),
     /// Pass the argument after casting it, to either
     /// a single uniform or a pair of registers.
@@ -434,28 +440,49 @@ pub struct ArgAbi<'a, Ty> {
 }
 
 impl<'a, Ty> ArgAbi<'a, Ty> {
-    pub fn new(layout: TyAndLayout<'a, Ty>) -> Self {
-        ArgAbi { layout, pad: None, mode: PassMode::Direct(ArgAttributes::new()) }
+    pub fn new(
+        cx: &impl HasDataLayout,
+        layout: TyAndLayout<'a, Ty>,
+        scalar_attrs: impl Fn(&TyAndLayout<'a, Ty>, &abi::Scalar, Size) -> ArgAttributes,
+    ) -> Self {
+        let mode = match &layout.abi {
+            Abi::Uninhabited => PassMode::Ignore,
+            Abi::Scalar(scalar) => PassMode::Direct(scalar_attrs(&layout, scalar, Size::ZERO)),
+            Abi::ScalarPair(a, b) => PassMode::Pair(
+                scalar_attrs(&layout, a, Size::ZERO),
+                scalar_attrs(&layout, b, a.value.size(cx).align_to(b.value.align(cx).abi)),
+            ),
+            Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()),
+            Abi::Aggregate { .. } => Self::indirect_pass_mode(&layout),
+        };
+        ArgAbi { layout, pad: None, mode }
     }
 
-    pub fn make_indirect(&mut self) {
-        assert_eq!(self.mode, PassMode::Direct(ArgAttributes::new()));
-
-        // Start with fresh attributes for the pointer.
+    fn indirect_pass_mode(layout: &TyAndLayout<'a, Ty>) -> PassMode {
         let mut attrs = ArgAttributes::new();
 
         // For non-immediate arguments the callee gets its own copy of
         // the value on the stack, so there are no aliases. It's also
         // program-invisible so can't possibly capture
         attrs.set(ArgAttribute::NoAlias).set(ArgAttribute::NoCapture).set(ArgAttribute::NonNull);
-        attrs.pointee_size = self.layout.size;
+        attrs.pointee_size = layout.size;
         // FIXME(eddyb) We should be doing this, but at least on
         // i686-pc-windows-msvc, it results in wrong stack offsets.
-        // attrs.pointee_align = Some(self.layout.align.abi);
+        // attrs.pointee_align = Some(layout.align.abi);
+
+        let extra_attrs = layout.is_unsized().then_some(ArgAttributes::new());
 
-        let extra_attrs = self.layout.is_unsized().then_some(ArgAttributes::new());
+        PassMode::Indirect { attrs, extra_attrs, on_stack: false }
+    }
+
+    pub fn make_indirect(&mut self) {
+        match self.mode {
+            PassMode::Direct(_) | PassMode::Pair(_, _) => {}
+            PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: false } => return,
+            _ => panic!("Tried to make {:?} indirect", self.mode),
+        }
 
-        self.mode = PassMode::Indirect { attrs, extra_attrs, on_stack: false };
+        self.mode = Self::indirect_pass_mode(&self.layout);
     }
 
     pub fn make_indirect_byval(&mut self) {
@@ -486,7 +513,6 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
     }
 
     pub fn cast_to<T: Into<CastTarget>>(&mut self, target: T) {
-        assert_eq!(self.mode, PassMode::Direct(ArgAttributes::new()));
         self.mode = PassMode::Cast(target.into());
     }