Skip to content

Commit c1901af

Browse files
SpriteOvOSpriteOvO
committed
Introduce adjust_for_rust_abi in rustc_target 
1 parent bca5fde commit c1901af

File tree

3 files changed

+161
-138
lines changed

3 files changed

+161
-138
lines changed

‎compiler/rustc_target/src/callconv/mod.rs

+114-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
1-
use std::fmt;
21
use std::str::FromStr;
2+
use std::{fmt, iter};
33

4+
use rustc_abi::AddressSpace;
5+
use rustc_abi::Primitive::Pointer;
46
pub use rustc_abi::{Reg, RegKind};
57
use rustc_macros::HashStable_Generic;
68
use rustc_span::Symbol;
79

810
use crate::abi::{self, Abi, Align, HasDataLayout, Size, TyAbiInterface, TyAndLayout};
11+
use crate::spec::abi::Abi as SpecAbi;
912
use crate::spec::{self, HasTargetSpec, HasWasmCAbiOpt, HasX86AbiOpt, WasmCAbi};
1013

1114
mod aarch64;
@@ -720,6 +723,116 @@ impl<'a, Ty> FnAbi<'a, Ty> {
720723

721724
Ok(())
722725
}
726+
727+
pub fn adjust_for_rust_abi<C>(&mut self, cx: &C, abi: SpecAbi)
728+
where
729+
Ty: TyAbiInterface<'a, C> + Copy,
730+
C: HasDataLayout + HasTargetSpec,
731+
{
732+
let spec = cx.target_spec();
733+
match &spec.arch[..] {
734+
"x86" => x86::compute_rust_abi_info(cx, self, abi),
735+
_ => {}
736+
};
737+
738+
for (arg_idx, arg) in self
739+
.args
740+
.iter_mut()
741+
.enumerate()
742+
.map(|(idx, arg)| (Some(idx), arg))
743+
.chain(iter::once((None, &mut self.ret)))
744+
{
745+
if arg.is_ignore() {
746+
continue;
747+
}
748+
749+
if arg_idx.is_none() && arg.layout.size > Pointer(AddressSpace::DATA).size(cx) * 2 {
750+
// Return values larger than 2 registers using a return area
751+
// pointer. LLVM and Cranelift disagree about how to return
752+
// values that don't fit in the registers designated for return
753+
// values. LLVM will force the entire return value to be passed
754+
// by return area pointer, while Cranelift will look at each IR level
755+
// return value independently and decide to pass it in a
756+
// register or not, which would result in the return value
757+
// being passed partially in registers and partially through a
758+
// return area pointer.
759+
//
760+
// While Cranelift may need to be fixed as the LLVM behavior is
761+
// generally more correct with respect to the surface language,
762+
// forcing this behavior in rustc itself makes it easier for
763+
// other backends to conform to the Rust ABI and for the C ABI
764+
// rustc already handles this behavior anyway.
765+
//
766+
// In addition LLVM's decision to pass the return value in
767+
// registers or using a return area pointer depends on how
768+
// exactly the return type is lowered to an LLVM IR type. For
769+
// example `Option<u128>` can be lowered as `{ i128, i128 }`
770+
// in which case the x86_64 backend would use a return area
771+
// pointer, or it could be passed as `{ i32, i128 }` in which
772+
// case the x86_64 backend would pass it in registers by taking
773+
// advantage of an LLVM ABI extension that allows using 3
774+
// registers for the x86_64 sysv call conv rather than the
775+
// officially specified 2 registers.
776+
//
777+
// FIXME: Technically we should look at the amount of available
778+
// return registers rather than guessing that there are 2
779+
// registers for return values. In practice only a couple of
780+
// architectures have less than 2 return registers. None of
781+
// which supported by Cranelift.
782+
//
783+
// NOTE: This adjustment is only necessary for the Rust ABI as
784+
// for other ABI's the calling convention implementations in
785+
// rustc_target already ensure any return value which doesn't
786+
// fit in the available amount of return registers is passed in
787+
// the right way for the current target.
788+
arg.make_indirect();
789+
continue;
790+
}
791+
792+
match arg.layout.abi {
793+
Abi::Aggregate { .. } => {}
794+
795+
// This is a fun case! The gist of what this is doing is
796+
// that we want callers and callees to always agree on the
797+
// ABI of how they pass SIMD arguments. If we were to *not*
798+
// make these arguments indirect then they'd be immediates
799+
// in LLVM, which means that they'd used whatever the
800+
// appropriate ABI is for the callee and the caller. That
801+
// means, for example, if the caller doesn't have AVX
802+
// enabled but the callee does, then passing an AVX argument
803+
// across this boundary would cause corrupt data to show up.
804+
//
805+
// This problem is fixed by unconditionally passing SIMD
806+
// arguments through memory between callers and callees
807+
// which should get them all to agree on ABI regardless of
808+
// target feature sets. Some more information about this
809+
// issue can be found in #44367.
810+
//
811+
// Note that the intrinsic ABI is exempt here as
812+
// that's how we connect up to LLVM and it's unstable
813+
// anyway, we control all calls to it in libstd.
814+
Abi::Vector { .. } if abi != SpecAbi::RustIntrinsic && spec.simd_types_indirect => {
815+
arg.make_indirect();
816+
continue;
817+
}
818+
819+
_ => continue,
820+
}
821+
// Compute `Aggregate` ABI.
822+
823+
let is_indirect_not_on_stack =
824+
matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
825+
assert!(is_indirect_not_on_stack);
826+
827+
let size = arg.layout.size;
828+
if !arg.layout.is_unsized() && size <= Pointer(AddressSpace::DATA).size(cx) {
829+
// We want to pass small aggregates as immediates, but using
830+
// an LLVM aggregate type for this leads to bad optimizations,
831+
// so we pick an appropriately sized integer type instead.
832+
arg.cast_to(Reg { kind: RegKind::Integer, size });
833+
}
834+
}
835+
}
723836
}
724837

725838
impl FromStr for Conv {

‎compiler/rustc_target/src/callconv/x86.rs

+39
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1+
use rustc_abi::Float::*;
2+
use rustc_abi::Primitive::Float;
3+
14
use crate::abi::call::{ArgAttribute, FnAbi, PassMode, Reg, RegKind};
25
use crate::abi::{Abi, Align, HasDataLayout, TyAbiInterface, TyAndLayout};
36
use crate::spec::HasTargetSpec;
7+
use crate::spec::abi::Abi as SpecAbi;
48

59
#[derive(PartialEq)]
610
pub(crate) enum Flavor {
@@ -207,3 +211,38 @@ pub(crate) fn fill_inregs<'a, Ty, C>(
207211
}
208212
}
209213
}
214+
215+
pub(crate) fn compute_rust_abi_info<'a, Ty, C>(_cx: &C, fn_abi: &mut FnAbi<'a, Ty>, abi: SpecAbi)
216+
where
217+
Ty: TyAbiInterface<'a, C> + Copy,
218+
C: HasDataLayout + HasTargetSpec,
219+
{
220+
// Avoid returning floats in x87 registers on x86 as loading and storing from x87
221+
// registers will quiet signalling NaNs.
222+
if !fn_abi.ret.is_ignore()
223+
// Intrinsics themselves are not actual "real" functions, so theres no need to change their ABIs.
224+
&& abi != SpecAbi::RustIntrinsic
225+
{
226+
match fn_abi.ret.layout.abi {
227+
// Handle similar to the way arguments with an `Abi::Aggregate` abi are handled
228+
// in `adjust_for_rust_abi`, by returning arguments up to the size of a pointer (32 bits on x86)
229+
// cast to an appropriately sized integer.
230+
Abi::Scalar(s) if s.primitive() == Float(F32) => {
231+
// Same size as a pointer, return in a register.
232+
fn_abi.ret.cast_to(Reg::i32());
233+
}
234+
Abi::Scalar(s) if s.primitive() == Float(F64) => {
235+
// Larger than a pointer, return indirectly.
236+
fn_abi.ret.make_indirect();
237+
}
238+
Abi::ScalarPair(s1, s2)
239+
if matches!(s1.primitive(), Float(F32 | F64))
240+
|| matches!(s2.primitive(), Float(F32 | F64)) =>
241+
{
242+
// Larger than a pointer, return indirectly.
243+
fn_abi.ret.make_indirect();
244+
}
245+
_ => {}
246+
};
247+
}
248+
}

‎compiler/rustc_ty_utils/src/abi.rs

+8-137
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
use std::iter;
22

3-
use rustc_abi::Float::*;
4-
use rustc_abi::Primitive::{Float, Pointer};
5-
use rustc_abi::{Abi, AddressSpace, PointerKind, Scalar, Size};
3+
use rustc_abi::Primitive::Pointer;
4+
use rustc_abi::{Abi, PointerKind, Scalar, Size};
65
use rustc_hir as hir;
76
use rustc_hir::lang_items::LangItem;
87
use rustc_middle::bug;
@@ -14,8 +13,7 @@ use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt};
1413
use rustc_session::config::OptLevel;
1514
use rustc_span::def_id::DefId;
1615
use rustc_target::abi::call::{
17-
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, Reg, RegKind,
18-
RiscvInterruptKind,
16+
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, RiscvInterruptKind,
1917
};
2018
use rustc_target::spec::abi::Abi as SpecAbi;
2119
use tracing::debug;
@@ -679,6 +677,8 @@ fn fn_abi_adjust_for_abi<'tcx>(
679677
let tcx = cx.tcx();
680678

681679
if abi == SpecAbi::Rust || abi == SpecAbi::RustCall || abi == SpecAbi::RustIntrinsic {
680+
fn_abi.adjust_for_rust_abi(cx, abi);
681+
682682
// Look up the deduced parameter attributes for this function, if we have its def ID and
683683
// we're optimizing in non-incremental mode. We'll tag its parameters with those attributes
684684
// as appropriate.
@@ -689,141 +689,17 @@ fn fn_abi_adjust_for_abi<'tcx>(
689689
&[]
690690
};
691691

692-
let fixup = |arg: &mut ArgAbi<'tcx, Ty<'tcx>>, arg_idx: Option<usize>| {
692+
for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() {
693693
if arg.is_ignore() {
694-
return;
695-
}
696-
697-
// Avoid returning floats in x87 registers on x86 as loading and storing from x87
698-
// registers will quiet signalling NaNs.
699-
if tcx.sess.target.arch == "x86"
700-
&& arg_idx.is_none()
701-
// Intrinsics themselves are not actual "real" functions, so theres no need to
702-
// change their ABIs.
703-
&& abi != SpecAbi::RustIntrinsic
704-
{
705-
match arg.layout.abi {
706-
// Handle similar to the way arguments with an `Abi::Aggregate` abi are handled
707-
// below, by returning arguments up to the size of a pointer (32 bits on x86)
708-
// cast to an appropriately sized integer.
709-
Abi::Scalar(s) if s.primitive() == Float(F32) => {
710-
// Same size as a pointer, return in a register.
711-
arg.cast_to(Reg::i32());
712-
return;
713-
}
714-
Abi::Scalar(s) if s.primitive() == Float(F64) => {
715-
// Larger than a pointer, return indirectly.
716-
arg.make_indirect();
717-
return;
718-
}
719-
Abi::ScalarPair(s1, s2)
720-
if matches!(s1.primitive(), Float(F32 | F64))
721-
|| matches!(s2.primitive(), Float(F32 | F64)) =>
722-
{
723-
// Larger than a pointer, return indirectly.
724-
arg.make_indirect();
725-
return;
726-
}
727-
_ => {}
728-
};
729-
}
730-
731-
if arg_idx.is_none() && arg.layout.size > Pointer(AddressSpace::DATA).size(cx) * 2 {
732-
// Return values larger than 2 registers using a return area
733-
// pointer. LLVM and Cranelift disagree about how to return
734-
// values that don't fit in the registers designated for return
735-
// values. LLVM will force the entire return value to be passed
736-
// by return area pointer, while Cranelift will look at each IR level
737-
// return value independently and decide to pass it in a
738-
// register or not, which would result in the return value
739-
// being passed partially in registers and partially through a
740-
// return area pointer.
741-
//
742-
// While Cranelift may need to be fixed as the LLVM behavior is
743-
// generally more correct with respect to the surface language,
744-
// forcing this behavior in rustc itself makes it easier for
745-
// other backends to conform to the Rust ABI and for the C ABI
746-
// rustc already handles this behavior anyway.
747-
//
748-
// In addition LLVM's decision to pass the return value in
749-
// registers or using a return area pointer depends on how
750-
// exactly the return type is lowered to an LLVM IR type. For
751-
// example `Option<u128>` can be lowered as `{ i128, i128 }`
752-
// in which case the x86_64 backend would use a return area
753-
// pointer, or it could be passed as `{ i32, i128 }` in which
754-
// case the x86_64 backend would pass it in registers by taking
755-
// advantage of an LLVM ABI extension that allows using 3
756-
// registers for the x86_64 sysv call conv rather than the
757-
// officially specified 2 registers.
758-
//
759-
// FIXME: Technically we should look at the amount of available
760-
// return registers rather than guessing that there are 2
761-
// registers for return values. In practice only a couple of
762-
// architectures have less than 2 return registers. None of
763-
// which supported by Cranelift.
764-
//
765-
// NOTE: This adjustment is only necessary for the Rust ABI as
766-
// for other ABI's the calling convention implementations in
767-
// rustc_target already ensure any return value which doesn't
768-
// fit in the available amount of return registers is passed in
769-
// the right way for the current target.
770-
arg.make_indirect();
771-
return;
772-
}
773-
774-
match arg.layout.abi {
775-
Abi::Aggregate { .. } => {}
776-
777-
// This is a fun case! The gist of what this is doing is
778-
// that we want callers and callees to always agree on the
779-
// ABI of how they pass SIMD arguments. If we were to *not*
780-
// make these arguments indirect then they'd be immediates
781-
// in LLVM, which means that they'd used whatever the
782-
// appropriate ABI is for the callee and the caller. That
783-
// means, for example, if the caller doesn't have AVX
784-
// enabled but the callee does, then passing an AVX argument
785-
// across this boundary would cause corrupt data to show up.
786-
//
787-
// This problem is fixed by unconditionally passing SIMD
788-
// arguments through memory between callers and callees
789-
// which should get them all to agree on ABI regardless of
790-
// target feature sets. Some more information about this
791-
// issue can be found in #44367.
792-
//
793-
// Note that the intrinsic ABI is exempt here as
794-
// that's how we connect up to LLVM and it's unstable
795-
// anyway, we control all calls to it in libstd.
796-
Abi::Vector { .. }
797-
if abi != SpecAbi::RustIntrinsic && tcx.sess.target.simd_types_indirect =>
798-
{
799-
arg.make_indirect();
800-
return;
801-
}
802-
803-
_ => return,
804-
}
805-
// Compute `Aggregate` ABI.
806-
807-
let is_indirect_not_on_stack =
808-
matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
809-
assert!(is_indirect_not_on_stack, "{:?}", arg);
810-
811-
let size = arg.layout.size;
812-
if !arg.layout.is_unsized() && size <= Pointer(AddressSpace::DATA).size(cx) {
813-
// We want to pass small aggregates as immediates, but using
814-
// an LLVM aggregate type for this leads to bad optimizations,
815-
// so we pick an appropriately sized integer type instead.
816-
arg.cast_to(Reg { kind: RegKind::Integer, size });
694+
continue;
817695
}
818696

819697
// If we deduced that this parameter was read-only, add that to the attribute list now.
820698
//
821699
// The `readonly` parameter only applies to pointers, so we can only do this if the
822700
// argument was passed indirectly. (If the argument is passed directly, it's an SSA
823701
// value, so it's implicitly immutable.)
824-
if let (Some(arg_idx), &mut PassMode::Indirect { ref mut attrs, .. }) =
825-
(arg_idx, &mut arg.mode)
826-
{
702+
if let &mut PassMode::Indirect { ref mut attrs, .. } = &mut arg.mode {
827703
// The `deduced_param_attrs` list could be empty if this is a type of function
828704
// we can't deduce any parameters for, so make sure the argument index is in
829705
// bounds.
@@ -834,11 +710,6 @@ fn fn_abi_adjust_for_abi<'tcx>(
834710
}
835711
}
836712
}
837-
};
838-
839-
fixup(&mut fn_abi.ret, None);
840-
for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() {
841-
fixup(arg, Some(arg_idx));
842713
}
843714
} else {
844715
fn_abi

0 commit comments

Comments
 (0)