From d39b4411296585895b1111d14c1459ea3e9bf9be Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 10:45:01 +0100 Subject: [PATCH 01/16] Start using ArgAbi --- src/abi/pass_mode.rs | 80 +++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 27 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 8e3682c86..957beab74 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -2,6 +2,7 @@ use crate::prelude::*; +use rustc_target::abi::call::{ArgAbi, ArgAttributes, PassMode as RustcPassMode}; pub(super) use EmptySinglePair::*; #[derive(Copy, Clone, Debug)] @@ -83,39 +84,64 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer PassMode::NoPass } else { - match &layout.abi { - Abi::Uninhabited => PassMode::NoPass, - Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), - Abi::ScalarPair(a, b) => { - let a = scalar_to_clif_type(tcx, a.clone()); - let b = scalar_to_clif_type(tcx, b.clone()); - if a == types::I128 && b == types::I128 { - // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are - // available on x86_64. Cranelift gets confused when too many return params - // are used. - PassMode::ByRef { - size: Some(layout.size), + let arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); + match arg_abi.mode { + RustcPassMode::Ignore => PassMode::NoPass, + RustcPassMode::Direct(_) => match &arg_abi.layout.abi { + Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + if let Some(vector_ty) = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout) { + PassMode::ByVal(vector_ty) + } else { + PassMode::ByRef { + size: Some(arg_abi.layout.size), + } } - } else { - PassMode::ByValPair(a, b) } - } - - // FIXME implement Vector Abi in a cg_llvm compatible way - Abi::Vector { .. } => { - if let Some(vector_ty) = crate::intrinsics::clif_vector_type(tcx, layout) { - PassMode::ByVal(vector_ty) - } else { - PassMode::ByRef { - size: Some(layout.size), + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + if a == types::I128 && b == types::I128 { + // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are + // available on x86_64. Cranelift gets confused when too many return params + // are used. + PassMode::ByRef { + size: Some(arg_abi.layout.size), + } + } else { + PassMode::ByValPair(a, b) } } + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Cast(_) | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: false, + } => PassMode::ByRef { + size: Some(arg_abi.layout.size), + }, + RustcPassMode::Indirect { + attrs: _, + extra_attrs, + on_stack: true, + } => { + assert!(extra_attrs.is_none()); + PassMode::ByRef { + size: Some(arg_abi.layout.size) + } } - - Abi::Aggregate { sized: true } => PassMode::ByRef { - size: Some(layout.size), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: false, + } => PassMode::ByRef { + size: None, }, - Abi::Aggregate { sized: false } => PassMode::ByRef { size: None }, } } } From 6170fc617ef61698705e3c3d8847ce72d92991ca Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 10:57:59 +0100 Subject: [PATCH 02/16] Split abi adjustments out --- src/abi/pass_mode.rs | 131 ++++++++++++++++++++++++------------------- 1 file changed, 74 insertions(+), 57 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 957beab74..2d2410f2d 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -80,69 +80,86 @@ impl PassMode { } pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> PassMode { + let mut arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); if layout.is_zst() { // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer - PassMode::NoPass - } else { - let arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); - match arg_abi.mode { - RustcPassMode::Ignore => PassMode::NoPass, - RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), - // FIXME implement Vector Abi in a cg_llvm compatible way - Abi::Vector { .. } => { - if let Some(vector_ty) = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout) { - PassMode::ByVal(vector_ty) - } else { - PassMode::ByRef { - size: Some(arg_abi.layout.size), - } - } - } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { - Abi::ScalarPair(a, b) => { - let a = scalar_to_clif_type(tcx, a.clone()); - let b = scalar_to_clif_type(tcx, b.clone()); - if a == types::I128 && b == types::I128 { - // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are - // available on x86_64. Cranelift gets confused when too many return params - // are used. - PassMode::ByRef { - size: Some(arg_abi.layout.size), - } - } else { - PassMode::ByValPair(a, b) - } + arg_abi.mode = RustcPassMode::Ignore; + } + match arg_abi.mode { + RustcPassMode::Ignore => {} + RustcPassMode::Direct(_) => match &arg_abi.layout.abi { + Abi::Scalar(_) => {}, + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() { + arg_abi.mode = RustcPassMode::Indirect { + attrs: ArgAttributes::new(), + extra_attrs: None, + on_stack: false, + }; } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Cast(_) | RustcPassMode::Indirect { - attrs: _, - extra_attrs: None, - on_stack: false, - } => PassMode::ByRef { - size: Some(arg_abi.layout.size), - }, - RustcPassMode::Indirect { - attrs: _, - extra_attrs, - on_stack: true, - } => { - assert!(extra_attrs.is_none()); - PassMode::ByRef { - size: Some(arg_abi.layout.size) + } + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + if a == types::I128 && b == types::I128 { + arg_abi.mode = RustcPassMode::Indirect { + attrs: ArgAttributes::new(), + extra_attrs: None, + on_stack: false, + }; } } - RustcPassMode::Indirect { - attrs: _, - extra_attrs: Some(_), - on_stack: false, - } => PassMode::ByRef { - size: None, - }, + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + _ => {} + } + match arg_abi.mode { + RustcPassMode::Ignore => PassMode::NoPass, + RustcPassMode::Direct(_) => match &arg_abi.layout.abi { + Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).unwrap(); + PassMode::ByVal(vector_ty) + } + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + PassMode::ByValPair(a, b) + } + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Cast(_) | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: false, + } => PassMode::ByRef { + size: Some(arg_abi.layout.size), + }, + RustcPassMode::Indirect { + attrs: _, + extra_attrs, + on_stack: true, + } => { + assert!(extra_attrs.is_none()); + PassMode::ByRef { + size: Some(arg_abi.layout.size) + } } + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: false, + } => PassMode::ByRef { + size: None, + }, } } From ff3304285a41c85486249c5db337e4561ef970c5 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 11:40:26 +0100 Subject: [PATCH 03/16] Rustup to rustc 1.51.0-nightly (1d0d76f8d 2021-01-24) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index ff530ab26..55ac079c0 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2021-01-21 +nightly-2021-01-25 From de713a80cac60164f435a4e6a7ca710f5fdccd45 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 15:37:49 +0100 Subject: [PATCH 04/16] Replace all uses of PassMode with ArgAbi --- src/abi/comments.rs | 5 +- src/abi/mod.rs | 61 +++--------- src/abi/pass_mode.rs | 230 +++++++++++++++++++++++++++---------------- src/abi/returning.rs | 95 ++++++++++++------ 4 files changed, 232 insertions(+), 159 deletions(-) diff --git a/src/abi/comments.rs b/src/abi/comments.rs index 01073d26e..af42e5445 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use rustc_middle::mir; +use rustc_target::abi::call::ArgAbi; use cranelift_codegen::entity::EntityRef; @@ -22,7 +23,7 @@ pub(super) fn add_arg_comment<'tcx>( local: Option, local_field: Option, params: EmptySinglePair, - pass_mode: PassMode, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ty: Ty<'tcx>, ) { let local = if let Some(local) = local { @@ -42,7 +43,7 @@ pub(super) fn add_arg_comment<'tcx>( Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)), }; - let pass_mode = format!("{:?}", pass_mode); + let pass_mode = format!("{:?}", arg_abi.mode); fx.add_global_comment(format!( "{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}", kind = kind, diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 76e198745..ddbef5ead 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -6,9 +6,10 @@ mod pass_mode; mod returning; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_target::abi::call::PassMode as RustcPassMode; use rustc_target::spec::abi::Abi; -use cranelift_codegen::ir::{AbiParam, ArgumentPurpose}; +use cranelift_codegen::ir::AbiParam; use self::pass_mode::*; use crate::prelude::*; @@ -96,7 +97,6 @@ fn clif_sig_from_fn_sig<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, sig: FnSig<'tcx>, - span: Span, is_vtable_fn: bool, requires_caller_location: bool, ) -> Signature { @@ -147,54 +147,26 @@ fn clif_sig_from_fn_sig<'tcx>( .layout_of(ParamEnv::reveal_all().and(tcx.mk_mut_ptr(tcx.mk_unit()))) .unwrap(); } - let pass_mode = get_pass_mode(tcx, layout); + let mut arg_abi = get_arg_abi(tcx, layout); if abi != Abi::Rust && abi != Abi::RustCall && abi != Abi::RustIntrinsic { - match pass_mode { - PassMode::NoPass | PassMode::ByVal(_) => {} - PassMode::ByRef { size: Some(size) } => { - let purpose = ArgumentPurpose::StructArgument(u32::try_from(size.bytes()).expect("struct too big to pass on stack")); - return EmptySinglePair::Single(AbiParam::special(pointer_ty(tcx), purpose)).into_iter(); - } - PassMode::ByValPair(_, _) | PassMode::ByRef { size: None } => { - tcx.sess.span_warn( - span, - &format!( - "Argument of type `{:?}` with pass mode `{:?}` is not yet supported \ - for non-rust abi `{}`. Calling this function may result in a crash.", - layout.ty, - pass_mode, - abi, - ), - ); - } + match arg_abi.mode { + RustcPassMode::Indirect { + ref mut on_stack, .. + } => *on_stack = true, + _ => {} } } - pass_mode.get_param_ty(tcx).map(AbiParam::new).into_iter() + arg_abi.get_abi_param(tcx).into_iter() }) .flatten(); - let (mut params, returns): (Vec<_>, Vec<_>) = match get_pass_mode( + let return_arg_abi = get_arg_abi( tcx, tcx.layout_of(ParamEnv::reveal_all().and(output)).unwrap(), - ) { - PassMode::NoPass => (inputs.collect(), vec![]), - PassMode::ByVal(ret_ty) => (inputs.collect(), vec![AbiParam::new(ret_ty)]), - PassMode::ByValPair(ret_ty_a, ret_ty_b) => ( - inputs.collect(), - vec![AbiParam::new(ret_ty_a), AbiParam::new(ret_ty_b)], - ), - PassMode::ByRef { size: Some(_) } => { - ( - Some(pointer_ty(tcx)) // First param is place to put return val - .into_iter() - .map(|ty| AbiParam::special(ty, ArgumentPurpose::StructReturn)) - .chain(inputs) - .collect(), - vec![], - ) - } - PassMode::ByRef { size: None } => todo!(), - }; + ); + let (return_ptr, returns) = return_arg_abi.get_abi_return(tcx); + // Sometimes the first param is an pointer to the place where the return value needs to be stored. + let mut params: Vec<_> = return_ptr.into_iter().chain(inputs).collect(); if requires_caller_location { params.push(AbiParam::new(pointer_ty(tcx))); @@ -226,7 +198,6 @@ pub(crate) fn get_function_name_and_sig<'tcx>( tcx, triple, fn_sig, - tcx.def_span(inst.def_id()), false, inst.def.requires_caller_location(tcx), ); @@ -584,7 +555,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( nop_inst, format!( "virtual call; self arg pass mode: {:?}", - get_pass_mode(fx.tcx, args[0].layout()) + get_arg_abi(fx.tcx, args[0].layout()).mode, ), ); } @@ -647,7 +618,6 @@ pub(crate) fn codegen_terminator_call<'tcx>( fx.tcx, fx.triple(), fn_sig, - span, is_virtual_call, false, // calls through function pointers never pass the caller location ); @@ -723,7 +693,6 @@ pub(crate) fn codegen_drop<'tcx>( fx.tcx, fx.triple(), fn_sig, - span, true, false, // `drop_in_place` is never `#[track_caller]` ); diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 2d2410f2d..51fc4ecd1 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -2,17 +2,10 @@ use crate::prelude::*; +use cranelift_codegen::ir::ArgumentPurpose; use rustc_target::abi::call::{ArgAbi, ArgAttributes, PassMode as RustcPassMode}; pub(super) use EmptySinglePair::*; -#[derive(Copy, Clone, Debug)] -pub(super) enum PassMode { - NoPass, - ByVal(Type), - ByValPair(Type, Type), - ByRef { size: Option }, -} - #[derive(Copy, Clone, Debug)] pub(super) enum EmptySinglePair { Empty, @@ -67,19 +60,126 @@ impl EmptySinglePair { } } -impl PassMode { - pub(super) fn get_param_ty(self, tcx: TyCtxt<'_>) -> EmptySinglePair { - match self { - PassMode::NoPass => Empty, - PassMode::ByVal(clif_type) => Single(clif_type), - PassMode::ByValPair(a, b) => Pair(a, b), - PassMode::ByRef { size: Some(_) } => Single(pointer_ty(tcx)), - PassMode::ByRef { size: None } => Pair(pointer_ty(tcx), pointer_ty(tcx)), +pub(super) trait ArgAbiExt<'tcx> { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair; + fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec); +} + +impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair { + match self.mode { + RustcPassMode::Ignore => EmptySinglePair::Empty, + RustcPassMode::Direct(_) => match &self.layout.abi { + Abi::Scalar(scalar) => { + EmptySinglePair::Single(AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))) + } + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); + EmptySinglePair::Single(AbiParam::new(vector_ty)) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Pair(_, _) => match &self.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + EmptySinglePair::Pair(AbiParam::new(a), AbiParam::new(b)) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack, + } => { + if on_stack { + let size = u32::try_from(self.layout.size.bytes()).unwrap(); + EmptySinglePair::Single(AbiParam::special( + pointer_ty(tcx), + ArgumentPurpose::StructArgument(size), + )) + } else { + EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))) + } + } + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack, + } => { + assert!(!on_stack); + EmptySinglePair::Pair( + AbiParam::new(pointer_ty(tcx)), + AbiParam::new(pointer_ty(tcx)), + ) + } + } + } + + fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec) { + match self.mode { + RustcPassMode::Ignore => (None, vec![]), + RustcPassMode::Direct(_) => match &self.layout.abi { + Abi::Scalar(scalar) => ( + None, + vec![AbiParam::new(scalar_to_clif_type( + tcx, + scalar.clone(), + ))], + ), + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); + (None, vec![AbiParam::new(vector_ty)]) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Pair(_, _) => match &self.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + ( + None, + vec![AbiParam::new(a), AbiParam::new(b)], + ) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Cast(_) => ( + Some(AbiParam::special( + pointer_ty(tcx), + ArgumentPurpose::StructReturn, + )), + vec![], + ), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack, + } => { + assert!(!on_stack); + ( + Some(AbiParam::special( + pointer_ty(tcx), + ArgumentPurpose::StructReturn, + )), + vec![], + ) + } + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), } } } -pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> PassMode { +pub(super) fn get_arg_abi<'tcx>( + tcx: TyCtxt<'tcx>, + layout: TyAndLayout<'tcx>, +) -> ArgAbi<'tcx, Ty<'tcx>> { let mut arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); if layout.is_zst() { // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer @@ -88,7 +188,7 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) match arg_abi.mode { RustcPassMode::Ignore => {} RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(_) => {}, + Abi::Scalar(_) => {} // FIXME implement Vector Abi in a cg_llvm compatible way Abi::Vector { .. } => { if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() { @@ -99,7 +199,7 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) }; } } - _ => unreachable!("{:?}", arg_abi.layout.abi) + _ => unreachable!("{:?}", arg_abi.layout.abi), }, RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { Abi::ScalarPair(a, b) => { @@ -113,54 +213,11 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) }; } } - _ => unreachable!("{:?}", arg_abi.layout.abi) + _ => unreachable!("{:?}", arg_abi.layout.abi), }, _ => {} } - match arg_abi.mode { - RustcPassMode::Ignore => PassMode::NoPass, - RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), - // FIXME implement Vector Abi in a cg_llvm compatible way - Abi::Vector { .. } => { - let vector_ty = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).unwrap(); - PassMode::ByVal(vector_ty) - } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { - Abi::ScalarPair(a, b) => { - let a = scalar_to_clif_type(tcx, a.clone()); - let b = scalar_to_clif_type(tcx, b.clone()); - PassMode::ByValPair(a, b) - } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Cast(_) | RustcPassMode::Indirect { - attrs: _, - extra_attrs: None, - on_stack: false, - } => PassMode::ByRef { - size: Some(arg_abi.layout.size), - }, - RustcPassMode::Indirect { - attrs: _, - extra_attrs, - on_stack: true, - } => { - assert!(extra_attrs.is_none()); - PassMode::ByRef { - size: Some(arg_abi.layout.size) - } - } - RustcPassMode::Indirect { - attrs: _, - extra_attrs: Some(_), - on_stack: false, - } => PassMode::ByRef { - size: None, - }, - } + arg_abi } /// Get a set of values to be passed as function arguments. @@ -168,14 +225,15 @@ pub(super) fn adjust_arg_for_abi<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, arg: CValue<'tcx>, ) -> EmptySinglePair { - match get_pass_mode(fx.tcx, arg.layout()) { - PassMode::NoPass => Empty, - PassMode::ByVal(_) => Single(arg.load_scalar(fx)), - PassMode::ByValPair(_, _) => { + let arg_abi = get_arg_abi(fx.tcx, arg.layout()); + match arg_abi.mode { + RustcPassMode::Ignore => Empty, + RustcPassMode::Direct(_) => Single(arg.load_scalar(fx)), + RustcPassMode::Pair(_, _) => { let (a, b) = arg.load_scalar_pair(fx); Pair(a, b) } - PassMode::ByRef { size: _ } => match arg.force_stack(fx) { + RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => match arg.force_stack(fx) { (ptr, None) => Single(ptr.get_addr(fx)), (ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta), }, @@ -192,14 +250,11 @@ pub(super) fn cvalue_for_param<'tcx>( arg_ty: Ty<'tcx>, ) -> Option> { let layout = fx.layout_of(arg_ty); - let pass_mode = get_pass_mode(fx.tcx, layout); + let arg_abi = get_arg_abi(fx.tcx, layout); - if let PassMode::NoPass = pass_mode { - return None; - } - - let clif_types = pass_mode.get_param_ty(fx.tcx); - let block_params = clif_types.map(|t| fx.bcx.append_block_param(start_block, t)); + let clif_types = arg_abi.get_abi_param(fx.tcx); + let block_params = + clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)); #[cfg(debug_assertions)] crate::abi::comments::add_arg_comment( @@ -208,22 +263,31 @@ pub(super) fn cvalue_for_param<'tcx>( local, local_field, block_params, - pass_mode, + &arg_abi, arg_ty, ); - match pass_mode { - PassMode::NoPass => unreachable!(), - PassMode::ByVal(_) => Some(CValue::by_val(block_params.assert_single(), layout)), - PassMode::ByValPair(_, _) => { + match arg_abi.mode { + RustcPassMode::Ignore => None, + RustcPassMode::Direct(_) => Some(CValue::by_val(block_params.assert_single(), layout)), + RustcPassMode::Pair(_, _) => { let (a, b) = block_params.assert_pair(); Some(CValue::by_val_pair(a, b, layout)) } - PassMode::ByRef { size: Some(_) } => Some(CValue::by_ref( + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => Some(CValue::by_ref( Pointer::new(block_params.assert_single()), layout, )), - PassMode::ByRef { size: None } => { + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => { let (ptr, meta) = block_params.assert_pair(); Some(CValue::by_ref_unsized(Pointer::new(ptr), meta, layout)) } diff --git a/src/abi/returning.rs b/src/abi/returning.rs index f6d40c880..9edaa1dd8 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -3,6 +3,8 @@ use crate::abi::pass_mode::*; use crate::prelude::*; +use rustc_target::abi::call::PassMode as RustcPassMode; + fn return_layout<'a, 'tcx>(fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> TyAndLayout<'tcx> { fx.layout_of(fx.monomorphize(&fx.mir.local_decls[RETURN_PLACE].ty)) } @@ -12,10 +14,10 @@ pub(crate) fn can_return_to_ssa_var<'tcx>( tcx: TyCtxt<'tcx>, dest_layout: TyAndLayout<'tcx>, ) -> bool { - match get_pass_mode(tcx, dest_layout) { - PassMode::NoPass | PassMode::ByVal(_) | PassMode::ByValPair(_, _) => true, - // FIXME Make it possible to return ByRef to an ssa var. - PassMode::ByRef { size: _ } => false, + match get_arg_abi(tcx, dest_layout).mode { + RustcPassMode::Ignore | RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => true, + // FIXME Make it possible to return Cast and Indirect to an ssa var. + RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => false, } } @@ -27,24 +29,33 @@ pub(super) fn codegen_return_param<'tcx>( start_block: Block, ) -> CPlace<'tcx> { let ret_layout = return_layout(fx); - let ret_pass_mode = get_pass_mode(fx.tcx, ret_layout); - let (ret_place, ret_param) = match ret_pass_mode { - PassMode::NoPass => (CPlace::no_place(ret_layout), Empty), - PassMode::ByVal(_) | PassMode::ByValPair(_, _) => { + let ret_arg_abi = get_arg_abi(fx.tcx, ret_layout); + let (ret_place, ret_param) = match ret_arg_abi.mode { + RustcPassMode::Ignore => (CPlace::no_place(ret_layout), Empty), + RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; ( super::make_local_place(fx, RETURN_PLACE, ret_layout, is_ssa), Empty, ) } - PassMode::ByRef { size: Some(_) } => { + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => { let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type); ( CPlace::for_ptr(Pointer::new(ret_param), ret_layout), Single(ret_param), ) } - PassMode::ByRef { size: None } => todo!(), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), }; #[cfg(not(debug_assertions))] @@ -57,7 +68,7 @@ pub(super) fn codegen_return_param<'tcx>( Some(RETURN_PLACE), None, ret_param, - ret_pass_mode, + &ret_arg_abi, ret_layout.ty, ); @@ -74,36 +85,54 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( ) -> (Inst, T) { let ret_layout = fx.layout_of(fn_sig.output()); - let output_pass_mode = get_pass_mode(fx.tcx, ret_layout); - let return_ptr = match output_pass_mode { - PassMode::NoPass => None, - PassMode::ByRef { size: Some(_) } => match ret_place { + let output_arg_abi = get_arg_abi(fx.tcx, ret_layout); + let return_ptr = match output_arg_abi.mode { + RustcPassMode::Ignore => None, + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => match ret_place { Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)), None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot }, - PassMode::ByRef { size: None } => todo!(), - PassMode::ByVal(_) | PassMode::ByValPair(_, _) => None, + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), + RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => None, }; let (call_inst, meta) = f(fx, return_ptr); - match output_pass_mode { - PassMode::NoPass => {} - PassMode::ByVal(_) => { + match output_arg_abi.mode { + RustcPassMode::Ignore => {} + RustcPassMode::Direct(_) => { if let Some(ret_place) = ret_place { let ret_val = fx.bcx.inst_results(call_inst)[0]; ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout)); } } - PassMode::ByValPair(_, _) => { + RustcPassMode::Pair(_, _) => { if let Some(ret_place) = ret_place { let ret_val_a = fx.bcx.inst_results(call_inst)[0]; let ret_val_b = fx.bcx.inst_results(call_inst)[1]; ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout)); } } - PassMode::ByRef { size: Some(_) } => {} - PassMode::ByRef { size: None } => todo!(), + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => {} + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), } (call_inst, meta) @@ -111,17 +140,27 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( /// Codegen a return instruction with the right return value(s) if any. pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { - match get_pass_mode(fx.tcx, return_layout(fx)) { - PassMode::NoPass | PassMode::ByRef { size: Some(_) } => { + match get_arg_abi(fx.tcx, return_layout(fx)).mode { + RustcPassMode::Ignore + | RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => { fx.bcx.ins().return_(&[]); } - PassMode::ByRef { size: None } => todo!(), - PassMode::ByVal(_) => { + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), + RustcPassMode::Direct(_) => { let place = fx.get_local_place(RETURN_PLACE); let ret_val = place.to_cvalue(fx).load_scalar(fx); fx.bcx.ins().return_(&[ret_val]); } - PassMode::ByValPair(_, _) => { + RustcPassMode::Pair(_, _) => { let place = fx.get_local_place(RETURN_PLACE); let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); From 2b58d8c187936567d92ab45be8a34be087071e05 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 17:12:16 +0100 Subject: [PATCH 05/16] Misc cleanups --- src/abi/comments.rs | 3 +-- src/abi/pass_mode.rs | 32 +++++--------------------------- src/abi/returning.rs | 1 - 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/src/abi/comments.rs b/src/abi/comments.rs index af42e5445..4847b007a 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -24,7 +24,6 @@ pub(super) fn add_arg_comment<'tcx>( local_field: Option, params: EmptySinglePair, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, - ty: Ty<'tcx>, ) { let local = if let Some(local) = local { Cow::Owned(format!("{:?}", local)) @@ -51,7 +50,7 @@ pub(super) fn add_arg_comment<'tcx>( local_field = local_field, params = params, pass_mode = pass_mode, - ty = ty, + ty = arg_abi.layout.ty, )); } diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 51fc4ecd1..aec321bd4 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -123,10 +123,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { RustcPassMode::Direct(_) => match &self.layout.abi { Abi::Scalar(scalar) => ( None, - vec![AbiParam::new(scalar_to_clif_type( - tcx, - scalar.clone(), - ))], + vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))], ), // FIXME implement Vector Abi in a cg_llvm compatible way Abi::Vector { .. } => { @@ -139,10 +136,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); - ( - None, - vec![AbiParam::new(a), AbiParam::new(b)], - ) + (None, vec![AbiParam::new(a), AbiParam::new(b)]) } _ => unreachable!("{:?}", self.layout.abi), }, @@ -192,11 +186,7 @@ pub(super) fn get_arg_abi<'tcx>( // FIXME implement Vector Abi in a cg_llvm compatible way Abi::Vector { .. } => { if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() { - arg_abi.mode = RustcPassMode::Indirect { - attrs: ArgAttributes::new(), - extra_attrs: None, - on_stack: false, - }; + arg_abi.make_indirect(); } } _ => unreachable!("{:?}", arg_abi.layout.abi), @@ -206,11 +196,7 @@ pub(super) fn get_arg_abi<'tcx>( let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); if a == types::I128 && b == types::I128 { - arg_abi.mode = RustcPassMode::Indirect { - attrs: ArgAttributes::new(), - extra_attrs: None, - on_stack: false, - }; + arg_abi.make_indirect(); } } _ => unreachable!("{:?}", arg_abi.layout.abi), @@ -257,15 +243,7 @@ pub(super) fn cvalue_for_param<'tcx>( clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)); #[cfg(debug_assertions)] - crate::abi::comments::add_arg_comment( - fx, - "arg", - local, - local_field, - block_params, - &arg_abi, - arg_ty, - ); + crate::abi::comments::add_arg_comment(fx, "arg", local, local_field, block_params, &arg_abi); match arg_abi.mode { RustcPassMode::Ignore => None, diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 9edaa1dd8..3a5f61315 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -69,7 +69,6 @@ pub(super) fn codegen_return_param<'tcx>( None, ret_param, &ret_arg_abi, - ret_layout.ty, ); ret_place From 4555737152c0f68df5596b16d6e996d19caf2a6a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 26 Jan 2021 15:11:03 +0100 Subject: [PATCH 06/16] Split symbol name and signature calculation --- src/abi/mod.rs | 12 ++++++------ src/base.rs | 3 ++- src/driver/jit.rs | 12 ++++-------- src/driver/mod.rs | 3 ++- src/main_shim.rs | 4 ++-- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index ddbef5ead..bc2111726 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -179,12 +179,12 @@ fn clif_sig_from_fn_sig<'tcx>( } } -pub(crate) fn get_function_name_and_sig<'tcx>( +pub(crate) fn get_function_sig<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, inst: Instance<'tcx>, support_vararg: bool, -) -> (String, Signature) { +) -> Signature { assert!(!inst.substs.needs_infer()); let fn_sig = tcx .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_sig_for_fn_abi(tcx, inst)); @@ -194,14 +194,13 @@ pub(crate) fn get_function_name_and_sig<'tcx>( "Variadic function definitions are not yet supported", ); } - let sig = clif_sig_from_fn_sig( + clif_sig_from_fn_sig( tcx, triple, fn_sig, false, inst.def.requires_caller_location(tcx), - ); - (tcx.symbol_name(inst).name.to_string(), sig) + ) } /// Instance must be monomorphized @@ -210,7 +209,8 @@ pub(crate) fn import_function<'tcx>( module: &mut impl Module, inst: Instance<'tcx>, ) -> FuncId { - let (name, sig) = get_function_name_and_sig(tcx, module.isa().triple(), inst, true); + let name = tcx.symbol_name(inst).name.to_string(); + let sig = get_function_sig(tcx, module.isa().triple(), inst, true); module .declare_function(&name, Linkage::Import, &sig) .unwrap() diff --git a/src/base.rs b/src/base.rs index 757915ba9..1fafc1215 100644 --- a/src/base.rs +++ b/src/base.rs @@ -19,7 +19,8 @@ pub(crate) fn codegen_fn<'tcx>( let mir = tcx.instance_mir(instance.def); // Declare function - let (name, sig) = get_function_name_and_sig(tcx, cx.module.isa().triple(), instance, false); + let name = tcx.symbol_name(instance).name.to_string(); + let sig = get_function_sig(tcx, cx.module.isa().triple(), instance, false); let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap(); cx.cached_context.clear(); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 9a42c675c..6a8792592 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -156,12 +156,8 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 let jit_module = jit_module.as_mut().unwrap(); let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); - let (name, sig) = crate::abi::get_function_name_and_sig( - tcx, - cx.module.isa().triple(), - instance, - true, - ); + let name = tcx.symbol_name(instance).name.to_string(); + let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance, true); let func_id = cx .module .declare_function(&name, Linkage::Export, &sig) @@ -246,8 +242,8 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: In let pointer_type = cx.module.target_config().pointer_type(); - let (name, sig) = - crate::abi::get_function_name_and_sig(tcx, cx.module.isa().triple(), inst, true); + let name = tcx.symbol_name(inst).name.to_string(); + let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst, true); let func_id = cx .module .declare_function(&name, Linkage::Export, &sig) diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 9f4ea9a38..e462f34a0 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -50,7 +50,8 @@ fn predefine_mono_items<'tcx>( for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { - let (name, sig) = get_function_name_and_sig( + let name = cx.tcx.symbol_name(instance).name.to_string(); + let sig= get_function_sig( cx.tcx, cx.module.isa().triple(), instance, diff --git a/src/main_shim.rs b/src/main_shim.rs index 6c472e677..7900abb32 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -69,8 +69,8 @@ pub(crate) fn maybe_create_entry_wrapper( let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); - let (main_name, main_sig) = - get_function_name_and_sig(tcx, m.isa().triple(), instance, false); + let main_name = tcx.symbol_name(instance).name.to_string(); + let main_sig = get_function_sig(tcx, m.isa().triple(), instance, false); let main_func_id = m .declare_function(&main_name, Linkage::Import, &main_sig) .unwrap(); From fc595f1a555d7f43802679511e9fdf1f64f2c49a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 26 Jan 2021 21:41:20 +0100 Subject: [PATCH 07/16] [WIP] Use FnAbi everywhere instead of our own abi calculations --- src/abi/comments.rs | 9 +- src/abi/mod.rs | 227 ++++++++++++++++++----------------------- src/abi/pass_mode.rs | 113 ++++++++------------ src/abi/returning.rs | 138 ++++++++++++++++--------- src/analyze.rs | 6 +- src/base.rs | 3 + src/common.rs | 57 +++++++++-- src/lib.rs | 2 +- src/value_and_place.rs | 117 ++++++++++----------- 9 files changed, 355 insertions(+), 317 deletions(-) diff --git a/src/abi/comments.rs b/src/abi/comments.rs index 4847b007a..41cb4c627 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -4,7 +4,7 @@ use std::borrow::Cow; use rustc_middle::mir; -use rustc_target::abi::call::ArgAbi; +use rustc_target::abi::call::PassMode; use cranelift_codegen::entity::EntityRef; @@ -23,7 +23,8 @@ pub(super) fn add_arg_comment<'tcx>( local: Option, local_field: Option, params: EmptySinglePair, - arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + arg_abi_mode: PassMode, + arg_layout: TyAndLayout<'tcx>, ) { let local = if let Some(local) = local { Cow::Owned(format!("{:?}", local)) @@ -42,7 +43,7 @@ pub(super) fn add_arg_comment<'tcx>( Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)), }; - let pass_mode = format!("{:?}", arg_abi.mode); + let pass_mode = format!("{:?}", arg_abi_mode); fx.add_global_comment(format!( "{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}", kind = kind, @@ -50,7 +51,7 @@ pub(super) fn add_arg_comment<'tcx>( local_field = local_field, params = params, pass_mode = pass_mode, - ty = arg_abi.layout.ty, + ty = arg_layout.ty, )); } diff --git a/src/abi/mod.rs b/src/abi/mod.rs index bc2111726..55ebd39e3 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -6,7 +6,8 @@ mod pass_mode; mod returning; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_target::abi::call::PassMode as RustcPassMode; +use rustc_middle::ty::layout::FnAbiExt; +use rustc_target::abi::call::{Conv, FnAbi}; use rustc_target::spec::abi::Abi; use cranelift_codegen::ir::AbiParam; @@ -16,6 +17,7 @@ use crate::prelude::*; pub(crate) use self::returning::{can_return_to_ssa_var, codegen_return}; +// FIXME remove // Copied from https://github.com/rust-lang/rust/blob/f52c72948aa1dd718cc1f168d21c91c584c0a662/src/librustc_middle/ty/layout.rs#L2301 #[rustfmt::skip] pub(crate) fn fn_sig_for_fn_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> ty::PolyFnSig<'tcx> { @@ -93,84 +95,38 @@ pub(crate) fn fn_sig_for_fn_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx } } -fn clif_sig_from_fn_sig<'tcx>( +fn clif_sig_from_fn_abi<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, - sig: FnSig<'tcx>, - is_vtable_fn: bool, - requires_caller_location: bool, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, ) -> Signature { - let abi = match sig.abi { - Abi::System => Abi::C, - abi => abi, - }; - let (call_conv, inputs, output): (CallConv, Vec>, Ty<'tcx>) = match abi { - Abi::Rust => ( - CallConv::triple_default(triple), - sig.inputs().to_vec(), - sig.output(), - ), - Abi::C | Abi::Unadjusted => ( - CallConv::triple_default(triple), - sig.inputs().to_vec(), - sig.output(), - ), - Abi::SysV64 => (CallConv::SystemV, sig.inputs().to_vec(), sig.output()), - Abi::RustCall => { - assert_eq!(sig.inputs().len(), 2); - let extra_args = match sig.inputs().last().unwrap().kind() { - ty::Tuple(ref tupled_arguments) => tupled_arguments, - _ => bug!("argument to function with \"rust-call\" ABI is not a tuple"), - }; - let mut inputs: Vec> = vec![sig.inputs()[0]]; - inputs.extend(extra_args.types()); - (CallConv::triple_default(triple), inputs, sig.output()) + let call_conv = match fn_abi.conv { + Conv::Rust | Conv::C => CallConv::triple_default(triple), + Conv::X86_64SysV => CallConv::SystemV, + Conv::X86_64Win64 => CallConv::WindowsFastcall, + Conv::ArmAapcs + | Conv::Msp430Intr + | Conv::PtxKernel + | Conv::X86Fastcall + | Conv::X86Intr + | Conv::X86Stdcall + | Conv::X86ThisCall + | Conv::X86VectorCall + | Conv::AmdGpuKernel + | Conv::AvrInterrupt + | Conv::AvrNonBlockingInterrupt => { + todo!("{:?}", fn_abi.conv) } - Abi::System => unreachable!(), - Abi::RustIntrinsic => ( - CallConv::triple_default(triple), - sig.inputs().to_vec(), - sig.output(), - ), - _ => unimplemented!("unsupported abi {:?}", sig.abi), }; - - let inputs = inputs - .into_iter() - .enumerate() - .map(|(i, ty)| { - let mut layout = tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap(); - if i == 0 && is_vtable_fn { - // Virtual calls turn their self param into a thin pointer. - // See https://github.com/rust-lang/rust/blob/37b6a5e5e82497caf5353d9d856e4eb5d14cbe06/src/librustc/ty/layout.rs#L2519-L2572 for more info - layout = tcx - .layout_of(ParamEnv::reveal_all().and(tcx.mk_mut_ptr(tcx.mk_unit()))) - .unwrap(); - } - let mut arg_abi = get_arg_abi(tcx, layout); - if abi != Abi::Rust && abi != Abi::RustCall && abi != Abi::RustIntrinsic { - match arg_abi.mode { - RustcPassMode::Indirect { - ref mut on_stack, .. - } => *on_stack = true, - _ => {} - } - } - arg_abi.get_abi_param(tcx).into_iter() - }) + let inputs = fn_abi + .args + .iter() + .map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()) .flatten(); - let return_arg_abi = get_arg_abi( - tcx, - tcx.layout_of(ParamEnv::reveal_all().and(output)).unwrap(), - ); - let (return_ptr, returns) = return_arg_abi.get_abi_return(tcx); + let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx); // Sometimes the first param is an pointer to the place where the return value needs to be stored. - let mut params: Vec<_> = return_ptr.into_iter().chain(inputs).collect(); - - if requires_caller_location { - params.push(AbiParam::new(pointer_ty(tcx))); - } + let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect(); Signature { params, @@ -194,12 +150,11 @@ pub(crate) fn get_function_sig<'tcx>( "Variadic function definitions are not yet supported", ); } - clif_sig_from_fn_sig( + + clif_sig_from_fn_abi( tcx, triple, - fn_sig, - false, - inst.def.requires_caller_location(tcx), + &FnAbi::of_instance(&RevealAllLayoutCx(tcx), inst, &[]), ) } @@ -337,6 +292,9 @@ pub(crate) fn codegen_fn_prelude<'tcx>( Spread(Vec>>), } + let fn_abi = fx.fn_abi.take().unwrap(); + let mut arg_abis_iter = fn_abi.args.iter(); + let func_params = fx .mir .args_iter() @@ -356,14 +314,16 @@ pub(crate) fn codegen_fn_prelude<'tcx>( }; let mut params = Vec::new(); - for (i, arg_ty) in tupled_arg_tys.types().enumerate() { - let param = cvalue_for_param(fx, start_block, Some(local), Some(i), arg_ty); + for (i, _arg_ty) in tupled_arg_tys.types().enumerate() { + let arg_abi = arg_abis_iter.next().unwrap(); + let param = cvalue_for_param(fx, start_block, Some(local), Some(i), arg_abi); params.push(param); } (local, ArgKind::Spread(params), arg_ty) } else { - let param = cvalue_for_param(fx, start_block, Some(local), None, arg_ty); + let arg_abi = arg_abis_iter.next().unwrap(); + let param = cvalue_for_param(fx, start_block, Some(local), None, arg_abi); (local, ArgKind::Normal(param), arg_ty) } }) @@ -372,11 +332,13 @@ pub(crate) fn codegen_fn_prelude<'tcx>( assert!(fx.caller_location.is_none()); if fx.instance.def.requires_caller_location(fx.tcx) { // Store caller location for `#[track_caller]`. - fx.caller_location = Some( - cvalue_for_param(fx, start_block, None, None, fx.tcx.caller_location_ty()).unwrap(), - ); + let arg_abi = arg_abis_iter.next().unwrap(); + fx.caller_location = Some(cvalue_for_param(fx, start_block, None, None, arg_abi).unwrap()); } + assert!(arg_abis_iter.next().is_none(), "ArgAbi left behind"); + fx.fn_abi = Some(fn_abi); + fx.bcx.switch_to_block(start_block); fx.bcx.ins().nop(); @@ -504,6 +466,21 @@ pub(crate) fn codegen_terminator_call<'tcx>( None }; + let extra_args = &args[fn_sig.inputs().len()..]; + let extra_args = extra_args + .iter() + .map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx))) + .collect::>(); + let fn_abi = if let Some(instance) = instance { + FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args) + } else { + FnAbi::of_fn_ptr( + &RevealAllLayoutCx(fx.tcx), + fn_ty.fn_sig(fx.tcx), + &extra_args, + ) + }; + let is_cold = instance .map(|inst| { fx.tcx @@ -541,8 +518,8 @@ pub(crate) fn codegen_terminator_call<'tcx>( // | indirect call target // | | the first argument to be passed - // v v v virtual calls are special cased below - let (func_ref, first_arg, is_virtual_call) = match instance { + // v v + let (func_ref, first_arg) = match instance { // Trait object call Some(Instance { def: InstanceDef::Virtual(_, idx), @@ -553,23 +530,19 @@ pub(crate) fn codegen_terminator_call<'tcx>( let nop_inst = fx.bcx.ins().nop(); fx.add_comment( nop_inst, - format!( - "virtual call; self arg pass mode: {:?}", - get_arg_abi(fx.tcx, args[0].layout()).mode, - ), + format!("virtual call; self arg pass mode: {:?}", &fn_abi.args[0],), ); } let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0], idx); - (Some(method), Single(ptr), true) + (Some(method), Single(ptr)) } // Normal call Some(_) => ( None, args.get(0) - .map(|arg| adjust_arg_for_abi(fx, *arg)) + .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0])) .unwrap_or(Empty), - false, ), // Indirect call @@ -583,23 +556,27 @@ pub(crate) fn codegen_terminator_call<'tcx>( ( Some(func), args.get(0) - .map(|arg| adjust_arg_for_abi(fx, *arg)) + .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0])) .unwrap_or(Empty), - false, ) } }; let ret_place = destination.map(|(place, _)| place); - let (call_inst, call_args) = - self::returning::codegen_with_call_return_arg(fx, fn_sig, ret_place, |fx, return_ptr| { + let (call_inst, call_args) = self::returning::codegen_with_call_return_arg( + fx, + &fn_abi.ret, + ret_place, + |fx, return_ptr| { + let regular_args_count = args.len(); let mut call_args: Vec = return_ptr .into_iter() .chain(first_arg.into_iter()) .chain( args.into_iter() + .enumerate() .skip(1) - .map(|arg| adjust_arg_for_abi(fx, arg).into_iter()) + .map(|(i, arg)| adjust_arg_for_abi(fx, arg, &fn_abi.args[i]).into_iter()) .flatten(), ) .collect::>(); @@ -610,17 +587,17 @@ pub(crate) fn codegen_terminator_call<'tcx>( { // Pass the caller location for `#[track_caller]`. let caller_location = fx.get_caller_location(span); - call_args.extend(adjust_arg_for_abi(fx, caller_location).into_iter()); + call_args.extend( + adjust_arg_for_abi(fx, caller_location, &fn_abi.args[regular_args_count]) + .into_iter(), + ); + assert_eq!(fn_abi.args.len(), regular_args_count + 1); + } else { + assert_eq!(fn_abi.args.len(), regular_args_count); } let call_inst = if let Some(func_ref) = func_ref { - let sig = clif_sig_from_fn_sig( - fx.tcx, - fx.triple(), - fn_sig, - is_virtual_call, - false, // calls through function pointers never pass the caller location - ); + let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi); let sig = fx.bcx.import_signature(sig); fx.bcx.ins().call_indirect(sig, func_ref, &call_args) } else { @@ -630,7 +607,8 @@ pub(crate) fn codegen_terminator_call<'tcx>( }; (call_inst, call_args) - }); + }, + ); // FIXME find a cleaner way to support varargs if fn_sig.c_variadic { @@ -671,36 +649,33 @@ pub(crate) fn codegen_drop<'tcx>( drop_place: CPlace<'tcx>, ) { let ty = drop_place.layout().ty; - let drop_fn = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx); + let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx); - if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def { + if let ty::InstanceDef::DropGlue(_, None) = drop_instance.def { // we don't actually need to drop anything } else { - let drop_fn_ty = drop_fn.ty(fx.tcx, ParamEnv::reveal_all()); - let fn_sig = fx.tcx.normalize_erasing_late_bound_regions( - ParamEnv::reveal_all(), - drop_fn_ty.fn_sig(fx.tcx), - ); - assert_eq!(fn_sig.output(), fx.tcx.mk_unit()); - match ty.kind() { ty::Dynamic(..) => { let (ptr, vtable) = drop_place.to_ptr_maybe_unsized(); let ptr = ptr.get_addr(fx); let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap()); - let sig = clif_sig_from_fn_sig( - fx.tcx, - fx.triple(), - fn_sig, - true, - false, // `drop_in_place` is never `#[track_caller]` - ); + // FIXME(eddyb) perhaps move some of this logic into + // `Instance::resolve_drop_in_place`? + let virtual_drop = Instance { + def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0), + substs: drop_instance.substs, + }; + let fn_abi = FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), virtual_drop, &[]); + + let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi); let sig = fx.bcx.import_signature(sig); fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]); } _ => { - assert!(!matches!(drop_fn.def, InstanceDef::Virtual(_, _))); + assert!(!matches!(drop_instance.def, InstanceDef::Virtual(_, _))); + + let fn_abi = FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), drop_instance, &[]); let arg_value = drop_place.place_ref( fx, @@ -712,17 +687,19 @@ pub(crate) fn codegen_drop<'tcx>( }, )), ); - let arg_value = adjust_arg_for_abi(fx, arg_value); + let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0]); let mut call_args: Vec = arg_value.into_iter().collect::>(); - if drop_fn.def.requires_caller_location(fx.tcx) { + if drop_instance.def.requires_caller_location(fx.tcx) { // Pass the caller location for `#[track_caller]`. let caller_location = fx.get_caller_location(span); - call_args.extend(adjust_arg_for_abi(fx, caller_location).into_iter()); + call_args.extend( + adjust_arg_for_abi(fx, caller_location, &fn_abi.args[1]).into_iter(), + ); } - let func_ref = fx.get_function_ref(drop_fn); + let func_ref = fx.get_function_ref(drop_instance); fx.bcx.ins().call(func_ref, &call_args); } } diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index aec321bd4..e2b78bfea 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -1,9 +1,10 @@ //! Argument passing use crate::prelude::*; +use crate::value_and_place::assert_assignable; use cranelift_codegen::ir::ArgumentPurpose; -use rustc_target::abi::call::{ArgAbi, ArgAttributes, PassMode as RustcPassMode}; +use rustc_target::abi::call::{ArgAbi, PassMode}; pub(super) use EmptySinglePair::*; #[derive(Copy, Clone, Debug)] @@ -68,8 +69,8 @@ pub(super) trait ArgAbiExt<'tcx> { impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair { match self.mode { - RustcPassMode::Ignore => EmptySinglePair::Empty, - RustcPassMode::Direct(_) => match &self.layout.abi { + PassMode::Ignore => EmptySinglePair::Empty, + PassMode::Direct(_) => match &self.layout.abi { Abi::Scalar(scalar) => { EmptySinglePair::Single(AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))) } @@ -79,7 +80,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - RustcPassMode::Pair(_, _) => match &self.layout.abi { + PassMode::Pair(_, _) => match &self.layout.abi { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); @@ -87,8 +88,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - RustcPassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))), - RustcPassMode::Indirect { + PassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))), + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack, @@ -103,7 +104,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))) } } - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack, @@ -119,8 +120,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec) { match self.mode { - RustcPassMode::Ignore => (None, vec![]), - RustcPassMode::Direct(_) => match &self.layout.abi { + PassMode::Ignore => (None, vec![]), + PassMode::Direct(_) => match &self.layout.abi { Abi::Scalar(scalar) => ( None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))], @@ -132,7 +133,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - RustcPassMode::Pair(_, _) => match &self.layout.abi { + PassMode::Pair(_, _) => match &self.layout.abi { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); @@ -140,14 +141,14 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - RustcPassMode::Cast(_) => ( + PassMode::Cast(_) => ( Some(AbiParam::special( pointer_ty(tcx), ArgumentPurpose::StructReturn, )), vec![], ), - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack, @@ -161,7 +162,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { vec![], ) } - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, @@ -170,56 +171,21 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } } -pub(super) fn get_arg_abi<'tcx>( - tcx: TyCtxt<'tcx>, - layout: TyAndLayout<'tcx>, -) -> ArgAbi<'tcx, Ty<'tcx>> { - let mut arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); - if layout.is_zst() { - // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer - arg_abi.mode = RustcPassMode::Ignore; - } - match arg_abi.mode { - RustcPassMode::Ignore => {} - RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(_) => {} - // FIXME implement Vector Abi in a cg_llvm compatible way - Abi::Vector { .. } => { - if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() { - arg_abi.make_indirect(); - } - } - _ => unreachable!("{:?}", arg_abi.layout.abi), - }, - RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { - Abi::ScalarPair(a, b) => { - let a = scalar_to_clif_type(tcx, a.clone()); - let b = scalar_to_clif_type(tcx, b.clone()); - if a == types::I128 && b == types::I128 { - arg_abi.make_indirect(); - } - } - _ => unreachable!("{:?}", arg_abi.layout.abi), - }, - _ => {} - } - arg_abi -} - /// Get a set of values to be passed as function arguments. pub(super) fn adjust_arg_for_abi<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, arg: CValue<'tcx>, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ) -> EmptySinglePair { - let arg_abi = get_arg_abi(fx.tcx, arg.layout()); + assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty); match arg_abi.mode { - RustcPassMode::Ignore => Empty, - RustcPassMode::Direct(_) => Single(arg.load_scalar(fx)), - RustcPassMode::Pair(_, _) => { + PassMode::Ignore => Empty, + PassMode::Direct(_) => Single(arg.load_scalar(fx)), + PassMode::Pair(_, _) => { let (a, b) = arg.load_scalar_pair(fx); Pair(a, b) } - RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => match arg.force_stack(fx) { + PassMode::Cast(_) | PassMode::Indirect { .. } => match arg.force_stack(fx) { (ptr, None) => Single(ptr.get_addr(fx)), (ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta), }, @@ -233,41 +199,52 @@ pub(super) fn cvalue_for_param<'tcx>( start_block: Block, #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option, #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option, - arg_ty: Ty<'tcx>, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ) -> Option> { - let layout = fx.layout_of(arg_ty); - let arg_abi = get_arg_abi(fx.tcx, layout); - let clif_types = arg_abi.get_abi_param(fx.tcx); let block_params = clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)); #[cfg(debug_assertions)] - crate::abi::comments::add_arg_comment(fx, "arg", local, local_field, block_params, &arg_abi); + crate::abi::comments::add_arg_comment( + fx, + "arg", + local, + local_field, + block_params, + arg_abi.mode, + arg_abi.layout, + ); match arg_abi.mode { - RustcPassMode::Ignore => None, - RustcPassMode::Direct(_) => Some(CValue::by_val(block_params.assert_single(), layout)), - RustcPassMode::Pair(_, _) => { + PassMode::Ignore => None, + PassMode::Direct(_) => { + Some(CValue::by_val(block_params.assert_single(), arg_abi.layout)) + } + PassMode::Pair(_, _) => { let (a, b) = block_params.assert_pair(); - Some(CValue::by_val_pair(a, b, layout)) + Some(CValue::by_val_pair(a, b, arg_abi.layout)) } - RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => Some(CValue::by_ref( Pointer::new(block_params.assert_single()), - layout, + arg_abi.layout, )), - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, } => { let (ptr, meta) = block_params.assert_pair(); - Some(CValue::by_ref_unsized(Pointer::new(ptr), meta, layout)) + Some(CValue::by_ref_unsized( + Pointer::new(ptr), + meta, + arg_abi.layout, + )) } } } diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 3a5f61315..d7a82e0c3 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -3,21 +3,55 @@ use crate::abi::pass_mode::*; use crate::prelude::*; -use rustc_target::abi::call::PassMode as RustcPassMode; - -fn return_layout<'a, 'tcx>(fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> TyAndLayout<'tcx> { - fx.layout_of(fx.monomorphize(&fx.mir.local_decls[RETURN_PLACE].ty)) -} +use rustc_middle::ty::layout::FnAbiExt; +use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; /// Can the given type be returned into an ssa var or does it need to be returned on the stack. pub(crate) fn can_return_to_ssa_var<'tcx>( - tcx: TyCtxt<'tcx>, - dest_layout: TyAndLayout<'tcx>, + fx: &FunctionCx<'_, 'tcx, impl Module>, + func: &mir::Operand<'tcx>, + args: &[mir::Operand<'tcx>], ) -> bool { - match get_arg_abi(tcx, dest_layout).mode { - RustcPassMode::Ignore | RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => true, + let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx)); + let fn_sig = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx)); + + // Handle special calls like instrinsics and empty drop glue. + let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() { + let instance = ty::Instance::resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs) + .unwrap() + .unwrap() + .polymorphize(fx.tcx); + + match instance.def { + InstanceDef::Intrinsic(_) | InstanceDef::DropGlue(_, _) => { + return true; + } + _ => Some(instance), + } + } else { + None + }; + + let extra_args = &args[fn_sig.inputs().len()..]; + let extra_args = extra_args + .iter() + .map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx))) + .collect::>(); + let fn_abi = if let Some(instance) = instance { + FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args) + } else { + FnAbi::of_fn_ptr( + &RevealAllLayoutCx(fx.tcx), + fn_ty.fn_sig(fx.tcx), + &extra_args, + ) + }; + match fn_abi.ret.mode { + PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => true, // FIXME Make it possible to return Cast and Indirect to an ssa var. - RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => false, + PassMode::Cast(_) | PassMode::Indirect { .. } => false, } } @@ -28,30 +62,39 @@ pub(super) fn codegen_return_param<'tcx>( ssa_analyzed: &rustc_index::vec::IndexVec, start_block: Block, ) -> CPlace<'tcx> { - let ret_layout = return_layout(fx); - let ret_arg_abi = get_arg_abi(fx.tcx, ret_layout); - let (ret_place, ret_param) = match ret_arg_abi.mode { - RustcPassMode::Ignore => (CPlace::no_place(ret_layout), Empty), - RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => { + let (ret_place, ret_param) = match fx.fn_abi.as_ref().unwrap().ret.mode { + PassMode::Ignore => ( + CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout), + Empty, + ), + PassMode::Direct(_) | PassMode::Pair(_, _) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; ( - super::make_local_place(fx, RETURN_PLACE, ret_layout, is_ssa), + super::make_local_place( + fx, + RETURN_PLACE, + fx.fn_abi.as_ref().unwrap().ret.layout, + is_ssa, + ), Empty, ) } - RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => { let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type); ( - CPlace::for_ptr(Pointer::new(ret_param), ret_layout), + CPlace::for_ptr( + Pointer::new(ret_param), + fx.fn_abi.as_ref().unwrap().ret.layout, + ), Single(ret_param), ) } - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, @@ -68,7 +111,8 @@ pub(super) fn codegen_return_param<'tcx>( Some(RETURN_PLACE), None, ret_param, - &ret_arg_abi, + fx.fn_abi.as_ref().unwrap().ret.mode, + fx.fn_abi.as_ref().unwrap().ret.layout, ); ret_place @@ -78,17 +122,14 @@ pub(super) fn codegen_return_param<'tcx>( /// returns the call return value(s) if any are written to the correct place. pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( fx: &mut FunctionCx<'_, 'tcx, M>, - fn_sig: FnSig<'tcx>, + ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ret_place: Option>, f: impl FnOnce(&mut FunctionCx<'_, 'tcx, M>, Option) -> (Inst, T), ) -> (Inst, T) { - let ret_layout = fx.layout_of(fn_sig.output()); - - let output_arg_abi = get_arg_abi(fx.tcx, ret_layout); - let return_ptr = match output_arg_abi.mode { - RustcPassMode::Ignore => None, - RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + let return_ptr = match ret_arg_abi.mode { + PassMode::Ignore => None, + PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, @@ -96,38 +137,41 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)), None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot }, - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, } => unreachable!("unsized return value"), - RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => None, + PassMode::Direct(_) | PassMode::Pair(_, _) => None, }; let (call_inst, meta) = f(fx, return_ptr); - match output_arg_abi.mode { - RustcPassMode::Ignore => {} - RustcPassMode::Direct(_) => { + match ret_arg_abi.mode { + PassMode::Ignore => {} + PassMode::Direct(_) => { if let Some(ret_place) = ret_place { let ret_val = fx.bcx.inst_results(call_inst)[0]; - ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout)); + ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_arg_abi.layout)); } } - RustcPassMode::Pair(_, _) => { + PassMode::Pair(_, _) => { if let Some(ret_place) = ret_place { let ret_val_a = fx.bcx.inst_results(call_inst)[0]; let ret_val_b = fx.bcx.inst_results(call_inst)[1]; - ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout)); + ret_place.write_cvalue( + fx, + CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout), + ); } } - RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => {} - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, @@ -139,27 +183,27 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( /// Codegen a return instruction with the right return value(s) if any. pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { - match get_arg_abi(fx.tcx, return_layout(fx)).mode { - RustcPassMode::Ignore - | RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + match fx.fn_abi.as_ref().unwrap().ret.mode { + PassMode::Ignore + | PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => { fx.bcx.ins().return_(&[]); } - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, } => unreachable!("unsized return value"), - RustcPassMode::Direct(_) => { + PassMode::Direct(_) => { let place = fx.get_local_place(RETURN_PLACE); let ret_val = place.to_cvalue(fx).load_scalar(fx); fx.bcx.ins().return_(&[ret_val]); } - RustcPassMode::Pair(_, _) => { + PassMode::Pair(_, _) => { let place = fx.get_local_place(RETURN_PLACE); let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); diff --git a/src/analyze.rs b/src/analyze.rs index adf5c7ac4..dc5e8a7e3 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -40,11 +40,9 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec { + TerminatorKind::Call { destination, func, args, .. } => { if let Some((dest_place, _dest_bb)) = destination { - let dest_layout = fx - .layout_of(fx.monomorphize(&dest_place.ty(&fx.mir.local_decls, fx.tcx).ty)); - if !crate::abi::can_return_to_ssa_var(fx.tcx, dest_layout) { + if !crate::abi::can_return_to_ssa_var(fx, func, args) { not_ssa(&mut flag_map, dest_place.local) } } diff --git a/src/base.rs b/src/base.rs index 1fafc1215..1eff0d4f5 100644 --- a/src/base.rs +++ b/src/base.rs @@ -2,6 +2,8 @@ use rustc_index::vec::IndexVec; use rustc_middle::ty::adjustment::PointerCast; +use rustc_middle::ty::layout::FnAbiExt; +use rustc_target::abi::call::FnAbi; use crate::prelude::*; @@ -51,6 +53,7 @@ pub(crate) fn codegen_fn<'tcx>( instance, mir, + fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])), bcx, block_map, diff --git a/src/common.rs b/src/common.rs index 1485d4451..fbee84e09 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,4 +1,5 @@ use rustc_index::vec::IndexVec; +use rustc_target::abi::call::FnAbi; use rustc_target::abi::{Integer, Primitive}; use rustc_target::spec::{HasTargetSpec, Target}; @@ -294,6 +295,7 @@ pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> { pub(crate) instance: Instance<'tcx>, pub(crate) mir: &'tcx Body<'tcx>, + pub(crate) fn_abi: Option>>, pub(crate) bcx: FunctionBuilder<'clif>, pub(crate) block_map: IndexVec, @@ -319,16 +321,7 @@ impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> { type TyAndLayout = TyAndLayout<'tcx>; fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> { - assert!(!ty.still_further_specializable()); - self.tcx - .layout_of(ParamEnv::reveal_all().and(&ty)) - .unwrap_or_else(|e| { - if let layout::LayoutError::SizeOverflow(_) = e { - self.tcx.sess.fatal(&e.to_string()) - } else { - bug!("failed to get layout for `{}`: {}", ty, e) - } - }) + RevealAllLayoutCx(self.tcx).layout_of(ty) } } @@ -442,3 +435,47 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> { self.bcx.ins().global_value(self.pointer_type, local_msg_id) } } + +pub(crate) struct RevealAllLayoutCx<'tcx>(pub(crate) TyCtxt<'tcx>); + +impl<'tcx> LayoutOf for RevealAllLayoutCx<'tcx> { + type Ty = Ty<'tcx>; + type TyAndLayout = TyAndLayout<'tcx>; + + fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> { + assert!(!ty.still_further_specializable()); + self.0 + .layout_of(ParamEnv::reveal_all().and(&ty)) + .unwrap_or_else(|e| { + if let layout::LayoutError::SizeOverflow(_) = e { + self.0.sess.fatal(&e.to_string()) + } else { + bug!("failed to get layout for `{}`: {}", ty, e) + } + }) + } +} + +impl<'tcx> layout::HasTyCtxt<'tcx> for RevealAllLayoutCx<'tcx> { + fn tcx<'b>(&'b self) -> TyCtxt<'tcx> { + self.0 + } +} + +impl<'tcx> rustc_target::abi::HasDataLayout for RevealAllLayoutCx<'tcx> { + fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout { + &self.0.data_layout + } +} + +impl<'tcx> layout::HasParamEnv<'tcx> for RevealAllLayoutCx<'tcx> { + fn param_env(&self) -> ParamEnv<'tcx> { + ParamEnv::reveal_all() + } +} + +impl<'tcx> HasTargetSpec for RevealAllLayoutCx<'tcx> { + fn target_spec(&self) -> &Target { + &self.0.sess.target + } +} diff --git a/src/lib.rs b/src/lib.rs index 9b5b7d805..ed7ee3b53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -90,7 +90,7 @@ mod prelude { pub(crate) use rustc_middle::mir::{self, *}; pub(crate) use rustc_middle::ty::layout::{self, TyAndLayout}; pub(crate) use rustc_middle::ty::{ - self, FnSig, Instance, InstanceDef, ParamEnv, Ty, TyCtxt, TypeAndMut, TypeFoldable, + self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt, TypeAndMut, TypeFoldable, }; pub(crate) use rustc_target::abi::{Abi, LayoutOf, Scalar, Size, VariantIdx}; diff --git a/src/value_and_place.rs b/src/value_and_place.rs index 5bcb11fd5..17cb09d55 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -450,64 +450,6 @@ impl<'tcx> CPlace<'tcx> { fx: &mut FunctionCx<'_, 'tcx, impl Module>, from: CValue<'tcx>, ) { - fn assert_assignable<'tcx>( - fx: &FunctionCx<'_, 'tcx, impl Module>, - from_ty: Ty<'tcx>, - to_ty: Ty<'tcx>, - ) { - match (from_ty.kind(), to_ty.kind()) { - (ty::Ref(_, a, _), ty::Ref(_, b, _)) - | ( - ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), - ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }), - ) => { - assert_assignable(fx, a, b); - } - (ty::FnPtr(_), ty::FnPtr(_)) => { - let from_sig = fx.tcx.normalize_erasing_late_bound_regions( - ParamEnv::reveal_all(), - from_ty.fn_sig(fx.tcx), - ); - let to_sig = fx.tcx.normalize_erasing_late_bound_regions( - ParamEnv::reveal_all(), - to_ty.fn_sig(fx.tcx), - ); - assert_eq!( - from_sig, to_sig, - "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", - from_sig, to_sig, fx, - ); - // fn(&T) -> for<'l> fn(&'l T) is allowed - } - (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => { - for (from, to) in from_traits.iter().zip(to_traits) { - let from = fx - .tcx - .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from); - let to = fx - .tcx - .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to); - assert_eq!( - from, to, - "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}", - from_traits, to_traits, fx, - ); - } - // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed - } - _ => { - assert_eq!( - from_ty, - to_ty, - "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}", - from_ty, - to_ty, - fx, - ); - } - } - } - assert_assignable(fx, from.layout().ty, self.layout().ty); self.write_cvalue_maybe_transmute(fx, from, "write_cvalue"); @@ -794,3 +736,62 @@ impl<'tcx> CPlace<'tcx> { } } } + +#[track_caller] +pub(crate) fn assert_assignable<'tcx>( + fx: &FunctionCx<'_, 'tcx, impl Module>, + from_ty: Ty<'tcx>, + to_ty: Ty<'tcx>, +) { + match (from_ty.kind(), to_ty.kind()) { + (ty::Ref(_, a, _), ty::Ref(_, b, _)) + | ( + ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), + ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }), + ) => { + assert_assignable(fx, a, b); + } + (ty::Ref(_, a, _), ty::RawPtr(TypeAndMut { ty: b, mutbl: _ })) + | (ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), ty::Ref(_, b, _)) => { + assert_assignable(fx, a, b); + } + (ty::FnPtr(_), ty::FnPtr(_)) => { + let from_sig = fx.tcx.normalize_erasing_late_bound_regions( + ParamEnv::reveal_all(), + from_ty.fn_sig(fx.tcx), + ); + let to_sig = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx)); + assert_eq!( + from_sig, to_sig, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); + // fn(&T) -> for<'l> fn(&'l T) is allowed + } + (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => { + for (from, to) in from_traits.iter().zip(to_traits) { + let from = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from); + let to = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to); + assert_eq!( + from, to, + "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}", + from_traits, to_traits, fx, + ); + } + // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed + } + _ => { + assert_eq!( + from_ty, to_ty, + "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}", + from_ty, to_ty, fx, + ); + } + } +} From d2634478781be741c872f33c7793797c081dc16f Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 27 Jan 2021 10:24:31 +0100 Subject: [PATCH 08/16] Replace EmptySinglePair with SmallVec --- Cargo.lock | 1 + Cargo.toml | 1 + src/abi/comments.rs | 17 ++++-- src/abi/mod.rs | 7 ++- src/abi/pass_mode.rs | 128 ++++++++++++++----------------------------- src/abi/returning.rs | 12 ++-- 6 files changed, 66 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 431e80686..5495cfa5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -333,6 +333,7 @@ dependencies = [ "indexmap", "libloading", "object", + "smallvec", "target-lexicon", ] diff --git a/Cargo.toml b/Cargo.toml index 4558da2de..3820fce6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ object = { version = "0.22.0", default-features = false, features = ["std", "rea ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" } indexmap = "1.0.2" libloading = { version = "0.6.0", optional = true } +smallvec = "1.6.1" # Uncomment to use local checkout of cranelift #[patch."https://github.com/bytecodealliance/wasmtime/"] diff --git a/src/abi/comments.rs b/src/abi/comments.rs index 41cb4c627..9aab45b62 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -8,7 +8,6 @@ use rustc_target::abi::call::PassMode; use cranelift_codegen::entity::EntityRef; -use crate::abi::pass_mode::*; use crate::prelude::*; pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) { @@ -22,7 +21,7 @@ pub(super) fn add_arg_comment<'tcx>( kind: &str, local: Option, local_field: Option, - params: EmptySinglePair, + params: &[Value], arg_abi_mode: PassMode, arg_layout: TyAndLayout<'tcx>, ) { @@ -38,9 +37,17 @@ pub(super) fn add_arg_comment<'tcx>( }; let params = match params { - Empty => Cow::Borrowed("-"), - Single(param) => Cow::Owned(format!("= {:?}", param)), - Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)), + [] => Cow::Borrowed("-"), + [param] => Cow::Owned(format!("= {:?}", param)), + [param_a, param_b] => Cow::Owned(format!("= {:?},{:?}", param_a, param_b)), + params => Cow::Owned(format!( + "= {}", + params + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + )), }; let pass_mode = format!("{:?}", arg_abi_mode); diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 55ebd39e3..bc35ca2de 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -11,6 +11,7 @@ use rustc_target::abi::call::{Conv, FnAbi}; use rustc_target::spec::abi::Abi; use cranelift_codegen::ir::AbiParam; +use smallvec::smallvec; use self::pass_mode::*; use crate::prelude::*; @@ -534,7 +535,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( ); } let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0], idx); - (Some(method), Single(ptr)) + (Some(method), smallvec![ptr]) } // Normal call @@ -542,7 +543,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( None, args.get(0) .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0])) - .unwrap_or(Empty), + .unwrap_or(smallvec![]), ), // Indirect call @@ -557,7 +558,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( Some(func), args.get(0) .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0])) - .unwrap_or(Empty), + .unwrap_or(smallvec![]), ) } }; diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index e2b78bfea..e047ddceb 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -5,78 +5,24 @@ use crate::value_and_place::assert_assignable; use cranelift_codegen::ir::ArgumentPurpose; use rustc_target::abi::call::{ArgAbi, PassMode}; -pub(super) use EmptySinglePair::*; - -#[derive(Copy, Clone, Debug)] -pub(super) enum EmptySinglePair { - Empty, - Single(T), - Pair(T, T), -} - -impl EmptySinglePair { - pub(super) fn into_iter(self) -> EmptySinglePairIter { - EmptySinglePairIter(self) - } - - pub(super) fn map(self, mut f: impl FnMut(T) -> U) -> EmptySinglePair { - match self { - Empty => Empty, - Single(v) => Single(f(v)), - Pair(a, b) => Pair(f(a), f(b)), - } - } -} - -pub(super) struct EmptySinglePairIter(EmptySinglePair); - -impl Iterator for EmptySinglePairIter { - type Item = T; - - fn next(&mut self) -> Option { - match std::mem::replace(&mut self.0, Empty) { - Empty => None, - Single(v) => Some(v), - Pair(a, b) => { - self.0 = Single(b); - Some(a) - } - } - } -} - -impl EmptySinglePair { - pub(super) fn assert_single(self) -> T { - match self { - Single(v) => v, - _ => panic!("Called assert_single on {:?}", self), - } - } - - pub(super) fn assert_pair(self) -> (T, T) { - match self { - Pair(a, b) => (a, b), - _ => panic!("Called assert_pair on {:?}", self), - } - } -} +use smallvec::{smallvec, SmallVec}; pub(super) trait ArgAbiExt<'tcx> { - fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair; + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]>; fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec); } impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { - fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> { match self.mode { - PassMode::Ignore => EmptySinglePair::Empty, + PassMode::Ignore => smallvec![], PassMode::Direct(_) => match &self.layout.abi { Abi::Scalar(scalar) => { - EmptySinglePair::Single(AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))) + smallvec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))] } Abi::Vector { .. } => { let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); - EmptySinglePair::Single(AbiParam::new(vector_ty)) + smallvec![AbiParam::new(vector_ty)] } _ => unreachable!("{:?}", self.layout.abi), }, @@ -84,11 +30,11 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); - EmptySinglePair::Pair(AbiParam::new(a), AbiParam::new(b)) + smallvec![AbiParam::new(a), AbiParam::new(b)] } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))), + PassMode::Cast(_) => smallvec![AbiParam::new(pointer_ty(tcx))], PassMode::Indirect { attrs: _, extra_attrs: None, @@ -96,12 +42,12 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } => { if on_stack { let size = u32::try_from(self.layout.size.bytes()).unwrap(); - EmptySinglePair::Single(AbiParam::special( + smallvec![AbiParam::special( pointer_ty(tcx), ArgumentPurpose::StructArgument(size), - )) + )] } else { - EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))) + smallvec![AbiParam::new(pointer_ty(tcx))] } } PassMode::Indirect { @@ -110,10 +56,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { on_stack, } => { assert!(!on_stack); - EmptySinglePair::Pair( + smallvec![ AbiParam::new(pointer_ty(tcx)), AbiParam::new(pointer_ty(tcx)), - ) + ] } } } @@ -176,18 +122,18 @@ pub(super) fn adjust_arg_for_abi<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, arg: CValue<'tcx>, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, -) -> EmptySinglePair { +) -> SmallVec<[Value; 2]> { assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty); match arg_abi.mode { - PassMode::Ignore => Empty, - PassMode::Direct(_) => Single(arg.load_scalar(fx)), + PassMode::Ignore => smallvec![], + PassMode::Direct(_) => smallvec![arg.load_scalar(fx)], PassMode::Pair(_, _) => { let (a, b) = arg.load_scalar_pair(fx); - Pair(a, b) + smallvec![a, b] } PassMode::Cast(_) | PassMode::Indirect { .. } => match arg.force_stack(fx) { - (ptr, None) => Single(ptr.get_addr(fx)), - (ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta), + (ptr, None) => smallvec![ptr.get_addr(fx)], + (ptr, Some(meta)) => smallvec![ptr.get_addr(fx), meta], }, } } @@ -202,8 +148,10 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ) -> Option> { let clif_types = arg_abi.get_abi_param(fx.tcx); - let block_params = - clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)); + let block_params = clif_types + .into_iter() + .map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)) + .collect::>(); #[cfg(debug_assertions)] crate::abi::comments::add_arg_comment( @@ -211,7 +159,7 @@ pub(super) fn cvalue_for_param<'tcx>( "arg", local, local_field, - block_params, + &block_params, arg_abi.mode, arg_abi.layout, ); @@ -219,30 +167,38 @@ pub(super) fn cvalue_for_param<'tcx>( match arg_abi.mode { PassMode::Ignore => None, PassMode::Direct(_) => { - Some(CValue::by_val(block_params.assert_single(), arg_abi.layout)) + assert_eq!(block_params.len(), 1, "{:?}", block_params); + Some(CValue::by_val(block_params[0], arg_abi.layout)) } PassMode::Pair(_, _) => { - let (a, b) = block_params.assert_pair(); - Some(CValue::by_val_pair(a, b, arg_abi.layout)) + assert_eq!(block_params.len(), 2, "{:?}", block_params); + Some(CValue::by_val_pair( + block_params[0], + block_params[1], + arg_abi.layout, + )) } PassMode::Cast(_) | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, - } => Some(CValue::by_ref( - Pointer::new(block_params.assert_single()), - arg_abi.layout, - )), + } => { + assert_eq!(block_params.len(), 1, "{:?}", block_params); + Some(CValue::by_ref( + Pointer::new(block_params[0]), + arg_abi.layout, + )) + } PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, } => { - let (ptr, meta) = block_params.assert_pair(); + assert_eq!(block_params.len(), 2, "{:?}", block_params); Some(CValue::by_ref_unsized( - Pointer::new(ptr), - meta, + Pointer::new(block_params[0]), + block_params[1], arg_abi.layout, )) } diff --git a/src/abi/returning.rs b/src/abi/returning.rs index d7a82e0c3..8376f8457 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -1,10 +1,10 @@ //! Return value handling -use crate::abi::pass_mode::*; use crate::prelude::*; use rustc_middle::ty::layout::FnAbiExt; use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; +use smallvec::{SmallVec, smallvec}; /// Can the given type be returned into an ssa var or does it need to be returned on the stack. pub(crate) fn can_return_to_ssa_var<'tcx>( @@ -62,10 +62,10 @@ pub(super) fn codegen_return_param<'tcx>( ssa_analyzed: &rustc_index::vec::IndexVec, start_block: Block, ) -> CPlace<'tcx> { - let (ret_place, ret_param) = match fx.fn_abi.as_ref().unwrap().ret.mode { + let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode { PassMode::Ignore => ( CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout), - Empty, + smallvec![], ), PassMode::Direct(_) | PassMode::Pair(_, _) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; @@ -76,7 +76,7 @@ pub(super) fn codegen_return_param<'tcx>( fx.fn_abi.as_ref().unwrap().ret.layout, is_ssa, ), - Empty, + smallvec![], ) } PassMode::Cast(_) @@ -91,7 +91,7 @@ pub(super) fn codegen_return_param<'tcx>( Pointer::new(ret_param), fx.fn_abi.as_ref().unwrap().ret.layout, ), - Single(ret_param), + smallvec![ret_param], ) } PassMode::Indirect { @@ -110,7 +110,7 @@ pub(super) fn codegen_return_param<'tcx>( "ret", Some(RETURN_PLACE), None, - ret_param, + &ret_param, fx.fn_abi.as_ref().unwrap().ret.mode, fx.fn_abi.as_ref().unwrap().ret.layout, ); From aa23f862dc020a568278dc4ad92c455a0a3ced46 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 27 Jan 2021 10:32:56 +0100 Subject: [PATCH 09/16] Remove vararg support check This check wasn't very useful and removing it simplifies the code. --- src/abi/mod.rs | 12 +----------- src/base.rs | 2 +- src/driver/jit.rs | 4 ++-- src/driver/mod.rs | 7 +------ src/main_shim.rs | 2 +- 5 files changed, 6 insertions(+), 21 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index bc35ca2de..a27d5b8ab 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -140,18 +140,8 @@ pub(crate) fn get_function_sig<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, inst: Instance<'tcx>, - support_vararg: bool, ) -> Signature { assert!(!inst.substs.needs_infer()); - let fn_sig = tcx - .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_sig_for_fn_abi(tcx, inst)); - if fn_sig.c_variadic && !support_vararg { - tcx.sess.span_fatal( - tcx.def_span(inst.def_id()), - "Variadic function definitions are not yet supported", - ); - } - clif_sig_from_fn_abi( tcx, triple, @@ -166,7 +156,7 @@ pub(crate) fn import_function<'tcx>( inst: Instance<'tcx>, ) -> FuncId { let name = tcx.symbol_name(inst).name.to_string(); - let sig = get_function_sig(tcx, module.isa().triple(), inst, true); + let sig = get_function_sig(tcx, module.isa().triple(), inst); module .declare_function(&name, Linkage::Import, &sig) .unwrap() diff --git a/src/base.rs b/src/base.rs index 1eff0d4f5..e81aa5278 100644 --- a/src/base.rs +++ b/src/base.rs @@ -22,7 +22,7 @@ pub(crate) fn codegen_fn<'tcx>( // Declare function let name = tcx.symbol_name(instance).name.to_string(); - let sig = get_function_sig(tcx, cx.module.isa().triple(), instance, false); + let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap(); cx.cached_context.clear(); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 6a8792592..2d14ff2c0 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -157,7 +157,7 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); let name = tcx.symbol_name(instance).name.to_string(); - let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance, true); + let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance); let func_id = cx .module .declare_function(&name, Linkage::Export, &sig) @@ -243,7 +243,7 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: In let pointer_type = cx.module.target_config().pointer_type(); let name = tcx.symbol_name(inst).name.to_string(); - let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst, true); + let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); let func_id = cx .module .declare_function(&name, Linkage::Export, &sig) diff --git a/src/driver/mod.rs b/src/driver/mod.rs index e462f34a0..752c3f747 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -51,12 +51,7 @@ fn predefine_mono_items<'tcx>( match mono_item { MonoItem::Fn(instance) => { let name = cx.tcx.symbol_name(instance).name.to_string(); - let sig= get_function_sig( - cx.tcx, - cx.module.isa().triple(), - instance, - false, - ); + let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); cx.module.declare_function(&name, linkage, &sig).unwrap(); } diff --git a/src/main_shim.rs b/src/main_shim.rs index 7900abb32..b193cea87 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -70,7 +70,7 @@ pub(crate) fn maybe_create_entry_wrapper( let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); let main_name = tcx.symbol_name(instance).name.to_string(); - let main_sig = get_function_sig(tcx, m.isa().triple(), instance, false); + let main_sig = get_function_sig(tcx, m.isa().triple(), instance); let main_func_id = m .declare_function(&main_name, Linkage::Import, &main_sig) .unwrap(); From e564a0ad319c8fabddc3e62616be0dbfd761ecec Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 27 Jan 2021 10:33:06 +0100 Subject: [PATCH 10/16] Rustfmt --- src/abi/returning.rs | 2 +- src/analyze.rs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 8376f8457..3acfae3e1 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -4,7 +4,7 @@ use crate::prelude::*; use rustc_middle::ty::layout::FnAbiExt; use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; -use smallvec::{SmallVec, smallvec}; +use smallvec::{smallvec, SmallVec}; /// Can the given type be returned into an ssa var or does it need to be returned on the stack. pub(crate) fn can_return_to_ssa_var<'tcx>( diff --git a/src/analyze.rs b/src/analyze.rs index dc5e8a7e3..62fbcfe3f 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -40,7 +40,12 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec { + TerminatorKind::Call { + destination, + func, + args, + .. + } => { if let Some((dest_place, _dest_bb)) = destination { if !crate::abi::can_return_to_ssa_var(fx, func, args) { not_ssa(&mut flag_map, dest_place.local) From 268d7bc459d436d28171e37050edec287f950bfe Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 27 Jan 2021 10:36:40 +0100 Subject: [PATCH 11/16] Remove fn_sig_for_fn_abi --- src/abi/mod.rs | 78 ---------------------------------------------- src/pretty_clif.rs | 9 +++--- 2 files changed, 4 insertions(+), 83 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index a27d5b8ab..92d6b3897 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -18,84 +18,6 @@ use crate::prelude::*; pub(crate) use self::returning::{can_return_to_ssa_var, codegen_return}; -// FIXME remove -// Copied from https://github.com/rust-lang/rust/blob/f52c72948aa1dd718cc1f168d21c91c584c0a662/src/librustc_middle/ty/layout.rs#L2301 -#[rustfmt::skip] -pub(crate) fn fn_sig_for_fn_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> ty::PolyFnSig<'tcx> { - use rustc_middle::ty::subst::Subst; - - // FIXME(davidtwco,eddyb): A `ParamEnv` should be passed through to this function. - let ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); - match *ty.kind() { - ty::FnDef(..) => { - // HACK(davidtwco,eddyb): This is a workaround for polymorphization considering - // parameters unused if they show up in the signature, but not in the `mir::Body` - // (i.e. due to being inside a projection that got normalized, see - // `src/test/ui/polymorphization/normalized_sig_types.rs`), and codegen not keeping - // track of a polymorphization `ParamEnv` to allow normalizing later. - let mut sig = match *ty.kind() { - ty::FnDef(def_id, substs) => tcx - .normalize_erasing_regions(tcx.param_env(def_id), tcx.fn_sig(def_id)) - .subst(tcx, substs), - _ => unreachable!(), - }; - - if let ty::InstanceDef::VtableShim(..) = instance.def { - // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`. - sig = sig.map_bound(|mut sig| { - let mut inputs_and_output = sig.inputs_and_output.to_vec(); - inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]); - sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output); - sig - }); - } - sig - } - ty::Closure(def_id, substs) => { - let sig = substs.as_closure().sig(); - - let env_ty = tcx.closure_env_ty(def_id, substs).unwrap(); - sig.map_bound(|sig| { - tcx.mk_fn_sig( - std::iter::once(env_ty.skip_binder()).chain(sig.inputs().iter().cloned()), - sig.output(), - sig.c_variadic, - sig.unsafety, - sig.abi, - ) - }) - } - ty::Generator(_, substs, _) => { - let sig = substs.as_generator().poly_sig(); - - let env_region = ty::ReLateBound(ty::INNERMOST, ty::BoundRegion { kind: ty::BrEnv }); - let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty); - - let pin_did = tcx.require_lang_item(rustc_hir::LangItem::Pin, None); - let pin_adt_ref = tcx.adt_def(pin_did); - let pin_substs = tcx.intern_substs(&[env_ty.into()]); - let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs); - - sig.map_bound(|sig| { - let state_did = tcx.require_lang_item(rustc_hir::LangItem::GeneratorState, None); - let state_adt_ref = tcx.adt_def(state_did); - let state_substs = - tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]); - let ret_ty = tcx.mk_adt(state_adt_ref, state_substs); - - tcx.mk_fn_sig( - [env_ty, sig.resume_ty].iter(), - &ret_ty, - false, - rustc_hir::Unsafety::Normal, - rustc_target::spec::abi::Abi::Rust, - ) - }) - } - _ => bug!("unexpected type {:?} in Instance::fn_sig", ty), - } -} - fn clif_sig_from_fn_abi<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index 22c94fec8..f4a15ab12 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -61,7 +61,9 @@ use cranelift_codegen::{ write::{FuncWriter, PlainWriter}, }; +use rustc_middle::ty::layout::FnAbiExt; use rustc_session::config::OutputType; +use rustc_target::abi::call::FnAbi; use crate::prelude::*; @@ -78,11 +80,8 @@ impl CommentWriter { format!("symbol {}", tcx.symbol_name(instance).name), format!("instance {:?}", instance), format!( - "sig {:?}", - tcx.normalize_erasing_late_bound_regions( - ParamEnv::reveal_all(), - crate::abi::fn_sig_for_fn_abi(tcx, instance) - ) + "abi {:?}", + FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[]) ), String::new(), ] From e8f48e4bae83295816f035474a726a5d92056453 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 29 Jan 2021 22:15:15 +0100 Subject: [PATCH 12/16] [WIP] Implement PassMode::Cast --- src/abi/mod.rs | 25 +++++-- src/abi/pass_mode.rs | 162 ++++++++++++++++++++++++++++++++++++++----- src/abi/returning.rs | 38 +++++++--- 3 files changed, 191 insertions(+), 34 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 92d6b3897..c227bdd53 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -191,12 +191,24 @@ pub(crate) fn codegen_fn_prelude<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, start_block: Block, ) { + fx.bcx.append_block_params_for_function_params(start_block); + + fx.bcx.switch_to_block(start_block); + fx.bcx.ins().nop(); + let ssa_analyzed = crate::analyze::analyze(fx); #[cfg(debug_assertions)] self::comments::add_args_header_comment(fx); - let ret_place = self::returning::codegen_return_param(fx, &ssa_analyzed, start_block); + let mut block_params_iter = fx + .bcx + .func + .dfg + .block_params(start_block) + .to_vec() + .into_iter(); + let ret_place = self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); // None means pass_mode == NoPass @@ -229,14 +241,14 @@ pub(crate) fn codegen_fn_prelude<'tcx>( let mut params = Vec::new(); for (i, _arg_ty) in tupled_arg_tys.types().enumerate() { let arg_abi = arg_abis_iter.next().unwrap(); - let param = cvalue_for_param(fx, start_block, Some(local), Some(i), arg_abi); + let param = cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); params.push(param); } (local, ArgKind::Spread(params), arg_ty) } else { let arg_abi = arg_abis_iter.next().unwrap(); - let param = cvalue_for_param(fx, start_block, Some(local), None, arg_abi); + let param = cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); (local, ArgKind::Normal(param), arg_ty) } }) @@ -246,14 +258,13 @@ pub(crate) fn codegen_fn_prelude<'tcx>( if fx.instance.def.requires_caller_location(fx.tcx) { // Store caller location for `#[track_caller]`. let arg_abi = arg_abis_iter.next().unwrap(); - fx.caller_location = Some(cvalue_for_param(fx, start_block, None, None, arg_abi).unwrap()); + fx.caller_location = + Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap()); } assert!(arg_abis_iter.next().is_none(), "ArgAbi left behind"); fx.fn_abi = Some(fn_abi); - - fx.bcx.switch_to_block(start_block); - fx.bcx.ins().nop(); + assert!(block_params_iter.next().is_none(), "arg_value left behind"); #[cfg(debug_assertions)] self::comments::add_locals_header_comment(fx); diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index e047ddceb..6f27fa52d 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -4,7 +4,7 @@ use crate::prelude::*; use crate::value_and_place::assert_assignable; use cranelift_codegen::ir::ArgumentPurpose; -use rustc_target::abi::call::{ArgAbi, PassMode}; +use rustc_target::abi::call::{ArgAbi, CastTarget, PassMode, Reg, RegKind}; use smallvec::{smallvec, SmallVec}; pub(super) trait ArgAbiExt<'tcx> { @@ -12,6 +12,78 @@ pub(super) trait ArgAbiExt<'tcx> { fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec); } +fn reg_to_abi_param(reg: Reg) -> AbiParam { + let clif_ty = match (reg.kind, reg.size.bytes()) { + (RegKind::Integer, 1) => types::I8, + (RegKind::Integer, 2) => types::I16, + (RegKind::Integer, 4) => types::I32, + (RegKind::Integer, 8) => types::I64, + (RegKind::Integer, 16) => types::I128, + (RegKind::Float, 4) => types::F32, + (RegKind::Float, 8) => types::F64, + (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(), + _ => unreachable!("{:?}", reg), + }; + AbiParam::new(clif_ty) +} + +fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> { + let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 { + (0, 0) + } else { + ( + cast.rest.total.bytes() / cast.rest.unit.size.bytes(), + cast.rest.total.bytes() % cast.rest.unit.size.bytes(), + ) + }; + + if cast.prefix.iter().all(|x| x.is_none()) { + // Simplify to a single unit when there is no prefix and size <= unit size + if cast.rest.total <= cast.rest.unit.size { + let clif_ty = match (cast.rest.unit.kind, cast.rest.unit.size.bytes()) { + (RegKind::Integer, 1) => types::I8, + (RegKind::Integer, 2) => types::I16, + (RegKind::Integer, 3..=4) => types::I32, + (RegKind::Integer, 5..=8) => types::I64, + (RegKind::Integer, 9..=16) => types::I128, + (RegKind::Float, 4) => types::F32, + (RegKind::Float, 8) => types::F64, + (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(), + _ => unreachable!("{:?}", cast.rest.unit), + }; + return smallvec![AbiParam::new(clif_ty)]; + } + } + + // Create list of fields in the main structure + let mut args = cast + .prefix + .iter() + .flatten() + .map(|&kind| { + reg_to_abi_param(Reg { + kind, + size: cast.prefix_chunk_size, + }) + }) + .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit))) + .collect::>(); + + // Append final integer + if rem_bytes != 0 { + // Only integers can be really split further. + assert_eq!(cast.rest.unit.kind, RegKind::Integer); + args.push(reg_to_abi_param(Reg { + kind: RegKind::Integer, + size: Size::from_bytes(rem_bytes), + })); + } + + args +} + +// FIXME respect argument extension mode + impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> { match self.mode { @@ -34,7 +106,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(_) => smallvec![AbiParam::new(pointer_ty(tcx))], + PassMode::Cast(cast) => cast_target_to_abi_params(cast), PassMode::Indirect { attrs: _, extra_attrs: None, @@ -87,13 +159,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(_) => ( - Some(AbiParam::special( - pointer_ty(tcx), - ArgumentPurpose::StructReturn, - )), - vec![], - ), + PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()), PassMode::Indirect { attrs: _, extra_attrs: None, @@ -117,6 +183,60 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } } +pub(super) fn to_casted_value<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + arg: CValue<'tcx>, + cast: CastTarget, +) -> SmallVec<[Value; 2]> { + let (ptr, meta) = arg.force_stack(fx); + assert!(meta.is_none()); + let mut offset = 0; + cast_target_to_abi_params(cast) + .into_iter() + .map(|param| { + let val = ptr + .offset_i64(fx, offset) + .load(fx, param.value_type, MemFlags::new()); + offset += i64::from(param.value_type.bytes()); + val + }) + .collect() +} + +pub(super) fn from_casted_value<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + block_params: &[Value], + layout: TyAndLayout<'tcx>, + cast: CastTarget, +) -> CValue<'tcx> { + let abi_params = cast_target_to_abi_params(cast); + let size = abi_params + .iter() + .map(|param| param.value_type.bytes()) + .sum(); + // Stack slot size may be bigger for for example `[u8; 3]` which is packed into an `i32`. + assert!(u64::from(size) >= layout.size.bytes()); + let stack_slot = fx.bcx.create_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + size, + offset: None, + }); + let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0)); + let mut offset = 0; + let mut block_params_iter = block_params.into_iter().copied(); + for param in abi_params { + let val = ptr.offset_i64(fx, offset).store( + fx, + block_params_iter.next().unwrap(), + MemFlags::new(), + ); + offset += i64::from(param.value_type.bytes()); + val + } + assert_eq!(block_params_iter.next(), None, "Leftover block param"); + CValue::by_ref(ptr, layout) +} + /// Get a set of values to be passed as function arguments. pub(super) fn adjust_arg_for_abi<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, @@ -131,7 +251,8 @@ pub(super) fn adjust_arg_for_abi<'tcx>( let (a, b) = arg.load_scalar_pair(fx); smallvec![a, b] } - PassMode::Cast(_) | PassMode::Indirect { .. } => match arg.force_stack(fx) { + PassMode::Cast(cast) => to_casted_value(fx, arg, cast), + PassMode::Indirect { .. } => match arg.force_stack(fx) { (ptr, None) => smallvec![ptr.get_addr(fx)], (ptr, Some(meta)) => smallvec![ptr.get_addr(fx), meta], }, @@ -142,15 +263,22 @@ pub(super) fn adjust_arg_for_abi<'tcx>( /// as necessary. pub(super) fn cvalue_for_param<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, - start_block: Block, #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option, #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + block_params_iter: &mut impl Iterator, ) -> Option> { - let clif_types = arg_abi.get_abi_param(fx.tcx); - let block_params = clif_types + let block_params = arg_abi + .get_abi_param(fx.tcx) .into_iter() - .map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)) + .map(|abi_param| { + let block_param = block_params_iter.next().unwrap(); + assert_eq!( + fx.bcx.func.dfg.value_type(block_param), + abi_param.value_type + ); + block_param + }) .collect::>(); #[cfg(debug_assertions)] @@ -178,8 +306,10 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, )) } - PassMode::Cast(_) - | PassMode::Indirect { + PassMode::Cast(cast) => { + Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) + } + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 3acfae3e1..a382963bf 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -60,14 +60,14 @@ pub(crate) fn can_return_to_ssa_var<'tcx>( pub(super) fn codegen_return_param<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, ssa_analyzed: &rustc_index::vec::IndexVec, - start_block: Block, + block_params_iter: &mut impl Iterator, ) -> CPlace<'tcx> { let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode { PassMode::Ignore => ( CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout), smallvec![], ), - PassMode::Direct(_) | PassMode::Pair(_, _) => { + PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; ( super::make_local_place( @@ -79,13 +79,13 @@ pub(super) fn codegen_return_param<'tcx>( smallvec![], ) } - PassMode::Cast(_) - | PassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => { - let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type); + let ret_param = block_params_iter.next().unwrap(); + assert_eq!(fx.bcx.func.dfg.value_type(ret_param), pointer_ty(fx.tcx)); ( CPlace::for_ptr( Pointer::new(ret_param), @@ -128,8 +128,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( ) -> (Inst, T) { let return_ptr = match ret_arg_abi.mode { PassMode::Ignore => None, - PassMode::Cast(_) - | PassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, @@ -142,7 +141,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( extra_attrs: Some(_), on_stack: _, } => unreachable!("unsized return value"), - PassMode::Direct(_) | PassMode::Pair(_, _) => None, + PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => None, }; let (call_inst, meta) = f(fx, return_ptr); @@ -165,8 +164,20 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( ); } } - PassMode::Cast(_) - | PassMode::Indirect { + PassMode::Cast(cast) => { + if let Some(ret_place) = ret_place { + let results = fx + .bcx + .inst_results(call_inst) + .into_iter() + .copied() + .collect::>(); + let result = + super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast); + ret_place.write_cvalue(fx, result); + } + } + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, @@ -185,7 +196,6 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { match fx.fn_abi.as_ref().unwrap().ret.mode { PassMode::Ignore - | PassMode::Cast(_) | PassMode::Indirect { attrs: _, extra_attrs: None, @@ -208,5 +218,11 @@ pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); } + PassMode::Cast(cast) => { + let place = fx.get_local_place(RETURN_PLACE); + let ret_val = place.to_cvalue(fx); + let ret_vals = super::pass_mode::to_casted_value(fx, ret_val, cast); + fx.bcx.ins().return_(&ret_vals); + } } } From 139a6d12de65f81a69e345731f0547c824730a19 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 11:02:24 +0100 Subject: [PATCH 13/16] Fix 128bit checked math intrinsic calls --- src/abi/mod.rs | 25 ++++++++++-------- src/base.rs | 6 ++++- src/codegen_i128.rs | 62 +++++++++++++++++++++++---------------------- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index c227bdd53..6a025f2e8 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -102,13 +102,13 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> { pub(crate) fn lib_call( &mut self, name: &str, - input_tys: Vec, - output_tys: Vec, + params: Vec, + returns: Vec, args: &[Value], ) -> &[Value] { let sig = Signature { - params: input_tys.iter().cloned().map(AbiParam::new).collect(), - returns: output_tys.iter().cloned().map(AbiParam::new).collect(), + params, + returns, call_conv: CallConv::triple_default(self.triple()), }; let func_id = self @@ -140,16 +140,18 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> { .iter() .map(|arg| { ( - self.clif_type(arg.layout().ty).unwrap(), + AbiParam::new(self.clif_type(arg.layout().ty).unwrap()), arg.load_scalar(self), ) }) .unzip(); let return_layout = self.layout_of(return_ty); let return_tys = if let ty::Tuple(tup) = return_ty.kind() { - tup.types().map(|ty| self.clif_type(ty).unwrap()).collect() + tup.types() + .map(|ty| AbiParam::new(self.clif_type(ty).unwrap())) + .collect() } else { - vec![self.clif_type(return_ty).unwrap()] + vec![AbiParam::new(self.clif_type(return_ty).unwrap())] }; let ret_vals = self.lib_call(name, input_tys, return_tys, &args); match *ret_vals { @@ -208,7 +210,8 @@ pub(crate) fn codegen_fn_prelude<'tcx>( .block_params(start_block) .to_vec() .into_iter(); - let ret_place = self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); + let ret_place = + self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); // None means pass_mode == NoPass @@ -241,14 +244,16 @@ pub(crate) fn codegen_fn_prelude<'tcx>( let mut params = Vec::new(); for (i, _arg_ty) in tupled_arg_tys.types().enumerate() { let arg_abi = arg_abis_iter.next().unwrap(); - let param = cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); + let param = + cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); params.push(param); } (local, ArgKind::Spread(params), arg_ty) } else { let arg_abi = arg_abis_iter.next().unwrap(); - let param = cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); + let param = + cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); (local, ArgKind::Normal(param), arg_ty) } }) diff --git a/src/base.rs b/src/base.rs index e81aa5278..4842628a9 100644 --- a/src/base.rs +++ b/src/base.rs @@ -1060,7 +1060,11 @@ pub(crate) fn codegen_panic_inner<'tcx>( fx.lib_call( &*symbol_name, - vec![fx.pointer_type, fx.pointer_type, fx.pointer_type], + vec![ + AbiParam::new(fx.pointer_type), + AbiParam::new(fx.pointer_type), + AbiParam::new(fx.pointer_type), + ], vec![], args, ); diff --git a/src/codegen_i128.rs b/src/codegen_i128.rs index d6a38bdaf..866ba90e4 100644 --- a/src/codegen_i128.rs +++ b/src/codegen_i128.rs @@ -1,5 +1,7 @@ //! Replaces 128-bit operators with lang item calls where necessary +use cranelift_codegen::ir::ArgumentPurpose; + use crate::prelude::*; pub(crate) fn maybe_codegen<'tcx>( @@ -24,41 +26,41 @@ pub(crate) fn maybe_codegen<'tcx>( None } BinOp::Add | BinOp::Sub if !checked => None, - BinOp::Add => { - let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); - return Some(if is_signed { - fx.easy_call("__rust_i128_addo", &[lhs, rhs], out_ty) + BinOp::Mul if !checked => { + let val_ty = if is_signed { + fx.tcx.types.i128 } else { - fx.easy_call("__rust_u128_addo", &[lhs, rhs], out_ty) - }); + fx.tcx.types.u128 + }; + Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty)) } - BinOp::Sub => { + BinOp::Add | BinOp::Sub | BinOp::Mul => { + assert!(checked); let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); - return Some(if is_signed { - fx.easy_call("__rust_i128_subo", &[lhs, rhs], out_ty) - } else { - fx.easy_call("__rust_u128_subo", &[lhs, rhs], out_ty) - }); - } - BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), - BinOp::Mul => { - let res = if checked { - let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); - if is_signed { - fx.easy_call("__rust_i128_mulo", &[lhs, rhs], out_ty) - } else { - fx.easy_call("__rust_u128_mulo", &[lhs, rhs], out_ty) - } - } else { - let val_ty = if is_signed { - fx.tcx.types.i128 - } else { - fx.tcx.types.u128 - }; - fx.easy_call("__multi3", &[lhs, rhs], val_ty) + let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty)); + let param_types = vec![ + AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn), + AbiParam::new(types::I128), + AbiParam::new(types::I128), + ]; + let args = [ + out_place.to_ptr().get_addr(fx), + lhs.load_scalar(fx), + rhs.load_scalar(fx), + ]; + let name = match (bin_op, is_signed) { + (BinOp::Add, false) => "__rust_u128_addo", + (BinOp::Add, true) => "__rust_i128_addo", + (BinOp::Sub, false) => "__rust_u128_subo", + (BinOp::Sub, true) => "__rust_i128_subo", + (BinOp::Mul, false) => "__rust_u128_mulo", + (BinOp::Mul, true) => "__rust_i128_mulo", + _ => unreachable!(), }; - Some(res) + fx.lib_call(name, param_types, vec![], &args); + Some(out_place.to_cvalue(fx)) } + BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), BinOp::Div => { assert!(!checked); if is_signed { From b5ddb76f84f3e6df242e8c766b5015a124a50c15 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 12:11:40 +0100 Subject: [PATCH 14/16] Force stack slot size to be a multiple of 16 This ensures that all stack slots are aligned to 16 bytes. Without this linking against crates compiled with cg_llvm may cause a crash due to simd instructions requiring a 16 byte alignment. --- src/abi/pass_mode.rs | 11 +++++------ src/value_and_place.rs | 8 ++++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 6f27fa52d..b3231e94a 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -144,7 +144,6 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))], ), - // FIXME implement Vector Abi in a cg_llvm compatible way Abi::Vector { .. } => { let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); (None, vec![AbiParam::new(vector_ty)]) @@ -210,7 +209,7 @@ pub(super) fn from_casted_value<'tcx>( cast: CastTarget, ) -> CValue<'tcx> { let abi_params = cast_target_to_abi_params(cast); - let size = abi_params + let size: u32 = abi_params .iter() .map(|param| param.value_type.bytes()) .sum(); @@ -218,7 +217,9 @@ pub(super) fn from_casted_value<'tcx>( assert!(u64::from(size) >= layout.size.bytes()); let stack_slot = fx.bcx.create_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, - size, + // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to + // specify stack slot alignment. + size: (size + 15) / 16 * 16, offset: None, }); let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0)); @@ -306,9 +307,7 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, )) } - PassMode::Cast(cast) => { - Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) - } + PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)), PassMode::Indirect { attrs: _, extra_attrs: None, diff --git a/src/value_and_place.rs b/src/value_and_place.rs index 17cb09d55..765604e0f 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -334,7 +334,9 @@ impl<'tcx> CPlace<'tcx> { let stack_slot = fx.bcx.create_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, - size: u32::try_from(layout.size.bytes()).unwrap(), + // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to + // specify stack slot alignment. + size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16, offset: None, }); CPlace { @@ -498,7 +500,9 @@ impl<'tcx> CPlace<'tcx> { // FIXME do something more efficient for transmutes between vectors and integers. let stack_slot = fx.bcx.create_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, - size: src_ty.bytes(), + // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to + // specify stack slot alignment. + size: (src_ty.bytes() + 15) / 16 * 16, offset: None, }); let ptr = Pointer::stack_slot(stack_slot); From 2d6981756da4a21cb54c2b76378a66b326df209e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 12:25:20 +0100 Subject: [PATCH 15/16] Handle argument extension mode --- src/abi/pass_mode.rs | 52 +++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index b3231e94a..2f91e8338 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -3,8 +3,10 @@ use crate::prelude::*; use crate::value_and_place::assert_assignable; -use cranelift_codegen::ir::ArgumentPurpose; -use rustc_target::abi::call::{ArgAbi, CastTarget, PassMode, Reg, RegKind}; +use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose}; +use rustc_target::abi::call::{ + ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, Reg, RegKind, +}; use smallvec::{smallvec, SmallVec}; pub(super) trait ArgAbiExt<'tcx> { @@ -27,6 +29,15 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam { AbiParam::new(clif_ty) } +fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -> AbiParam { + match arg_attrs.arg_ext { + RustcArgExtension::None => {} + RustcArgExtension::Zext => param.extension = ArgumentExtension::Uext, + RustcArgExtension::Sext => param.extension = ArgumentExtension::Sext, + } + param +} + fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> { let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 { (0, 0) @@ -82,15 +93,16 @@ fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> { args } -// FIXME respect argument extension mode - impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> { match self.mode { PassMode::Ignore => smallvec![], - PassMode::Direct(_) => match &self.layout.abi { + PassMode::Direct(attrs) => match &self.layout.abi { Abi::Scalar(scalar) => { - smallvec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))] + smallvec![apply_arg_attrs_to_abi_param( + AbiParam::new(scalar_to_clif_type(tcx, scalar.clone())), + attrs + )] } Abi::Vector { .. } => { let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); @@ -98,39 +110,45 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Pair(_, _) => match &self.layout.abi { + PassMode::Pair(attrs_a, attrs_b) => match &self.layout.abi { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); - smallvec![AbiParam::new(a), AbiParam::new(b)] + smallvec![ + apply_arg_attrs_to_abi_param(AbiParam::new(a), attrs_a), + apply_arg_attrs_to_abi_param(AbiParam::new(b), attrs_b), + ] } _ => unreachable!("{:?}", self.layout.abi), }, PassMode::Cast(cast) => cast_target_to_abi_params(cast), PassMode::Indirect { - attrs: _, + attrs, extra_attrs: None, on_stack, } => { if on_stack { let size = u32::try_from(self.layout.size.bytes()).unwrap(); - smallvec![AbiParam::special( - pointer_ty(tcx), - ArgumentPurpose::StructArgument(size), + smallvec![apply_arg_attrs_to_abi_param( + AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructArgument(size),), + attrs )] } else { - smallvec![AbiParam::new(pointer_ty(tcx))] + smallvec![apply_arg_attrs_to_abi_param( + AbiParam::new(pointer_ty(tcx)), + attrs + )] } } PassMode::Indirect { - attrs: _, - extra_attrs: Some(_), + attrs, + extra_attrs: Some(extra_attrs), on_stack, } => { assert!(!on_stack); smallvec![ - AbiParam::new(pointer_ty(tcx)), - AbiParam::new(pointer_ty(tcx)), + apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs), + apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), extra_attrs), ] } } From 7fcf59f19e711b89285505785f97dd57f80cbe2e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 14:19:13 +0100 Subject: [PATCH 16/16] Add print on panic for predefining of a function --- src/driver/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 752c3f747..2497f9dfd 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -51,6 +51,7 @@ fn predefine_mono_items<'tcx>( match mono_item { MonoItem::Fn(instance) => { let name = cx.tcx.symbol_name(instance).name.to_string(); + let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); cx.module.declare_function(&name, linkage, &sig).unwrap();