diff --git a/Cargo.toml b/Cargo.toml index 9066e4dbb..51bb5d9db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ smallvec = "1.8.1" unstable-features = ["jit", "inline_asm_sym"] jit = ["cranelift-jit", "libloading"] inline_asm_sym = [] +unwinding = [] # Not yet included in unstable-features for performance reasons [package.metadata.rust-analyzer] rustc_private = true diff --git a/Readme.md b/Readme.md index 4d1e4d843..9989cc7e7 100644 --- a/Readme.md +++ b/Readme.md @@ -104,7 +104,7 @@ See [rustc_testing.md](docs/rustc_testing.md). ## Not yet supported * SIMD ([tracked here](https://github.com/rust-lang/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported) -* Unwinding on panics ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1677), `-Cpanic=abort` is enabled by default) +* Unwinding on panics ([experimental and not supported on Windows and macOS](https://github.com/rust-lang/rustc_codegen_cranelift/issues/1567), `-Cpanic=abort` is enabled by default) ## License diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index 43025137b..5a393a217 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -19,6 +19,7 @@ pub(crate) fn run( cg_clif_dylib: &CodegenBackend, rustup_toolchain_name: Option<&str>, bootstrap_host_compiler: &Compiler, + panic_unwind_support: bool, ) { std::fs::create_dir_all(&dirs.download_dir).unwrap(); ABI_CAFE_REPO.fetch(dirs); @@ -32,6 +33,7 @@ pub(crate) fn run( bootstrap_host_compiler, rustup_toolchain_name, bootstrap_host_compiler.triple.clone(), + panic_unwind_support, ); eprintln!("Running abi-cafe"); diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs index bf7cf1c0a..a1f19a1af 100644 --- a/build_system/build_backend.rs +++ b/build_system/build_backend.rs @@ -12,6 +12,7 @@ pub(crate) fn build_backend( dirs: &Dirs, bootstrap_host_compiler: &Compiler, use_unstable_features: bool, + panic_unwind_support: bool, ) -> PathBuf { let _group = LogGroup::guard("Build backend"); @@ -31,6 +32,10 @@ pub(crate) fn build_backend( cmd.arg("--features").arg("unstable-features"); } + if panic_unwind_support { + cmd.arg("--features").arg("unwinding"); + } + cmd.arg("--release"); eprintln!("[BUILD] rustc_codegen_cranelift"); diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 00955998e..fec6ff241 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -17,6 +17,7 @@ pub(crate) fn build_sysroot( bootstrap_host_compiler: &Compiler, rustup_toolchain_name: Option<&str>, target_triple: String, + panic_unwind_support: bool, ) -> Compiler { let _guard = LogGroup::guard("Build sysroot"); @@ -52,6 +53,9 @@ pub(crate) fn build_sysroot( .arg("-o") .arg(&wrapper_path) .arg("-Cstrip=debuginfo"); + if panic_unwind_support { + build_cargo_wrapper_cmd.arg("--cfg").arg("support_panic_unwind"); + } if let Some(rustup_toolchain_name) = &rustup_toolchain_name { build_cargo_wrapper_cmd .env("TOOLCHAIN_NAME", rustup_toolchain_name) @@ -77,6 +81,7 @@ pub(crate) fn build_sysroot( bootstrap_host_compiler.clone(), &cg_clif_dylib_path, sysroot_kind, + panic_unwind_support, ); host.install_into_sysroot(dist_dir); @@ -91,6 +96,7 @@ pub(crate) fn build_sysroot( }, &cg_clif_dylib_path, sysroot_kind, + panic_unwind_support, ) .install_into_sysroot(dist_dir); } @@ -141,12 +147,15 @@ fn build_sysroot_for_triple( compiler: Compiler, cg_clif_dylib_path: &CodegenBackend, sysroot_kind: SysrootKind, + panic_unwind_support: bool, ) -> SysrootTarget { match sysroot_kind { SysrootKind::None => build_rtstartup(dirs, &compiler) .unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }), SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler), - SysrootKind::Clif => build_clif_sysroot_for_triple(dirs, compiler, cg_clif_dylib_path), + SysrootKind::Clif => { + build_clif_sysroot_for_triple(dirs, compiler, cg_clif_dylib_path, panic_unwind_support) + } } } @@ -188,6 +197,7 @@ fn build_clif_sysroot_for_triple( dirs: &Dirs, mut compiler: Compiler, cg_clif_dylib_path: &CodegenBackend, + panic_unwind_support: bool, ) -> SysrootTarget { let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; @@ -206,7 +216,10 @@ fn build_clif_sysroot_for_triple( } // Build sysroot - let mut rustflags = vec!["-Zforce-unstable-if-unmarked".to_owned(), "-Cpanic=abort".to_owned()]; + let mut rustflags = vec!["-Zforce-unstable-if-unmarked".to_owned()]; + if !panic_unwind_support { + rustflags.push("-Cpanic=abort".to_owned()); + } match cg_clif_dylib_path { CodegenBackend::Local(path) => { rustflags.push(format!("-Zcodegen-backend={}", path.to_str().unwrap())); diff --git a/build_system/main.rs b/build_system/main.rs index 3ff9751a3..fc0093128 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -83,6 +83,7 @@ fn main() { let mut download_dir = None; let mut sysroot_kind = SysrootKind::Clif; let mut use_unstable_features = true; + let mut panic_unwind_support = false; let mut frozen = false; let mut skip_tests = vec![]; let mut use_backend = None; @@ -108,6 +109,7 @@ fn main() { } } "--no-unstable-features" => use_unstable_features = false, + "--panic-unwind-support" => panic_unwind_support = true, "--frozen" => frozen = true, "--skip-test" => { // FIXME check that all passed in tests actually exist @@ -201,6 +203,7 @@ fn main() { &dirs, &bootstrap_host_compiler, use_unstable_features, + panic_unwind_support, )) }; match command { @@ -212,6 +215,7 @@ fn main() { &dirs, sysroot_kind, use_unstable_features, + panic_unwind_support, &skip_tests.iter().map(|test| &**test).collect::>(), &cg_clif_dylib, &bootstrap_host_compiler, @@ -230,6 +234,7 @@ fn main() { &cg_clif_dylib, rustup_toolchain_name.as_deref(), &bootstrap_host_compiler, + panic_unwind_support, ); } Command::Build => { @@ -240,6 +245,7 @@ fn main() { &bootstrap_host_compiler, rustup_toolchain_name.as_deref(), target_triple, + panic_unwind_support, ); } Command::Bench => { @@ -250,6 +256,7 @@ fn main() { &bootstrap_host_compiler, rustup_toolchain_name.as_deref(), target_triple, + panic_unwind_support, ); bench::benchmark(&dirs, &compiler); } diff --git a/build_system/tests.rs b/build_system/tests.rs index eec89c026..8cd84a79f 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -233,6 +233,7 @@ pub(crate) fn run_tests( dirs: &Dirs, sysroot_kind: SysrootKind, use_unstable_features: bool, + panic_unwind_support: bool, skip_tests: &[&str], cg_clif_dylib: &CodegenBackend, bootstrap_host_compiler: &Compiler, @@ -251,12 +252,14 @@ pub(crate) fn run_tests( bootstrap_host_compiler, rustup_toolchain_name, target_triple.clone(), + panic_unwind_support, ); let runner = TestRunner::new( dirs.clone(), target_compiler, use_unstable_features, + panic_unwind_support, skip_tests, bootstrap_host_compiler.triple == target_triple, stdlib_source.clone(), @@ -283,12 +286,14 @@ pub(crate) fn run_tests( bootstrap_host_compiler, rustup_toolchain_name, target_triple.clone(), + panic_unwind_support, ); let mut runner = TestRunner::new( dirs.clone(), target_compiler, use_unstable_features, + panic_unwind_support, skip_tests, bootstrap_host_compiler.triple == target_triple, stdlib_source, @@ -314,6 +319,7 @@ pub(crate) fn run_tests( struct TestRunner<'a> { is_native: bool, jit_supported: bool, + panic_unwind_support: bool, skip_tests: &'a [&'a str], dirs: Dirs, target_compiler: Compiler, @@ -325,6 +331,7 @@ impl<'a> TestRunner<'a> { dirs: Dirs, mut target_compiler: Compiler, use_unstable_features: bool, + panic_unwind_support: bool, skip_tests: &'a [&'a str], is_native: bool, stdlib_source: PathBuf, @@ -335,7 +342,15 @@ impl<'a> TestRunner<'a> { let jit_supported = use_unstable_features && is_native && !target_compiler.triple.contains("windows"); - Self { is_native, jit_supported, skip_tests, dirs, target_compiler, stdlib_source } + Self { + is_native, + jit_supported, + panic_unwind_support, + skip_tests, + dirs, + target_compiler, + stdlib_source, + } } fn run_testsuite(&self, tests: &[TestCase]) { @@ -404,7 +419,9 @@ impl<'a> TestRunner<'a> { cmd.arg("-Cdebuginfo=2"); cmd.arg("--target"); cmd.arg(&self.target_compiler.triple); - cmd.arg("-Cpanic=abort"); + if !self.panic_unwind_support { + cmd.arg("-Cpanic=abort"); + } cmd.arg("--check-cfg=cfg(jit)"); cmd.args(args); cmd diff --git a/build_system/usage.txt b/build_system/usage.txt index 5c333fe2d..6c98087e5 100644 --- a/build_system/usage.txt +++ b/build_system/usage.txt @@ -25,6 +25,10 @@ OPTIONS: Some features are not yet ready for production usage. This option will disable these features. This includes the JIT mode and inline assembly support. + --panic-unwind-support + Enable support for unwinding when -Cpanic=unwind is used. This currently regresses build + performance. + --frozen Require Cargo.lock and cache are up to date diff --git a/scripts/cargo-clif.rs b/scripts/cargo-clif.rs index e6c63bf5e..e391cc7f7 100644 --- a/scripts/cargo-clif.rs +++ b/scripts/cargo-clif.rs @@ -12,7 +12,11 @@ fn main() { sysroot = sysroot.parent().unwrap(); } - let mut rustflags = vec!["-Cpanic=abort".to_owned(), "-Zpanic-abort-tests".to_owned()]; + let mut rustflags = vec![]; + if !cfg!(support_panic_unwind) { + rustflags.push("-Cpanic=abort".to_owned()); + rustflags.push("-Zpanic-abort-tests".to_owned()); + } if let Some(name) = option_env!("BUILTIN_BACKEND") { rustflags.push(format!("-Zcodegen-backend={name}")); } else { diff --git a/scripts/rustc-clif.rs b/scripts/rustc-clif.rs index 528031af8..15d929d0f 100644 --- a/scripts/rustc-clif.rs +++ b/scripts/rustc-clif.rs @@ -17,8 +17,10 @@ fn main() { let passed_args = std::env::args_os().skip(1).collect::>(); let mut args = vec![]; - args.push(OsString::from("-Cpanic=abort")); - args.push(OsString::from("-Zpanic-abort-tests")); + if !cfg!(support_panic_unwind) { + args.push(OsString::from("-Cpanic=abort")); + args.push(OsString::from("-Zpanic-abort-tests")); + } if let Some(name) = option_env!("BUILTIN_BACKEND") { args.push(OsString::from(format!("-Zcodegen-backend={name}"))) } else { diff --git a/scripts/rustdoc-clif.rs b/scripts/rustdoc-clif.rs index 6ebe060d8..dc5bef18c 100644 --- a/scripts/rustdoc-clif.rs +++ b/scripts/rustdoc-clif.rs @@ -17,8 +17,10 @@ fn main() { let passed_args = std::env::args_os().skip(1).collect::>(); let mut args = vec![]; - args.push(OsString::from("-Cpanic=abort")); - args.push(OsString::from("-Zpanic-abort-tests")); + if !cfg!(support_panic_unwind) { + args.push(OsString::from("-Cpanic=abort")); + args.push(OsString::from("-Zpanic-abort-tests")); + } if let Some(name) = option_env!("BUILTIN_BACKEND") { args.push(OsString::from(format!("-Zcodegen-backend={name}"))) } else { diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 4c6fd9078..23fe0c328 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -7,7 +7,7 @@ mod returning; use std::borrow::Cow; use std::mem; -use cranelift_codegen::ir::{ArgumentPurpose, SigRef}; +use cranelift_codegen::ir::{ArgumentPurpose, BlockArg, ExceptionTableData, ExceptionTag, SigRef}; use cranelift_codegen::isa::CallConv; use cranelift_module::ModuleError; use rustc_abi::{CanonAbi, ExternAbi, X86Call}; @@ -20,10 +20,12 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_session::Session; use rustc_span::source_map::Spanned; use rustc_target::callconv::{FnAbi, PassMode}; -use smallvec::SmallVec; +use smallvec::{SmallVec, smallvec}; use self::pass_mode::*; pub(crate) use self::returning::codegen_return; +use crate::base::codegen_unwind_terminate; +use crate::debuginfo::EXCEPTION_HANDLER_CLEANUP; use crate::prelude::*; fn clif_sig_from_fn_abi<'tcx>( @@ -380,7 +382,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( args: &[Spanned>], destination: Place<'tcx>, target: Option, - _unwind: UnwindAction, + unwind: UnwindAction, ) { let func = codegen_operand(fx, func); let fn_sig = func.layout().ty.fn_sig(fx.tcx); @@ -515,12 +517,6 @@ pub(crate) fn codegen_terminator_call<'tcx>( let args = args; assert_eq!(fn_abi.args.len(), args.len()); - #[derive(Copy, Clone)] - enum CallTarget { - Direct(FuncRef), - Indirect(SigRef, Value), - } - let (func_ref, first_arg_override) = match instance { // Trait object call Some(Instance { def: InstanceKind::Virtual(_, idx), .. }) => { @@ -582,18 +578,12 @@ pub(crate) fn codegen_terminator_call<'tcx>( adjust_call_for_c_variadic(fx, &fn_abi, source_info, func_ref, &mut call_args); } - let call_inst = match func_ref { - CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args), - CallTarget::Indirect(sig, func_ptr) => { - fx.bcx.ins().call_indirect(sig, func_ptr, &call_args) - } - }; - if fx.clif_comments.enabled() { - with_no_trimmed_paths!(fx.add_comment(call_inst, format!("abi: {:?}", fn_abi))); + let nop_inst = fx.bcx.ins().nop(); + with_no_trimmed_paths!(fx.add_post_comment(nop_inst, format!("abi: {:?}", fn_abi))); } - fx.bcx.func.dfg.inst_results(call_inst).iter().copied().collect::>() + codegen_call_with_unwind_action(fx, source_info.span, func_ref, unwind, &call_args, None) }); if let Some(dest) = target { @@ -703,7 +693,7 @@ pub(crate) fn codegen_drop<'tcx>( source_info: mir::SourceInfo, drop_place: CPlace<'tcx>, target: BasicBlock, - _unwind: UnwindAction, + unwind: UnwindAction, ) { let ty = drop_place.layout().ty; let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty); @@ -749,9 +739,14 @@ pub(crate) fn codegen_drop<'tcx>( let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); let sig = fx.bcx.import_signature(sig); - // FIXME implement cleanup on exceptions - fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]); - fx.bcx.ins().jump(ret_block, &[]); + codegen_call_with_unwind_action( + fx, + source_info.span, + CallTarget::Indirect(sig, drop_fn), + unwind, + &[ptr], + Some(ret_block), + ); } ty::Dynamic(_, _, ty::DynStar) => { // IN THIS ARM, WE HAVE: @@ -794,9 +789,14 @@ pub(crate) fn codegen_drop<'tcx>( let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); let sig = fx.bcx.import_signature(sig); - fx.bcx.ins().call_indirect(sig, drop_fn, &[data]); - // FIXME implement cleanup on exceptions - fx.bcx.ins().jump(ret_block, &[]); + codegen_call_with_unwind_action( + fx, + source_info.span, + CallTarget::Indirect(sig, drop_fn), + unwind, + &[data], + Some(ret_block), + ); } _ => { assert!(!matches!(drop_instance.def, InstanceKind::Virtual(_, _))); @@ -821,9 +821,137 @@ pub(crate) fn codegen_drop<'tcx>( } let func_ref = fx.get_function_ref(drop_instance); - fx.bcx.ins().call(func_ref, &call_args); - // FIXME implement cleanup on exceptions - fx.bcx.ins().jump(ret_block, &[]); + codegen_call_with_unwind_action( + fx, + source_info.span, + CallTarget::Direct(func_ref), + unwind, + &call_args, + Some(ret_block), + ); + } + } + } +} + +#[derive(Copy, Clone)] +pub(crate) enum CallTarget { + Direct(FuncRef), + Indirect(SigRef, Value), +} + +pub(crate) fn codegen_call_with_unwind_action( + fx: &mut FunctionCx<'_, '_, '_>, + span: Span, + func_ref: CallTarget, + mut unwind: UnwindAction, + call_args: &[Value], + target_block: Option, +) -> SmallVec<[Value; 2]> { + let sig_ref = match func_ref { + CallTarget::Direct(func_ref) => fx.bcx.func.dfg.ext_funcs[func_ref].signature, + CallTarget::Indirect(sig_ref, _func_ptr) => sig_ref, + }; + + if target_block.is_some() { + assert!(fx.bcx.func.dfg.signatures[sig_ref].returns.is_empty()); + } + + if cfg!(not(feature = "unwinding")) { + unwind = UnwindAction::Unreachable; + } + + match unwind { + UnwindAction::Continue | UnwindAction::Unreachable => { + let call_inst = match func_ref { + CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args), + CallTarget::Indirect(sig, func_ptr) => { + fx.bcx.ins().call_indirect(sig, func_ptr, &call_args) + } + }; + + if let Some(target_block) = target_block { + fx.bcx.ins().jump(target_block, &[]); + smallvec![] + } else { + fx.bcx + .func + .dfg + .inst_results(call_inst) + .iter() + .copied() + .collect::>() + } + } + UnwindAction::Cleanup(_) | UnwindAction::Terminate(_) => { + let returns_types = fx.bcx.func.dfg.signatures[sig_ref] + .returns + .iter() + .map(|return_param| return_param.value_type) + .collect::>(); + + let fallthrough_block = fx.bcx.create_block(); + let fallthrough_block_call_args = returns_types + .iter() + .enumerate() + .map(|(i, _)| BlockArg::TryCallRet(i.try_into().unwrap())) + .collect::>(); + let fallthrough_block_call = fx.bcx.func.dfg.block_call( + target_block.unwrap_or(fallthrough_block), + &fallthrough_block_call_args, + ); + let pre_cleanup_block = fx.bcx.create_block(); + let pre_cleanup_block_call = + fx.bcx.func.dfg.block_call(pre_cleanup_block, &[BlockArg::TryCallExn(0)]); + let exception_table = fx.bcx.func.dfg.exception_tables.push(ExceptionTableData::new( + sig_ref, + fallthrough_block_call, + [( + Some(ExceptionTag::with_number(EXCEPTION_HANDLER_CLEANUP).unwrap()), + pre_cleanup_block_call, + )], + )); + + match func_ref { + CallTarget::Direct(func_ref) => { + fx.bcx.ins().try_call(func_ref, &call_args, exception_table); + } + CallTarget::Indirect(_sig, func_ptr) => { + fx.bcx.ins().try_call_indirect(func_ptr, &call_args, exception_table); + } + } + + fx.bcx.seal_block(pre_cleanup_block); + fx.bcx.switch_to_block(pre_cleanup_block); + fx.bcx.set_cold_block(pre_cleanup_block); + match unwind { + UnwindAction::Continue | UnwindAction::Unreachable => unreachable!(), + UnwindAction::Cleanup(cleanup) => { + let exception_ptr = + fx.bcx.append_block_param(pre_cleanup_block, fx.pointer_type); + fx.bcx.def_var(fx.exception_slot, exception_ptr); + let cleanup_block = fx.get_block(cleanup); + fx.bcx.ins().jump(cleanup_block, &[]); + } + UnwindAction::Terminate(reason) => { + // FIXME dedup terminate blocks + fx.bcx.append_block_param(pre_cleanup_block, fx.pointer_type); + + codegen_unwind_terminate(fx, span, reason); + } + } + + if target_block.is_none() { + fx.bcx.seal_block(fallthrough_block); + fx.bcx.switch_to_block(fallthrough_block); + let returns = returns_types + .into_iter() + .map(|ty| fx.bcx.append_block_param(fallthrough_block, ty)) + .collect(); + fx.bcx.ins().nop(); + returns + } else { + smallvec![] } } } diff --git a/src/base.rs b/src/base.rs index 0b641ba64..93d68720b 100644 --- a/src/base.rs +++ b/src/base.rs @@ -2,7 +2,7 @@ use cranelift_codegen::CodegenError; use cranelift_codegen::ir::UserFuncName; -use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; use cranelift_module::ModuleError; use rustc_ast::InlineAsmOptions; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; @@ -83,6 +83,9 @@ pub(crate) fn codegen_fn<'tcx>( None }; + let exception_slot = Variable::from_u32(0); + bcx.declare_var(exception_slot, pointer_type); + let mut fx = FunctionCx { cx, module, @@ -101,9 +104,10 @@ pub(crate) fn codegen_fn<'tcx>( block_map, local_map: IndexVec::with_capacity(mir.local_decls.len()), caller_location: None, // set by `codegen_fn_prelude` + exception_slot, clif_comments, - next_ssa_var: 0, + next_ssa_var: 1, // var0 is used for the exception slot }; tcx.prof.generic_activity("codegen clif ir").run(|| codegen_fn_body(&mut fx, start_block)); @@ -296,11 +300,11 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { } if bb_data.is_cleanup { - // Unwinding after panicking is not supported - continue; + if cfg!(not(feature = "unwinding")) { + continue; + } - // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do - // so for cleanup blocks. + fx.bcx.set_cold_block(block); } fx.bcx.ins().nop(); @@ -534,7 +538,15 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { codegen_unwind_terminate(fx, source_info.span, *reason); } TerminatorKind::UnwindResume => { - // FIXME implement unwinding + if cfg!(feature = "unwinding") { + let exception_ptr = fx.bcx.use_var(fx.exception_slot); + fx.lib_call( + "_Unwind_Resume", + vec![AbiParam::new(fx.pointer_type)], + vec![], + &[exception_ptr], + ); + } fx.bcx.ins().trap(TrapCode::user(1 /* unreachable */).unwrap()); } TerminatorKind::Unreachable => { @@ -1101,7 +1113,7 @@ fn codegen_panic_inner<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, lang_item: rustc_hir::LangItem, args: &[Value], - _unwind: UnwindAction, + unwind: UnwindAction, span: Span, ) { fx.bcx.set_cold_block(fx.bcx.current_block().unwrap()); @@ -1117,14 +1129,23 @@ fn codegen_panic_inner<'tcx>( let symbol_name = fx.tcx.symbol_name(instance).name; - // FIXME implement cleanup on exceptions + let sig = Signature { + params: args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(), + returns: vec![], + call_conv: fx.target_config.default_call_conv, + }; + let func_id = fx.module.declare_function(symbol_name, Linkage::Import, &sig).unwrap(); + let func_ref = fx.module.declare_func_in_func(func_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(func_ref, format!("{:?}", symbol_name)); + } - fx.lib_call( - symbol_name, - args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(), - vec![], - args, - ); + let nop_inst = fx.bcx.ins().nop(); + if fx.clif_comments.enabled() { + fx.add_comment(nop_inst, format!("panic {}", symbol_name)); + } + + codegen_call_with_unwind_action(fx, span, CallTarget::Direct(func_ref), unwind, &args, None); fx.bcx.ins().trap(TrapCode::user(1 /* unreachable */).unwrap()); } diff --git a/src/common.rs b/src/common.rs index 2f11b2d2d..c222bfca9 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,5 +1,5 @@ use cranelift_codegen::isa::TargetFrontendConfig; -use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; use rustc_abi::{Float, Integer, Primitive}; use rustc_index::IndexVec; use rustc_middle::ty::TypeFoldable; @@ -288,6 +288,9 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { /// When `#[track_caller]` is used, the implicit caller location is stored in this variable. pub(crate) caller_location: Option>, + /// During cleanup the exception pointer will be stored in this variable. + pub(crate) exception_slot: Variable, + pub(crate) clif_comments: crate::pretty_clif::CommentWriter, /// This should only be accessed by `CPlace::new_var`. diff --git a/src/debuginfo/gcc_except_table.rs b/src/debuginfo/gcc_except_table.rs new file mode 100644 index 000000000..b8e602c76 --- /dev/null +++ b/src/debuginfo/gcc_except_table.rs @@ -0,0 +1,272 @@ +use gimli::write::{Address, Writer}; +use gimli::{DW_EH_PE_omit, DW_EH_PE_uleb128, Encoding, LittleEndian}; + +pub(super) struct GccExceptTable { + pub call_sites: CallSiteTable, + pub actions: ActionTable, + pub type_info: TypeInfoTable, +} + +impl GccExceptTable { + pub(super) fn write( + &self, + w: &mut W, + encoding: Encoding, + ) -> gimli::write::Result<()> { + // lpStartEncoding + w.write_u8(DW_EH_PE_omit.0)?; + // lpStart (omitted) + let type_info_padding = if self.type_info.type_info.is_empty() { + // ttypeEncoding + w.write_u8(DW_EH_PE_omit.0)?; + None + } else { + // ttypeEncoding + w.write_u8(self.type_info.ttype_encoding.0)?; + + // classInfoOffset + let class_info_offset_field_offset = w.len() as u64; + + // Note: The offset in classInfoOffset is relative to position right after classInfoOffset + // itself. + let class_info_offset_no_padding = self.call_sites.encoded_size() + + self.actions.encoded_size() + + self.type_info.encoded_size(encoding); + + let type_info_is_aligned = |type_info_padding: u64| { + (class_info_offset_field_offset + + gimli::leb128::write::uleb128_size( + class_info_offset_no_padding + type_info_padding, + ) as u64 + + self.call_sites.encoded_size() + + self.actions.encoded_size() + + type_info_padding) + % 4 + == 0 + }; + + let mut type_info_padding = 0; + while !type_info_is_aligned(type_info_padding) { + type_info_padding += 1; + } + + w.write_uleb128(class_info_offset_no_padding + type_info_padding)?; + + Some(type_info_padding) + }; + + // call site table + self.call_sites.write(w)?; + + // action table + self.actions.write(w)?; + + // align to 4 bytes + if let Some(type_info_padding) = type_info_padding { + for _ in 0..type_info_padding { + w.write_u8(0)?; + } + // In this case we calculated the expected padding amount and used it to write the + // classInfoOffset field. Assert that the expected value matched the actual value to catch + // any inconsistency. + assert!(w.len() % 4 == 0, "type_info must be aligned to 4 bytes"); + } else { + while w.len() % 4 != 0 { + w.write_u8(0)?; + } + } + + // type_info + self.type_info.write(w, encoding)?; + + // exception specs (unused for rust) + + // align to 4 bytes + while w.len() % 4 != 0 { + w.write_u8(0)?; + } + + Ok(()) + } +} + +pub(super) struct CallSiteTable(pub Vec); + +impl CallSiteTable { + fn encoded_size(&self) -> u64 { + let mut len = LenWriter(0); + self.write(&mut len).unwrap(); + len.0 as u64 + } + + fn write(&self, w: &mut W) -> gimli::write::Result<()> { + let callsite_table_length = self.0.iter().map(|call_site| call_site.encoded_size()).sum(); + + // callsiteEncoding + w.write_u8(DW_EH_PE_uleb128.0)?; + // callsiteTableLength + w.write_uleb128(callsite_table_length)?; + + for call_site in &self.0 { + call_site.write(w)?; + } + + Ok(()) + } +} + +pub(super) struct CallSite { + pub start: u64, + pub length: u64, + pub landing_pad: u64, + pub action_entry: Option, +} + +impl CallSite { + fn encoded_size(&self) -> u64 { + let mut len = LenWriter(0); + self.write(&mut len).unwrap(); + len.0 as u64 + } + + fn write(&self, w: &mut W) -> gimli::write::Result<()> { + w.write_uleb128(self.start)?; + w.write_uleb128(self.length)?; + w.write_uleb128(self.landing_pad)?; + w.write_uleb128(match self.action_entry { + Some(action_offset) => action_offset.0 + 1, + None => 0, + })?; + Ok(()) + } +} + +pub(super) struct ActionTable { + actions: Vec, + encoded_length: u64, +} + +impl ActionTable { + pub(super) fn new() -> ActionTable { + ActionTable { actions: vec![], encoded_length: 0 } + } + + pub(super) fn add(&mut self, action: Action) -> ActionOffset { + let id = ActionOffset(self.encoded_length); + self.encoded_length += action.encoded_size(self.encoded_length); + self.actions.push(action); + id + } + + fn encoded_size(&self) -> u64 { + let mut len = LenWriter(0); + self.write(&mut len).unwrap(); + len.0 as u64 + } + + fn write(&self, w: &mut W) -> gimli::write::Result<()> { + let action_table_start = w.len() as u64; + for action in &self.actions { + action.write(w, w.len() as u64 - action_table_start)?; + } + + Ok(()) + } +} + +#[derive(Copy, Clone)] +pub(super) struct ActionOffset(u64); + +pub(super) struct Action { + pub(super) kind: ActionKind, + pub(super) next_action: Option, +} + +impl Action { + fn encoded_size(&self, action_table_offset: u64) -> u64 { + let mut len = LenWriter(0); + self.write(&mut len, action_table_offset).unwrap(); + len.0 as u64 + } + + fn write(&self, w: &mut W, action_table_offset: u64) -> gimli::write::Result<()> { + // ttypeIndex + let ttype_index = match self.kind { + ActionKind::Catch(type_info_id) => type_info_id.0 as i64 + 1, + }; + w.write_sleb128(ttype_index)?; + // actionOffset + let action_offset_field_offset = + action_table_offset + gimli::leb128::write::sleb128_size(ttype_index) as u64; + w.write_sleb128(match self.next_action { + Some(next_action_offset) => { + next_action_offset.0 as i64 - action_offset_field_offset as i64 + } + None => 0, + })?; + Ok(()) + } +} + +#[derive(Copy, Clone)] +pub(super) enum ActionKind { + Catch(TypeInfoId), +} + +pub(super) struct TypeInfoTable { + ttype_encoding: gimli::DwEhPe, + type_info: Vec
, +} + +impl TypeInfoTable { + pub(super) fn new(ttype_encoding: gimli::DwEhPe) -> TypeInfoTable { + TypeInfoTable { ttype_encoding, type_info: vec![] } + } + + pub(super) fn add(&mut self, type_info: Address) -> TypeInfoId { + let id = TypeInfoId(self.type_info.len() as u64); + self.type_info.push(type_info); + id + } + + fn encoded_size(&self, encoding: Encoding) -> u64 { + let mut len = LenWriter(0); + self.write(&mut len, encoding).unwrap(); + len.0 as u64 + } + + fn write(&self, w: &mut W, encoding: Encoding) -> gimli::write::Result<()> { + for &type_info in self.type_info.iter().rev() { + w.write_eh_pointer(type_info, self.ttype_encoding, encoding.address_size)?; + } + + Ok(()) + } +} + +#[derive(Copy, Clone)] +pub(super) struct TypeInfoId(u64); + +struct LenWriter(usize); + +impl Writer for LenWriter { + type Endian = LittleEndian; + + fn endian(&self) -> LittleEndian { + LittleEndian + } + + fn len(&self) -> usize { + self.0 + } + + fn write(&mut self, bytes: &[u8]) -> gimli::write::Result<()> { + self.0 += bytes.len(); + Ok(()) + } + + fn write_at(&mut self, offset: usize, bytes: &[u8]) -> gimli::write::Result<()> { + assert!(offset + bytes.len() < self.0); + Ok(()) + } +} diff --git a/src/debuginfo/mod.rs b/src/debuginfo/mod.rs index 286e02b98..35f926099 100644 --- a/src/debuginfo/mod.rs +++ b/src/debuginfo/mod.rs @@ -1,6 +1,7 @@ //! Handling of everything related to debuginfo. mod emit; +mod gcc_except_table; mod line_info; mod object; mod types; @@ -24,7 +25,7 @@ use rustc_target::callconv::FnAbi; pub(crate) use self::emit::{DebugReloc, DebugRelocName}; pub(crate) use self::types::TypeDebugContext; -pub(crate) use self::unwind::UnwindContext; +pub(crate) use self::unwind::{EXCEPTION_HANDLER_CATCH, EXCEPTION_HANDLER_CLEANUP, UnwindContext}; use crate::debuginfo::emit::{address_for_data, address_for_func}; use crate::prelude::*; diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index 74b82a713..ff1148119 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -2,14 +2,21 @@ use cranelift_codegen::ir::Endianness; use cranelift_codegen::isa::unwind::UnwindInfo; +use cranelift_module::DataId; use cranelift_object::ObjectProduct; -use gimli::RunTimeEndian; -use gimli::write::{CieId, EhFrame, FrameTable, Section}; +use gimli::write::{Address, CieId, EhFrame, FrameTable, Section}; +use gimli::{Encoding, Format, RunTimeEndian}; -use super::emit::address_for_func; +use super::emit::{DebugRelocName, address_for_data, address_for_func}; +use super::gcc_except_table::{ + Action, ActionKind, ActionTable, CallSite, CallSiteTable, GccExceptTable, TypeInfoTable, +}; use super::object::WriteDebugInfo; use crate::prelude::*; +pub(crate) const EXCEPTION_HANDLER_CLEANUP: u32 = 0; +pub(crate) const EXCEPTION_HANDLER_CATCH: u32 = 1; + pub(crate) struct UnwindContext { endian: RunTimeEndian, frame_table: FrameTable, @@ -25,10 +32,79 @@ impl UnwindContext { let mut frame_table = FrameTable::default(); let cie_id = if let Some(mut cie) = module.isa().create_systemv_cie() { - if pic_eh_frame { - cie.fde_address_encoding = - gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0); + let ptr_encoding = if pic_eh_frame { + gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0) + } else { + gimli::DW_EH_PE_absptr + }; + + cie.fde_address_encoding = ptr_encoding; + + // FIXME only add personality function and lsda when necessary: https://github.com/rust-lang/rust/blob/1f76d219c906f0112bb1872f33aa977164c53fa6/compiler/rustc_codegen_ssa/src/mir/mod.rs#L200-L204 + if cfg!(feature = "unwinding") { + let code_ptr_encoding = if pic_eh_frame { + if module.isa().triple().architecture == target_lexicon::Architecture::X86_64 { + gimli::DwEhPe( + gimli::DW_EH_PE_indirect.0 + | gimli::DW_EH_PE_pcrel.0 + | gimli::DW_EH_PE_sdata4.0, + ) + } else if let target_lexicon::Architecture::Aarch64(_) = + module.isa().triple().architecture + { + gimli::DwEhPe( + gimli::DW_EH_PE_indirect.0 + | gimli::DW_EH_PE_pcrel.0 + | gimli::DW_EH_PE_sdata8.0, + ) + } else { + todo!() + } + } else { + gimli::DwEhPe(gimli::DW_EH_PE_indirect.0 | gimli::DW_EH_PE_absptr.0) + }; + + cie.lsda_encoding = Some(ptr_encoding); + + // FIXME use eh_personality lang item instead + let personality = module + .declare_function( + "rust_eh_personality", + Linkage::Import, + &Signature { + params: vec![ + AbiParam::new(types::I32), + AbiParam::new(types::I32), + AbiParam::new(types::I64), + AbiParam::new(module.target_config().pointer_type()), + AbiParam::new(module.target_config().pointer_type()), + ], + returns: vec![AbiParam::new(types::I32)], + call_conv: module.target_config().default_call_conv, + }, + ) + .unwrap(); + + // Use indirection here to support PIC the case where rust_eh_personality is defined in + // another DSO. + let personality_ref = module + .declare_data("DW.ref.rust_eh_personality", Linkage::Local, false, false) + .unwrap(); + + let mut personality_ref_data = DataDescription::new(); + // Note: Must not use define_zeroinit. The unwinder can't handle this being in the .bss + // section. + let pointer_bytes = usize::from(module.target_config().pointer_bytes()); + personality_ref_data.define(vec![0; pointer_bytes].into_boxed_slice()); + let personality_func_ref = + module.declare_func_in_data(personality, &mut personality_ref_data); + personality_ref_data.write_function_addr(0, personality_func_ref); + + module.define_data(personality_ref, &personality_ref_data).unwrap(); + + cie.personality = Some((code_ptr_encoding, address_for_data(personality_ref))); } + Some(frame_table.add_cie(cie)) } else { None @@ -63,8 +139,95 @@ impl UnwindContext { match unwind_info { UnwindInfo::SystemV(unwind_info) => { - self.frame_table - .add_fde(self.cie_id.unwrap(), unwind_info.to_fde(address_for_func(func_id))); + let mut fde = unwind_info.to_fde(address_for_func(func_id)); + + // FIXME only add personality function and lsda when necessary: https://github.com/rust-lang/rust/blob/1f76d219c906f0112bb1872f33aa977164c53fa6/compiler/rustc_codegen_ssa/src/mir/mod.rs#L200-L204 + if cfg!(feature = "unwinding") { + // FIXME use unique symbol name derived from function name + let lsda = module.declare_anonymous_data(false, false).unwrap(); + + let encoding = Encoding { + format: Format::Dwarf32, + version: 1, + address_size: module.isa().frontend_config().pointer_bytes(), + }; + + let mut gcc_except_table_data = GccExceptTable { + call_sites: CallSiteTable(vec![]), + actions: ActionTable::new(), + type_info: TypeInfoTable::new(gimli::DW_EH_PE_udata4), + }; + + let catch_type = gcc_except_table_data.type_info.add(Address::Constant(0)); + let catch_action = gcc_except_table_data + .actions + .add(Action { kind: ActionKind::Catch(catch_type), next_action: None }); + + for call_site in context.compiled_code().unwrap().buffer.call_sites() { + if call_site.exception_handlers.is_empty() { + gcc_except_table_data.call_sites.0.push(CallSite { + start: u64::from(call_site.ret_addr - 1), + length: 1, + landing_pad: 0, + action_entry: None, + }); + } + for &(tag, landingpad) in call_site.exception_handlers { + match tag.expand().unwrap().as_u32() { + EXCEPTION_HANDLER_CLEANUP => { + gcc_except_table_data.call_sites.0.push(CallSite { + start: u64::from(call_site.ret_addr - 1), + length: 1, + landing_pad: u64::from(landingpad), + action_entry: None, + }) + } + EXCEPTION_HANDLER_CATCH => { + gcc_except_table_data.call_sites.0.push(CallSite { + start: u64::from(call_site.ret_addr - 1), + length: 1, + landing_pad: u64::from(landingpad), + action_entry: Some(catch_action), + }) + } + _ => unreachable!(), + } + } + } + + let mut gcc_except_table = super::emit::WriterRelocate::new(self.endian); + + gcc_except_table_data.write(&mut gcc_except_table, encoding).unwrap(); + + let mut data = DataDescription::new(); + data.define(gcc_except_table.writer.into_vec().into_boxed_slice()); + data.set_segment_section("", ".gcc_except_table"); + + for reloc in &gcc_except_table.relocs { + match reloc.name { + DebugRelocName::Section(_id) => unreachable!(), + DebugRelocName::Symbol(id) => { + let id = id.try_into().unwrap(); + if id & 1 << 31 == 0 { + let func_ref = module + .declare_func_in_data(FuncId::from_u32(id), &mut data); + data.write_function_addr(reloc.offset, func_ref); + } else { + let gv = module.declare_data_in_data( + DataId::from_u32(id & !(1 << 31)), + &mut data, + ); + data.write_data_addr(reloc.offset, gv, 0); + } + } + }; + } + + module.define_data(lsda, &data).unwrap(); + fde.lsda = Some(address_for_data(lsda)); + } + + self.frame_table.add_fde(self.cie_id.unwrap(), fde); } UnwindInfo::WindowsX64(_) | UnwindInfo::WindowsArm64(_) => { // Windows does not have debug info for its unwind info. diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 120d6ff9e..37a0bdaa3 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -827,6 +827,7 @@ fn call_inline_asm<'tcx>( } let stack_slot_addr = stack_slot.get_addr(fx); + // FIXME use try_call once unwinding inline assembly is supported fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]); for (offset, place) in outputs { diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index df5748c34..1fcd58714 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -17,17 +17,19 @@ mod llvm_aarch64; mod llvm_x86; mod simd; -use cranelift_codegen::ir::AtomicRmwOp; +use cranelift_codegen::ir::{AtomicRmwOp, BlockArg, ExceptionTableData, ExceptionTag}; use rustc_middle::ty; use rustc_middle::ty::GenericArgsRef; use rustc_middle::ty::layout::ValidityRequirement; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_span::source_map::Spanned; use rustc_span::{Symbol, sym}; +use rustc_target::spec::PanicStrategy; pub(crate) use self::llvm::codegen_llvm_intrinsic_call; use crate::cast::clif_intcast; use crate::codegen_f16_f128; +use crate::debuginfo::EXCEPTION_HANDLER_CATCH; use crate::prelude::*; fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! { @@ -1352,23 +1354,75 @@ fn codegen_regular_intrinsic_call<'tcx>( } sym::catch_unwind => { + let ret_block = fx.get_block(destination.unwrap()); + intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic); let f = f.load_scalar(fx); let data = data.load_scalar(fx); - let _catch_fn = catch_fn.load_scalar(fx); + let catch_fn = catch_fn.load_scalar(fx); - // FIXME once unwinding is supported, change this to actually catch panics let f_sig = fx.bcx.func.import_signature(Signature { call_conv: fx.target_config.default_call_conv, params: vec![AbiParam::new(pointer_ty(fx.tcx))], returns: vec![], }); - fx.bcx.ins().call_indirect(f_sig, f, &[data]); + if cfg!(not(feature = "unwinding")) + || fx.tcx.sess.panic_strategy() == PanicStrategy::Abort + { + fx.bcx.ins().call_indirect(f_sig, f, &[data]); + + let layout = fx.layout_of(fx.tcx.types.i32); + let ret_val = CValue::by_val(fx.bcx.ins().iconst(types::I32, 0), layout); + ret.write_cvalue(fx, ret_val); + + fx.bcx.ins().jump(ret_block, &[]); + } else { + let catch_fn_sig = fx.bcx.func.import_signature(Signature { + call_conv: fx.target_config.default_call_conv, + params: vec![ + AbiParam::new(pointer_ty(fx.tcx)), + AbiParam::new(pointer_ty(fx.tcx)), + ], + returns: vec![], + }); + + let fallthrough_block = fx.bcx.create_block(); + let fallthrough_block_call = fx.bcx.func.dfg.block_call(fallthrough_block, &[]); + let catch_block = fx.bcx.create_block(); + let catch_block_call = + fx.bcx.func.dfg.block_call(catch_block, &[BlockArg::TryCallExn(0)]); + let exception_table = + fx.bcx.func.dfg.exception_tables.push(ExceptionTableData::new( + f_sig, + fallthrough_block_call, + [( + Some(ExceptionTag::with_number(EXCEPTION_HANDLER_CATCH).unwrap()), + catch_block_call, + )], + )); + + fx.bcx.ins().try_call_indirect(f, &[data], exception_table); + + fx.bcx.seal_block(fallthrough_block); + fx.bcx.switch_to_block(fallthrough_block); + let layout = fx.layout_of(fx.tcx.types.i32); + let ret_val = CValue::by_val(fx.bcx.ins().iconst(types::I32, 0), layout); + ret.write_cvalue(fx, ret_val); + fx.bcx.ins().jump(ret_block, &[]); + + fx.bcx.seal_block(catch_block); + fx.bcx.switch_to_block(catch_block); + fx.bcx.set_cold_block(catch_block); + let exception = fx.bcx.append_block_param(catch_block, pointer_ty(fx.tcx)); + fx.bcx.ins().call_indirect(catch_fn_sig, catch_fn, &[data, exception]); + let layout = fx.layout_of(fx.tcx.types.i32); + let ret_val = CValue::by_val(fx.bcx.ins().iconst(types::I32, 1), layout); + ret.write_cvalue(fx, ret_val); + fx.bcx.ins().jump(ret_block, &[]); + } - let layout = fx.layout_of(fx.tcx.types.i32); - let ret_val = CValue::by_val(fx.bcx.ins().iconst(types::I32, 0), layout); - ret.write_cvalue(fx, ret_val); + return Ok(()); } sym::fadd_fast