From 5fad627bc199ad6bcc97c7fd0f0b4545ba2d2a1b Mon Sep 17 00:00:00 2001 From: Matt Ickstadt Date: Sat, 24 Dec 2016 21:38:48 -0600 Subject: [PATCH 1/6] Starting to implement a frame allocator for Rustc --- configure | 1 + src/Cargo.lock | 10 ++ src/bootstrap/config.rs | 5 + src/bootstrap/config.toml.example | 3 + src/bootstrap/lib.rs | 6 + src/liballoc_frame/Cargo.toml | 13 ++ src/liballoc_frame/lib.rs | 191 ++++++++++++++++++++++++++++++ src/libstd/Cargo.toml | 1 + src/libstd/lib.rs | 2 +- src/rustc/Cargo.toml | 1 + src/rustc/rustc.rs | 5 + src/rustc/rustdoc.rs | 5 + src/rustc/std_shim/Cargo.toml | 1 + src/tools/tidy/src/pal.rs | 1 + 14 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 src/liballoc_frame/Cargo.toml create mode 100644 src/liballoc_frame/lib.rs diff --git a/configure b/configure index 4f1e8f656ae22..0514cb3e2d75b 100755 --- a/configure +++ b/configure @@ -637,6 +637,7 @@ opt codegen-tests 1 "run the src/test/codegen tests" opt option-checking 1 "complain about unrecognized options in this configure script" opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)" opt vendor 0 "enable usage of vendored Rust crates" +opt frame-alloc 0 "enable the frame allocator" # Optimization and debugging options. These may be overridden by the release channel, etc. opt_nosave optimize 1 "build optimized rust code" diff --git a/src/Cargo.lock b/src/Cargo.lock index 9cd77e71b82dd..ccaff65ba3333 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -13,6 +13,14 @@ dependencies = [ "core 0.0.0", ] +[[package]] +name = "alloc_frame" +version = "0.0.0" +dependencies = [ + "core 0.0.0", + "libc 0.0.0", +] + [[package]] name = "alloc_jemalloc" version = "0.0.0" @@ -259,6 +267,7 @@ dependencies = [ name = "rustc-main" version = "0.0.0" dependencies = [ + "alloc_frame 0.0.0", "rustc_back 0.0.0", "rustc_driver 0.0.0", "rustdoc 0.0.0", @@ -581,6 +590,7 @@ name = "std" version = "0.0.0" dependencies = [ "alloc 0.0.0", + "alloc_frame 0.0.0", "alloc_jemalloc 0.0.0", "alloc_system 0.0.0", "build_helper 0.1.0", diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 6b86e537b7d22..d6558b7e21d1d 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -77,6 +77,7 @@ pub struct Config { // libstd features pub debug_jemalloc: bool, pub use_jemalloc: bool, + pub use_alloc_frame: bool, pub backtrace: bool, // support for RUST_BACKTRACE // misc @@ -176,6 +177,7 @@ struct Rust { debuginfo_lines: Option, debug_jemalloc: Option, use_jemalloc: Option, + use_alloc_frame: Option, backtrace: Option, default_linker: Option, default_ar: Option, @@ -203,6 +205,7 @@ impl Config { let mut config = Config::default(); config.llvm_optimize = true; config.use_jemalloc = true; + config.use_alloc_frame = false; config.backtrace = true; config.rust_optimize = true; config.rust_optimize_tests = true; @@ -298,6 +301,7 @@ impl Config { set(&mut config.rust_rpath, rust.rpath); set(&mut config.debug_jemalloc, rust.debug_jemalloc); set(&mut config.use_jemalloc, rust.use_jemalloc); + set(&mut config.use_alloc_frame, rust.use_alloc_frame); set(&mut config.backtrace, rust.backtrace); set(&mut config.channel, rust.channel.clone()); config.rustc_default_linker = rust.default_linker.clone(); @@ -385,6 +389,7 @@ impl Config { ("DEBUGINFO_LINES", self.rust_debuginfo_lines), ("JEMALLOC", self.use_jemalloc), ("DEBUG_JEMALLOC", self.debug_jemalloc), + ("FRAME_ALLOC", self.use_alloc_frame), ("RPATH", self.rust_rpath), ("OPTIMIZE_TESTS", self.rust_optimize_tests), ("DEBUGINFO_TESTS", self.rust_debuginfo_tests), diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index 5fc095137c793..0ac5aa62f5401 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -137,6 +137,9 @@ # Whether or not jemalloc is built with its debug option set #debug-jemalloc = false +# Whether or not the frame allocator is built and enabled +#use-alloc-frame = false + # Whether or not `panic!`s generate backtraces (RUST_BACKTRACE) #backtrace = true diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 665e0c67b7f6c..e5919499a89ca 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -609,6 +609,9 @@ impl Build { if self.config.use_jemalloc { features.push_str(" jemalloc"); } + if self.config.use_alloc_frame { + features.push_str(" alloc_frame"); + } if self.config.backtrace { features.push_str(" backtrace"); } @@ -621,6 +624,9 @@ impl Build { if self.config.use_jemalloc { features.push_str(" jemalloc"); } + if self.config.use_alloc_frame { + features.push_str(" alloc_frame"); + } return features } diff --git a/src/liballoc_frame/Cargo.toml b/src/liballoc_frame/Cargo.toml new file mode 100644 index 0000000000000..b5c34cc2dd025 --- /dev/null +++ b/src/liballoc_frame/Cargo.toml @@ -0,0 +1,13 @@ +[package] +authors = ["The Rust Project Developers"] +name = "alloc_frame" +version = "0.0.0" + +[lib] +name = "alloc_frame" +path = "lib.rs" +test = false + +[dependencies] +core = { path = "../libcore" } +libc = { path = "../rustc/libc_shim" } diff --git a/src/liballoc_frame/lib.rs b/src/liballoc_frame/lib.rs new file mode 100644 index 0000000000000..1cb82f5fba97e --- /dev/null +++ b/src/liballoc_frame/lib.rs @@ -0,0 +1,191 @@ +// Copyright 2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![crate_name = "alloc_frame"] +#![crate_type = "rlib"] +#![no_std] +#![allocator] +#![cfg_attr(not(stage0), deny(warnings))] +#![unstable(feature = "alloc_frame", + reason = "this library is unlikely to be stabilized in its current \ + form or name", + issue = "0")] +#![feature(allocator)] +#![feature(const_fn)] +#![feature(staged_api)] +#![feature(libc)] +#![cfg_attr(any(unix, target_os = "redox"), feature(libc))] + +extern crate libc; + +use core::ptr; + +// The minimum alignment guaranteed by the architecture. This value is used to +// add fast paths for low alignment values. In practice, the alignment is a +// constant at the call site and the branch will be optimized out. +#[cfg(all(any(target_arch = "x86", + target_arch = "arm", + target_arch = "mips", + target_arch = "powerpc", + target_arch = "powerpc64", + target_arch = "asmjs", + target_arch = "wasm32")))] +const MIN_ALIGN: usize = 8; +#[cfg(all(any(target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "mips64", + target_arch = "s390x")))] +const MIN_ALIGN: usize = 16; + +// The size of the chunk which is actually allocated +const CHUNK_SIZE: usize = 4096 * 16; +const CHUNK_ALIGN: usize = 4096; + +static mut HEAP: *mut u8 = ptr::null_mut(); +static mut HEAP_LEFT: usize = 0; + +#[no_mangle] +pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 { + let new_align = if align < MIN_ALIGN { MIN_ALIGN } else { align }; + let new_size = (size + new_align - 1) & !(new_align - 1); + + unsafe { + if new_size < HEAP_LEFT { + HEAP_LEFT -= new_size; + let p = HEAP; + HEAP = HEAP.offset(new_size as isize); + return p; + } else if new_size > CHUNK_SIZE { + return imp::allocate(size, align); + } else { + HEAP_LEFT = CHUNK_SIZE - new_size; + let p = imp::allocate(CHUNK_SIZE, CHUNK_ALIGN); + HEAP = p.offset(new_size as isize); + return p; + } + } +} + +#[no_mangle] +pub extern "C" fn __rust_deallocate(_ptr: *mut u8, _old_size: usize, _align: usize) { +} + +#[no_mangle] +pub extern "C" fn __rust_reallocate(ptr: *mut u8, + old_size: usize, + size: usize, + align: usize) + -> *mut u8 { + let new_ptr = __rust_allocate(size, align); + unsafe { libc::memcpy(new_ptr as *mut _, ptr as *mut _, old_size); } + new_ptr +} + +#[no_mangle] +pub extern "C" fn __rust_reallocate_inplace(_ptr: *mut u8, + old_size: usize, + _size: usize, + _align: usize) + -> usize { + old_size +} + +#[no_mangle] +pub extern "C" fn __rust_usable_size(size: usize, _align: usize) -> usize { + size +} + +#[cfg(any(unix, target_os = "redox"))] +mod imp { + use core::cmp; + use MIN_ALIGN; + + pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 { + if align <= MIN_ALIGN { + libc::malloc(size as libc::size_t) as *mut u8 + } else { + aligned_malloc(size, align) + } + } + + #[cfg(any(target_os = "android", target_os = "redox"))] + unsafe fn aligned_malloc(size: usize, align: usize) -> *mut u8 { + // On android we currently target API level 9 which unfortunately + // doesn't have the `posix_memalign` API used below. Instead we use + // `memalign`, but this unfortunately has the property on some systems + // where the memory returned cannot be deallocated by `free`! + // + // Upon closer inspection, however, this appears to work just fine with + // Android, so for this platform we should be fine to call `memalign` + // (which is present in API level 9). Some helpful references could + // possibly be chromium using memalign [1], attempts at documenting that + // memalign + free is ok [2] [3], or the current source of chromium + // which still uses memalign on android [4]. + // + // [1]: https://codereview.chromium.org/10796020/ + // [2]: https://code.google.com/p/android/issues/detail?id=35391 + // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579 + // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/ + // /memory/aligned_memory.cc + libc::memalign(align as libc::size_t, size as libc::size_t) as *mut u8 + } + + #[cfg(not(any(target_os = "android", target_os = "redox")))] + unsafe fn aligned_malloc(size: usize, align: usize) -> *mut u8 { + let mut out = ptr::null_mut(); + let ret = libc::posix_memalign(&mut out, align as libc::size_t, size as libc::size_t); + if ret != 0 { + ptr::null_mut() + } else { + out as *mut u8 + } + } +} + +#[cfg(windows)] +#[allow(bad_style)] +mod imp { + use MIN_ALIGN; + + type LPVOID = *mut u8; + type HANDLE = LPVOID; + type SIZE_T = usize; + type DWORD = u32; + + extern "system" { + fn GetProcessHeap() -> HANDLE; + fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID; + } + + #[repr(C)] + struct Header(*mut u8); + + unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header { + &mut *(ptr as *mut Header).offset(-1) + } + + unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 { + let aligned = ptr.offset((align - (ptr as usize & (align - 1))) as isize); + *get_header(aligned) = Header(ptr); + aligned + } + + pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 { + if align <= MIN_ALIGN { + HeapAlloc(GetProcessHeap(), 0, size as SIZE_T) as *mut u8 + } else { + let ptr = HeapAlloc(GetProcessHeap(), 0, (size + align) as SIZE_T) as *mut u8; + if ptr.is_null() { + return ptr; + } + align_ptr(ptr, align) + } + } +} diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml index fcf84cb716917..aa92003f28c17 100644 --- a/src/libstd/Cargo.toml +++ b/src/libstd/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["dylib", "rlib"] alloc = { path = "../liballoc" } alloc_jemalloc = { path = "../liballoc_jemalloc", optional = true } alloc_system = { path = "../liballoc_system" } +alloc_frame = { path = "../liballoc_frame", optional = true } panic_unwind = { path = "../libpanic_unwind", optional = true } panic_abort = { path = "../libpanic_abort" } collections = { path = "../libcollections" } diff --git a/src/libstd/lib.rs b/src/libstd/lib.rs index fc5c6968544e8..3205b80ff7b63 100644 --- a/src/libstd/lib.rs +++ b/src/libstd/lib.rs @@ -331,7 +331,7 @@ extern crate libc; // We always need an unwinder currently for backtraces extern crate unwind; -#[cfg(stage0)] +#[cfg(all(stage0, not(alloc_frame)))] extern crate alloc_system; // compiler-rt intrinsics diff --git a/src/rustc/Cargo.toml b/src/rustc/Cargo.toml index dce1a0a8ec858..0f495755aa72f 100644 --- a/src/rustc/Cargo.toml +++ b/src/rustc/Cargo.toml @@ -17,6 +17,7 @@ path = "rustdoc.rs" rustc_back = { path = "../librustc_back" } rustc_driver = { path = "../librustc_driver" } rustdoc = { path = "../librustdoc" } +alloc_frame = { path = "../liballoc_frame", optional = true } [features] jemalloc = ["rustc_back/jemalloc"] diff --git a/src/rustc/rustc.rs b/src/rustc/rustc.rs index bfd01146d2c46..aac912c25a025 100644 --- a/src/rustc/rustc.rs +++ b/src/rustc/rustc.rs @@ -9,7 +9,12 @@ // except according to those terms. #![feature(rustc_private)] +#![cfg_attr(feature = "alloc_frame", feature(alloc_frame))] extern crate rustc_driver; +// Use the frame allocator to speed up runtime +#[cfg(feature = "alloc_frame")] +extern crate alloc_frame; + fn main() { rustc_driver::main() } diff --git a/src/rustc/rustdoc.rs b/src/rustc/rustdoc.rs index 6fecd3a27a8a4..046f2c0ef9bdb 100644 --- a/src/rustc/rustdoc.rs +++ b/src/rustc/rustdoc.rs @@ -9,7 +9,12 @@ // except according to those terms. #![feature(rustdoc)] +#![cfg_attr(feature = "alloc_frame", feature(alloc_frame))] extern crate rustdoc; +// Use the frame allocator to speed up runtime +#[cfg(feature = "alloc_frame")] +extern crate alloc_frame; + fn main() { rustdoc::main() } diff --git a/src/rustc/std_shim/Cargo.toml b/src/rustc/std_shim/Cargo.toml index 18680dc4fd9a1..c3e09a1685b95 100644 --- a/src/rustc/std_shim/Cargo.toml +++ b/src/rustc/std_shim/Cargo.toml @@ -37,4 +37,5 @@ core = { path = "../../libcore" } backtrace = ["std/backtrace"] debug-jemalloc = ["std/debug-jemalloc"] jemalloc = ["std/jemalloc"] +alloc_frame = ["std/alloc_frame"] panic-unwind = ["std/panic-unwind"] diff --git a/src/tools/tidy/src/pal.rs b/src/tools/tidy/src/pal.rs index a5e4e5a4c2672..4287fb6ba826e 100644 --- a/src/tools/tidy/src/pal.rs +++ b/src/tools/tidy/src/pal.rs @@ -53,6 +53,7 @@ const EXCEPTION_PATHS: &'static [&'static str] = &[ // std crates "src/liballoc_jemalloc", "src/liballoc_system", + "src/liballoc_frame", "src/liblibc", "src/libpanic_abort", "src/libpanic_unwind", From 553f5bee9b54c3515bf59fa1aef93cf1d6bb1bea Mon Sep 17 00:00:00 2001 From: Matt Ickstadt Date: Sun, 25 Dec 2016 15:30:24 -0600 Subject: [PATCH 2/6] Hack to enable alloc_frame for stage1+ rustc --- configure | 8 +++++++- src/Cargo.lock | 1 - src/bootstrap/config.rs | 10 +++++----- src/bootstrap/config.toml.example | 5 ++++- src/bootstrap/lib.rs | 7 ++----- src/librustc_driver/Cargo.toml | 3 +++ src/librustc_metadata/Cargo.toml | 3 +++ src/librustc_metadata/creader.rs | 5 ++++- src/librustdoc/Cargo.toml | 3 +++ src/libstd/Cargo.toml | 1 - src/libstd/lib.rs | 2 +- src/rustc/Cargo.toml | 1 + src/rustc/rustc.rs | 5 +++-- src/rustc/rustdoc.rs | 5 +++-- src/rustc/std_shim/Cargo.toml | 1 - src/tools/tidy/src/pal.rs | 1 + 16 files changed, 40 insertions(+), 21 deletions(-) diff --git a/configure b/configure index 0514cb3e2d75b..213c105efb1c2 100755 --- a/configure +++ b/configure @@ -637,7 +637,7 @@ opt codegen-tests 1 "run the src/test/codegen tests" opt option-checking 1 "complain about unrecognized options in this configure script" opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)" opt vendor 0 "enable usage of vendored Rust crates" -opt frame-alloc 0 "enable the frame allocator" +opt rustc-alloc-frame 0 "enable the frame allocator in rustc" # Optimization and debugging options. These may be overridden by the release channel, etc. opt_nosave optimize 1 "build optimized rust code" @@ -1065,6 +1065,12 @@ else CFG_USING_LIBCPP="0" fi +# Disable jemalloc if using alloc frame. +if [ -n "$CFG_ENABLE_RUSTC_ALLOC_FRAME" ] +then + CFG_DISABLE_JEMALLOC=1 +fi + # Same with jemalloc. save the setting here. if [ -n "$CFG_DISABLE_JEMALLOC" ] then diff --git a/src/Cargo.lock b/src/Cargo.lock index ccaff65ba3333..d25232213da8a 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -590,7 +590,6 @@ name = "std" version = "0.0.0" dependencies = [ "alloc 0.0.0", - "alloc_frame 0.0.0", "alloc_jemalloc 0.0.0", "alloc_system 0.0.0", "build_helper 0.1.0", diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index d6558b7e21d1d..f7fae9f3bfcd6 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -77,10 +77,10 @@ pub struct Config { // libstd features pub debug_jemalloc: bool, pub use_jemalloc: bool, - pub use_alloc_frame: bool, pub backtrace: bool, // support for RUST_BACKTRACE // misc + pub rustc_alloc_frame: bool, pub channel: String, pub quiet_tests: bool, // Fallback musl-root for all targets @@ -177,7 +177,7 @@ struct Rust { debuginfo_lines: Option, debug_jemalloc: Option, use_jemalloc: Option, - use_alloc_frame: Option, + rustc_alloc_frame: Option, backtrace: Option, default_linker: Option, default_ar: Option, @@ -205,7 +205,7 @@ impl Config { let mut config = Config::default(); config.llvm_optimize = true; config.use_jemalloc = true; - config.use_alloc_frame = false; + config.rustc_alloc_frame = false; config.backtrace = true; config.rust_optimize = true; config.rust_optimize_tests = true; @@ -301,9 +301,9 @@ impl Config { set(&mut config.rust_rpath, rust.rpath); set(&mut config.debug_jemalloc, rust.debug_jemalloc); set(&mut config.use_jemalloc, rust.use_jemalloc); - set(&mut config.use_alloc_frame, rust.use_alloc_frame); set(&mut config.backtrace, rust.backtrace); set(&mut config.channel, rust.channel.clone()); + set(&mut config.rustc_alloc_frame, rust.rustc_alloc_frame); config.rustc_default_linker = rust.default_linker.clone(); config.rustc_default_ar = rust.default_ar.clone(); config.musl_root = rust.musl_root.clone().map(PathBuf::from); @@ -389,7 +389,7 @@ impl Config { ("DEBUGINFO_LINES", self.rust_debuginfo_lines), ("JEMALLOC", self.use_jemalloc), ("DEBUG_JEMALLOC", self.debug_jemalloc), - ("FRAME_ALLOC", self.use_alloc_frame), + ("RUSTC_ALLOC_FRAME", self.rustc_alloc_frame), ("RPATH", self.rust_rpath), ("OPTIMIZE_TESTS", self.rust_optimize_tests), ("DEBUGINFO_TESTS", self.rust_debuginfo_tests), diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index 0ac5aa62f5401..717083ec75d59 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -137,9 +137,12 @@ # Whether or not jemalloc is built with its debug option set #debug-jemalloc = false -# Whether or not the frame allocator is built and enabled +# Whether or not the frame allocator is built and enabled in std #use-alloc-frame = false +# Whether or not the frame allocator is built and enabled in rustc +#rustc-use-alloc-frame = false + # Whether or not `panic!`s generate backtraces (RUST_BACKTRACE) #backtrace = true diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index e5919499a89ca..2bfd38218aa18 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -609,9 +609,6 @@ impl Build { if self.config.use_jemalloc { features.push_str(" jemalloc"); } - if self.config.use_alloc_frame { - features.push_str(" alloc_frame"); - } if self.config.backtrace { features.push_str(" backtrace"); } @@ -624,8 +621,8 @@ impl Build { if self.config.use_jemalloc { features.push_str(" jemalloc"); } - if self.config.use_alloc_frame { - features.push_str(" alloc_frame"); + if self.config.rustc_alloc_frame { + features.push_str(" rustc_alloc_frame"); } return features } diff --git a/src/librustc_driver/Cargo.toml b/src/librustc_driver/Cargo.toml index 99d3e155e8936..8c2e7f4c90446 100644 --- a/src/librustc_driver/Cargo.toml +++ b/src/librustc_driver/Cargo.toml @@ -36,3 +36,6 @@ serialize = { path = "../libserialize" } syntax = { path = "../libsyntax" } syntax_ext = { path = "../libsyntax_ext" } syntax_pos = { path = "../libsyntax_pos" } + +[features] +rustc_alloc_frame = ["rustc_metadata/rustc_alloc_frame"] diff --git a/src/librustc_metadata/Cargo.toml b/src/librustc_metadata/Cargo.toml index 6f7f03ca216b9..16a8760e722c8 100644 --- a/src/librustc_metadata/Cargo.toml +++ b/src/librustc_metadata/Cargo.toml @@ -22,3 +22,6 @@ serialize = { path = "../libserialize" } syntax = { path = "../libsyntax" } syntax_ext = { path = "../libsyntax_ext" } syntax_pos = { path = "../libsyntax_pos" } + +[features] +rustc_alloc_frame = [] diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index a9af4118c5957..3907c29fb4bff 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -820,7 +820,10 @@ impl<'a> CrateLoader<'a> { // * Binaries use jemalloc // * Staticlibs and Rust dylibs use system malloc // * Rust dylibs used as dependencies to rust use jemalloc - let name = if need_lib_alloc && !self.sess.opts.cg.prefer_dynamic { + let name = if cfg!(rustc_alloc_frame) && (cfg!(stage0) || cfg!(stage1)) { + // HACK to make stage1/2 with alloc_frame + Symbol::intern(&"alloc_frame") + } else if need_lib_alloc && !self.sess.opts.cg.prefer_dynamic { Symbol::intern(&self.sess.target.target.options.lib_allocation_crate) } else { Symbol::intern(&self.sess.target.target.options.exe_allocation_crate) diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index d66d2001f2304..66dcc37481f02 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -30,3 +30,6 @@ log = { path = "../liblog" } [build-dependencies] build_helper = { path = "../build_helper" } gcc = "0.3.27" + +[features] +rustc_alloc_frame = ["rustc_driver/rustc_alloc_frame", "rustc_metadata/rustc_alloc_frame"] diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml index aa92003f28c17..fcf84cb716917 100644 --- a/src/libstd/Cargo.toml +++ b/src/libstd/Cargo.toml @@ -13,7 +13,6 @@ crate-type = ["dylib", "rlib"] alloc = { path = "../liballoc" } alloc_jemalloc = { path = "../liballoc_jemalloc", optional = true } alloc_system = { path = "../liballoc_system" } -alloc_frame = { path = "../liballoc_frame", optional = true } panic_unwind = { path = "../libpanic_unwind", optional = true } panic_abort = { path = "../libpanic_abort" } collections = { path = "../libcollections" } diff --git a/src/libstd/lib.rs b/src/libstd/lib.rs index 3205b80ff7b63..fc5c6968544e8 100644 --- a/src/libstd/lib.rs +++ b/src/libstd/lib.rs @@ -331,7 +331,7 @@ extern crate libc; // We always need an unwinder currently for backtraces extern crate unwind; -#[cfg(all(stage0, not(alloc_frame)))] +#[cfg(stage0)] extern crate alloc_system; // compiler-rt intrinsics diff --git a/src/rustc/Cargo.toml b/src/rustc/Cargo.toml index 0f495755aa72f..9f867f246d2b1 100644 --- a/src/rustc/Cargo.toml +++ b/src/rustc/Cargo.toml @@ -21,3 +21,4 @@ alloc_frame = { path = "../liballoc_frame", optional = true } [features] jemalloc = ["rustc_back/jemalloc"] +rustc_alloc_frame = ["alloc_frame", "rustc_driver/rustc_alloc_frame"] diff --git a/src/rustc/rustc.rs b/src/rustc/rustc.rs index aac912c25a025..5489288e74c30 100644 --- a/src/rustc/rustc.rs +++ b/src/rustc/rustc.rs @@ -9,12 +9,13 @@ // except according to those terms. #![feature(rustc_private)] -#![cfg_attr(feature = "alloc_frame", feature(alloc_frame))] +#![feature(staged_api)] +#![cfg_attr(all(feature = "rustc_alloc_frame", not(stage0)), feature(alloc_frame))] extern crate rustc_driver; // Use the frame allocator to speed up runtime -#[cfg(feature = "alloc_frame")] +#[cfg(all(feature = "rustc_alloc_frame", not(stage0)))] extern crate alloc_frame; fn main() { rustc_driver::main() } diff --git a/src/rustc/rustdoc.rs b/src/rustc/rustdoc.rs index 046f2c0ef9bdb..4a58b63b5f209 100644 --- a/src/rustc/rustdoc.rs +++ b/src/rustc/rustdoc.rs @@ -9,12 +9,13 @@ // except according to those terms. #![feature(rustdoc)] -#![cfg_attr(feature = "alloc_frame", feature(alloc_frame))] +#![feature(staged_api)] +#![cfg_attr(all(feature = "rustc_alloc_frame", not(stage0)), feature(alloc_frame))] extern crate rustdoc; // Use the frame allocator to speed up runtime -#[cfg(feature = "alloc_frame")] +#[cfg(all(feature = "rustc_alloc_frame", not(stage0)))] extern crate alloc_frame; fn main() { rustdoc::main() } diff --git a/src/rustc/std_shim/Cargo.toml b/src/rustc/std_shim/Cargo.toml index c3e09a1685b95..18680dc4fd9a1 100644 --- a/src/rustc/std_shim/Cargo.toml +++ b/src/rustc/std_shim/Cargo.toml @@ -37,5 +37,4 @@ core = { path = "../../libcore" } backtrace = ["std/backtrace"] debug-jemalloc = ["std/debug-jemalloc"] jemalloc = ["std/jemalloc"] -alloc_frame = ["std/alloc_frame"] panic-unwind = ["std/panic-unwind"] diff --git a/src/tools/tidy/src/pal.rs b/src/tools/tidy/src/pal.rs index 4287fb6ba826e..9314d49bedf87 100644 --- a/src/tools/tidy/src/pal.rs +++ b/src/tools/tidy/src/pal.rs @@ -28,6 +28,7 @@ //! - core may not have platform-specific code //! - liballoc_system may have platform-specific code //! - liballoc_jemalloc may have platform-specific code +//! - liballoc_frame may have platform-specific code //! - libpanic_abort may have platform-specific code //! - libpanic_unwind may have platform-specific code //! - libunwind may have platform-specific code From 5c4aadf4f629a63412416f5642315434fe77d1be Mon Sep 17 00:00:00 2001 From: Matt Ickstadt Date: Fri, 30 Dec 2016 17:19:59 -0600 Subject: [PATCH 3/6] Working?! --- src/Cargo.lock | 1 + src/liballoc_frame/lib.rs | 4 ++-- src/librustc_metadata/creader.rs | 12 ++++++++---- src/libstd/Cargo.toml | 1 + src/rustc/rustc.rs | 4 ++-- src/rustc/rustdoc.rs | 4 ++-- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index d25232213da8a..ccaff65ba3333 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -590,6 +590,7 @@ name = "std" version = "0.0.0" dependencies = [ "alloc 0.0.0", + "alloc_frame 0.0.0", "alloc_jemalloc 0.0.0", "alloc_system 0.0.0", "build_helper 0.1.0", diff --git a/src/liballoc_frame/lib.rs b/src/liballoc_frame/lib.rs index 1cb82f5fba97e..271f3a80fd45e 100644 --- a/src/liballoc_frame/lib.rs +++ b/src/liballoc_frame/lib.rs @@ -21,7 +21,6 @@ #![feature(const_fn)] #![feature(staged_api)] #![feature(libc)] -#![cfg_attr(any(unix, target_os = "redox"), feature(libc))] extern crate libc; @@ -104,7 +103,8 @@ pub extern "C" fn __rust_usable_size(size: usize, _align: usize) -> usize { #[cfg(any(unix, target_os = "redox"))] mod imp { - use core::cmp; + use libc; + use core::ptr; use MIN_ALIGN; pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 { diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index 3907c29fb4bff..1245d4c352a1e 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -771,14 +771,17 @@ impl<'a> CrateLoader<'a> { // also bail out as we don't need to implicitly inject one. let mut needs_allocator = false; let mut found_required_allocator = false; + println!("injecting allocator for {}", self.local_crate_name); self.cstore.iter_crate_data(|cnum, data| { needs_allocator = needs_allocator || data.needs_allocator(); if data.is_allocator() { - info!("{} required by rlib and is an allocator", data.name()); + println!("{} required by rlib and is an allocator", data.name()); self.inject_dependency_if(cnum, "an allocator", &|data| data.needs_allocator()); - found_required_allocator = found_required_allocator || - data.dep_kind.get() == DepKind::Explicit; + let explicit_dep = data.dep_kind.get() == DepKind::Explicit; + println!("{} {} an explicit dependency", data.name(), + if explicit_dep {"is"} else {"is not"}); + found_required_allocator = found_required_allocator || explicit_dep; } }); if !needs_allocator || found_required_allocator { return } @@ -820,7 +823,7 @@ impl<'a> CrateLoader<'a> { // * Binaries use jemalloc // * Staticlibs and Rust dylibs use system malloc // * Rust dylibs used as dependencies to rust use jemalloc - let name = if cfg!(rustc_alloc_frame) && (cfg!(stage0) || cfg!(stage1)) { + let name = if cfg!(feature = "rustc_alloc_frame") && cfg!(stage0) { // HACK to make stage1/2 with alloc_frame Symbol::intern(&"alloc_frame") } else if need_lib_alloc && !self.sess.opts.cg.prefer_dynamic { @@ -828,6 +831,7 @@ impl<'a> CrateLoader<'a> { } else { Symbol::intern(&self.sess.target.target.options.exe_allocation_crate) }; + println!("Injecting {} for {}", name, self.local_crate_name); let dep_kind = DepKind::Implicit; let (cnum, data) = self.resolve_crate(&None, name, name, None, DUMMY_SP, PathKind::Crate, dep_kind); diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml index fcf84cb716917..34b1fe457dc1e 100644 --- a/src/libstd/Cargo.toml +++ b/src/libstd/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["dylib", "rlib"] alloc = { path = "../liballoc" } alloc_jemalloc = { path = "../liballoc_jemalloc", optional = true } alloc_system = { path = "../liballoc_system" } +alloc_frame = { path = "../liballoc_frame" } panic_unwind = { path = "../libpanic_unwind", optional = true } panic_abort = { path = "../libpanic_abort" } collections = { path = "../libcollections" } diff --git a/src/rustc/rustc.rs b/src/rustc/rustc.rs index 5489288e74c30..c65a2e8abbf12 100644 --- a/src/rustc/rustc.rs +++ b/src/rustc/rustc.rs @@ -10,12 +10,12 @@ #![feature(rustc_private)] #![feature(staged_api)] -#![cfg_attr(all(feature = "rustc_alloc_frame", not(stage0)), feature(alloc_frame))] +#![cfg_attr(all(feature = "rustc_alloc_frame", stage1), feature(alloc_frame))] extern crate rustc_driver; // Use the frame allocator to speed up runtime -#[cfg(all(feature = "rustc_alloc_frame", not(stage0)))] +#[cfg(all(feature = "rustc_alloc_frame", stage1))] extern crate alloc_frame; fn main() { rustc_driver::main() } diff --git a/src/rustc/rustdoc.rs b/src/rustc/rustdoc.rs index 4a58b63b5f209..1fe87275222cb 100644 --- a/src/rustc/rustdoc.rs +++ b/src/rustc/rustdoc.rs @@ -10,12 +10,12 @@ #![feature(rustdoc)] #![feature(staged_api)] -#![cfg_attr(all(feature = "rustc_alloc_frame", not(stage0)), feature(alloc_frame))] +#![cfg_attr(all(feature = "rustc_alloc_frame", stage1), feature(alloc_frame))] extern crate rustdoc; // Use the frame allocator to speed up runtime -#[cfg(all(feature = "rustc_alloc_frame", not(stage0)))] +#[cfg(all(feature = "rustc_alloc_frame", stage1))] extern crate alloc_frame; fn main() { rustdoc::main() } From 6eec9415648f6902d4899576e876b83965d6e4fc Mon Sep 17 00:00:00 2001 From: Matt Ickstadt Date: Fri, 30 Dec 2016 18:24:49 -0600 Subject: [PATCH 4/6] Remove debug print statemtnts --- src/librustc_metadata/creader.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index 1245d4c352a1e..a47f6c7335632 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -771,17 +771,14 @@ impl<'a> CrateLoader<'a> { // also bail out as we don't need to implicitly inject one. let mut needs_allocator = false; let mut found_required_allocator = false; - println!("injecting allocator for {}", self.local_crate_name); self.cstore.iter_crate_data(|cnum, data| { needs_allocator = needs_allocator || data.needs_allocator(); if data.is_allocator() { - println!("{} required by rlib and is an allocator", data.name()); + info!("{} required by rlib and is an allocator", data.name()); self.inject_dependency_if(cnum, "an allocator", &|data| data.needs_allocator()); - let explicit_dep = data.dep_kind.get() == DepKind::Explicit; - println!("{} {} an explicit dependency", data.name(), - if explicit_dep {"is"} else {"is not"}); - found_required_allocator = found_required_allocator || explicit_dep; + found_required_allocator = found_required_allocator || + data.dep_kind.get() == DepKind::Explicit; } }); if !needs_allocator || found_required_allocator { return } @@ -824,14 +821,13 @@ impl<'a> CrateLoader<'a> { // * Staticlibs and Rust dylibs use system malloc // * Rust dylibs used as dependencies to rust use jemalloc let name = if cfg!(feature = "rustc_alloc_frame") && cfg!(stage0) { - // HACK to make stage1/2 with alloc_frame + // HACK to make stage1 with alloc_frame Symbol::intern(&"alloc_frame") } else if need_lib_alloc && !self.sess.opts.cg.prefer_dynamic { Symbol::intern(&self.sess.target.target.options.lib_allocation_crate) } else { Symbol::intern(&self.sess.target.target.options.exe_allocation_crate) }; - println!("Injecting {} for {}", name, self.local_crate_name); let dep_kind = DepKind::Implicit; let (cnum, data) = self.resolve_crate(&None, name, name, None, DUMMY_SP, PathKind::Crate, dep_kind); From 8be59fa2882a4eb98d43fdce1e4f74e5bc495554 Mon Sep 17 00:00:00 2001 From: Matt Ickstadt Date: Fri, 30 Dec 2016 19:02:42 -0600 Subject: [PATCH 5/6] Use atomic operations for statics --- src/liballoc_frame/lib.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/liballoc_frame/lib.rs b/src/liballoc_frame/lib.rs index 271f3a80fd45e..81da27d403f97 100644 --- a/src/liballoc_frame/lib.rs +++ b/src/liballoc_frame/lib.rs @@ -25,6 +25,7 @@ extern crate libc; use core::ptr; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; // The minimum alignment guaranteed by the architecture. This value is used to // add fast paths for low alignment values. In practice, the alignment is a @@ -47,8 +48,8 @@ const MIN_ALIGN: usize = 16; const CHUNK_SIZE: usize = 4096 * 16; const CHUNK_ALIGN: usize = 4096; -static mut HEAP: *mut u8 = ptr::null_mut(); -static mut HEAP_LEFT: usize = 0; +static HEAP: AtomicPtr = AtomicPtr::new(ptr::null_mut()); +static HEAP_LEFT: AtomicUsize = AtomicUsize::new(0); #[no_mangle] pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 { @@ -56,17 +57,20 @@ pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 { let new_size = (size + new_align - 1) & !(new_align - 1); unsafe { - if new_size < HEAP_LEFT { - HEAP_LEFT -= new_size; - let p = HEAP; - HEAP = HEAP.offset(new_size as isize); - return p; - } else if new_size > CHUNK_SIZE { + if new_size > CHUNK_SIZE { return imp::allocate(size, align); + } + + let heap = HEAP.load(Ordering::SeqCst); + let heap_left = HEAP_LEFT.load(Ordering::SeqCst); + if new_size < heap_left { + HEAP_LEFT.store(heap_left - new_size, Ordering::SeqCst); + HEAP.store(heap.offset(new_size as isize), Ordering::SeqCst); + return heap; } else { - HEAP_LEFT = CHUNK_SIZE - new_size; + HEAP_LEFT.store(CHUNK_SIZE - new_size, Ordering::SeqCst); let p = imp::allocate(CHUNK_SIZE, CHUNK_ALIGN); - HEAP = p.offset(new_size as isize); + HEAP.store(p.offset(new_size as isize), Ordering::SeqCst); return p; } } From 39af02b1328e31d8a06288bce04d6bb411ac76d7 Mon Sep 17 00:00:00 2001 From: Matt Ickstadt Date: Fri, 30 Dec 2016 19:48:04 -0600 Subject: [PATCH 6/6] Use an atomic spinlock for the heap --- src/liballoc_frame/lib.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/liballoc_frame/lib.rs b/src/liballoc_frame/lib.rs index 81da27d403f97..8a7a8e77e5e77 100644 --- a/src/liballoc_frame/lib.rs +++ b/src/liballoc_frame/lib.rs @@ -25,7 +25,7 @@ extern crate libc; use core::ptr; -use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicBool, Ordering}; // The minimum alignment guaranteed by the architecture. This value is used to // add fast paths for low alignment values. In practice, the alignment is a @@ -48,8 +48,9 @@ const MIN_ALIGN: usize = 16; const CHUNK_SIZE: usize = 4096 * 16; const CHUNK_ALIGN: usize = 4096; -static HEAP: AtomicPtr = AtomicPtr::new(ptr::null_mut()); -static HEAP_LEFT: AtomicUsize = AtomicUsize::new(0); +static mut HEAP: *mut u8 = ptr::null_mut(); +static mut HEAP_LEFT: usize = 0; +static HEAP_MUTEX: AtomicBool = AtomicBool::new(false); #[no_mangle] pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 { @@ -61,16 +62,19 @@ pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 { return imp::allocate(size, align); } - let heap = HEAP.load(Ordering::SeqCst); - let heap_left = HEAP_LEFT.load(Ordering::SeqCst); - if new_size < heap_left { - HEAP_LEFT.store(heap_left - new_size, Ordering::SeqCst); - HEAP.store(heap.offset(new_size as isize), Ordering::SeqCst); - return heap; + while HEAP_MUTEX.compare_and_swap(false, true, Ordering::SeqCst) {} + + if new_size < HEAP_LEFT { + let p = HEAP; + HEAP = p.offset(new_size as isize); + HEAP_LEFT -= new_size; + HEAP_MUTEX.store(false, Ordering::SeqCst); + return p; } else { - HEAP_LEFT.store(CHUNK_SIZE - new_size, Ordering::SeqCst); let p = imp::allocate(CHUNK_SIZE, CHUNK_ALIGN); - HEAP.store(p.offset(new_size as isize), Ordering::SeqCst); + HEAP = p.offset(new_size as isize); + HEAP_LEFT = CHUNK_SIZE - new_size; + HEAP_MUTEX.store(false, Ordering::SeqCst); return p; } }