diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index 854942dad3ded..3e241a4558a2d 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -13,7 +13,10 @@ #![feature(core_intrinsics)] #![feature(dropck_eyepatch)] +#![feature(iter_once_with)] +#![feature(ptr_offset_from)] #![feature(raw_vec_internals)] +#![feature(untagged_unions)] #![cfg_attr(test, feature(test))] #![allow(deprecated)] @@ -21,12 +24,11 @@ extern crate alloc; use rustc_data_structures::cold_path; -use rustc_data_structures::sync::MTLock; +use rustc_data_structures::sync::{SharedWorkerLocal, WorkerLocal, Lock}; use smallvec::SmallVec; use std::cell::{Cell, RefCell}; use std::cmp; -use std::intrinsics; use std::marker::{PhantomData, Send}; use std::mem; use std::ptr; @@ -34,149 +36,364 @@ use std::slice; use alloc::raw_vec::RawVec; -/// An arena that can hold objects of only one type. -pub struct TypedArena { - /// A pointer to the next object to be allocated. - ptr: Cell<*mut T>, +trait ChunkBackend: Sized { + type ChunkVecType: Sized; - /// A pointer to the end of the allocated area. When this pointer is - /// reached, a new chunk is allocated. - end: Cell<*mut T>, + /// Create new vec. + fn new() -> Self; - /// A vector of arena chunks. - chunks: RefCell>>, + /// Create new vec. + fn new_vec() -> Self::ChunkVecType; - /// Marker indicating that dropping the arena causes its owned - /// instances of `T` to be dropped. - _own: PhantomData, + /// Check current chunk has enough space for next allocation. + fn can_allocate(&self, len: usize, align: usize) -> bool; + + /// Allocate a new chunk and point to it. + fn grow(&self, len: usize, align: usize, chunks: &mut Self::ChunkVecType); + + /// Allocate a slice from this chunk. Panic if space lacks. + unsafe fn alloc_raw_slice(&self, len: usize, align: usize) -> *mut T; + + /// Clear the arena. + fn clear(&self, chunks: &mut Self::ChunkVecType); } -struct TypedArenaChunk { - /// The raw storage for the arena chunk. - storage: RawVec, - /// The number of valid entries in the chunk. - entries: usize, +/// Chunk used for zero-sized dropless types. +/// These don't require work, hence the trivial implementation. +struct NOPCurrentChunk { + phantom: PhantomData<*mut T>, } -impl TypedArenaChunk { +impl ChunkBackend for NOPCurrentChunk { + type ChunkVecType = (); + #[inline] - unsafe fn new(capacity: usize) -> TypedArenaChunk { - TypedArenaChunk { - storage: RawVec::with_capacity(capacity), - entries: 0, + fn new() -> Self { + let phantom = PhantomData; + NOPCurrentChunk { phantom } + } + + #[inline] + fn new_vec() {} + + #[inline] + fn can_allocate(&self, _len: usize, _align: usize) -> bool + { true } + + #[inline] + fn grow(&self, _len: usize, _align: usize, _chunks: &mut Self::ChunkVecType) + {} + + #[inline] + unsafe fn alloc_raw_slice(&self, _len: usize, align: usize) -> *mut T { + assert!(align >= mem::align_of::()); + align as *mut T + } + + #[inline] + fn clear(&self, _chunk: &mut Self::ChunkVecType) + {} +} + +/// Chunk used for zero-sized droppable types. +/// The only thing to do is counting the number of allocated items +/// in order to drop them at the end. +struct ZSTCurrentChunk { + counter: Cell, + phantom: PhantomData<*mut T>, +} + +impl ChunkBackend for ZSTCurrentChunk { + type ChunkVecType = (); + + #[inline] + fn new() -> Self { + ZSTCurrentChunk { + counter: Cell::new(0), + phantom: PhantomData, } } - /// Destroys this arena chunk. #[inline] - unsafe fn destroy(&mut self, len: usize) { - // The branch on needs_drop() is an -O1 performance optimization. - // Without the branch, dropping TypedArena takes linear time. - if mem::needs_drop::() { - let mut start = self.start(); - // Destroy all allocated objects. - for _ in 0..len { - ptr::drop_in_place(start); - start = start.offset(1); - } + fn new_vec() {} + + #[inline] + fn can_allocate(&self, _len: usize, _align: usize) -> bool + { true } + + #[inline] + fn grow(&self, _len: usize, _align: usize, _chunks: &mut Self::ChunkVecType) + {} + + #[inline] + unsafe fn alloc_raw_slice(&self, len: usize, align: usize) -> *mut T { + assert!(align >= mem::align_of::()); + let count = self.counter.get(); + self.counter.set(count+len); + align as *mut T + } + + #[inline] + fn clear(&self, _chunks: &mut Self::ChunkVecType) { + assert!(mem::needs_drop::()); + + let count = self.counter.get(); + for _ in 0..count { + let ptr = mem::align_of::() as *mut T; + unsafe { ptr::drop_in_place(ptr) } } + + self.counter.set(0) } +} + +/// Chunk used for droppable types. +struct TypedCurrentChunk { + /// A pointer to the start of the allocatable area. + /// When this pointer is reached, a new chunk is allocated. + start: Cell<*mut T>, + + /// A pointer to the end of the allocated area. + ptr: Cell<*mut T>, +} + +impl ChunkBackend for TypedCurrentChunk { + type ChunkVecType = Vec>; - // Returns a pointer to the first allocated object. #[inline] - fn start(&self) -> *mut T { - self.storage.ptr() + fn new() -> Self { + TypedCurrentChunk { + start: Cell::new(ptr::null_mut()), + ptr: Cell::new(ptr::null_mut()), + } } - // Returns a pointer to the end of the allocated space. #[inline] - fn end(&self) -> *mut T { + fn new_vec() -> Self::ChunkVecType { + vec![] + } + + #[inline] + fn can_allocate(&self, len: usize, align: usize) -> bool { + assert!(mem::size_of::() > 0); + assert!(mem::align_of::() == align); + let available_capacity = unsafe { self.ptr.get().offset_from(self.start.get()) }; + assert!(available_capacity >= 0); + let available_capacity = available_capacity as usize; + available_capacity >= len + } + + /// Grows the arena. + #[inline(never)] + #[cold] + fn grow(&self, len: usize, align: usize, chunks: &mut Self::ChunkVecType) { + assert!(mem::size_of::() > 0); + assert!(mem::align_of::() == align); + assert!(!self.can_allocate(len, align)); unsafe { - if mem::size_of::() == 0 { - // A pointer as large as possible for zero-sized elements. - !0 as *mut T + let mut new_capacity: usize; + if let Some(last_chunk) = chunks.last_mut() { + let currently_used_cap = last_chunk.end().offset_from(self.ptr.get()); + assert!(currently_used_cap >= 0); + let currently_used_cap = currently_used_cap as usize; + last_chunk.entries = currently_used_cap; + + new_capacity = last_chunk.storage.capacity(); + loop { + new_capacity = new_capacity.checked_mul(2).unwrap(); + if new_capacity >= currently_used_cap + len { + break; + } + } } else { - self.start().add(self.storage.capacity()) + let elem_size = mem::size_of::(); + new_capacity = cmp::max(len, PAGE / elem_size); + } + + let chunk = TypedArenaChunk::new(new_capacity); + self.start.set(chunk.start()); + self.ptr.set(chunk.end()); + chunks.push(chunk); + } + } + + #[inline] + unsafe fn alloc_raw_slice(&self, len: usize, align: usize) -> *mut T { + assert!(mem::size_of::() > 0); + assert!(align == mem::align_of::()); + + let ptr = self.ptr.get(); + let ptr = ptr.sub(len); + assert!(ptr >= self.start.get()); + + self.ptr.set(ptr); + ptr + } + + fn clear(&self, chunks: &mut Self::ChunkVecType) { + assert!(mem::needs_drop::()); + assert!(mem::size_of::() > 0); + + if let Some(last_chunk) = chunks.last_mut() { + // Clear the last chunk, which is partially filled. + unsafe { + let end = last_chunk.end(); + let len = end.offset_from(self.ptr.get()); + assert!(len >= 0); + let len = len as usize; + for i in 0..len { + ptr::drop_in_place(end.sub(i + 1)) + } + self.ptr.set(end); + } + + let len = chunks.len(); + // If `T` is ZST, code below has no effect. + for mut chunk in chunks.drain(..len-1) { + unsafe { chunk.destroy(chunk.entries) } } } } } -const PAGE: usize = 4096; +/// Chunk used for types with trivial drop. +/// It is spearated from `TypedCurrentChunk` for code reuse in `DroplessArena`. +struct DroplessCurrentChunk { + /// A pointer to the next object to be allocated. + /// When this pointer is reached, a new chunk is allocated. + start: Cell<*mut u8>, -impl Default for TypedArena { - /// Creates a new `TypedArena`. - fn default() -> TypedArena { - TypedArena { - // We set both `ptr` and `end` to 0 so that the first call to + /// A pointer to the end of the allocated area. + ptr: Cell<*mut u8>, + + /// Ensure correct semantics. + _own: PhantomData<*mut T>, +} + +impl ChunkBackend for DroplessCurrentChunk { + type ChunkVecType = Vec>; + + #[inline] + fn new() -> Self { + DroplessCurrentChunk { + // We set both `start` and `end` to 0 so that the first call to // alloc() will trigger a grow(). + start: Cell::new(ptr::null_mut()), ptr: Cell::new(ptr::null_mut()), - end: Cell::new(ptr::null_mut()), - chunks: RefCell::new(vec![]), _own: PhantomData, } } -} - -impl TypedArena { - pub fn in_arena(&self, ptr: *const T) -> bool { - let ptr = ptr as *const T as *mut T; - self.chunks.borrow().iter().any(|chunk| chunk.start() <= ptr && ptr < chunk.end()) + #[inline] + fn new_vec() -> Self::ChunkVecType { + vec![] } - /// Allocates an object in the `TypedArena`, returning a reference to it. + #[inline] - pub fn alloc(&self, object: T) -> &mut T { - if self.ptr == self.end { - self.grow(1) - } + fn can_allocate(&self, len: usize, _align: usize) -> bool { + let len = len * mem::size_of::(); + // start is always aligned, so removing len from ptr cannot go below it + (unsafe { self.start.get().add(len) }) <= self.ptr.get() + } + /// Grows the arena. + #[inline(never)] + #[cold] + fn grow(&self, len: usize, _align: usize, chunks: &mut Self::ChunkVecType) { + let len = len * mem::size_of::(); unsafe { - if mem::size_of::() == 0 { - self.ptr - .set(intrinsics::arith_offset(self.ptr.get() as *mut u8, 1) - as *mut T); - let ptr = mem::align_of::() as *mut T; - // Don't drop the object. This `write` is equivalent to `forget`. - ptr::write(ptr, object); - &mut *ptr + let mut new_capacity; + if let Some(last_chunk) = chunks.last_mut() { + let currently_used_cap = last_chunk.end().offset_from(self.ptr.get()); + let currently_used_cap = currently_used_cap as usize; + last_chunk.entries = currently_used_cap; + if last_chunk.storage.reserve_in_place(currently_used_cap, len) { + self.ptr.set(last_chunk.end()); + return; + } else { + new_capacity = last_chunk.storage.capacity(); + loop { + new_capacity = new_capacity.checked_mul(2).unwrap(); + if new_capacity >= currently_used_cap + len { + break; + } + } + } } else { - let ptr = self.ptr.get(); - // Advance the pointer. - self.ptr.set(self.ptr.get().offset(1)); - // Write into uninitialized memory. - ptr::write(ptr, object); - &mut *ptr + new_capacity = cmp::max(len, PAGE); } + + let chunk = TypedArenaChunk::new(new_capacity); + self.start.set(chunk.start()); + self.ptr.set(chunk.end()); + chunks.push(chunk); } } #[inline] - fn can_allocate(&self, len: usize) -> bool { - let available_capacity_bytes = self.end.get() as usize - self.ptr.get() as usize; - let at_least_bytes = len.checked_mul(mem::size_of::()).unwrap(); - available_capacity_bytes >= at_least_bytes + unsafe fn alloc_raw_slice(&self, len: usize, align: usize) -> *mut T { + assert!(self.can_allocate(len, align)); + let len = len * mem::size_of::(); + let ptr = self.ptr.get().sub(len); + let ptr = ((ptr as usize) & !(align - 1)) as *mut u8; + assert!(ptr >= self.start.get()); + + self.ptr.set(ptr); + ptr as *mut T } - /// Ensures there's enough space in the current chunk to fit `len` objects. #[inline] - fn ensure_capacity(&self, len: usize) { - if !self.can_allocate(len) { - self.grow(len); - debug_assert!(self.can_allocate(len)); + fn clear(&self, chunks: &mut Self::ChunkVecType) { + if let Some(last_chunk) = chunks.last_mut() { + // Clear the last chunk, which is partially filled. + self.ptr.set(last_chunk.end()) } } +} + +struct GenericArena> { + /// Current chunk for next allocation. + current: Chunk, + + /// A vector of arena chunks. + chunks: RefCell, + /// Marker indicating that dropping the arena causes its owned + /// instances of `T` to be dropped. + _own: PhantomData, +} + +impl> Default for GenericArena { + fn default() -> Self { + let current = Chunk::new(); + let chunks = RefCell::new(Chunk::new_vec()); + GenericArena { + current, chunks, + _own: PhantomData, + } + } +} + +impl> GenericArena { #[inline] unsafe fn alloc_raw_slice(&self, len: usize) -> *mut T { - assert!(mem::size_of::() != 0); - assert!(len != 0); + let align = mem::align_of::(); + if !self.current.can_allocate(len, align) { + self.current.grow(len, align, &mut *self.chunks.borrow_mut()); + debug_assert!(self.current.can_allocate(len, align)); + } - self.ensure_capacity(len); + self.current.alloc_raw_slice(len, align) + } - let start_ptr = self.ptr.get(); - self.ptr.set(start_ptr.add(len)); - start_ptr + /// Allocates an object in the `GenericArena`, returning a reference to it. + #[inline] + pub fn alloc(&self, object: T) -> &mut T { + unsafe { + let ptr = self.alloc_raw_slice(1); + ptr::write(ptr, object); + &mut *ptr + } } /// Allocates a slice of objects that are copied into the `TypedArena`, returning a mutable @@ -201,207 +418,183 @@ impl TypedArena { #[inline] pub fn alloc_from_iter>(&self, iter: I) -> &mut [T] { - assert!(mem::size_of::() != 0); - let mut vec: SmallVec<[_; 8]> = iter.into_iter().collect(); - if vec.is_empty() { - return &mut []; - } - // Move the content to the arena by copying it and then forgetting - // the content of the SmallVec unsafe { - let len = vec.len(); - let start_ptr = self.alloc_raw_slice(len); - vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); - vec.set_len(0); - slice::from_raw_parts_mut(start_ptr, len) - } - } - - /// Grows the arena. - #[inline(never)] - #[cold] - fn grow(&self, n: usize) { - unsafe { - let mut chunks = self.chunks.borrow_mut(); - let (chunk, mut new_capacity); - if let Some(last_chunk) = chunks.last_mut() { - let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; - let currently_used_cap = used_bytes / mem::size_of::(); - last_chunk.entries = currently_used_cap; - if last_chunk.storage.reserve_in_place(currently_used_cap, n) { - self.end.set(last_chunk.end()); - return; - } else { - new_capacity = last_chunk.storage.capacity(); - loop { - new_capacity = new_capacity.checked_mul(2).unwrap(); - if new_capacity >= currently_used_cap + n { - break; - } - } - } - } else { - let elem_size = cmp::max(1, mem::size_of::()); - new_capacity = cmp::max(n, PAGE / elem_size); - } - chunk = TypedArenaChunk::::new(new_capacity); - self.ptr.set(chunk.start()); - self.end.set(chunk.end()); - chunks.push(chunk); + alloc_from_iter_buffer( + move |len| self.alloc_raw_slice(len), + iter.into_iter(), + ) } } /// Clears the arena. Deallocates all but the longest chunk which may be reused. pub fn clear(&mut self) { - unsafe { - // Clear the last chunk, which is partially filled. - let mut chunks_borrow = self.chunks.borrow_mut(); - if let Some(mut last_chunk) = chunks_borrow.last_mut() { - self.clear_last_chunk(&mut last_chunk); - let len = chunks_borrow.len(); - // If `T` is ZST, code below has no effect. - for mut chunk in chunks_borrow.drain(..len-1) { - chunk.destroy(chunk.entries); - } - } + self.current.clear(&mut *self.chunks.borrow_mut()) + } +} + +unsafe impl<#[may_dangle] T, Chunk: ChunkBackend> Drop for GenericArena { + fn drop(&mut self) { + self.clear() + // RawVec handles deallocation of `last_chunk` and `self.chunks`. + } +} + +// The implementation relies on switching between the different `GenericArena` backends. +// The switch is done using the `needs_drop` and `size_of` information. +pub union TypedArena { + nop: mem::ManuallyDrop>>, + zst: mem::ManuallyDrop>>, + dropless: mem::ManuallyDrop>>, + typed: mem::ManuallyDrop>>, + _own: PhantomData, +} + +impl Default for TypedArena { + fn default() -> Self { + match (mem::needs_drop::(), mem::size_of::() > 0) { + (true, true) => Self { typed: mem::ManuallyDrop::new(GenericArena::default()) }, + (true, false) => Self { zst: mem::ManuallyDrop::new(GenericArena::default()) }, + (false, true) => Self { dropless: mem::ManuallyDrop::new(GenericArena::default()) }, + (false, false) => Self { nop: mem::ManuallyDrop::new(GenericArena::default()) }, } } +} - // Drops the contents of the last chunk. The last chunk is partially empty, unlike all other - // chunks. - fn clear_last_chunk(&self, last_chunk: &mut TypedArenaChunk) { - // Determine how much was filled. - let start = last_chunk.start() as usize; - // We obtain the value of the pointer to the first uninitialized element. - let end = self.ptr.get() as usize; - // We then calculate the number of elements to be dropped in the last chunk, - // which is the filled area's length. - let diff = if mem::size_of::() == 0 { - // `T` is ZST. It can't have a drop flag, so the value here doesn't matter. We get - // the number of zero-sized values in the last and only chunk, just out of caution. - // Recall that `end` was incremented for each allocated value. - end - start - } else { - (end - start) / mem::size_of::() - }; - // Pass that to the `destroy` method. - unsafe { - last_chunk.destroy(diff); +macro_rules! forward_impl { + ($t:ty, &$self:ident; $call:ident($($e:expr),*)) => { + match (mem::needs_drop::<$t>(), mem::size_of::<$t>() > 0) { + (true, true) => unsafe { &*$self.typed }.$call($($e),*), + (true, false) => unsafe { &*$self.zst }.$call($($e),*), + (false, true) => unsafe { &*$self.dropless }.$call($($e),*), + (false, false) => unsafe { &*$self.nop }.$call($($e),*), } - // Reset the chunk. - self.ptr.set(last_chunk.start()); + }; + ($t:ty, &mut $self:ident; $call:ident($($e:expr),*)) => { + match (mem::needs_drop::<$t>(), mem::size_of::<$t>() > 0) { + (true, true) => unsafe { &mut*$self.typed }.$call($($e),*), + (true, false) => unsafe { &mut*$self.zst }.$call($($e),*), + (false, true) => unsafe { &mut*$self.dropless }.$call($($e),*), + (false, false) => unsafe { &mut*$self.nop }.$call($($e),*), + } + }; +} + +impl TypedArena { + /// Allocates an object in the `GenericArena`, returning a reference to it. + #[inline] + pub fn alloc(&self, object: T) -> &mut T { + forward_impl!(T, &self; alloc(object)) + } + + /// Allocates a slice of objects that are copied into the `TypedArena`, returning a mutable + /// reference to it. Will panic if passed a zero-sized types. + /// + /// Panics: + /// + /// - Zero-sized types + /// - Zero-length slices + #[inline] + pub fn alloc_slice(&self, slice: &[T]) -> &mut [T] + where + T: Copy, + { + forward_impl!(T, &self; alloc_slice(slice)) + } + + #[inline] + pub fn alloc_from_iter>(&self, iter: I) -> &mut [T] { + forward_impl!(T, &self; alloc_from_iter(iter)) + } + + /// Clears the arena. Deallocates all but the longest chunk which may be reused. + pub fn clear(&mut self) { + forward_impl!(T, &mut self; clear()) } } unsafe impl<#[may_dangle] T> Drop for TypedArena { fn drop(&mut self) { - unsafe { - // Determine how much was filled. - let mut chunks_borrow = self.chunks.borrow_mut(); - if let Some(mut last_chunk) = chunks_borrow.pop() { - // Drop the contents of the last chunk. - self.clear_last_chunk(&mut last_chunk); - // The last chunk will be dropped. Destroy all other chunks. - for chunk in chunks_borrow.iter_mut() { - chunk.destroy(chunk.entries); - } - } - // RawVec handles deallocation of `last_chunk` and `self.chunks`. + match (mem::needs_drop::(), mem::size_of::() > 0) { + (true, true) => unsafe { let _ = mem::ManuallyDrop::drop(&mut self.typed); }, + (true, false) => unsafe { let _ = mem::ManuallyDrop::drop(&mut self.zst); }, + (false, true) => unsafe { let _ = mem::ManuallyDrop::drop(&mut self.dropless); }, + (false, false) => unsafe { let _ = mem::ManuallyDrop::drop(&mut self.nop); }, } } } unsafe impl Send for TypedArena {} -pub struct DroplessArena { - /// A pointer to the next object to be allocated. - ptr: Cell<*mut u8>, - - /// A pointer to the end of the allocated area. When this pointer is - /// reached, a new chunk is allocated. - end: Cell<*mut u8>, - - /// A vector of arena chunks. - chunks: RefCell>>, +struct TypedArenaChunk { + /// The raw storage for the arena chunk. + storage: RawVec, + /// The number of valid entries in the chunk. + entries: usize, } -unsafe impl Send for DroplessArena {} - -impl Default for DroplessArena { +impl TypedArenaChunk { #[inline] - fn default() -> DroplessArena { - DroplessArena { - ptr: Cell::new(ptr::null_mut()), - end: Cell::new(ptr::null_mut()), - chunks: Default::default(), + unsafe fn new(capacity: usize) -> TypedArenaChunk { + TypedArenaChunk { + storage: RawVec::with_capacity(capacity), + entries: 0, } } -} -impl DroplessArena { - pub fn in_arena(&self, ptr: *const T) -> bool { - let ptr = ptr as *const u8 as *mut u8; - - self.chunks.borrow().iter().any(|chunk| chunk.start() <= ptr && ptr < chunk.end()) + /// Destroys this arena chunk. + #[inline] + unsafe fn destroy(&mut self, len: usize) { + // The branch on needs_drop() is an -O1 performance optimization. + // Without the branch, dropping TypedArena takes linear time. + if mem::needs_drop::() { + let mut start = self.end().sub(len); + // Destroy all allocated objects. + for _ in 0..len { + ptr::drop_in_place(start); + start = start.offset(1); + } + } } + // Returns a pointer to the first allocated object. #[inline] - fn align(&self, align: usize) { - let final_address = ((self.ptr.get() as usize) + align - 1) & !(align - 1); - self.ptr.set(final_address as *mut u8); - assert!(self.ptr <= self.end); + fn start(&self) -> *mut T { + self.storage.ptr() } - #[inline(never)] - #[cold] - fn grow(&self, needed_bytes: usize) { + // Returns a pointer to the end of the allocated space. + #[inline] + fn end(&self) -> *mut T { unsafe { - let mut chunks = self.chunks.borrow_mut(); - let (chunk, mut new_capacity); - if let Some(last_chunk) = chunks.last_mut() { - let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; - if last_chunk - .storage - .reserve_in_place(used_bytes, needed_bytes) - { - self.end.set(last_chunk.end()); - return; - } else { - new_capacity = last_chunk.storage.capacity(); - loop { - new_capacity = new_capacity.checked_mul(2).unwrap(); - if new_capacity >= used_bytes + needed_bytes { - break; - } - } - } + if mem::size_of::() == 0 { + // A pointer as large as possible for zero-sized elements. + !0 as *mut T } else { - new_capacity = cmp::max(needed_bytes, PAGE); + self.start().add(self.storage.capacity()) } - chunk = TypedArenaChunk::::new(new_capacity); - self.ptr.set(chunk.start()); - self.end.set(chunk.end()); - chunks.push(chunk); } } +} - #[inline] - pub fn alloc_raw(&self, bytes: usize, align: usize) -> &mut [u8] { - unsafe { - assert!(bytes != 0); +const PAGE: usize = 4096; - self.align(align); +#[derive(Default)] +pub struct DroplessArena { + backend: GenericArena>, +} - let future_end = intrinsics::arith_offset(self.ptr.get(), bytes as isize); - if (future_end as *mut u8) >= self.end.get() { - self.grow(bytes); - } +unsafe impl Send for DroplessArena {} - let ptr = self.ptr.get(); - // Set the pointer past ourselves - self.ptr.set( - intrinsics::arith_offset(self.ptr.get(), bytes as isize) as *mut u8, - ); +impl DroplessArena { + #[inline] + pub fn alloc_raw(&self, bytes: usize, align: usize) -> &mut [u8] { + if !self.backend.current.can_allocate(bytes, align) { + self.backend.current.grow(bytes, align, &mut *self.backend.chunks.borrow_mut()); + debug_assert!(self.backend.current.can_allocate(bytes, align)); + } + + unsafe { + let ptr = self.backend.current.alloc_raw_slice(bytes, align); slice::from_raw_parts_mut(ptr, bytes) } } @@ -448,133 +641,221 @@ impl DroplessArena { } } - #[inline] - unsafe fn write_from_iter>( - &self, - mut iter: I, - len: usize, - mem: *mut T, - ) -> &mut [T] { - let mut i = 0; - // Use a manual loop since LLVM manages to optimize it better for - // slice iterators - loop { - let value = iter.next(); - if i >= len || value.is_none() { - // We only return as many items as the iterator gave us, even - // though it was supposed to give us `len` - return slice::from_raw_parts_mut(mem, i); - } - ptr::write(mem.add(i), value.unwrap()); - i += 1; - } - } - #[inline] pub fn alloc_from_iter>(&self, iter: I) -> &mut [T] { - let iter = iter.into_iter(); assert!(mem::size_of::() != 0); assert!(!mem::needs_drop::()); - let size_hint = iter.size_hint(); - - match size_hint { - (min, Some(max)) if min == max => { - // We know the exact number of elements the iterator will produce here - let len = min; - - if len == 0 { - return &mut [] - } - let size = len.checked_mul(mem::size_of::()).unwrap(); - let mem = self.alloc_raw(size, mem::align_of::()) as *mut _ as *mut T; - unsafe { - self.write_from_iter(iter, len, mem) - } - } - (_, _) => { - cold_path(move || -> &mut [T] { - let mut vec: SmallVec<[_; 8]> = iter.collect(); - if vec.is_empty() { - return &mut []; - } - // Move the content to the arena by copying it and then forgetting - // the content of the SmallVec - unsafe { - let len = vec.len(); - let start_ptr = self.alloc_raw( - len * mem::size_of::(), - mem::align_of::() - ) as *mut _ as *mut T; - vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); - vec.set_len(0); - slice::from_raw_parts_mut(start_ptr, len) - } - }) - } + unsafe { + alloc_from_iter_dropless( + move |len| self.alloc_raw( + len * mem::size_of::(), + mem::align_of::(), + ) as *mut _ as *mut T, + iter.into_iter(), + ) } } } -#[derive(Default)] -// FIXME(@Zoxc): this type is entirely unused in rustc -pub struct SyncTypedArena { - lock: MTLock>, -} - -impl SyncTypedArena { - #[inline(always)] - pub fn alloc(&self, object: T) -> &mut T { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc(object) as *mut T) } - } - - #[inline(always)] - pub fn alloc_slice(&self, slice: &[T]) -> &mut [T] - where - T: Copy, - { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc_slice(slice) as *mut [T]) } - } +pub struct SyncDroplessArena { + /// Current chunk for next allocation. + current: WorkerLocal>, - #[inline(always)] - pub fn clear(&mut self) { - self.lock.get_mut().clear(); - } + /// A vector of arena chunks. + chunks: Lock>>>, } -#[derive(Default)] -pub struct SyncDroplessArena { - lock: MTLock, +impl Default for SyncDroplessArena { + #[inline] + fn default() -> SyncDroplessArena { + SyncDroplessArena { + current: WorkerLocal::new(|_| DroplessCurrentChunk::new()), + chunks: Default::default(), + } + } } impl SyncDroplessArena { - #[inline(always)] pub fn in_arena(&self, ptr: *const T) -> bool { - self.lock.lock().in_arena(ptr) + let ptr = ptr as *const u8 as *mut u8; + + self.chunks.lock().iter().any(|chunks| chunks.iter().any(|chunk| { + chunk.start() <= ptr && ptr < chunk.end() + })) } - #[inline(always)] + #[inline] pub fn alloc_raw(&self, bytes: usize, align: usize) -> &mut [u8] { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc_raw(bytes, align) as *mut [u8]) } + if !self.current.can_allocate(bytes, align) { + self.current.grow(bytes, align, &mut *self.chunks.borrow_mut()); + debug_assert!(self.current.can_allocate(bytes, align)); + } + + unsafe { + let ptr = self.current.alloc_raw_slice(bytes, align); + slice::from_raw_parts_mut(ptr, bytes) + } } - #[inline(always)] + #[inline] pub fn alloc(&self, object: T) -> &mut T { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc(object) as *mut T) } + assert!(!mem::needs_drop::()); + + let mem = self.alloc_raw( + mem::size_of::(), + mem::align_of::()) as *mut _ as *mut T; + + unsafe { + // Write into uninitialized memory. + ptr::write(mem, object); + &mut *mem + } } - #[inline(always)] + /// Allocates a slice of objects that are copied into the `DroplessArena`, returning a mutable + /// reference to it. Will panic if passed a zero-sized type. + /// + /// Panics: + /// + /// - Zero-sized types + /// - Zero-length slices + #[inline] pub fn alloc_slice(&self, slice: &[T]) -> &mut [T] where T: Copy, { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc_slice(slice) as *mut [T]) } + assert!(!mem::needs_drop::()); + assert!(mem::size_of::() != 0); + assert!(!slice.is_empty()); + + let mem = self.alloc_raw( + slice.len() * mem::size_of::(), + mem::align_of::()) as *mut _ as *mut T; + + unsafe { + let arena_slice = slice::from_raw_parts_mut(mem, slice.len()); + arena_slice.copy_from_slice(slice); + arena_slice + } } + + #[inline] + pub fn alloc_from_iter>(&self, iter: I) -> &mut [T] { + assert!(mem::size_of::() != 0); + assert!(!mem::needs_drop::()); + + unsafe { + alloc_from_iter_dropless( + move |len| self.alloc_raw( + len * mem::size_of::(), + mem::align_of::(), + ) as *mut _ as *mut T, + iter.into_iter(), + ) + } + } + + /// Clears the arena. Deallocates all but the longest chunk which may be reused. + pub fn clear(&mut self) { + self.current.clear(&mut *self.chunks.borrow_mut()) + } +} + +impl Drop for SyncDroplessArena { + fn drop(&mut self) { + self.clear() + // RawVec handles deallocation of `last_chunk` and `self.chunks`. + } +} + +/// Helper method for slice allocation from iterators. +unsafe fn alloc_from_iter_buffer<'a, T, F, I>(alloc: F, iter: I) -> &'a mut [T] + where F: 'a + FnOnce(usize) -> *mut T, + I: Iterator, +{ + assert!(mem::size_of::() != 0); + let mut vec: SmallVec<[_; 8]> = iter.into_iter().collect(); + if vec.is_empty() { + return &mut []; + } + + // Move the content to the arena by copying it and then forgetting + // the content of the SmallVec + let len = vec.len(); + let start_ptr = alloc(len); + vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); + vec.set_len(0); + slice::from_raw_parts_mut(start_ptr, len) +} + +/// Helper method for slice allocation from iterators. +/// Dropless case, optimized for known-length iterators. +unsafe fn alloc_from_iter_dropless<'a, T, F, I>(alloc: F, iter: I) -> &'a mut [T] + where F: 'a + Fn(usize) -> *mut T, + I: Iterator, +{ + assert!(!mem::needs_drop::()); + + let size_hint = iter.size_hint(); + let mut vec: SmallVec<[T; 8]> = match size_hint { + (min, Some(max)) if min == max => { + // We know the exact number of elements the iterator will produce here + let len = min; + + let mut iter = iter.peekable(); + if iter.peek().is_none() { + return &mut [] + } + + let mem = alloc(len); + + // Use a manual loop since LLVM manages to optimize it better for + // slice iterators + for i in 0..len { + let value = iter.next(); + if value.is_none() { + // We only return as many items as the iterator gave us, even + // though it was supposed to give us `len` + return slice::from_raw_parts_mut(mem, i); + } + // The iterator is not supposed to give us more than `len`. + assert!(i < len); + ptr::write(mem.add(i), value.unwrap()); + } + + if iter.peek().is_none() { + return slice::from_raw_parts_mut(mem, len) + } + + // The iterator has lied to us, and produces more elements than advertised. + // In order to handle the rest, we create a buffer, move what we already have inside, + // and extend it with the rest of the items. + cold_path(move || { + let mut vec = SmallVec::with_capacity(len + iter.size_hint().0); + + mem.copy_to_nonoverlapping(vec.as_mut_ptr(), len); + vec.set_len(len); + vec.extend(iter); + vec + }) + } + (_, _) => iter.collect() + }; + + cold_path(move || -> &mut [T] { + if vec.is_empty() { + return &mut []; + } + + // Move the content to the arena by copying it and then forgetting + // the content of the SmallVec + let len = vec.len(); + let start_ptr = alloc(len); + vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); + vec.set_len(0); + slice::from_raw_parts_mut(start_ptr, len) + }) } #[cfg(test)] diff --git a/src/libarena/tests.rs b/src/libarena/tests.rs index fa4189409d0e8..bf3a031a415b2 100644 --- a/src/libarena/tests.rs +++ b/src/libarena/tests.rs @@ -1,7 +1,8 @@ extern crate test; use test::Bencher; -use super::TypedArena; +use super::{TypedArena, DroplessArena, SyncDroplessArena}; use std::cell::Cell; +use std::iter; #[allow(dead_code)] #[derive(Debug, Eq, PartialEq)] @@ -12,13 +13,7 @@ struct Point { } #[test] -pub fn test_unused() { - let arena: TypedArena = TypedArena::default(); - assert!(arena.chunks.borrow().is_empty()); -} - -#[test] -fn test_arena_alloc_nested() { +fn test_arena_alloc_nested_typed() { struct Inner { value: u8, } @@ -60,12 +55,163 @@ fn test_arena_alloc_nested() { assert_eq!(result.inner.value, 10); } +#[test] +fn test_arena_alloc_nested_dropless() { + struct Inner { + value: u8, + } + struct Outer<'a> { + inner: &'a Inner, + } + enum EI<'e> { + I(Inner), + O(Outer<'e>), + } + + struct Wrap(DroplessArena); + + impl Wrap { + fn alloc_inner Inner>(&self, f: F) -> &Inner { + let r: &EI<'_> = self.0.alloc(EI::I(f())); + if let &EI::I(ref i) = r { + i + } else { + panic!("mismatch"); + } + } + fn alloc_outer<'a, F: Fn() -> Outer<'a>>(&'a self, f: F) -> &Outer<'a> { + let r: &EI<'_> = self.0.alloc(EI::O(f())); + if let &EI::O(ref o) = r { + o + } else { + panic!("mismatch"); + } + } + } + + let arena = Wrap(DroplessArena::default()); + + let result = arena.alloc_outer(|| Outer { + inner: arena.alloc_inner(|| Inner { value: 10 }), + }); + + assert_eq!(result.inner.value, 10); +} + +#[test] +fn test_arena_alloc_nested_sync() { + struct Inner { + value: u8, + } + struct Outer<'a> { + inner: &'a Inner, + } + enum EI<'e> { + I(Inner), + O(Outer<'e>), + } + + struct Wrap(SyncDroplessArena); + + impl Wrap { + fn alloc_inner Inner>(&self, f: F) -> &Inner { + let r: &EI<'_> = self.0.alloc(EI::I(f())); + if let &EI::I(ref i) = r { + i + } else { + panic!("mismatch"); + } + } + fn alloc_outer<'a, F: Fn() -> Outer<'a>>(&'a self, f: F) -> &Outer<'a> { + let r: &EI<'_> = self.0.alloc(EI::O(f())); + if let &EI::O(ref o) = r { + o + } else { + panic!("mismatch"); + } + } + } + + let arena = Wrap(SyncDroplessArena::default()); + + let result = arena.alloc_outer(|| Outer { + inner: arena.alloc_inner(|| Inner { value: 10 }), + }); + + assert_eq!(result.inner.value, 10); +} + +#[test] +fn test_arena_alloc_nested_iter() { + struct Inner { + value: u8, + } + struct Outer<'a> { + inner: &'a Inner, + } + enum EI<'e> { + I(Inner), + O(Outer<'e>), + } + + struct Wrap<'a>(TypedArena>); + + impl<'a> Wrap<'a> { + fn alloc_inner Inner>(&self, f: F) -> &Inner { + let r: &[EI<'_>] = self.0.alloc_from_iter(iter::once_with(|| EI::I(f()))); + if let &[EI::I(ref i)] = r { + i + } else { + panic!("mismatch"); + } + } + fn alloc_outer Outer<'a>>(&self, f: F) -> &Outer<'_> { + let r: &[EI<'_>] = self.0.alloc_from_iter(iter::once_with(|| EI::O(f()))); + if let &[EI::O(ref o)] = r { + o + } else { + panic!("mismatch"); + } + } + } + + let arena = Wrap(TypedArena::default()); + + let result = arena.alloc_outer(|| Outer { + inner: arena.alloc_inner(|| Inner { value: 10 }), + }); + + assert_eq!(result.inner.value, 10); +} + #[test] pub fn test_copy() { let arena = TypedArena::default(); for _ in 0..100000 { arena.alloc(Point { x: 1, y: 2, z: 3 }); } + + let arena = DroplessArena::default(); + for _ in 0..100000 { + arena.alloc(Point { x: 1, y: 2, z: 3 }); + } + + let arena = SyncDroplessArena::default(); + for _ in 0..100000 { + arena.alloc(Point { x: 1, y: 2, z: 3 }); + } +} + +#[test] +pub fn test_align() { + #[repr(align(32))] + struct AlignedPoint(Point); + + let arena = TypedArena::default(); + for _ in 0..100000 { + let ptr = arena.alloc(AlignedPoint(Point { x: 1, y: 2, z: 3 })); + assert_eq!((ptr as *const _ as usize) & 31, 0); + } } #[bench] @@ -165,6 +311,31 @@ fn test_typed_arena_drop_on_clear() { } } +struct DropOrder<'a> { + rank: u32, + count: &'a Cell, +} + +impl Drop for DropOrder<'_> { + fn drop(&mut self) { + assert_eq!(self.rank, self.count.get()); + self.count.set(self.count.get() + 1); + } +} + +#[test] +fn test_typed_arena_drop_order() { + let counter = Cell::new(0); + { + let arena: TypedArena> = TypedArena::default(); + for rank in 0..100 { + // Allocate something with drop glue to make sure it doesn't leak. + arena.alloc(DropOrder { rank, count: &counter }); + } + }; + assert_eq!(counter.get(), 100); +} + thread_local! { static DROP_COUNTER: Cell = Cell::new(0) } diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index 6a19f52897e5d..5a69cfbcf4ce9 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -266,6 +266,45 @@ cfg_if! { } } + #[derive(Debug, Default)] + pub struct SharedWorkerLocal(T); + + impl SharedWorkerLocal { + /// Creates a new worker local where the `initial` closure computes the + /// value this worker local should take for each thread in the thread pool. + #[inline] + pub fn new T>(mut f: F) -> SharedWorkerLocal { + SharedWorkerLocal(f(0)) + } + + #[inline] + pub fn iter(&self) -> impl Iterator { + Some(&self.0).into_iter() + } + + /// Returns the worker-local value for each thread + #[inline] + pub fn into_inner(self) -> Vec { + vec![self.0] + } + } + + impl Deref for SharedWorkerLocal { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + &self.0 + } + } + + impl DerefMut for SharedWorkerLocal { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.0 + } + } + pub type MTRef<'a, T> = &'a mut T; #[derive(Debug, Default)] @@ -387,6 +426,55 @@ cfg_if! { } pub use rayon_core::WorkerLocal; + pub use rayon_core::Registry; + use rayon_core::current_thread_index; + use rayon_core::current_num_threads; + + #[derive(Debug)] + pub struct SharedWorkerLocal(Vec); + + impl SharedWorkerLocal { + /// Creates a new worker local where the `initial` closure computes the + /// value this worker local should take for each thread in the thread pool. + #[inline] + pub fn new T>(mut f: F) -> SharedWorkerLocal { + SharedWorkerLocal((0..current_num_threads()).map(|i| f(i)).collect()) + } + + #[inline] + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + /// Returns the worker-local value for each thread + #[inline] + pub fn into_inner(self) -> Vec { + self.0 + } + } + + impl Default for SharedWorkerLocal { + #[inline] + fn default() -> Self { + SharedWorkerLocal::new(|_| Default::default()) + } + } + + impl Deref for SharedWorkerLocal { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + &self.0[current_thread_index().unwrap()] + } + } + + impl DerefMut for SharedWorkerLocal { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.0[current_thread_index().unwrap()] + } + } pub use rayon::iter::ParallelIterator; use rayon::iter::IntoParallelIterator;