diff --git a/Cargo.toml b/Cargo.toml index 59f93599ca..ef7fd37965 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ itertools = "0.14.0" jemalloc-sys = { version = "0.5.3", features = ["disable_initial_exec_tls"], optional = true } lazy_static = "1.1" libc = "0.2" -log = { version = "0.4", features = ["max_level_trace", "release_max_level_off"] } +log = { version = "0.4", features = ["max_level_trace"] } memoffset = "0.9" mimalloc-sys = { version = "0.1.6", optional = true } # MMTk macros - we have to specify a version here in order to publish the crate, even though we use the dependency from a local path. @@ -188,6 +188,9 @@ work_packet_stats = [] # Count the malloc'd memory into the heap size malloc_counted_size = [] +# Dump memory stats about the plan (live bytes, live lines, live blocks, and used pages) +dump_memory_stats = [] + # Workaround a problem where bpftrace scripts (see tools/tracing/timeline/capture.bt) cannot # capture the type names of work packets. bpftrace_workaround = [] diff --git a/src/plan/global.rs b/src/plan/global.rs index 062cdc969b..f5be027b01 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -331,6 +331,10 @@ pub trait Plan: 'static + HasSpaces + Sync + Downcast { space.verify_side_metadata_sanity(&mut side_metadata_sanity_checker); }) } + + /// Dump memory stats for the plan + #[cfg(feature = "dump_memory_stats")] + fn dump_memory_stats(&self) {} } impl_downcast!(Plan assoc VM); diff --git a/src/plan/immix/global.rs b/src/plan/immix/global.rs index 1fcced92ef..fa3e4b1176 100644 --- a/src/plan/immix/global.rs +++ b/src/plan/immix/global.rs @@ -126,6 +126,12 @@ impl Plan for Immix { fn common(&self) -> &CommonPlan { &self.common } + + #[cfg(feature = "dump_memory_stats")] + fn dump_memory_stats(&self) { + self.immix_space.dump_memory_stats(); + self.common.los.dump_memory_stats(); + } } impl Immix { diff --git a/src/plan/sticky/immix/global.rs b/src/plan/sticky/immix/global.rs index 73a21d2c0e..ed43fc1c79 100644 --- a/src/plan/sticky/immix/global.rs +++ b/src/plan/sticky/immix/global.rs @@ -84,6 +84,12 @@ impl Plan for StickyImmix { self.immix.common() } + #[cfg(feature = "dump_memory_stats")] + fn dump_memory_stats(&self) { + self.immix.immix_space.dump_memory_stats(); + self.common().los.dump_memory_stats(); + } + fn schedule_collection(&'static self, scheduler: &crate::scheduler::GCWorkScheduler) { let is_full_heap = self.requires_full_heap_collection(); self.gc_full_heap.store(is_full_heap, Ordering::SeqCst); @@ -260,9 +266,11 @@ impl crate::plan::generational::global::GenerationalPlanExt f "Immix nursery object {} is being traced without moving", object ); - self.immix - .immix_space - .trace_object_without_moving(queue, object) + self.immix.immix_space.trace_object_without_moving( + queue, + object, + KIND == TRACE_KIND_TRANSITIVE_PIN, + ) } else if self.immix.immix_space.prefer_copy_on_nursery_gc() { let ret = self.immix.immix_space.trace_object_with_opportunistic_copy( queue, @@ -290,7 +298,7 @@ impl crate::plan::generational::global::GenerationalPlanExt f ); self.immix .immix_space - .trace_object_without_moving(queue, object) + .trace_object_without_moving(queue, object, false) }; return object; diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index 838ae4aa35..08c1d2cf01 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -34,6 +34,80 @@ use std::sync::{atomic::AtomicU8, atomic::AtomicUsize, Arc}; pub(crate) const TRACE_KIND_FAST: TraceKind = 0; pub(crate) const TRACE_KIND_DEFRAG: TraceKind = 1; +#[cfg(feature = "dump_memory_stats")] +#[derive(Default)] +/// Keeping track of the number of traced/copied/tpinned objects and live bytes +struct ImmixSpaceStats { + live_bytes: AtomicUsize, + traced_objects: AtomicUsize, + pinned_objects: AtomicUsize, + tpinned_objects: AtomicUsize, + copied_objects: AtomicUsize, +} + +#[cfg(feature = "dump_memory_stats")] +impl ImmixSpaceStats { + pub fn get_live_bytes(&self) -> usize { + self.live_bytes.load(Ordering::SeqCst) + } + + pub fn set_live_bytes(&self, size: usize) { + self.live_bytes.store(size, Ordering::SeqCst) + } + + pub fn increase_live_bytes(&self, size: usize) { + self.live_bytes.fetch_add(size, Ordering::SeqCst); + } + + pub fn get_traced_objects(&self) -> usize { + self.traced_objects.load(Ordering::SeqCst) + } + + pub fn set_traced_objects(&self, size: usize) { + self.traced_objects.store(size, Ordering::SeqCst) + } + + pub fn increase_traced_objects(&self, size: usize) { + self.traced_objects.fetch_add(size, Ordering::SeqCst); + } + + pub fn get_copied_objects(&self) -> usize { + self.copied_objects.load(Ordering::SeqCst) + } + + pub fn set_copied_objects(&self, size: usize) { + self.copied_objects.store(size, Ordering::SeqCst) + } + + pub fn increase_copied_objects(&self, size: usize) { + self.copied_objects.fetch_add(size, Ordering::SeqCst); + } + + pub fn get_pinned_objects(&self) -> usize { + self.pinned_objects.load(Ordering::SeqCst) + } + + pub fn set_pinned_objects(&self, size: usize) { + self.pinned_objects.store(size, Ordering::SeqCst) + } + + pub fn increase_pinned_objects(&self, size: usize) { + self.pinned_objects.fetch_add(size, Ordering::SeqCst); + } + + pub fn get_tpinned_objects(&self) -> usize { + self.tpinned_objects.load(Ordering::SeqCst) + } + + pub fn set_tpinned_objects(&self, size: usize) { + self.tpinned_objects.store(size, Ordering::SeqCst) + } + + pub fn increase_tpinned_objects(&self, size: usize) { + self.tpinned_objects.fetch_add(size, Ordering::SeqCst); + } +} + pub struct ImmixSpace { common: CommonSpace, pr: BlockPageResource, @@ -55,6 +129,9 @@ pub struct ImmixSpace { scheduler: Arc>, /// Some settings for this space space_args: ImmixSpaceArgs, + /// Keeping track of immix stats + #[cfg(feature = "dump_memory_stats")] + immix_stats: ImmixSpaceStats, } /// Some arguments for Immix Space. @@ -199,7 +276,7 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace worker: &mut GCWorker, ) -> ObjectReference { if KIND == TRACE_KIND_TRANSITIVE_PIN { - self.trace_object_without_moving(queue, object) + self.trace_object_without_moving(queue, object, true) } else if KIND == TRACE_KIND_DEFRAG { if Block::containing(object).is_defrag_source() { debug_assert!(self.in_defrag()); @@ -216,10 +293,10 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace false, ) } else { - self.trace_object_without_moving(queue, object) + self.trace_object_without_moving(queue, object, false) } } else if KIND == TRACE_KIND_FAST { - self.trace_object_without_moving(queue, object) + self.trace_object_without_moving(queue, object, false) } else { unreachable!() } @@ -230,6 +307,20 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace debug_assert!(self.in_space(object)); self.mark_lines(object); } + + // count the bytes for each object + #[cfg(feature = "dump_memory_stats")] + self.immix_stats + .increase_live_bytes(VM::VMObjectModel::get_current_size(object)); + + // increase the number of objects scanned + #[cfg(feature = "dump_memory_stats")] + { + self.immix_stats.increase_traced_objects(1); + if self.is_pinned(object) { + self.immix_stats.increase_pinned_objects(1); + } + } } fn may_move_objects() -> bool { @@ -341,6 +432,8 @@ impl ImmixSpace { mark_state: Self::MARKED_STATE, scheduler: scheduler.clone(), space_args, + #[cfg(feature = "dump_memory_stats")] + immix_stats: Default::default(), } } @@ -471,6 +564,15 @@ impl ImmixSpace { self.scheduler.work_buckets[WorkBucketStage::ClearVOBits].bulk_add(work_packets); } } + + #[cfg(feature = "dump_memory_stats")] + { + self.immix_stats.set_live_bytes(0); + self.immix_stats.set_traced_objects(0); + self.immix_stats.set_copied_objects(0); + self.immix_stats.set_tpinned_objects(0); + self.immix_stats.set_pinned_objects(0); + } } /// Release for the immix space. @@ -505,6 +607,98 @@ impl ImmixSpace { did_defrag } + #[cfg(feature = "dump_memory_stats")] + pub(crate) fn dump_memory_stats(&self) { + use std::time::{SystemTime, UNIX_EPOCH}; + + #[derive(Default)] + struct Dist { + live_blocks: usize, + live_lines: usize, + } + let mut dist = Dist::default(); + + for chunk in self.chunk_map.all_chunks() { + if !self.address_in_space(chunk.start()) { + continue; + } + + for block in chunk + .iter_region::() + .filter(|b| b.get_state() != BlockState::Unallocated) + { + dist.live_blocks += 1; + + let line_mark_state = self.line_mark_state.load(Ordering::Acquire); + let mut live_lines_in_table = 0; + let mut live_lines_from_block_state = 0; + + for line in block.lines() { + if line.is_marked(line_mark_state) { + live_lines_in_table += 1; + } + } + + match block.get_state() { + BlockState::Marked => { + panic!("At this point the block should have been swept already"); + } + BlockState::Unmarked => { + // Block is unmarked and cannot be reused (has no holes) + dist.live_lines += Block::LINES; + live_lines_from_block_state += Block::LINES; + } + BlockState::Reusable { unavailable_lines } => { + dist.live_lines += unavailable_lines as usize; + live_lines_from_block_state += unavailable_lines as usize; + } + BlockState::Unallocated => {} + } + + // assert_eq!(live_lines_in_table, live_lines_from_block_state); + } + } + + let start = SystemTime::now(); + let since_the_epoch = start + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); + + println!("{:?} mmtk_immixspace", since_the_epoch.as_millis()); + println!( + "\t#Live objects = {}", + self.immix_stats.get_traced_objects() + ); + println!( + "\t#Copied objects = {}", + self.immix_stats.get_copied_objects() + ); + println!( + "\t#Pinned objects = {}", + self.immix_stats.get_pinned_objects() + ); + println!( + "\t#Transitively pinned objects = {}", + self.immix_stats.get_tpinned_objects() + ); + println!("\tLive bytes = {}", self.immix_stats.get_live_bytes()); + println!("\tReserved pages = {}", self.reserved_pages()); + println!( + "\tReserved pages (bytes) = {}", + self.reserved_pages() << LOG_BYTES_IN_PAGE + ); + println!("\tLive blocks = {}", dist.live_blocks); + println!( + "\tLive blocks (bytes) = {}", + dist.live_blocks << Block::LOG_BYTES + ); + println!("\tLive lines = {}", dist.live_lines); + println!( + "\tLive lines (bytes) = {}", + dist.live_lines << Line::LOG_BYTES + ); + } + /// Generate chunk sweep tasks fn generate_sweep_tasks(&self) -> Vec>> { self.defrag.mark_histograms.lock().clear(); @@ -581,6 +775,7 @@ impl ImmixSpace { &self, queue: &mut impl ObjectQueue, object: ObjectReference, + _is_tpinned: bool, ) -> ObjectReference { #[cfg(feature = "vo_bit")] vo_bit::helper::on_trace_object::(object); @@ -601,6 +796,14 @@ impl ImmixSpace { // Visit node queue.enqueue(object); self.unlog_object_if_needed(object); + + #[cfg(feature = "dump_memory_stats")] + if _is_tpinned { + // increase the number of objects being tpinned + #[cfg(feature = "dump_memory_stats")] + self.immix_stats.increase_tpinned_objects(1); + } + return object; } object @@ -681,6 +884,9 @@ impl ImmixSpace { semantics, copy_context, |_new_object| { + // increase the number of objects being moved + #[cfg(feature = "dump_memory_stats")] + self.immix_stats.increase_copied_objects(1); #[cfg(feature = "vo_bit")] vo_bit::helper::on_object_forwarded::(_new_object); }, diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index effb39febd..8dff759984 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -6,6 +6,8 @@ use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::space::{CommonSpace, Space}; use crate::util::constants::BYTES_IN_PAGE; +#[cfg(feature = "dump_memory_stats")] +use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::heap::{FreeListPageResource, PageResource}; use crate::util::metadata; use crate::util::object_enum::ObjectEnumerator; @@ -14,6 +16,8 @@ use crate::util::treadmill::TreadMill; use crate::util::{Address, ObjectReference}; use crate::vm::ObjectModel; use crate::vm::VMBinding; +#[cfg(feature = "dump_memory_stats")] +use std::sync::atomic::AtomicUsize; #[allow(unused)] const PAGE_MASK: usize = !(BYTES_IN_PAGE - 1); @@ -29,6 +33,9 @@ pub struct LargeObjectSpace { mark_state: u8, in_nursery_gc: bool, treadmill: TreadMill, + /// Keeping track of live bytes + #[cfg(feature = "dump_memory_stats")] + live_bytes: AtomicUsize, } impl SFT for LargeObjectSpace { @@ -223,6 +230,8 @@ impl LargeObjectSpace { mark_state: 0, in_nursery_gc: false, treadmill: TreadMill::new(), + #[cfg(feature = "dump_memory_stats")] + live_bytes: AtomicUsize::new(0), } } @@ -233,6 +242,8 @@ impl LargeObjectSpace { } self.treadmill.flip(full_heap); self.in_nursery_gc = !full_heap; + #[cfg(feature = "dump_memory_stats")] + self.set_live_bytes(0); } pub fn release(&mut self, full_heap: bool) { @@ -348,6 +359,8 @@ impl LargeObjectSpace { break; } } + #[cfg(feature = "dump_memory_stats")] + self.increase_live_bytes(VM::VMObjectModel::get_current_size(object)); true } @@ -369,6 +382,39 @@ impl LargeObjectSpace { ) & NURSERY_BIT == NURSERY_BIT } + + #[cfg(feature = "dump_memory_stats")] + pub fn get_live_bytes(&self) -> usize { + self.live_bytes.load(Ordering::SeqCst) + } + + #[cfg(feature = "dump_memory_stats")] + pub fn set_live_bytes(&self, size: usize) { + self.live_bytes.store(size, Ordering::SeqCst) + } + + #[cfg(feature = "dump_memory_stats")] + pub fn increase_live_bytes(&self, size: usize) { + self.live_bytes.fetch_add(size, Ordering::SeqCst); + } + + #[cfg(feature = "dump_memory_stats")] + pub(crate) fn dump_memory_stats(&self) { + use std::time::{SystemTime, UNIX_EPOCH}; + + let start = SystemTime::now(); + let since_the_epoch = start + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); + + println!("{} mmtk_los", since_the_epoch.as_millis()); + println!("\tLive bytes = {}", self.get_live_bytes()); + println!("\tReserved pages = {}", self.reserved_pages()); + println!( + "\tReserved pages (bytes) = {}", + self.reserved_pages() << LOG_BYTES_IN_PAGE + ); + } } fn get_super_page(cell: Address) -> Address { diff --git a/src/scheduler/scheduler.rs b/src/scheduler/scheduler.rs index 1df5488b37..5641b8c57a 100644 --- a/src/scheduler/scheduler.rs +++ b/src/scheduler/scheduler.rs @@ -554,15 +554,13 @@ impl GCWorkScheduler { probe!(mmtk, gc_end); if *mmtk.get_options().count_live_bytes_in_gc { - // Aggregate the live bytes let live_bytes = mmtk .scheduler .worker_group .get_and_clear_worker_live_bytes(); - let mut live_bytes_in_last_gc = mmtk.state.live_bytes_in_last_gc.borrow_mut(); - *live_bytes_in_last_gc = mmtk.aggregate_live_bytes_in_last_gc(live_bytes); - // Logging - for (space_name, &stats) in live_bytes_in_last_gc.iter() { + let mut stats = mmtk.state.live_bytes_in_last_gc.borrow_mut(); + *stats = mmtk.aggregate_live_bytes_in_last_gc(live_bytes); + for (space_name, &stats) in stats.iter() { info!( "{} = {} pages ({:.1}% live)", space_name, @@ -572,6 +570,9 @@ impl GCWorkScheduler { } } + #[cfg(feature = "dump_memory_stats")] + mmtk.get_plan().dump_memory_stats(); + #[cfg(feature = "extreme_assertions")] if crate::util::slot_logger::should_check_duplicate_slots(mmtk.get_plan()) { // reset the logging info at the end of each GC diff --git a/src/scheduler/worker.rs b/src/scheduler/worker.rs index 40a54e4046..8849e9afe9 100644 --- a/src/scheduler/worker.rs +++ b/src/scheduler/worker.rs @@ -69,9 +69,10 @@ impl GCWorkerShared { // The live bytes of the object let bytes = VM::VMObjectModel::get_current_size(object); - // Get the space index from descriptor - let space_descriptor = VM_MAP.get_descriptor_for_address(object.to_raw_address()); - if space_descriptor != crate::util::heap::space_descriptor::SpaceDescriptor::UNINITIALIZED { + // Check if the object is in our heap. If it is not in our heap (e.g. in the VM space), we cannot use VM map to get its descriptor. + if crate::util::heap::vm_layout::vm_layout().in_available_range(object.to_raw_address()) { + // Get the space index from descriptor + let space_descriptor = VM_MAP.get_descriptor_for_address(object.to_raw_address()); let space_index = space_descriptor.get_index(); debug_assert!( space_index < MAX_SPACES, diff --git a/src/util/heap/layout/vm_layout.rs b/src/util/heap/layout/vm_layout.rs index 720237e6fb..60987acc54 100644 --- a/src/util/heap/layout/vm_layout.rs +++ b/src/util/heap/layout/vm_layout.rs @@ -63,6 +63,10 @@ impl VMLayout { pub const fn available_end(&self) -> Address { self.heap_end } + /// Is the given address within the address space between available start and available end? + pub fn in_available_range(&self, addr: Address) -> bool { + addr >= self.available_start() && addr < self.available_end() + } /// Size of the address space available to the MMTk heap. pub const fn available_bytes(&self) -> usize { self.available_end().get_extent(self.available_start())