Object stats

src/snmalloc/backend_helpers/statsrange.h

@@ -16,8 +16,7 @@ namespace snmalloc
     {
       using ContainsParent<ParentRange>::parent;
 
-      static inline stl::Atomic<size_t> current_usage{};
-      static inline stl::Atomic<size_t> peak_usage{};
+      static inline Stat usage{};
 
     public:
       static constexpr bool Aligned = ParentRange::Aligned;
@@ -30,34 +29,26 @@ namespace snmalloc
 
       CapPtr<void, ChunkBounds> alloc_range(size_t size)
       {
-        auto result = parent.alloc_range(size);
-        if (result != nullptr)
-        {
-          auto prev = current_usage.fetch_add(size);
-          auto curr = peak_usage.load();
-          while (curr < prev + size)
-          {
-            if (peak_usage.compare_exchange_weak(curr, prev + size))
-              break;
-          }
-        }
-        return result;
+        auto r = parent.alloc_range(size);
+        if (r != nullptr)
+          usage += size;
+        return r;
       }
 
       void dealloc_range(CapPtr<void, ChunkBounds> base, size_t size)
       {
-        current_usage -= size;
+        usage -= size;
         parent.dealloc_range(base, size);
       }
 
       size_t get_current_usage()
       {
-        return current_usage.load();
+        return usage.get_curr();
       }
 
       size_t get_peak_usage()
       {
-        return peak_usage.load();
+        return usage.get_peak();
       }
     };
   };

src/snmalloc/ds_aal/ds_aal.h

@@ -7,4 +7,5 @@
 #include "../aal/aal.h"
 #include "flaglock.h"
 #include "prevent_fork.h"
+#include "seqset.h"
 #include "singleton.h"

src/snmalloc/ds_core/seqset.h → src/snmalloc/ds_aal/seqset.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "../aal/aal.h"
 #include "../ds_core/ds_core.h"
 #include "snmalloc/stl/type_traits.h"
 #include "snmalloc/stl/utility.h"

src/snmalloc/ds_core/ds_core.h

@@ -15,5 +15,5 @@
 #include "mitigations.h"
 #include "ptrwrap.h"
 #include "redblacktree.h"
-#include "seqset.h"
-#include "tid.h"
+#include "stats.h"
+#include "tid.h"

src/snmalloc/ds_core/stats.h

@@ -0,0 +1,101 @@
+#pragma once
+
+#include "defines.h"
+#include "snmalloc/stl/atomic.h"
+#include "stddef.h"
+
+namespace snmalloc
+{
+  /**
+   * Very basic statistic that tracks current and peak values.
+   */
+  class Stat
+  {
+  private:
+    stl::Atomic<size_t> curr{0};
+    stl::Atomic<size_t> peak{0};
+
+  public:
+    void increase(size_t amount)
+    {
+      size_t old = curr.fetch_add(amount);
+      size_t c = old + amount;
+      size_t p = peak.load(stl::memory_order_relaxed);
+      while (c > p)
+      {
+        if (peak.compare_exchange_strong(p, c))
+          break;
+      }
+    }
+
+    void decrease(size_t amount)
+    {
+      size_t prev = curr.fetch_sub(amount);
+      SNMALLOC_ASSERT_MSG(
+        prev >= amount, "prev = {}, amount = {}", prev, amount);
+      UNUSED(prev);
+    }
+
+    size_t get_curr()
+    {
+      return curr.load(stl::memory_order_relaxed);
+    }
+
+    size_t get_peak()
+    {
+      return peak.load(stl::memory_order_relaxed);
+    }
+
+    void operator+=(size_t amount)
+    {
+      increase(amount);
+    }
+
+    void operator-=(size_t amount)
+    {
+      decrease(amount);
+    }
+
+    void operator++()
+    {
+      increase(1);
+    }
+
+    void operator--()
+    {
+      decrease(1);
+    }
+  };
+
+  /**
+   * Very basic statistic that can only grow.  Not thread-safe.
+   */
+  class MonotoneLocalStat
+  {
+    stl::Atomic<size_t> value{0};
+
+  public:
+    void operator++(int)
+    {
+      auto old = value.load(stl::memory_order_relaxed);
+      value.store(old + 1, stl::memory_order_relaxed);
+    }
+
+    void operator+=(const MonotoneLocalStat& other)
+    {
+      auto v = other.value.load(stl::memory_order_relaxed);
+      value.fetch_add(v, stl::memory_order_relaxed);
+    }
+
+    void operator+=(size_t v)
+    {
+      auto old = value.load(stl::memory_order_relaxed);
+      value.store(old + v, stl::memory_order_relaxed);
+    }
+
+    size_t operator*()
+    {
+      return value.load(stl::memory_order_relaxed);
+    }
+  };
+} // namespace snmalloc

src/snmalloc/global/globalalloc.h

@@ -84,6 +84,15 @@ namespace snmalloc
       }
     }
 
+    if (
+      result == nullptr &&
+      RemoteDeallocCache<Config_>::remote_inflight.get_curr() != 0)
+    {
+      report_fatal_error(
+        "debug_check_empty: remote inflight deallocations left {}}",
+        RemoteDeallocCache<Config_>::remote_inflight.get_curr());
+    }
+
     if (result != nullptr)
     {
       *result = okay;
@@ -128,6 +137,81 @@ namespace snmalloc
     }
   }
 
+  template<SNMALLOC_CONCEPT(IsConfig) Config_ = Config>
+  inline static void get_stats(AllocStats& stats)
+  {
+    auto alloc = AllocPool<Config>::iterate();
+    while (alloc != nullptr)
+    {
+      stats += alloc->get_stats();
+      alloc = AllocPool<Config>::iterate(alloc);
+    }
+  }
+
+  template<SNMALLOC_CONCEPT(IsConfig) Config_ = Config>
+  inline static void print_alloc_stats()
+  {
+    static stl::Atomic<size_t> dump{0};
+
+    auto l_dump = dump++;
+    if (l_dump == 0)
+    {
+      message<1024>(
+        "snmalloc_allocs,dumpid,sizeclass,size,allocated,deallocated,in_use,"
+        "bytes,slabs allocated,slabs deallocated,slabs in_use,slabs bytes");
+      message<1024>(
+        "snmalloc_totals,dumpid,backend bytes,peak backend "
+        "bytes,requested,slabs requested bytes,remote inflight bytes,allocator "
+        "count");
+    }
+
+    AllocStats stats;
+    snmalloc::get_stats<Config>(stats);
+    size_t total_live{0};
+    size_t total_live_slabs{0};
+    for (size_t i = 0; i < snmalloc::SIZECLASS_REP_SIZE; i++)
+    {
+      auto sc = snmalloc::sizeclass_t::from_raw(i);
+      auto allocated = *stats[sc].objects_allocated;
+      auto deallocated = *stats[sc].objects_deallocated;
+      auto slabs_allocated = *stats[sc].slabs_allocated;
+      auto slabs_deallocated = *stats[sc].slabs_deallocated;
+      if (allocated == 0 && deallocated == 0)
+        continue;
+      auto size = snmalloc::sizeclass_full_to_size(sc);
+      auto slab_size = snmalloc::sizeclass_full_to_slab_size(sc);
+      auto in_use = allocated - deallocated;
+      auto amount = in_use * size;
+      total_live += amount;
+      auto in_use_slabs = slabs_allocated - slabs_deallocated;
+      auto amount_slabs = in_use_slabs * slab_size;
+      total_live_slabs += amount_slabs;
+
+      snmalloc::message<1024>(
+        "snmalloc_allocs,{},{},{},{},{},{},{},{},{},{},{}",
+        l_dump,
+        i,
+        size,
+        allocated,
+        deallocated,
+        in_use,
+        amount,
+        slabs_allocated,
+        slabs_deallocated,
+        in_use_slabs,
+        amount_slabs);
+    }
+    snmalloc::message<1024>(
+      "snmalloc_totals,{},{},{},{},{},{},{}",
+      l_dump,
+      Config::Backend::get_current_usage(),
+      Config::Backend::get_peak_usage(),
+      total_live,
+      total_live_slabs,
+      RemoteDeallocCache<Config>::remote_inflight.get_curr(),
+      Config::pool().get_count());
+  }
+
   /**
    * Returns the number of remaining bytes in an object.
    *