Introduce heap_graph_stats stdlib module, dmabuf support and rebase the metric

Android.bp

@@ -15323,13 +15323,15 @@ genrule {
         "src/trace_processor/perfetto_sql/stdlib/android/job_scheduler_states.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/kernel_wakelocks.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/dmabuf.sql",
+        "src/trace_processor/perfetto_sql/stdlib/android/memory/dmabuf_spans.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/class_relationship.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/class_summary_tree.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/class_tree.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/dominator_class_tree.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/dominator_tree.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/excluded_refs.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/heap_graph_class_aggregation.sql",
+        "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/heap_graph_stats.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/helpers.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/raw_dominator_tree.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_profile/callstacks.sql",

BUILD

@@ -3359,6 +3359,7 @@ perfetto_filegroup(
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/dominator_tree.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/excluded_refs.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/heap_graph_class_aggregation.sql",
+        "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/heap_graph_stats.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/helpers.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/raw_dominator_tree.sql",
     ],
@@ -3378,6 +3379,7 @@ perfetto_filegroup(
     name = "src_trace_processor_perfetto_sql_stdlib_android_memory_memory",
     srcs = [
         "src/trace_processor/perfetto_sql/stdlib/android/memory/dmabuf.sql",
+        "src/trace_processor/perfetto_sql/stdlib/android/memory/dmabuf_spans.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/lmk.sql",
         "src/trace_processor/perfetto_sql/stdlib/android/memory/process.sql",
     ],

protos/perfetto/metrics/android/java_heap_stats.proto

@@ -20,13 +20,10 @@ package perfetto.protos;
 import "protos/perfetto/metrics/android/process_metadata.proto";
 
 message JavaHeapStats {
-  message HeapRoots {
-    optional string root_type = 1;
-    optional string type_name = 2;
-    optional int64 obj_count = 3;
-  }
+  // Was repeated HeapRoots roots = 7;
+  reserved 7;
 
-  // Next id: 12
+  // Next id: 13
   message Sample {
     optional int64 ts = 1;
     // Size of the Java heap in bytes
@@ -42,9 +39,8 @@ message JavaHeapStats {
     optional int64 reachable_obj_count = 5;
     // Sum of anonymous RSS + swap pages in bytes.
     optional int64 anon_rss_and_swap_size = 6;
+    optional int64 dmabuf_rss_size = 12;
 
-    // ART root objects
-    repeated HeapRoots roots = 7;
     // OOM adjustment score
     optional int64 oom_score_adj = 10;
     // Process uptime in millis at the time of the heap dump

protos/perfetto/metrics/perfetto_merged_metrics.proto

@@ -1897,13 +1897,10 @@ message JavaHeapHistogram {
 // Begin of protos/perfetto/metrics/android/java_heap_stats.proto
 
 message JavaHeapStats {
-  message HeapRoots {
-    optional string root_type = 1;
-    optional string type_name = 2;
-    optional int64 obj_count = 3;
-  }
+  // Was repeated HeapRoots roots = 7;
+  reserved 7;
 
-  // Next id: 12
+  // Next id: 13
   message Sample {
     optional int64 ts = 1;
     // Size of the Java heap in bytes
@@ -1919,9 +1916,8 @@ message JavaHeapStats {
     optional int64 reachable_obj_count = 5;
     // Sum of anonymous RSS + swap pages in bytes.
     optional int64 anon_rss_and_swap_size = 6;
+    optional int64 dmabuf_rss_size = 12;
 
-    // ART root objects
-    repeated HeapRoots roots = 7;
     // OOM adjustment score
     optional int64 oom_score_adj = 10;
     // Process uptime in millis at the time of the heap dump

python/perfetto/trace_processor/metrics.descriptor

@@ -870,14 +870,10 @@ type_count (2-.perfetto.protos.JavaHeapClassStats.TypeCountR	typeCount
 upid
 
(RupidA
 process 
(2'.perfetto.protos.AndroidProcessMetadataRprocessD
 samples (2*.perfetto.protos.JavaHeapClassStats.SampleRsamples
-�
-5protos/perfetto/metrics/android/java_heap_stats.protoperfetto.protos6protos/perfetto/metrics/android/process_metadata.proto"�
+�
+5protos/perfetto/metrics/android/java_heap_stats.protoperfetto.protos6protos/perfetto/metrics/android/process_metadata.proto"�
 JavaHeapStatsS
-instance_stats
 (2,.perfetto.protos.JavaHeapStats.InstanceStatsRinstanceStatsb
-	HeapRoots
-	root_type
 
(	RrootType
-	type_name 
(	RtypeName
-	obj_count 
(RobjCount�
+instance_stats
 (2,.perfetto.protos.JavaHeapStats.InstanceStatsRinstanceStats�
 Sample
 ts
 
(Rts
 	heap_size 
(RheapSize(
@@ -886,15 +882,15 @@ type_count (2-.perfetto.protos.JavaHeapClassStats.TypeCountR	typeCount
 reachable_heap_size 
(RreachableHeapSize;
 reachable_heap_native_size	 
(RreachableHeapNativeSize.
 reachable_obj_count 
(RreachableObjCount2
-anon_rss_and_swap_size 
(RanonRssAndSwapSize>
-roots (2(.perfetto.protos.JavaHeapStats.HeapRootsRroots"
+anon_rss_and_swap_size 
(RanonRssAndSwapSize&
+dmabuf_rss_size 
(RdmabufRssSize"
 oom_score_adj
  
(RoomScoreAdj*
 process_uptime_ms 
(RprocessUptimeMs�

 InstanceStats
 upid
 
(RupidA
 process 
(2'.perfetto.protos.AndroidProcessMetadataRprocess?
-samples (2%.perfetto.protos.JavaHeapStats.SampleRsamples
+samples (2%.perfetto.protos.JavaHeapStats.SampleRsamplesJ
 �
 0protos/perfetto/metrics/android/lmk_metric.protoperfetto.protos"�

 AndroidLmkMetric

src/trace_processor/metrics/sql/android/java_heap_stats.sql

@@ -14,119 +14,30 @@
 -- limitations under the License.
 --
 
+INCLUDE PERFETTO MODULE android.memory.heap_graph.heap_graph_stats;
 SELECT RUN_METRIC('android/process_metadata.sql');
-SELECT RUN_METRIC('android/process_mem.sql');
 
 DROP VIEW IF EXISTS java_heap_stats_output;
 CREATE PERFETTO VIEW java_heap_stats_output AS
 WITH
--- Base view
-base_stat_counts AS (
-  SELECT
-    upid,
-    graph_sample_ts,
-    SUM(self_size) AS total_size,
-    SUM(native_size) AS total_native_size,
-    COUNT(1) AS total_obj_count,
-    SUM(IIF(reachable, self_size, 0)) AS reachable_size,
-    SUM(IIF(reachable, native_size, 0)) AS reachable_native_size,
-    SUM(IIF(reachable, 1, 0)) AS reachable_obj_count
-  FROM heap_graph_object
-  GROUP BY 1, 2
-),
-heap_roots AS (
-  SELECT
-    upid,
-    graph_sample_ts,
-    root_type,
-    IFNULL(t.deobfuscated_name, t.name) AS type_name,
-    COUNT(1) AS obj_count
-  FROM heap_graph_object o
-  JOIN heap_graph_class t ON o.type_id = t.id
-  -- Classes are going to be particularly spammy and uninteresting
-  -- from a memory analysis perspective (compared e.g. to local jni roots)
-  WHERE root_type IS NOT NULL AND root_type != 'ROOT_STICKY_CLASS'
-  GROUP BY 1, 2, 3, 4
-  ORDER BY obj_count DESC
-),
-heap_roots_proto AS (
-  SELECT
-    upid,
-    graph_sample_ts,
-    RepeatedField(JavaHeapStats_HeapRoots(
-      'root_type', root_type,
-      'type_name', type_name,
-      'obj_count', obj_count
-    )) AS roots
-  FROM heap_roots
-  GROUP BY 1, 2
-),
-base_stats AS (
-  SELECT * FROM base_stat_counts JOIN heap_roots_proto USING (upid, graph_sample_ts)
-),
--- Find closest value
-closest_anon_swap_oom AS (
-  SELECT
-    upid,
-    graph_sample_ts,
-    (
-      SELECT anon_swap_val
-      FROM (
-        SELECT
-          ts, dur,
-          CAST(anon_and_swap_val AS INTEGER) AS anon_swap_val,
-          ABS(ts - base_stats.graph_sample_ts) AS diff
-        FROM anon_and_swap_span
-        WHERE upid = base_stats.upid)
-      WHERE
-        (graph_sample_ts >= ts AND graph_sample_ts < ts + dur)
-        -- If the first memory sample for the UPID comes *after* the heap profile
-        -- accept it if close (500ms)
-        OR (graph_sample_ts < ts AND diff <= 500 * 1e6)
-      ORDER BY diff LIMIT 1
-    ) AS anon_swap_val,
-    (
-      SELECT oom_score_val
-      FROM (
-        SELECT
-          ts, dur,
-          oom_score_val,
-          ABS(ts - base_stats.graph_sample_ts) AS diff
-        FROM oom_score_span
-        WHERE upid = base_stats.upid)
-      WHERE
-        (graph_sample_ts >= ts AND graph_sample_ts < ts + dur)
-        -- If the first memory sample for the UPID comes *after* the heap profile
-        -- accept it if close (500ms)
-        OR (graph_sample_ts < ts AND diff <= 500 * 1e6)
-      ORDER BY diff LIMIT 1
-    ) AS oom_score_val
-  FROM base_stats
-),
 -- Group by upid
 heap_graph_sample_protos AS (
   SELECT
-    base_stats.upid,
+    upid,
     RepeatedField(JavaHeapStats_Sample(
       'ts', graph_sample_ts,
-      'process_uptime_ms',
-        CASE WHEN process.start_ts IS NOT NULL
-        THEN (graph_sample_ts - process.start_ts) / 1000000
-        ELSE NULL
-        END,
-      'heap_size', total_size,
-      'heap_native_size', total_native_size,
+      'process_uptime_ms', process_uptime / 1e6,
+      'heap_size', total_heap_size,
+      'heap_native_size', total_native_alloc_registry_size,
       'obj_count', total_obj_count,
-      'reachable_heap_size', reachable_size,
-      'reachable_heap_native_size', reachable_native_size,
+      'reachable_heap_size', reachable_heap_size,
+      'reachable_heap_native_size', reachable_native_alloc_registry_size,
       'reachable_obj_count', reachable_obj_count,
-      'roots', roots,
-      'anon_rss_and_swap_size', closest_anon_swap_oom.anon_swap_val,
-      'oom_score_adj', closest_anon_swap_oom.oom_score_val
+      'oom_score_adj', oom_score_adj,
+      'anon_rss_and_swap_size', anon_rss_and_swap_size,
+      'dmabuf_rss_size', dmabuf_rss_size
     )) AS sample_protos
-  FROM base_stats
-  JOIN process USING (upid)
-  LEFT JOIN closest_anon_swap_oom USING (upid, graph_sample_ts)
+  FROM android_heap_graph_stats
   GROUP BY 1
 )
 SELECT JavaHeapStats(

src/trace_processor/perfetto_sql/stdlib/android/memory/BUILD.gn

@@ -21,6 +21,7 @@ perfetto_sql_source_set("memory") {
   ]
   sources = [
     "dmabuf.sql",
+    "dmabuf_spans.sql",
     "lmk.sql",
     "process.sql",
   ]

src/trace_processor/perfetto_sql/stdlib/android/memory/dmabuf_spans.sql

@@ -0,0 +1,35 @@
+--
+-- Copyright 2025 The Android Open Source Project
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     https://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+INCLUDE PERFETTO MODULE counters.intervals;
+
+CREATE PERFETTO TABLE _dmabuf_spans AS
+WITH
+  dmabuf_track AS (
+    SELECT
+      counter.*
+    FROM counter
+    JOIN counter_track AS track
+      ON track.id = counter.track_id AND track.name = 'mem.dmabuf_rss'
+  )
+SELECT
+  upid,
+  ts,
+  dur,
+  CAST(value AS INTEGER) AS dmabuf_rss
+FROM counter_leading_intervals!(dmabuf_track) AS dmabuf_counter
+JOIN process_counter_track
+  ON dmabuf_counter.track_id = process_counter_track.id;

src/trace_processor/perfetto_sql/stdlib/android/memory/heap_graph/BUILD.gn

@@ -23,6 +23,7 @@ perfetto_sql_source_set("heap_graph") {
     "dominator_tree.sql",
     "excluded_refs.sql",
     "heap_graph_class_aggregation.sql",
+    "heap_graph_stats.sql",
     "helpers.sql",
     "raw_dominator_tree.sql",
   ]