Merge pull request #1904 from DLR-AMR/enhancement-bigarray_timing

Add timing of big arrays in forest

Author: sandro-elsweijer

src/t8_forest/t8_forest.cxx

@@ -3796,6 +3796,9 @@ t8_forest_compute_profile (t8_forest_t forest)
     sc_stats_set1 (&forest->stats[11], profile->ghost_waittime, "forest: Ghost waittime.");
     sc_stats_set1 (&forest->stats[12], profile->balance_runtime, "forest: Balance runtime.");
     sc_stats_set1 (&forest->stats[13], profile->balance_rounds, "forest: Balance rounds.");
+    sc_stats_set1 (&forest->stats[14], profile->balance_rounds, "forest: Tree offset runtime.");
+    sc_stats_set1 (&forest->stats[15], profile->balance_rounds, "forest: offset runtime.");
+    sc_stats_set1 (&forest->stats[16], profile->balance_rounds, "forest: first descendant runtime.");
     /* compute stats */
     sc_stats_compute (sc_MPI_COMM_WORLD, T8_PROFILE_NUM_STATS, forest->stats);
     forest->stats_computed = 1;
@@ -3936,6 +3939,35 @@ t8_forest_profile_get_balance (t8_forest_t forest, int *balance_rounds)
   }
   return 0;
 }
+double
+t8_forest_profile_get_cmesh_offset_runtime (t8_forest_t forest)
+{
+  T8_ASSERT (t8_forest_is_committed (forest));
+  if (forest->profile != NULL) {
+    return forest->profile->cmesh_offsets_runtime;
+  }
+  return 0;
+}
+
+double
+t8_forest_profile_get_forest_offset_runtime (t8_forest_t forest)
+{
+  T8_ASSERT (t8_forest_is_committed (forest));
+  if (forest->profile != NULL) {
+    return forest->profile->forest_offsets_runtime;
+  }
+  return 0;
+}
+
+double
+t8_forest_profile_get_first_descendant_runtime (t8_forest_t forest)
+{
+  T8_ASSERT (t8_forest_is_committed (forest));
+  if (forest->profile != NULL) {
+    return forest->profile->first_descendant_runtime;
+  }
+  return 0;
+}
 
 void
 t8_forest_compute_elements_offset (t8_forest_t forest)

src/t8_forest/t8_forest_partition.cxx

@@ -90,6 +90,11 @@ t8_forest_compute_first_local_element_id (t8_forest_t forest)
 void
 t8_forest_partition_create_offsets (t8_forest_t forest)
 {
+  if (forest->profile != NULL) {
+    /* If profiling is enabled, we measure the runtime of partition */
+    forest->profile->forest_offsets_runtime = -sc_MPI_Wtime ();
+  }
+
   sc_MPI_Comm comm;
   t8_gloidx_t first_local_element;
 
@@ -113,6 +118,10 @@ t8_forest_partition_create_offsets (t8_forest_t forest)
     t8_shmem_array_set_gloidx (forest->element_offsets, forest->mpisize, forest->global_num_leaf_elements);
   }
   t8_shmem_array_end_writing (forest->element_offsets);
+  if (forest->profile != NULL) {
+    /* If profiling is enabled, we measure the runtime of partition */
+    forest->profile->forest_offsets_runtime += sc_MPI_Wtime ();
+  }
 }
 
 #if T8_ENABLE_DEBUG
@@ -255,6 +264,10 @@ t8_forest_partition_test_boundary_element ([[maybe_unused]] const t8_forest_t fo
 void
 t8_forest_partition_create_first_desc (t8_forest_t forest)
 {
+  if (forest->profile != NULL) {
+    /* If profiling is enabled, we measure the runtime of partition */
+    forest->profile->first_descendant_runtime = -sc_MPI_Wtime ();
+  }
   sc_MPI_Comm comm;
   t8_linearidx_t local_first_desc;
   t8_element_t *first_desc = NULL;
@@ -333,11 +346,21 @@ t8_forest_partition_create_first_desc (t8_forest_t forest)
   }
   t8_forest_partition_test_desc (forest);
 #endif
+  if (forest->profile != NULL) {
+    /* If profiling is enabled, we measure the runtime of partition */
+    forest->profile->first_descendant_runtime += sc_MPI_Wtime ();
+  }
 }
 
 void
 t8_forest_partition_create_tree_offsets (t8_forest_t forest)
 {
+
+  if (forest->profile != NULL) {
+    /* If profiling is enabled, we measure the runtime of partition */
+    forest->profile->cmesh_offsets_runtime = -sc_MPI_Wtime ();
+  }
+
   t8_gloidx_t tree_offset;
   sc_MPI_Comm comm;
   int is_empty, has_empty;
@@ -401,6 +424,10 @@ t8_forest_partition_create_tree_offsets (t8_forest_t forest)
     /* Communicate the new tree offsets */
     t8_shmem_array_allgather (&tree_offset, 1, T8_MPI_GLOIDX, forest->tree_offsets, 1, T8_MPI_GLOIDX);
   }
+  if (forest->profile != NULL) {
+    /* If profiling is enabled, we measure the runtime of partition */
+    forest->profile->cmesh_offsets_runtime += sc_MPI_Wtime ();
+  }
 }
 
 /* Calculate the new element_offset for forest from

src/t8_forest/t8_forest_profiling.h

@@ -185,6 +185,31 @@ t8_forest_profile_get_ghost_time (t8_forest_t forest, t8_locidx_t *ghosts_sent);
  */
 double
 t8_forest_profile_get_ghostexchange_waittime (t8_forest_t forest);
+
+/** Get the runtime of the last call to \ref t8_forest_partition_create_tree_offsets.
+ * \param [in]   forest         The forest.
+ * \return                      The time
+ * \a forest must be committed before calling this function.
+ * \see t8_forest_set_profiling
+ */
+double
+t8_forest_profile_get_cmesh_offsets_runtime (t8_forest_t forest);
+/** Get the runtime of the last call to \ref t8_forest_partition_create_offsets.
+ * \param [in]   forest         The forest.
+ * \return                      The time
+ * \a forest must be committed before calling this function.
+ * \see t8_forest_set_profiling
+ */
+double
+t8_forest_profile_get_forest_offsets_runtime (t8_forest_t forest);
+/** Get the waittime of the last call to \ref t8_forest_partition_create_first_desc.
+ * \param [in]   forest         The forest.
+ * \return                      The time
+ * \a forest must be committed before calling this function.
+ * \see t8_forest_set_profiling
+ */
+double
+t8_forest_profile_get_first_descendant_runtime (t8_forest_t forest);
 T8_EXTERN_C_END ();
 
 #endif /* !T8_FOREST_PROFILING_H */

src/t8_forest/t8_forest_types.h

@@ -62,7 +62,7 @@ typedef int8_t t8_forest_from_t;
 #define T8_FOREST_BALANCE_NO_REPART 2 /**< Value of forest->set_balance if balancing without repartitioning */
 
 /** The number of statistics collected by a profile struct. */
-#define T8_PROFILE_NUM_STATS 14
+#define T8_PROFILE_NUM_STATS 17
 
 /** This structure is private to the implementation. */
 typedef struct t8_forest
@@ -157,9 +157,6 @@ typedef struct t8_tree
  * \see t8_cmesh_set_profiling and \see t8_cmesh_print_profile
  */
 
-/** The number of statistics collected by a profile struct. */
-#define T8_PROFILE_NUM_STATS 14
-
 /**
  * This struct holds profiling information, such as timings or statistics about communication.
 */
@@ -179,14 +176,17 @@ typedef struct t8_profile
   int ghosts_remotes;                     /**< The number of processes this process have sent ghost elements to
                                                   (and received from). */
   int balance_rounds;                     /**< The number of iterations during balance. */
-  double adapt_runtime;     /**< The runtime of the last call to \a t8_forest_adapt (not counting adaptation
+  double adapt_runtime;            /**< The runtime of the last call to \a t8_forest_adapt (not counting adaptation
                                                   in t8_forest_balance). */
-  double partition_runtime; /**< The runtime of the last call to \a t8_cmesh_partition (not count in
+  double partition_runtime;        /**< The runtime of the last call to \a t8_cmesh_partition (not count in
                                                   partition in t8_forest_balance). */
-  double ghost_runtime;     /**< The runtime of the last call to \a t8_forest_ghost_create. */
-  double ghost_waittime;    /**< Amount of synchronisation time in ghost. */
-  double balance_runtime;   /**< The runtime of the last call to \a t8_forest_balance. */
-  double commit_runtime;    /**< The runtime of the last call to \a t8_cmesh_commit. */
+  double ghost_runtime;            /**< The runtime of the last call to \a t8_forest_ghost_create. */
+  double ghost_waittime;           /**< Amount of synchronisation time in ghost. */
+  double balance_runtime;          /**< The runtime of the last call to \a t8_forest_balance. */
+  double commit_runtime;           /**< The runtime of the last call to \a t8_cmesh_commit. */
+  double cmesh_offsets_runtime;    /**< The runtime of the last call to \a t8_forest_partition_create_tree_offsets. */
+  double forest_offsets_runtime;   /**< The runtime of the last call to \a t8_forest_partition_create_offsets. */
+  double first_descendant_runtime; /**< The runtime of the last call to \a t8_forest_partition_create_first_desc. */
 
 } t8_profile_struct_t;