Merge pull request #2026 from DLR-AMR/feature_pfc_revisited

Feature: Partition for Coarsening (PFC)

Author: Davknapp

src/CMakeLists.txt

@@ -172,6 +172,8 @@ target_sources( T8 PRIVATE
     t8_data/t8_containers.cxx 
     t8_forest/t8_forest_adapt.cxx 
     t8_forest/t8_forest_partition.cxx 
+    t8_forest/t8_forest_partition_for_coarsening.cxx
+    t8_forest/t8_forest_pfc_helper.cxx 
     t8_forest/t8_forest.cxx 
     t8_forest/t8_forest_private.cxx
     t8_forest/t8_forest_ghost.cxx

src/t8.h

@@ -98,7 +98,8 @@ typedef int32_t t8_locidx_t;
 #define T8_LOCIDX_MAX INT32_MAX
 /** Comparison function for t8_locidx_t */
 #define t8_compare_locidx(v, w) sc_int32_compare (v, w)
-
+/** A type for holding process ids. */
+typedef int t8_procidx_t;
 /** A type for global indexing that holds really big numbers. */
 typedef int64_t t8_gloidx_t;
 /** The MPI Datatype of t8_gloidx_t */
@@ -146,7 +147,8 @@ typedef enum {
   T8_MPI_CMESH_UNIFORM_BOUNDS_START,    /**< Used for cmesh uniform bounds computation. */
   T8_MPI_CMESH_UNIFORM_BOUNDS_END,      /**< Used for cmesh uniform bounds computation. */
   T8_MPI_TEST_ELEMENT_PACK_TAG,         /**< Used for testing mpi pack and unpack functionality */
-  T8_MPI_TAG_LAST                       /**< Dummy last MPT tag. */
+  T8_MPI_PFC_TAG,                       /**< Used for data exchange during partition for coarsening. */
+  T8_MPI_TAG_LAST                       /**< Dummy last MPI tag. */
 } t8_MPI_tag_t;
 
 /** Query the package identity as registered in libsc.

src/t8_eclass.h

@@ -80,6 +80,8 @@ typedef enum t8_eclass {
 #define T8_ECLASS_MAX_CORNERS 8
 /** The maximal possible dimension for an eclass */
 #define T8_ECLASS_MAX_DIM 3
+/** The maximal number of children any element may have.*/
+#define T8_ECLASS_MAX_CHILDREN 10
 
 /* clang-format off */
 

src/t8_forest/t8_forest.cxx

@@ -1243,7 +1243,8 @@ t8_forest_tree_shared ([[maybe_unused]] t8_forest_t forest, [[maybe_unused]] int
   T8_ASSERT (forest != NULL);
   T8_ASSERT (forest->first_local_tree > -1);
   T8_ASSERT (forest->first_local_tree <= forest->global_num_trees);
-  T8_ASSERT (forest->last_local_tree < forest->global_num_trees);
+  T8_ASSERT (forest->first_local_tree <= forest->last_local_tree + 1);
+  T8_ASSERT (forest->last_local_tree <= forest->global_num_trees);
 #if T8_ENABLE_DEBUG
   if (forest->first_local_tree == 0 && forest->last_local_tree == -1) {
     T8_ASSERT (forest->last_local_tree < 0);

src/t8_forest/t8_forest_partition.cxx

@@ -28,6 +28,7 @@
 #include <t8_forest/t8_forest_types.h>
 #include <t8_forest/t8_forest_private.h>
 #include <t8_forest/t8_forest_general.h>
+#include <t8_forest/t8_forest_partition_for_coarsening.h>
 #include <t8_cmesh/t8_cmesh_internal/t8_cmesh_offset.h>
 #include <t8_schemes/t8_scheme.hxx>
 
@@ -540,6 +541,12 @@ t8_forest_partition_compute_new_offset (t8_forest_t forest)
     }
   }
   t8_shmem_array_end_writing (forest->element_offsets);
+
+  // In case the partition-for-coarsening flag is set, correct the partitioning if a family of elements is
+  // split across process boundaries
+  if (forest->set_for_coarsening != 0) {
+    t8_forest_pfc_correction_offsets (forest);
+  }
 }
 
 /* Find the owner of a given element.

src/t8_forest/t8_forest_partition_for_coarsening.cxx

@@ -0,0 +1,51 @@
+/*
+  This file is part of t8code.
+  t8code is a C library to manage a collection (a forest) of multiple
+  connected adaptive space-trees of general element classes in parallel.
+
+  Copyright (C) 2025 the developers
+
+  t8code is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  t8code is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with t8code; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+/** \file t8_forest_partition_for_coarsening.h
+ * adjust the partition of the receiving forest so that all elements from a family
+ * belong to the same process before applying the partition
+ */
+
+#ifndef T8_FOREST_PFC_H
+#define T8_FOREST_ITERATE_H
+
+#include <t8.h>
+#include <t8_forest/t8_forest_general.h>
+
+T8_EXTERN_C_BEGIN ();
+
+/** \brief Correct the partitioning if element families are split accorss process boundaries.
+ *
+ *  The default partitioning distributes the elements into equally-sized partitions. For coarsening, however,
+ *  all elements of a family have to be on the same process in order to be coarsened into their parent element.
+ *  This function corrects the partitioning such that no families are split across process boundaries.
+ *  The price to be paid is a slight deviation from the optimal balance of elements among processors.
+ *
+ * \param [in,out] forest   the forest. On input, it has been partitioned into equally-sized element partitions.
+ *                                      On output, the partitioning has been adjusted such that no element families
+ *                                      are split across the process boundaries.
+*/
+void
+t8_forest_pfc_correction_offsets (t8_forest_t forest);
+
+T8_EXTERN_C_END ();
+#endif /* T8_FOREST_ITERATE_H */

src/t8_forest/t8_forest_partition_for_coarsening.h

@@ -0,0 +1,122 @@
+/*
+  This file is part of t8code.
+  t8code is a C library to manage a collection (a forest) of multiple
+  connected adaptive space-trees of general element classes in parallel.
+
+  Copyright (C) 2025 the developers
+
+  t8code is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  t8code is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with t8code; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+/**
+ * \file t8_forest_pfc_helper.cxx
+ * This file defines some helper functions used for the partition-for-coarsening feature.
+*/
+
+#include <t8_forest/t8_forest_general.h>
+#include <t8_forest/t8_forest_private.h>
+#include <t8_forest/t8_forest_types.h>
+#include <t8_forest/t8_forest_pfc_helper.hxx>
+#include <t8_schemes/t8_scheme.h>
+#include <t8_schemes/t8_scheme.hxx>
+#include <t8_element.h>
+#include <algorithm>
+
+t8_locidx_t
+t8_forest_pfc_extreme_local_sibling (const t8_scheme_c *scheme, const t8_tree_t tree,
+                                     const t8_locidx_t start_element_id_in_tree, const int signed_increment)
+{
+  // The increment may either be +1 or -1.
+  T8_ASSERT (abs (signed_increment) == 1);
+
+  // Initialization and memory allocation.
+  const t8_eclass_t tree_class = tree->eclass;
+  t8_element_t *parent_possible_sibling, *parent_start;
+  t8_element_new (scheme, tree_class, 1, &parent_possible_sibling);
+  t8_element_new (scheme, tree_class, 1, &parent_start);
+
+  // Determine start element from tree and start ID within tree.
+  const t8_element_t *start_element = t8_forest_get_tree_leaf_element (tree, start_element_id_in_tree);
+
+  // If the start element is of level zero, i.e., the root, it does not have any siblings.
+  if (scheme->element_get_level (tree_class, start_element) == 0) {
+    return start_element_id_in_tree;
+  }
+
+  // Get parent of start element.
+  scheme->element_get_parent (tree_class, start_element, parent_start);
+
+  // Determine the parent's number of children.
+  const int num_children = scheme->element_get_num_children (tree_class, parent_start);
+
+  // Determine bound to be used within element loop (and make sure to stay within tree element range);
+  t8_locidx_t extreme_check_id_in_tree = start_element_id_in_tree + signed_increment * num_children;
+  extreme_check_id_in_tree
+    = SC_MAX (0, SC_MIN (extreme_check_id_in_tree, t8_forest_get_tree_leaf_element_count (tree) - 1));
+
+  // Initialize extreme_sibling_id_in_tree.
+  t8_locidx_t extreme_sibling_id_in_tree = start_element_id_in_tree;
+
+  // Loop over local IDs of all elements that may form a family
+  for (t8_locidx_t ielem = start_element_id_in_tree; (ielem - extreme_check_id_in_tree) * signed_increment <= 0;
+       ielem += signed_increment) {
+
+    // Get element from iteration index.
+    const t8_element_t *possible_sibling = t8_forest_get_tree_leaf_element (tree, ielem);
+
+    // Determine parent and check whether it matches parent_start:
+    // - if it does, the current element is a sibling of start_element. Thus, extreme_sibling_id_in_tree is updated.
+    scheme->element_get_parent (tree_class, possible_sibling, parent_possible_sibling);
+    if (scheme->element_is_equal (tree_class, parent_start, parent_possible_sibling)) {
+      extreme_sibling_id_in_tree = ielem;
+    }
+    // - else, the iteration has left the family and we can exit.
+    else {
+      break;
+    }
+  }
+
+  // Deallocation
+  t8_element_destroy (scheme, tree_class, 1, &parent_possible_sibling);
+  t8_element_destroy (scheme, tree_class, 1, &parent_start);
+
+  // Return extreme sibling ID.
+  return extreme_sibling_id_in_tree;
+}
+
+void
+t8_forest_pfc_helper_index_in_tree_from_globalid (const t8_forest_t forest, const t8_gloidx_t gelement_id,
+                                                  t8_gloidx_t &gtree_id, t8_tree_t &tree, t8_locidx_t &index_in_tree,
+                                                  t8_element_t *&element)
+{
+  // Determine local element ID by subtracting given and the process' first global ID.
+  const t8_gloidx_t global_id_of_first_local_element = t8_forest_get_first_local_leaf_element_id (forest);
+  const t8_locidx_t lelement_id = (t8_locidx_t) (gelement_id - global_id_of_first_local_element);
+
+  // Determine the element (as pointer) and the local tree ID.
+  t8_locidx_t ltree_id;
+  element = t8_forest_get_leaf_element (forest, lelement_id, &ltree_id);
+
+  // From local tree ID, get a pointer to the tree.
+  tree = t8_forest_get_tree (forest, ltree_id);
+
+  // Determine the index within the tree - and run sanity check.
+  index_in_tree = lelement_id - tree->elements_offset;
+  T8_ASSERT (element == t8_forest_get_tree_leaf_element (tree, index_in_tree));
+
+  // Compute global tree ID as the local one plus the process' first tree ID.
+  const t8_locidx_t first_local_tree_id = t8_forest_get_first_local_tree_id (forest);
+  gtree_id = first_local_tree_id + ltree_id;
+}

src/t8_forest/t8_forest_pfc_helper.cxx

@@ -0,0 +1,62 @@
+/*
+  This file is part of t8code.
+  t8code is a C library to manage a collection (a forest) of multiple
+  connected adaptive space-trees of general element classes in parallel.
+
+  Copyright (C) 2025 the developers
+
+  t8code is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  t8code is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with t8code; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+/**
+ * \file t8_forest_pfc_helper.hxx
+ * This file declares some helper functions used for the partition-for-coarsening feature.
+*/
+#ifndef T8_FOREST_PFC_HELPER_H
+#define T8_FOREST_PFC_HELPER_H
+
+#include <t8.h>
+#include <t8_data/t8_shmem.h>
+#include <t8_eclass.h>
+
+/** Determine the sibling with the biggest difference in IDs (in the given direction).
+ *
+ * \param[in] scheme                    the refinement scheme
+ * \param[in] tree                      the considered tree
+ * \param[in] start_element_id_in_tree  the tree-internal ID of the considered element
+ * \param[in] signed_increment          increment determining the search direction. Has to be
+ *                                      either -1 (decreasing IDs) or +1 (increasing IDs).
+ *
+ * \return The extreme sibling ID within tree, i.e., the tree-internal ID of the sibling
+ *         with the biggest difference to start_element_id_in_tree.
+*/
+t8_locidx_t
+t8_forest_pfc_extreme_local_sibling (const t8_scheme_c *scheme, t8_tree_t tree,
+                                     const t8_locidx_t start_element_id_in_tree, const int signed_increment);
+
+/** Helper function for PFC that computes multiple indices for a given global element ID.
+ *
+ * \param[in]   forest        the forest
+ * \param[in]   gelement_id   the global element ID
+ * \param[out]  gtree_id      the global ID of the tree holding the element
+ * \param[out]  tree          the tree holding the element
+ * \param[out]  index_in_tree index of the element within the tree
+ * \param[out]  element       pointer to the considered element
+*/
+void
+t8_forest_pfc_helper_index_in_tree_from_globalid (const t8_forest_t forest, const t8_gloidx_t gelement_id,
+                                                  t8_gloidx_t &gtree_id, t8_tree_t &tree, t8_locidx_t &index_in_tree,
+                                                  t8_element_t *&element);
+#endif