From c402f6ff0d83c8c7087f0c73a2c7e6c4bed18d1a Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 12 Mar 2024 11:32:29 +0100
Subject: [PATCH 01/90] treat local_mtx and non_local_mtx as single interface
 mtx

---
 DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 75f838306..f3545a9c8 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -178,16 +178,17 @@ struct MatrixInitFunctor {
         auto exec = exec_.get_ref_exec();
 
         auto num_rows = partition_.get_local_size();
-        gko::device_matrix_data<scalar, label> A_data(
+        std::vector<device_matrix_data> A_data_v;
+        A_data_v.emplace_back(
             exec, gko::dim<2>(num_rows, num_rows), *rows.get(), *cols.get(),
             *coeffs.get());
 
-        gko::device_matrix_data<scalar, label> non_local_A_data(
+        std::vector<device_matrix_data> non_local_A_data_v;
+        non_local_A_data.emplace_back(
             exec, gko::dim<2>(num_rows, non_local_cols->get_size()),
             *non_local_rows.get(), *non_local_cols.get(),
             *non_local_coeffs.get());
 
-
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
             exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm());
@@ -201,7 +202,8 @@ struct MatrixInitFunctor {
         auto send_sizes = sparse_comm->get_send_sizes();
         auto send_offs = sparse_comm->get_send_offsets();
 
-        dist_A->read_distributed(A_data, non_local_A_data, sparse_comm);
+        // TODO this needs a std::vector version
+        dist_A->read_distributed(A_data_v, non_local_A_data_v, sparse_comm);
 
         auto device_mat = generate_dist_mtx_with_inner_type(
             exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());

From 8abaa113cf80a1389308adf902d2c04d58d2bf97 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 12 Mar 2024 11:45:40 +0100
Subject: [PATCH 02/90] update remode branch

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 895020167..90ba391e6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -49,7 +49,7 @@ if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO))
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "fc86d48b78cebd2b2c5833a2dcf0fe40f615cf19"
+    "sparse-communicator-cmake-modernization-combination-interface-ogl"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)

From 2488c0c1ba427866d4408ccc72b9d72670747a34 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 8 Mar 2024 10:55:39 +0100
Subject: [PATCH 03/90] move to FetchContent Build of OF

---
 CMakeLists.txt                    | 12 --------
 third_party/ginkgo/CMakeLists.txt | 51 +++++++++++++++++++------------
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 90ba391e6..f86fabdd8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -63,18 +63,6 @@ ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.7.0)
 
 add_subdirectory(third_party)
 
-# If ginkgo is not found at this point, look for the third_party installation
-if(NOT ${OGL_USE_EXTERNAL_GINKGO})
-  find_package(
-    Ginkgo
-    CONFIG
-    1.7.0
-    REQUIRED
-    PATHS
-    $ENV{FOAM_USER_LIBBIN}/cmake/Ginkgo
-    ${CMAKE_CURRENT_BINARY_DIR}/third_party/ginkgo/build/install/lib/cmake)
-endif()
-
 # C++ 14 standard or later is required to interface with Ginkgo OGL itself uses
 # some c++ 17 features
 
diff --git a/third_party/ginkgo/CMakeLists.txt b/third_party/ginkgo/CMakeLists.txt
index 758716bd3..db1c9670a 100644
--- a/third_party/ginkgo/CMakeLists.txt
+++ b/third_party/ginkgo/CMakeLists.txt
@@ -3,22 +3,35 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 # TODO pass flags from main CMAKE file
-ginkgo_load_git_package(
-  ginkgo_external
-  "https://github.com/ginkgo-project/ginkgo.git"
-  ${GINKGO_CHECKOUT_VERSION}
-  "-DGINKGO_BUILD_CUDA=${GINKGO_BUILD_CUDA}"
-  "-DGINKGO_BUILD_HIP=${GINKGO_BUILD_HIP}"
-  "-DGINKGO_BUILD_OMP=${GINKGO_BUILD_OMP}"
-  "-DGINKGO_BUILD_MPI=ON"
-  "-DGINKGO_BUILD_TESTS=OFF"
-  "-DGINKGO_BUILD_EXAMPLES=OFF"
-  "-DGINKGO_BUILD_BENCHMARKS=OFF"
-  "-DGINKGO_BUILD_REFERENCE=${GINKGO_BUILD_REFERENCE}"
-  "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
-  "-DCMAKE_CUDA_ARCHITECTURES=${OGL_CUDA_ARCHITECTURES}"
-  "-DGINKGO_BUILD_HWLOC=OFF"
-  "-DGINKGO_FORCE_GPU_AWARE_MPI=${GINKGO_FORCE_GPU_AWARE_MPI}"
-  "-DCMAKE_INSTALL_PREFIX=$ENV{FOAM_USER_LIBBIN}/.."
-  "-DBUILD_SHARED_LIBS=ON"
-  "-DCMAKE_CXX_STANDARD=17")
+include(FetchContent)
+
+if(NOT ${OGL_USE_EXTERNAL_GINKGO})
+	FetchContent_Declare(
+	Ginkgo
+	QUITE
+	GIT_SHALLOW ON
+	GIT_REPOSITORY  "https://github.com/ginkgo-project/ginkgo.git"
+	GIT_TAG ${GINKGO_CHECKOUT_VERSION}
+	)
+	FetchContent_MakeAvailable(Ginkgo)
+endif()
+
+#ginkgo_load_git_package(
+#  ginkgo_external
+#  "https://github.com/ginkgo-project/ginkgo.git"
+#  ${GINKGO_CHECKOUT_VERSION}
+#  "-DGINKGO_BUILD_CUDA=${GINKGO_BUILD_CUDA}"
+#  "-DGINKGO_BUILD_HIP=${GINKGO_BUILD_HIP}"
+#  "-DGINKGO_BUILD_OMP=${GINKGO_BUILD_OMP}"
+#  "-DGINKGO_BUILD_MPI=ON"
+#  "-DGINKGO_BUILD_TESTS=OFF"
+#  "-DGINKGO_BUILD_EXAMPLES=OFF"
+#  "-DGINKGO_BUILD_BENCHMARKS=OFF"
+#  "-DGINKGO_BUILD_REFERENCE=${GINKGO_BUILD_REFERENCE}"
+#  "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+#  "-DCMAKE_CUDA_ARCHITECTURES=${OGL_CUDA_ARCHITECTURES}"
+#  "-DGINKGO_BUILD_HWLOC=OFF"
+#  "-DGINKGO_FORCE_GPU_AWARE_MPI=${GINKGO_FORCE_GPU_AWARE_MPI}"
+#  "-DCMAKE_INSTALL_PREFIX=$ENV{FOAM_USER_LIBBIN}/.."
+#  "-DBUILD_SHARED_LIBS=ON"
+#  "-DCMAKE_CXX_STANDARD=17")

From a489a112d2218ebb9131d4d60c6fa00706ab2bac Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 8 Mar 2024 22:04:59 +0100
Subject: [PATCH 04/90] add a version that compiles with recent cmake

---
 CMakeLists.txt                    | 40 +++++++++++++++++++------------
 CMakePresets.json                 |  3 ++-
 third_party/ginkgo/CMakeLists.txt | 37 ++++++++--------------------
 3 files changed, 37 insertions(+), 43 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f86fabdd8..2d17e2862 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,8 +14,11 @@ if(NOT DEFINED ENV{FOAM_SRC})
   message(FATAL_ERROR "You must source OpenFOAM before building OGL")
 endif()
 
-# Configuration options
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
+# Configuration options
 include(cmake/build_type_helpers.cmake)
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/")
 
@@ -26,17 +29,23 @@ option(OGL_USE_FOAM_FOUNDATION_VERSION
        "Build OGL for the OpenFOAM Foundation version" FALSE)
 option(OGL_BUILD_UNITTEST "Build OGL unittests" FALSE)
 option(OGL_CUDA_ARCHITECTURES "Which cuda architecture to build for" "native")
-option(GINKGO_BUILD_CUDA "Build Ginkgo with cuda backend" FALSE)
-option(GINKGO_BUILD_HIP "Build Ginkgo with hip backend" FALSE)
-option(GINKGO_BUILD_SYCL "Build Ginkgo with sycl backend" FALSE)
-option(GINKGO_BUILD_OMP "Build Ginkgo with omp backend" FALSE)
 option(GINKGO_BUILD_REFERENCE "Build Ginkgo with reference backend" TRUE)
 option(OGL_ALLOW_REFERENCE_ONLY
        "Enable builds of Ginkgo with reference backend only" FALSE)
-option(GINKGO_FORCE_GPU_AWARE_MPI "Build Ginkgo using device aware MPI" TRUE)
 option(GINKGO_WITH_OGL_EXTENSIONS "Whether ginkgo was build with OGL extension"
        FALSE)
 
+option(GINKGO_FORCE_GPU_AWARE_MPI "Build Ginkgo using device aware MPI" TRUE)
+option(GINKGO_BUILD_CUDA "Build Ginkgo with cuda backend" FALSE)
+option(GINKGO_BUILD_HIP "Build Ginkgo with hip backend" FALSE)
+option(GINKGO_BUILD_SYCL "Build Ginkgo with sycl backend" FALSE)
+option(GINKGO_BUILD_OMP "Build Ginkgo with omp backend" FALSE)
+
+set(GINKGO_BUILD_MPI ON CACHE INTERNAL "")
+set(GINKGO_BUILD_TESTS OFF CACHE INTERNAL "")
+set(GINKGO_BUILD_EXAMPLES OFF CACHE INTERNAL "")
+set(GINKGO_BUILD_BENCHMARKS OFF CACHE INTERNAL "")
+
 if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO))
   if((NOT GINKGO_BUILD_CUDA)
      AND (NOT GINKGO_BUILD_HIP)
@@ -63,12 +72,6 @@ ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.7.0)
 
 add_subdirectory(third_party)
 
-# C++ 14 standard or later is required to interface with Ginkgo OGL itself uses
-# some c++ 17 features
-
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0 -ggdb")
 set(CMAKE_CXX_FLAGS_RELEASE
@@ -191,8 +194,7 @@ target_include_directories(
 target_link_libraries(
   OGL
   PRIVATE $ENV{FOAM_LIBBIN}/libOpenFOAM.so $ENV{FOAM_LIBBIN}/libfiniteVolume.so
-          $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so
-  PUBLIC Ginkgo::ginkgo stdc++fs)
+          $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so Ginkgo::ginkgo stdc++fs)
 
 if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)
@@ -224,4 +226,12 @@ add_custom_target(
   COMMAND find . -name CMakeLists.txt -exec cmake-format -i {} +
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
 
-install(TARGETS OGL DESTINATION $ENV{FOAM_USER_LIBBIN})
+install(TARGETS OGL
+    ginkgo
+    ginkgo_device
+    ginkgo_hip
+    ginkgo_cuda
+    ginkgo_omp
+    ginkgo_dpcpp
+    ginkgo_reference
+    DESTINATION $ENV{FOAM_USER_LIBBIN})
diff --git a/CMakePresets.json b/CMakePresets.json
index 890dd7613..b62aed332 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -13,7 +13,8 @@
       "generator": "Ninja",
       "binaryDir": "${sourceDir}/build/Release",
       "cacheVariables": {
-        "CMAKE_BUILD_TYPE": "Release"
+        "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_INSTALL_PREFIX": "$env{FOAM_USER_LIBBIN}/.."
       }
     },
     {
diff --git a/third_party/ginkgo/CMakeLists.txt b/third_party/ginkgo/CMakeLists.txt
index db1c9670a..ca0facdfd 100644
--- a/third_party/ginkgo/CMakeLists.txt
+++ b/third_party/ginkgo/CMakeLists.txt
@@ -6,32 +6,15 @@
 include(FetchContent)
 
 if(NOT ${OGL_USE_EXTERNAL_GINKGO})
-	FetchContent_Declare(
-	Ginkgo
-	QUITE
-	GIT_SHALLOW ON
-	GIT_REPOSITORY  "https://github.com/ginkgo-project/ginkgo.git"
-	GIT_TAG ${GINKGO_CHECKOUT_VERSION}
-	)
-	FetchContent_MakeAvailable(Ginkgo)
+    FetchContent_Declare(
+    Ginkgo
+    QUITE
+    GIT_SHALLOW ON
+    GIT_REPOSITORY  "https://github.com/ginkgo-project/ginkgo.git"
+    GIT_TAG ${GINKGO_CHECKOUT_VERSION}
+    )
+
+    FetchContent_MakeAvailable(Ginkgo)
+
 endif()
 
-#ginkgo_load_git_package(
-#  ginkgo_external
-#  "https://github.com/ginkgo-project/ginkgo.git"
-#  ${GINKGO_CHECKOUT_VERSION}
-#  "-DGINKGO_BUILD_CUDA=${GINKGO_BUILD_CUDA}"
-#  "-DGINKGO_BUILD_HIP=${GINKGO_BUILD_HIP}"
-#  "-DGINKGO_BUILD_OMP=${GINKGO_BUILD_OMP}"
-#  "-DGINKGO_BUILD_MPI=ON"
-#  "-DGINKGO_BUILD_TESTS=OFF"
-#  "-DGINKGO_BUILD_EXAMPLES=OFF"
-#  "-DGINKGO_BUILD_BENCHMARKS=OFF"
-#  "-DGINKGO_BUILD_REFERENCE=${GINKGO_BUILD_REFERENCE}"
-#  "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
-#  "-DCMAKE_CUDA_ARCHITECTURES=${OGL_CUDA_ARCHITECTURES}"
-#  "-DGINKGO_BUILD_HWLOC=OFF"
-#  "-DGINKGO_FORCE_GPU_AWARE_MPI=${GINKGO_FORCE_GPU_AWARE_MPI}"
-#  "-DCMAKE_INSTALL_PREFIX=$ENV{FOAM_USER_LIBBIN}/.."
-#  "-DBUILD_SHARED_LIBS=ON"
-#  "-DCMAKE_CXX_STANDARD=17")

From f6f71819e67aae2c6924bd07d284d3e271a62ff9 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 13 Mar 2024 08:32:46 +0100
Subject: [PATCH 05/90] add in matrix, and create dist_mtx with Combination

---
 DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index f3545a9c8..61fb72d62 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -139,6 +139,9 @@ struct MatrixInitFunctor {
         std::shared_ptr<gko::Executor> exec,
         std::shared_ptr<gko::experimental::mpi::communicator> comm) const
     {
+        return dist_mtx::create(exec, *comm.get(),
+                                gko::with_matrix_type<gko::Combination>());
+
         if (matrix_format_ == "Csr") {
             return dist_mtx::create(exec, *comm.get(),
                                     gko::with_matrix_type<gko::matrix::Csr>());
@@ -203,7 +206,8 @@ struct MatrixInitFunctor {
         auto send_offs = sparse_comm->get_send_offsets();
 
         // TODO this needs a std::vector version
-        dist_A->read_distributed(A_data_v, non_local_A_data_v, sparse_comm);
+        dist_A->read_distributed<>(A_data_v, non_local_A_data_v, sparse_comm,
+                                 gko::matrix::Coo::create());
 
         auto device_mat = generate_dist_mtx_with_inner_type(
             exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());

From ba17e2f2403d437245df0d274cbd426a4e2db287 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 13 Mar 2024 13:20:52 +0100
Subject: [PATCH 06/90] add basic custom matrix implementation

---
 CMakeLists.txt                                |  7 ++--
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 41 ++++++++++---------
 DevicePersistent/Partition/Partition.H        |  2 +-
 .../LduMatrix}/GKOACG/GKOACG.C                |  0
 .../LduMatrix}/GKOACG/GKOACG.H                |  0
 lduLduBase/lduLduBase.H                       |  2 +-
 unitTests/test_HostMatrix.C                   |  3 +-
 7 files changed, 29 insertions(+), 26 deletions(-)
 rename {LduMatrix => MatrixWrapper/LduMatrix}/GKOACG/GKOACG.C (100%)
 rename {LduMatrix => MatrixWrapper/LduMatrix}/GKOACG/GKOACG.H (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2d17e2862..f213ab068 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -156,8 +156,8 @@ target_sources(
           Preconditioner/Preconditioner.C
           BaseWrapper/lduBase/GKOlduBase.C
           BaseWrapper/CoupledLduBase/GKOCoupledLduBase.C
-          HostMatrix/HostMatrix.C
-          HostMatrix/HostMatrixFreeFunctions.C
+          MatrixWrapper/HostMatrix/HostMatrix.C
+          MatrixWrapper/HostMatrix/HostMatrixFreeFunctions.C
           Solver/CG/GKOCG.C
           Solver/BiCGStab/GKOBiCGStab.C
           # Solver / IR / GKOIR.C Solver / Multigrid / GKOMultigrid.C
@@ -166,7 +166,8 @@ target_sources(
   PUBLIC common/common.H
          StoppingCriterion/StoppingCriterion.H
          lduLduBase/lduLduBase.H
-         HostMatrix/HostMatrix.H
+         MatrixWrapper/HostMatrix/HostMatrix.H
+         MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
          DevicePersistent/Base/Base.H
          DevicePersistent/Partition/Partition.H
          DevicePersistent/Array/Array.H
diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 61fb72d62..c09a609b8 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -9,6 +9,8 @@
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "DevicePersistent/Partition/Partition.H"
 
+#include "MatrixWrapper/GkoCombinationMatrix/GkoCombinatinoMatrix.H"
+
 #include "fvCFD.H"
 
 namespace Foam {
@@ -139,25 +141,28 @@ struct MatrixInitFunctor {
         std::shared_ptr<gko::Executor> exec,
         std::shared_ptr<gko::experimental::mpi::communicator> comm) const
     {
-        return dist_mtx::create(exec, *comm.get(),
-                                gko::with_matrix_type<gko::Combination>());
-
         if (matrix_format_ == "Csr") {
-            return dist_mtx::create(exec, *comm.get(),
-                                    gko::with_matrix_type<gko::matrix::Csr>());
+            return dist_mtx::create(
+                exec, *comm.get(),
+                gko::with_matrix_type<
+                    CombinationMatrix<scalar, label, gko::matrix::Csr>>());
         }
         if (matrix_format_ == "Ell") {
-            return dist_mtx::create(exec, *comm.get(),
-                                    gko::with_matrix_type<gko::matrix::Ell>());
+            return dist_mtx::create(
+                exec, *comm.get(),
+                gko::with_matrix_type <
+                    CombinationMatrix<scalar, label, gko::matrix::Ell>());
+        }
+        if (matrix_format_ == "Coo") {
+            return dist_mtx::create(
+                exec, *comm.get(),
+                gko::with_matrix_type <
+                    CombinationMatrix<scalar, label, gko::matrix::Coo>());
         }
         // if (matrix_format_ == "Hybrid") {
         //     return dist_mtx::create(
         //         *comm.get(), gko::with_matrix_type<gko::matrix::Hybrid>());
         // }
-        if (matrix_format_ == "Coo") {
-            return dist_mtx::create(exec, *comm.get(),
-                                    gko::with_matrix_type<gko::matrix::Coo>());
-        }
 
         FatalErrorInFunction << "Matrix format " << matrix_format_
                              << " not supported " << abort(FatalError);
@@ -181,16 +186,13 @@ struct MatrixInitFunctor {
         auto exec = exec_.get_ref_exec();
 
         auto num_rows = partition_.get_local_size();
-        std::vector<device_matrix_data> A_data_v;
-        A_data_v.emplace_back(
-            exec, gko::dim<2>(num_rows, num_rows), *rows.get(), *cols.get(),
-            *coeffs.get());
+        device_matrix_data A_data{exec, gko::dim<2>(num_rows, num_rows),
+                                  *rows.get(), *cols.get(), *coeffs.get()};
 
-        std::vector<device_matrix_data> non_local_A_data_v;
-        non_local_A_data.emplace_back(
+        device_matrix_data non_local_A_data{
             exec, gko::dim<2>(num_rows, non_local_cols->get_size()),
             *non_local_rows.get(), *non_local_cols.get(),
-            *non_local_coeffs.get());
+            *non_local_coeffs.get()};
 
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
@@ -206,8 +208,7 @@ struct MatrixInitFunctor {
         auto send_offs = sparse_comm->get_send_offsets();
 
         // TODO this needs a std::vector version
-        dist_A->read_distributed<>(A_data_v, non_local_A_data_v, sparse_comm,
-                                 gko::matrix::Coo::create());
+        dist_A->read_distributed(local_A_data, non_local_A_data, sparse_comm);
 
         auto device_mat = generate_dist_mtx_with_inner_type(
             exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 52ca51d77..7d57da673 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -10,7 +10,7 @@
 
 #include "DevicePersistent/Base/Base.H"
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
-#include "HostMatrix/HostMatrix.H"
+#include "MatrixWrapper/HostMatrix/HostMatrix.H"
 #include "common/common.H"
 
 namespace Foam {
diff --git a/LduMatrix/GKOACG/GKOACG.C b/MatrixWrapper/LduMatrix/GKOACG/GKOACG.C
similarity index 100%
rename from LduMatrix/GKOACG/GKOACG.C
rename to MatrixWrapper/LduMatrix/GKOACG/GKOACG.C
diff --git a/LduMatrix/GKOACG/GKOACG.H b/MatrixWrapper/LduMatrix/GKOACG/GKOACG.H
similarity index 100%
rename from LduMatrix/GKOACG/GKOACG.H
rename to MatrixWrapper/LduMatrix/GKOACG/GKOACG.H
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index cf77709fc..2a2b9a3a2 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -6,7 +6,7 @@
 
 #include "DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H"
 #include "DevicePersistent/Vector/Vector.H"
-#include "HostMatrix/HostMatrix.H"
+#include "MatrixWrapper/HostMatrix/HostMatrix.H"
 #include "Preconditioner/Preconditioner.H"
 #include "common/common.H"
 
diff --git a/unitTests/test_HostMatrix.C b/unitTests/test_HostMatrix.C
index 8b4dcd548..99438c2f5 100644
--- a/unitTests/test_HostMatrix.C
+++ b/unitTests/test_HostMatrix.C
@@ -2,7 +2,8 @@
 //
 // SPDX-License-Identifier: GPL-3.0-or-later
 
-#include "HostMatrix/HostMatrix.H"
+#include "MatrixWrapper/HostMatrix/HostMatrix.H"
+
 #include "gtest/gtest.h"
 
 TEST(HostMatrixConversion, symmetric_update)

From 6a9f12320c40281de06c38725d4ef9756ee95026 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 13 Mar 2024 13:22:54 +0100
Subject: [PATCH 07/90] add missing MatrixWrapper files

---
 .../GkoCombinationMatrix.H                    |  78 ++
 MatrixWrapper/HostMatrix/HostMatrix.C         | 764 ++++++++++++++++++
 MatrixWrapper/HostMatrix/HostMatrix.H         | 465 +++++++++++
 .../HostMatrix/HostMatrixFreeFunctions.C      | 199 +++++
 4 files changed, 1506 insertions(+)
 create mode 100644 MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
 create mode 100644 MatrixWrapper/HostMatrix/HostMatrix.C
 create mode 100644 MatrixWrapper/HostMatrix/HostMatrix.H
 create mode 100644 MatrixWrapper/HostMatrix/HostMatrixFreeFunctions.C

diff --git a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
new file mode 100644
index 000000000..53facf2f9
--- /dev/null
+++ b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
@@ -0,0 +1,78 @@
+#include <ginkgo/ginkgo.hpp>
+
+
+// The CombinationMatrix class is a wrapper around Ginkgos combination linear
+// operator. It provides an interface for creating linear combinations of a
+// giveen inner matrix type and a read implementation
+template <typename ValueType, typename IndexType, typename InnerMatrixType>
+class CombinationMatrix
+    : public gko::EnableLinOp<CombinationMatrix<ValueType>>,
+      public gko::EnableCreateMethod<CombinationMatrix<ValueType>>,
+      public gko::ReadableFromMatrixData<ValueType, IndexType> {
+public:
+    // This constructor will be called by the create method.
+    // Here we initialize, the interface_ranges to create the
+    // the linear combination during the read call
+    CombinationMatrix(std::shared_ptr<const gko::Executor> exec,
+                      std::vector<gko::span> interface_ranges)
+        : gko::EnableLinOp<CombinationMatrix>(exec, gko::dim<2>{size}),
+          interface_ranges_(interface_ranges),
+          comb_(gko::share(gko::Combination<ValueType>::create(exec, size)))
+    {}
+
+protected:
+    using vec = gko::matrix::Dense<ValueType>;
+    using coef_type = gko::array<ValueType>;
+
+    void read(const device_matrix_data &data) override
+    {
+        auto exec = this->get_executor();
+        for (auto &interface : interface_ranges_) {
+            auto op gko::share(InnerMatrixType::create(this->exec))
+
+                auto sub_array_view = [interface, exec](auto array_data) {
+                    make_const_array_view(exec, interface.length(),
+                                          array_data + interface.begin())
+                };
+
+            auto row_idxs = sub_array_view(data.get_const_row_idxs());
+            auto col_idxs = sub_array_view(data.get_const_col_idxs());
+            auto vals = sub_array_view(data.get_const_values());
+
+            device_matrix_data in(exec, this->get_size(), row_idxs, col_idxs,
+                                  vals);
+
+            op->read(std::move(interface_view))
+
+                combination->add_operators(
+                    gko::initialize<gko::matrix::Dense<ValueType>>({1}, exec),
+                    in);
+        }
+    }
+
+    // Here we implement the application of the linear operator, x = A * b.
+    // apply_impl will be called by the apply method, after the arguments have
+    // been moved to the correct executor and the operators checked for
+    // conforming sizes.
+    //
+    // For simplicity, we assume that there is always only one right hand side
+    // and the stride of consecutive elements in the vectors is 1 (both of these
+    // are always true in this example).
+    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override {
+        this->comb_->apply_impl(b,x);
+    }
+
+    // There is also a version of the apply function which does the operation
+    // x = alpha * A * b + beta * x. This function is commonly used and can
+    // often be better optimized than implementing it using x = A * b. However,
+    // for simplicity, we will implement it exactly like that in this example.
+    void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
+                    const gko::LinOp *beta, gko::LinOp *x) const override
+    {
+        this->comb_->apply_impl(alpha,b, beta, x);
+    }
+
+private:
+    std::vector<gko::span> interface_ranges_;
+    std::shared_ptr<gko::Combination> comb_;
+};
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
new file mode 100644
index 000000000..383d1739d
--- /dev/null
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -0,0 +1,764 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    HostMatrix.H
+
+\*---------------------------------------------------------------------------*/
+
+#include "HostMatrix.H"
+
+#include "cyclicFvPatchField.H"
+#include "lduMatrix.H"
+#include "processorFvPatch.H"
+
+#include <map>
+
+namespace Foam {
+
+template <class MatrixType>
+HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
+    const objectRegistry &db, const MatrixType &matrix,
+    // coeffs for cells on boundaries
+    const FieldField<Field, scalar> &interfaceBouCoeffs,
+    // coeffs for internal cells
+    const FieldField<Field, scalar> &interfaceIntCoeffs,
+    // pointers to interfaces can be used to access concrete
+    // functions such as transferring indices, patch neighbours etc
+    const lduInterfaceFieldPtrsList &interfaces,
+    const dictionary &solverControls, const word &fieldName)
+    : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
+                         interfaceIntCoeffs, interfaces, solverControls),
+      exec_{db, solverControls, fieldName},
+      device_id_guard_{db, fieldName, exec_.get_device_exec()},
+      verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      reorder_on_copy_(
+          solverControls.lookupOrDefault<Switch>("reorderOnHost", false)),
+      scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
+      nrows_(matrix.diag().size()),
+      local_interface_nnz_(count_interface_nnz(interfaces, false)),
+      upper_nnz_(matrix.lduAddr().upperAddr().size()),
+      non_diag_nnz_(2 * upper_nnz_),
+      local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
+      local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
+      local_sparsity_{
+          fieldName + "_local",           db,       exec_,
+          local_matrix_w_interfaces_nnz_, verbose_,
+      },
+      local_coeffs_{
+          fieldName + "_local_coeffs",
+          db,
+          exec_,
+          local_matrix_w_interfaces_nnz_,
+          verbose_,
+          true,              // needs to be updated
+          !reorder_on_copy_  // whether to init on device
+                             // if reorderOnHost is selected, the array needs
+                             // to be initialized on the host
+      },
+      non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
+      communication_pattern_(create_communication_pattern(interfaces)),
+      non_local_sparsity_{
+          fieldName + "_non_local", db, exec_, non_local_matrix_nnz_, verbose_,
+      },
+      non_local_coeffs_{
+          fieldName + "_non_local_coeffs",
+          db,
+          exec_,
+          non_local_matrix_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      permutation_matrix_name_{"PermutationMatrix"},
+      permutation_stored_{
+          db.template foundObject<regIOobject>(permutation_matrix_name_)},
+      P_{(permutation_stored_)
+             ? db.template lookupObjectRef<DevicePersistentBase<gko::LinOp>>(
+                     permutation_matrix_name_)
+                   .get_ptr()
+             : nullptr}
+{
+    if (!local_sparsity_.col_idxs_.get_stored() ||
+        local_sparsity_.col_idxs_.get_update()) {
+        TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
+                            this->fieldName(),
+                            init_local_sparsity_pattern(interfaces);)
+        TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
+                            this->fieldName(),
+                            init_non_local_sparsity_pattern(interfaces);)
+    }
+    if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
+        TIME_WITH_FIELDNAME(
+            verbose_, update_local_matrix_data, this->fieldName(),
+            update_local_matrix_data(interfaces, interfaceBouCoeffs);)
+        TIME_WITH_FIELDNAME(
+            verbose_, update_non_local_matrix_data, this->fieldName(),
+            update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
+    }
+}
+
+
+template <class MatrixType>
+HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
+    const objectRegistry &db, const MatrixType &matrix,
+    const dictionary &solverControls, const word &fieldName)
+    : MatrixType::solver(fieldName, matrix, solverControls),
+      exec_{db, solverControls, fieldName},
+      device_id_guard_{db, fieldName, exec_.get_device_exec()},
+      verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      reorder_on_copy_(
+          solverControls.lookupOrDefault<Switch>("reorderOnHost", true)),
+      scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
+      nrows_(matrix.diag().size()),
+      local_interface_nnz_(0),
+      upper_nnz_(matrix.lduAddr().upperAddr().size()),
+      non_diag_nnz_(2 * upper_nnz_),
+      local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
+      local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
+      local_sparsity_{
+          fieldName + "_cols", db, exec_, local_matrix_nnz_, verbose_,
+      },
+      local_coeffs_{
+          fieldName + "_coeffs",
+          db,
+          exec_,
+          local_matrix_nnz_,
+          verbose_,
+          true,              // needs to be updated
+          !reorder_on_copy_  // whether to init on device
+                             // if reorderOnHost is selected, the array needs
+                             // to be initialized on the host
+      },
+      non_local_matrix_nnz_(),
+      // proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
+      non_local_sparsity_{
+          fieldName + "_non_local", db, exec_, non_local_matrix_nnz_, verbose_,
+      },
+      non_local_coeffs_{
+          fieldName + "_non_local_coeffs",
+          db,
+          exec_,
+          non_local_matrix_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      permutation_matrix_name_{"PermutationMatrix"},
+      permutation_stored_{
+          db.template foundObject<regIOobject>(permutation_matrix_name_)},
+      P_{(permutation_stored_)
+             ? db.template lookupObjectRef<DevicePersistentBase<gko::LinOp>>(
+                     permutation_matrix_name_)
+                   .get_ptr()
+             : nullptr}
+{
+    FatalErrorInFunction << "This constructor is currently not implemented"
+                         << abort(FatalError);
+}
+
+template <class MatrixType>
+label HostMatrixWrapper<MatrixType>::count_interface_nnz(
+    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
+{
+    label ctr{0};
+    for (label i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+
+        bool count = (proc_interfaces)
+                         ? !!isA<processorLduInterface>(iface->interface())
+                         : !isA<processorLduInterface>(iface->interface());
+        if (count) {
+            ctr += iface->interface().faceCells().size();
+        }
+    }
+
+    return ctr;
+}
+
+template <class MatrixType>
+std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
+    const lduInterfaceFieldPtrsList &interfaces,
+    const FieldField<Field, scalar> &interfaceBouCoeffs, const bool local) const
+{
+    std::vector<scalar> ret{};
+    ret.reserve((local) ? local_interface_nnz_ : non_local_matrix_nnz_);
+
+    for (label i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+
+        bool collect = (local)
+                           ? !isA<processorLduInterface>(iface->interface())
+                           : !!isA<processorLduInterface>(iface->interface());
+
+        if (collect) {
+            ret.insert(ret.end(), interfaceBouCoeffs[i].begin(),
+                       interfaceBouCoeffs[i].end());
+        }
+    }
+
+    std::for_each(ret.begin(), ret.end(), [](scalar &c) { c = c * -1.0; });
+
+    return ret;
+}
+
+template <class Sel, class Func>
+void interface_iterator(const lduInterfaceFieldPtrsList &interfaces, Func func)
+{
+    label element_ctr = 0;
+    for (label i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+
+        if (isA<Sel>(iface->interface())) {
+            const Sel &patch = refCast<const Sel>(iface->interface());
+            func(element_ctr, interface_size, patch, iface);
+        }
+    }
+}
+
+
+/** Same as interface_iterator but checks if is *NOT* Sel
+ */
+template <class Sel, class Func>
+void neg_interface_iterator(const lduInterfaceFieldPtrsList &interfaces,
+                            Func func)
+{
+    label element_ctr = 0;
+    for (label i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+
+        if (!isA<Sel>(iface->interface())) {
+            func(element_ctr, interface_size, iface);
+        }
+    }
+}
+
+
+template <class MatrixType>
+CommunicationPattern
+HostMatrixWrapper<MatrixType>::create_communication_pattern(
+    const lduInterfaceFieldPtrsList &interfaces) const
+{
+    // temp map, mapping from neighbour rank interface cells
+    std::map<label, std::vector<label>> interface_cell_map{};
+
+    // iterate all interfaces, count number of neighbour procs
+    // and store rows to send to neighbour procs
+    label n_procs = 0;
+    interface_iterator<processorFvPatch>(
+        interfaces, [&](label, const label, const processorFvPatch &patch,
+                        const lduInterfaceField *iface) {
+            const auto &face_cells{iface->interface().faceCells()};
+            const label neighbProcNo = patch.neighbProcNo();
+
+            auto search = interface_cell_map.find(neighbProcNo);
+            if (search == interface_cell_map.end()) {
+                n_procs++;
+                interface_cell_map.insert(std::pair{
+                    neighbProcNo,
+                    std::vector<label>(face_cells.begin(), face_cells.end())});
+            } else {
+                auto &vec = search->second;
+                vec.insert(vec.end(), face_cells.begin(), face_cells.end());
+            }
+        });
+
+    using comm_size_type = CommunicationPattern::comm_size_type;
+
+    // create index_sets
+    std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
+    for (auto [proc, interface_cells] : interface_cell_map) {
+        auto exec = exec_.get_ref_exec();
+        send_idxs.push_back(std::pair<gko::array<label>, comm_size_type>(
+            gko::array<label>(exec, interface_cells.begin(),
+                              interface_cells.end()),
+            proc));
+    }
+
+    // convert to gko::array
+    gko::array<comm_size_type> target_ids{exec_.get_ref_exec(),
+                                          static_cast<comm_size_type>(n_procs)};
+    gko::array<comm_size_type> target_sizes{
+        exec_.get_ref_exec(), static_cast<comm_size_type>(n_procs)};
+
+    label iter = 0;
+    for (const auto &[proc, interface_cells] : interface_cell_map) {
+        target_ids.get_data()[iter] = proc;
+        target_sizes.get_data()[iter] = interface_cells.size();
+        iter++;
+    }
+
+    return CommunicationPattern{target_ids, target_sizes, send_idxs};
+}
+
+
+template <typename PatchType>
+void collect_local_interface_indices_impl(
+    label &element_ctr, const lduInterfaceField *iface,
+    const lduAddressing &addr,
+    std::vector<std::tuple<label, label, label>> &local_interface_idxs)
+{
+    if (isA<PatchType>(iface->interface())) {
+        const PatchType &patch = refCast<const PatchType>(iface->interface());
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+#ifdef WITH_ESI_VERSION
+        const label neighbPatchId = patch.neighbPatchID();
+#else
+        const label neighbPatchId = patch.nbrPatchID();
+#endif
+        const labelUList &cols = addr.patchAddr(neighbPatchId);
+        for (label cellI = 0; cellI < interface_size; cellI++) {
+            local_interface_idxs.push_back(
+                {element_ctr, face_cells[cellI], cols[cellI]});
+            element_ctr += 1;
+        }
+    }
+}
+
+void collect_local_interface_indices_impl_cyclicAMIFvPatch(
+    label &element_ctr, const lduInterfaceField *iface,
+    const lduAddressing &addr,
+    std::vector<std::tuple<label, label, label>> &local_interface_idxs)
+{
+    if (isA<cyclicAMIFvPatch>(iface->interface())) {
+        FatalErrorInFunction
+            << "Currently unsupported CyclicAMIFvPatch detected"
+            << exit(FatalError);
+        const cyclicAMIFvPatch &patch =
+            refCast<const cyclicAMIFvPatch>(iface->interface());
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+#ifdef WITH_ESI_VERSION
+        const label neighbPatchId = patch.cyclicAMIPatch().neighbPatchID();
+#else
+        const label neighbPatchId = patch.cyclicAMIPatch().nbrPatchID();
+#endif
+        const labelUList &cols = addr.patchAddr(neighbPatchId);
+        for (label cellI = 0; cellI < interface_size; cellI++) {
+            local_interface_idxs.push_back(
+                {element_ctr, face_cells[cellI], cols[cellI]});
+            element_ctr += 1;
+        }
+    }
+}
+
+#ifdef WITH_ESI_VERSION
+void collect_local_interface_indices_impl_cyclicACMIFvPatch(
+    label &element_ctr, const lduInterfaceField *iface,
+    const lduAddressing &addr,
+    std::vector<std::tuple<label, label, label>> &local_interface_idxs)
+{
+    if (isA<cyclicACMIFvPatch>(iface->interface())) {
+        FatalErrorInFunction
+            << "Currently unsupported CyclicACMIFvPatch detected"
+            << exit(FatalError);
+        const cyclicACMIFvPatch &patch =
+            refCast<const cyclicACMIFvPatch>(iface->interface());
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+        const labelUList &cols =
+            addr.patchAddr(patch.cyclicACMIPatch().neighbPatchID());
+        for (label cellI = 0; cellI < interface_size; cellI++) {
+            local_interface_idxs.push_back(
+                {element_ctr, face_cells[cellI], cols[cellI]});
+            element_ctr += 1;
+        }
+    }
+}
+#endif
+
+template <class MatrixType>
+std::vector<std::tuple<label, label, label>>
+HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
+    const lduInterfaceFieldPtrsList &interfaces) const
+{
+    std::vector<std::tuple<label, label, label>> local_interface_idxs{};
+    local_interface_idxs.reserve(local_interface_nnz_);
+
+    const auto &addr = this->matrix().lduAddr();
+    neg_interface_iterator<processorFvPatch>(
+        interfaces, [&](label &element_ctr, const label interface_size,
+                        const lduInterfaceField *iface) {
+            // check whether interface is either an cyclicFvPatch,
+            // cyclicAMIFvPatch or cyclicACMIFvPatch and collect local interface
+            // indices
+            collect_local_interface_indices_impl<cyclicFvPatch>(
+                element_ctr, iface, addr, local_interface_idxs);
+            collect_local_interface_indices_impl_cyclicAMIFvPatch(
+                element_ctr, iface, addr, local_interface_idxs);
+#ifdef WITH_ESI_VERSION
+            collect_local_interface_indices_impl_cyclicACMIFvPatch(
+                element_ctr, iface, addr, local_interface_idxs);
+#endif
+        });
+    return local_interface_idxs;
+}
+
+template <class MatrixType>
+std::vector<std::tuple<label, label>>
+HostMatrixWrapper<MatrixType>::collect_cells_on_non_local_interface(
+    const lduInterfaceFieldPtrsList &interfaces) const
+{
+    // vector of neighbour cell idx connected to interface
+    std::vector<std::tuple<label, label>> non_local_idxs{};
+    non_local_idxs.reserve(non_local_matrix_nnz_);
+    label interface_ctr = 0;
+    interface_iterator<processorFvPatch>(
+        interfaces,
+        [&](label, const label interface_size, const processorLduInterface &,
+            const lduInterfaceField *iface) {
+            const auto &face_cells{iface->interface().faceCells()};
+            for (label cellI = 0; cellI < interface_size; cellI++) {
+                auto local_row = face_cells[cellI];
+                non_local_idxs.push_back({interface_ctr, local_row});
+                interface_ctr += 1;
+            }
+        });
+
+    word msg = "done collecting neighbouring processor cell id";
+    LOG_2(verbose_, msg)
+    return non_local_idxs;
+}
+
+template <class MatrixType>
+void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
+    const lduInterfaceFieldPtrsList &interfaces) const
+{
+    auto non_local_row_indices =
+        collect_cells_on_non_local_interface(interfaces);
+    auto rows = non_local_sparsity_.row_idxs_.get_data();
+    auto cols = non_local_sparsity_.col_idxs_.get_data();
+    auto permute = non_local_sparsity_.ldu_mapping_.get_data();
+
+    // Sorting of the interfaces is still needed since we can we have
+    // multiple interfaces using the same rows/send_idxs
+    // if the resulting non_local_matrix is not in row major order
+    // we might get non converging solvers on GPU devices
+    std::sort(non_local_row_indices.begin(), non_local_row_indices.end(),
+              [&](const auto &a, const auto &b) {
+                  auto [interface_idx_a, row_a] = a;
+                  auto [interface_idx_b, row_b] = b;
+                  return row_a < row_b;
+              });
+
+    label element_ctr = 0;
+    for (auto [interface_idx, row] : non_local_row_indices) {
+        rows[element_ctr] = row;
+        cols[element_ctr] = interface_idx;
+        permute[element_ctr] = interface_idx;
+        element_ctr += 1;
+    }
+}
+
+template <class MatrixType>
+void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
+    const lduInterfaceFieldPtrsList &interfaces) const
+{
+    LOG_1(verbose_, "start init host sparsity pattern")
+    bool is_symmetric{this->matrix().symmetric()};
+
+    auto lower_local = idx_array::view(
+        exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
+        const_cast<label *>(&this->matrix().lduAddr().lowerAddr()[0]));
+
+    auto upper_local = idx_array::view(
+        exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
+        const_cast<label *>(&this->matrix().lduAddr().upperAddr()[0]));
+
+    // row of upper, col of lower
+    const auto lower = lower_local.get_const_data();
+    // col of upper, row of lower
+    const auto upper = upper_local.get_const_data();
+
+    auto rows = local_sparsity_.row_idxs_.get_data();
+    auto cols = local_sparsity_.col_idxs_.get_data();
+    const auto permute = local_sparsity_.ldu_mapping_.get_data();
+
+    // Scan through given rows and insert row and column indices into array
+    //
+    //  position after all local offdiagonal elements, needed for
+    //  permutation matrix
+    //
+    //  TODO in order to simplify when local interfaces exists set
+    //  local_sparsity to size of nrows_w_interfaces, if interfaces exist
+    //  local_sparsity is only valid till nrows_
+    label after_neighbours = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
+    init_local_sparsity(nrows_, upper_nnz_, is_symmetric, upper, lower, rows,
+                        cols, permute);
+
+    // if no local interfaces are present we are done here
+    // otherwise we need to add local interfaces in order
+    if (local_interface_nnz_) {
+        auto local_interfaces = collect_local_interface_indices(interfaces);
+        // sort local interfaces to be in row major order
+        // and keep original indices
+        std::sort(local_interfaces.begin(), local_interfaces.end(),
+                  [&](const auto &a, const auto &b) {
+                      auto [interface_idx_a, row_a, col_a] = a;
+                      auto [interface_idx_b, row_b, col_b] = b;
+                      return std::tie(row_a, col_a) < std::tie(row_b, col_b);
+                  });
+
+        // copy current rows and columns to tmp array
+        // TODO this has full length but is only valid till nrows_
+        auto rows_copy_a = gko::array<label>(
+            *local_sparsity_.row_idxs_.get_persistent_object().get());
+        auto cols_copy_a = gko::array<label>(
+            *local_sparsity_.col_idxs_.get_persistent_object().get());
+        auto permute_copy_a = gko::array<label>(
+            *local_sparsity_.ldu_mapping_.get_persistent_object().get());
+
+        auto rows_copy = rows_copy_a.get_data();
+        auto cols_copy = cols_copy_a.get_data();
+        auto permute_copy = permute_copy_a.get_data();
+
+        auto rows = local_sparsity_.row_idxs_.get_data();
+        auto cols = local_sparsity_.col_idxs_.get_data();
+        auto permute = local_sparsity_.ldu_mapping_.get_data();
+
+        label current_idx_ctr = 0;
+        label total_ctr = 0;
+        // iterate local interfaces i
+        // insert all coeffs for row < interface[i].row
+        // find local_interface with lowes row, column
+        for (auto const &interface : local_interfaces) {
+            auto [interface_idx, interface_row, interface_col] = interface;
+
+            // copy from existing matrix coefficients
+            while ([&]() {
+                // check for length
+                if (current_idx_ctr == local_matrix_nnz_) {
+                    return false;
+                }
+
+                if (rows_copy[current_idx_ctr] > interface_row) {
+                    return false;
+                }
+
+                // the copy rows are or equal
+                // in that case we need to check if the
+                // copy columns are lower
+                if (rows_copy[current_idx_ctr] == interface_row &&
+                    cols_copy[current_idx_ctr] > interface_col) {
+                    return false;
+                }
+                return true;
+            }()) {
+                // insert coeffs
+                rows[total_ctr] = rows_copy[current_idx_ctr];
+                cols[total_ctr] = cols_copy[current_idx_ctr];
+                permute[total_ctr] = permute_copy[current_idx_ctr];
+                current_idx_ctr++;
+                total_ctr++;
+            }
+            rows[total_ctr] = interface_row;
+            cols[total_ctr] = interface_col;
+            // store the original position of the contiguous interfaces
+            permute[total_ctr] = after_neighbours + nrows_ + interface_idx;
+            total_ctr++;
+        }
+
+        // post insert if local interfaces were consumed but stuff remains
+        // in rows_copy and cols_copy
+        for (label i = total_ctr; i < local_matrix_w_interfaces_nnz_; i++) {
+            rows[i] = rows_copy[current_idx_ctr];
+            cols[i] = cols_copy[current_idx_ctr];
+            permute[i] = permute_copy[current_idx_ctr];
+            current_idx_ctr++;
+        }
+    }
+
+    LOG_1(verbose_, "done init host matrix")
+}
+
+
+template <class MatrixType>
+void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
+    const lduInterfaceFieldPtrsList &interfaces,
+    const FieldField<Field, scalar> &interfaceBouCoeffs) const
+{
+    auto ref_exec = exec_.get_ref_exec();
+    auto upper = this->matrix().upper();
+    auto lower = this->matrix().lower();
+    auto diag = this->matrix().diag();
+
+    label diag_nnz = diag.size();
+    bool is_symmetric{this->matrix().symmetric()};
+
+    // TODO this does not work for Ell
+    auto permute = local_sparsity_.ldu_mapping_.get_data();
+
+    if (reorder_on_copy_) {
+        auto dense = local_coeffs_.get_data();
+        if (local_interface_nnz_) {
+            auto couple_coeffs =
+                collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
+            if (is_symmetric) {
+                symmetric_update_w_interface(
+                    local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
+                    permute, scaling_, diag.data(), upper.data(),
+                    couple_coeffs.data(), dense);
+            } else {
+                non_symmetric_update_w_interface(
+                    local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
+                    permute, scaling_, diag.data(), upper.data(), lower.data(),
+                    couple_coeffs.data(), dense);
+            }
+        } else {
+            if (is_symmetric) {
+                symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+                                 scaling_, diag.data(), upper.data(), dense);
+            } else {
+                non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+                                     scaling_, diag.data(), upper.data(),
+                                     lower.data(), dense);
+            }
+        }
+    } else {
+        // TODO
+        // - this should be moved to separate function to avoid making this
+        // too long
+
+        using dim_type = gko::dim<2>::dimension_type;
+
+        auto target_exec =
+            (reorder_on_copy_) ? ref_exec : exec_.get_device_exec();
+
+        // copy upper
+        auto upper = this->matrix().upper();
+        auto u_host_view =
+            gko::array<scalar>::view(ref_exec, upper_nnz_, &upper[0]);
+        auto u_device_view = gko::array<scalar>::view(target_exec, upper_nnz_,
+                                                      local_coeffs_.get_data());
+        u_device_view = u_host_view;
+
+        // for the non-symmetric case copy lower to the device
+        if (!is_symmetric) {
+            auto lower = this->matrix().lower();
+            auto l_host_view =
+                gko::array<scalar>::view(ref_exec, upper_nnz_, &lower[0]);
+            auto l_device_view = gko::array<scalar>::view(
+                target_exec, upper_nnz_, &local_coeffs_.get_data()[upper_nnz_]);
+            l_device_view = l_host_view;
+        }
+
+        // copy diag
+        auto diag = this->matrix().diag();
+        auto diag_host_view =
+            gko::array<scalar>::view(ref_exec, nrows_, &diag[0]);
+        const label diag_start = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
+        auto diag_device_view = gko::array<scalar>::view(
+            target_exec, nrows_, &local_coeffs_.get_data()[diag_start]);
+        diag_device_view = diag_host_view;
+
+        // copy local interfaces
+        if (local_interface_nnz_) {
+            const label interface_start = diag_start + nrows_;
+            auto couple_coeffs =
+                collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
+            auto interface_host_view = gko::array<scalar>::view(
+                ref_exec, local_interface_nnz_, couple_coeffs.data());
+            auto interface_device_view = gko::array<scalar>::view(
+                target_exec, local_interface_nnz_,
+                &local_coeffs_.get_data()[interface_start]);
+            interface_device_view = interface_host_view;
+        }
+
+        // permute and update local_coeffs
+        auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
+            target_exec,
+            gko::dim<2>{static_cast<dim_type>(local_matrix_w_interfaces_nnz_),
+                        1}));
+
+        auto dense_vec = vec::create(
+            target_exec,
+            gko::dim<2>{static_cast<dim_type>(local_matrix_w_interfaces_nnz_),
+                        1},
+            gko::array<scalar>::view(target_exec,
+                                     local_matrix_w_interfaces_nnz_,
+                                     local_coeffs_.get_data()),
+            1);
+
+        // TODO this needs to copy ldu_mapping to the device
+        dense_vec->row_gather(local_sparsity_.ldu_mapping_.get_array().get(),
+                              row_collection.get());
+
+        dense_vec->copy_from(row_collection);
+    }
+}
+
+
+template <class MatrixType>
+void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
+    const lduInterfaceFieldPtrsList &interfaces,
+    const FieldField<Field, scalar> &interfaceBouCoeffs) const
+{
+    auto ref_exec = exec_.get_ref_exec();
+    auto interface_coeffs =
+        collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
+    auto permute = non_local_sparsity_.ldu_mapping_.get_data();
+
+    // copy interfaces
+    auto tmp_contiguous_iface =
+        gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
+    auto contiguous_iface = tmp_contiguous_iface.get_data();
+
+    for (label element_ctr = 0; element_ctr < non_local_matrix_nnz_;
+         element_ctr++) {
+        contiguous_iface[element_ctr] = interface_coeffs[permute[element_ctr]];
+    }
+
+    // copy to persistent
+    auto i_device_view = gko::array<scalar>::view(
+        ref_exec, non_local_matrix_nnz_, non_local_coeffs_.get_data());
+    i_device_view = tmp_contiguous_iface;
+}
+
+template HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(
+    const objectRegistry &db, const lduMatrix &matrix,
+    // coeffs for cells on boundaries
+    const FieldField<Field, scalar> &interfaceBouCoeffs,
+    // coeffs for internal cells
+    const FieldField<Field, scalar> &interfaceIntCoeffs,
+    // pointers to interfaces can be used to access concrete
+    // functions such as transferring indices, patch neighbours etc
+    const lduInterfaceFieldPtrsList &interfaces,
+    const dictionary &solverControls, const word &fieldName);
+
+}  // namespace Foam
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.H b/MatrixWrapper/HostMatrix/HostMatrix.H
new file mode 100644
index 000000000..b0016e311
--- /dev/null
+++ b/MatrixWrapper/HostMatrix/HostMatrix.H
@@ -0,0 +1,465 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    HostMatrix.H
+
+\*---------------------------------------------------------------------------*/
+#ifndef OGL_HostMatrix_INCLUDED_H
+#define OGL_HostMatrix_INCLUDED_H
+#include <ginkgo/ginkgo.hpp>
+
+#include "fvCFD.H"
+#include "processorLduInterface.H"
+
+#include <vector>
+#include "DevicePersistent/Array/Array.H"
+#include "DevicePersistent/DeviceIdGuard/DeviceIdGuard.H"
+#include "DevicePersistent/IOGlobalIndex/gkoGlobalIndex.H"
+#include "common/common.H"
+
+
+namespace Foam {
+
+
+struct PersistentSparsityPattern {
+    PersistentSparsityPattern(
+      const word &fieldName,
+      const objectRegistry &db,
+      const ExecutorHandler &exec,
+      const label size,
+      const label verbose):
+    col_idxs_{
+        fieldName + "_cols",
+        db,
+        exec,
+        size,
+        verbose,
+        false,  // For now we assume columns and rows to be constant
+        false   // we don't init on device since the indices are written using the pointer
+                // to the data
+    },
+    row_idxs_{
+        fieldName + "_rows",
+        db,
+        exec,
+        size,
+        verbose,
+        false,  // For now we assume columns and rows to be constant
+        false   // same as for cols don't init on device
+    },
+    ldu_mapping_{
+        fieldName + "_ldu_map",
+        db,
+        exec,
+        size,
+        verbose,
+        false,
+        false // same as for rows and cols dont init on device
+    }
+    {}
+
+    mutable PersistentArray<label> col_idxs_;
+
+    mutable PersistentArray<label> row_idxs_;
+
+    // ldu_mapping[sorted_(csr)_position] =
+    // unsorted_(consecutive_ldu_)_position
+    mutable PersistentArray<label> ldu_mapping_;
+};
+
+struct CommunicationPattern {
+    using comm_size_type = int;
+    // an array storing to which rank to communicate
+    gko::array<comm_size_type> target_ids;
+
+    // an array storing how many elements to communicate
+    // to the corresponding target_id
+    gko::array<comm_size_type> target_sizes;
+
+    // send_idx stores the index_set ie which cells
+    // are owned by the interface and the corresponding target rank
+    std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
+};
+
+
+// Free functions
+const lduInterfaceField *interface_getter(
+    const lduInterfaceFieldPtrsList &interfaces, const label i);
+
+/** Write contiguous row and col indices from OpenFOAM lower and upper indices
+ ** For details on the lower triangular based indexing see
+ ** https://openfoamwiki.net/index.php/OpenFOAM_guide/Matrices_in_OpenFOA
+ ** Note that the order of the indices are depicted wrong on the wiki
+ ** In general the upper triangular matrix is traversed in row major
+ ** and the lower triangular matrix in column major order.
+ **
+ ** @param nrows number of rows
+ ** @param upper_nnz number of non zeros in the upper triangular matrix
+ ** @param is_symmetric whether matrix is symmetric, in the symmetric case
+ ** the lower elements indices in the permute array are computed differently
+ ** @param upper pointer to OFs rows array
+ ** @param lower pointer to OFs cols array
+ ** @param rows pointer to rows array
+ ** @param cols pointer to columns array
+ ** @param permute pointer to permuter array
+ */
+void init_local_sparsity(const label nrows, const label upper_nnz,
+                         const bool is_symmetric, const label *upper,
+                         const label *lower, label *rows, label *cols,
+                         label *permute);
+
+/** Perform update of the coefficients array in row major order from a symmetric
+ *ldu matrix
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ *upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param out pointer to data of output matrix
+ **/
+void symmetric_update(const label total_nnz, const label upper_nnz,
+                      const label *permute, const scalar scale,
+                      const scalar *diag, const scalar *upper, scalar *out);
+
+
+/** Perform update of the coefficients array in row major order from a
+ *non-symmetric ldu matrix
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ *upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for lower entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param lower pointer to lower triangular entries of the original ldu matrix
+ ** @param out pointer to data of output matrix
+ **/
+void non_symmetric_update(const label total_nnz, const label upper_nnz,
+                          const label *permute, const scalar scale,
+                          const scalar *diag, const scalar *upper,
+                          const scalar *lower, scalar *dense);
+
+
+/** Perform update of the coefficients array in row major order from a
+ ** symmetric ldu matrix including local interface data.
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ ** upper
+ **
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for interface entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param interface pointer to interface entries
+ ** @param out pointer to data of output matrix
+ **/
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
+                                  const label upper_nnz, const label *permute,
+                                  const scalar scale, const scalar *diag,
+                                  const scalar *upper, const scalar *interface,
+                                  scalar *dense);
+
+
+/** Perform update of the coefficients array in row major order from a
+ ** non-symmetric ldu matrix including local interface data.
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ ** upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for lower entries
+ **        2*upper_nnz + diag_nnz <= permute for interface entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param lower pointer to lower triangular entries of the original ldu matrix
+ ** @param interface pointer to interface entries
+ ** @param out pointer to data of output matrix
+ **/
+void non_symmetric_update_w_interface(
+    const label total_nnz, const label diag_nnz, const label upper_nnz,
+    const label *permute, const scalar scale, const scalar *diag,
+    const scalar *upper, const scalar *lower, const scalar *couple_coeffs,
+    scalar *dense);
+
+
+/* The HostMatrixWrapper class handles the conversion from OpenFOAMs lduMatrix
+ * format into Ginkgo array data structures
+ *
+ * It mainly generates:
+ * - sparsity pattern/device_matrix_data of the local_matrix and
+ * - non_local_matrix (with global columns)
+ * - CommunicationPattern
+ * */
+template <class MatrixType>
+class HostMatrixWrapper : public MatrixType::solver {
+private:
+    using vec = gko::matrix::Dense<scalar>;
+    using idx_array = gko::array<label>;
+
+    const ExecutorHandler exec_;
+
+    const DeviceIdGuardHandler device_id_guard_;
+
+    const label verbose_;
+
+    // Whether the matrix coefficients should be reordered
+    // during copy or on device
+    // NOTE Not implemented
+    const bool reorder_on_copy_;
+
+    // multiply the complete system by this factor, ie sAx=sb
+    // NOTE this could be also achieved by just fliping the sign
+    const scalar scaling_;
+
+    // number of local matrix rows
+    const label nrows_;
+
+    // number of local elements on interfaces
+    // ie number of interfaces which column_idx is < nrows_
+    // and have to be sorted into local matrix
+    const label local_interface_nnz_;
+
+    // number of local upper elements
+    // ie coefficients which column_idx < nrows_
+    const label upper_nnz_;
+
+    // total number of local upper and lower elements
+    // ie 2*upper_nnz_ since the sparsity pattern is symmetric
+    // TODO should this include the local_interface_nnz_ ?
+    const label non_diag_nnz_;
+
+    // nnz of local matrix wo local interfaces
+    const label local_matrix_nnz_;
+
+    // nnzs of local matrix including local interfaces
+    const label local_matrix_w_interfaces_nnz_;
+
+    mutable PersistentSparsityPattern local_sparsity_;
+
+    // matrix coefficients
+    mutable PersistentArray<scalar> local_coeffs_;
+
+    // non-local indices
+    const label non_local_matrix_nnz_;
+
+    const CommunicationPattern communication_pattern_;
+
+    mutable PersistentSparsityPattern non_local_sparsity_;
+
+    mutable PersistentArray<scalar> non_local_coeffs_;
+
+    // FIXME TODO unused
+    const word permutation_matrix_name_;
+
+    const bool permutation_stored_;
+
+    mutable std::shared_ptr<gko::LinOp> P_;
+
+
+    /* Iterates all interfaces and collects the coefficients into a vector
+    **
+    ** @param local whether local or non local coefficients should be collected
+    */
+    std::vector<scalar> collect_interface_coeffs(
+        const lduInterfaceFieldPtrsList &interfaces_,
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        const bool local) const;
+
+
+    /* Iterates all interfaces and counts the number of elements
+    **
+    ** @param interfaces The list of interfaces for the search
+    ** @param proc_interfaces Count only elements on (true)
+    *processorLduInterfaces or exclude processorLduInterfaces (false)
+    */
+    label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
+                              bool proc_interfaces) const;
+
+    /** Iterates all local interfaces and returns the relative order and
+    **corresponding row and column indices
+    **
+    ** @return vector of tuples containing the interface number, the local row,
+    *the local column
+    **/
+    std::vector<std::tuple<label, label, label>>
+    collect_local_interface_indices(
+        const lduInterfaceFieldPtrsList &interfaces_) const;
+
+
+    /** Iterates all interfaces and counts the number of unique neighbour
+     ** processors and number of interfaces in total for this processor
+     ** and collects all interface cells of this rank.
+     **
+     ** @param interfaces The list of interfaces for the search
+     ** @return the CommunicationPattern
+     */
+    CommunicationPattern create_communication_pattern(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    /** Iterates all interfaces and collect the corresponding cell id (row)
+    ** and a unique counter
+    **
+    ** @return vector of size nnz_non_local, with a running index, row
+    ** index, and a unique compressed column index
+    ** sections for each interface
+    ** ret = [(1,2,1),(2,20, 2), (3, 20, 2) ...]
+    **         i0   i1,   i...
+    */
+    std::vector<std::tuple<label, label>> collect_cells_on_non_local_interface(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    /** Based on OpenFOAMs ldu matrix format this function computes two
+     ** consecutive index arrays (local_sparsisty_.row_idxs and col_idxs) in row
+     ** major ordering and the permutation index (local_sparsity_.ldu_mapping),
+     ** which are required to create to a ginkgo matrix
+     **/
+    void init_local_sparsity_pattern(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    /** Based on OpenFOAMs interfaces this function computes two
+     ** consecutive index arrays (non_local_sparsisty_.row_idxs and col_idxs) in
+     ** row *major ordering and the permutation index
+     ** (non_local_sparsity_.ldu_mapping), which are required to create a ginkgo
+     ** matrix
+     **/
+    void init_non_local_sparsity_pattern(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    // updater
+
+    /** Copies the LDU matrix coefficients to local_coeffs without changing or
+     ** reinstantiating the sparsity pattern.
+     ** This uses the local_sparsity_.ldu_mapping to permute the data already
+     ** on the host or device to be in row major order.
+     **/
+    void update_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
+
+    /** Copies the interface matrix coefficients to non_local_coeffs without
+    ** changing or reinstantiating the sparsity pattern.
+    **/
+    void update_non_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
+
+public:
+    // segregated wrapper constructor
+    // TODO - compute local_interface_nnzs these are matrix coefficient
+    //      that are on an interface but not on a different rank
+    //      - then a new array/vector is needed to store local_interfaces
+    //      - if local interfaces are present init_local_sparsity_pattern
+    //      needs to be adapted
+    //      - local_sparsity_.ldu_mapping needs a new section for interfaces
+    HostMatrixWrapper(
+        const objectRegistry &db, const MatrixType &matrix,
+        // coeffs for cells on boundaries
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        // coeffs for internal cells
+        const FieldField<Field, scalar> &interfaceIntCoeffs,
+        // pointers to interfaces can be used to access concrete
+        // functions such as transferring indices, patch neighbours etc
+        const lduInterfaceFieldPtrsList &interfaces,
+        const dictionary &solverControls, const word &fieldName);
+
+    // TODO reimplement this at some point
+    // coupled wrapper constructor
+    HostMatrixWrapper(const objectRegistry &db, const MatrixType &matrix,
+                      const dictionary &solverControls, const word &fieldName);
+
+
+    // getter
+
+public:
+    bool get_verbose() const { return verbose_; }
+
+    const CommunicationPattern &get_communication_pattern() const
+    {
+        return communication_pattern_;
+    };
+
+    label get_local_nrows() const { return nrows_; }
+
+    const ExecutorHandler &get_exec_handler() const { return exec_; }
+
+    PersistentArray<scalar> &get_local_coeffs() const { return local_coeffs_; };
+
+    PersistentArray<label> &get_local_col_idxs() const
+    {
+        return local_sparsity_.col_idxs_;
+    };
+
+    PersistentArray<label> &get_local_row_idxs() const
+    {
+        return local_sparsity_.row_idxs_;
+    };
+
+    PersistentArray<scalar> &get_non_local_coeffs() const
+    {
+        return non_local_coeffs_;
+    };
+
+    PersistentArray<label> &get_non_local_col_idxs() const
+    {
+        return non_local_sparsity_.col_idxs_;
+    };
+
+    PersistentArray<label> &get_non_local_row_idxs() const
+    {
+        return non_local_sparsity_.row_idxs_;
+    };
+};
+
+
+}  // namespace Foam
+#endif
diff --git a/MatrixWrapper/HostMatrix/HostMatrixFreeFunctions.C b/MatrixWrapper/HostMatrix/HostMatrixFreeFunctions.C
new file mode 100644
index 000000000..4ec1649e1
--- /dev/null
+++ b/MatrixWrapper/HostMatrix/HostMatrixFreeFunctions.C
@@ -0,0 +1,199 @@
+#include "HostMatrix.H"
+
+namespace Foam {
+
+const lduInterfaceField *interface_getter(
+    const lduInterfaceFieldPtrsList &interfaces, const label i)
+{
+#ifdef WITH_ESI_VERSION
+    return interfaces.get(i);
+#else
+    return interfaces.operator()(i);
+#endif
+}
+
+
+// free functions - for unit testing
+void symmetric_update(const label total_nnz, const label upper_nnz,
+                      const label *permute, const scalar scale,
+                      const scalar *diag, const scalar *upper, scalar *out)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        out[i] =
+            scale * (pos >= upper_nnz) ? diag[pos - upper_nnz] : upper[pos];
+    }
+}
+
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
+                                  const label upper_nnz, const label *permute,
+                                  const scalar scale, const scalar *diag,
+                                  const scalar *upper, const scalar *interface,
+                                  scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        // where the element is stored in a combined array
+        const label pos{permute[i]};
+        scalar value;
+        // all values up to upper_nnz_ are upper_nnz_ values
+        if (pos < upper_nnz) {
+            value = upper[pos];
+        }
+        // all values up to upper_nnz_ are upper_nnz_ values
+        if (pos >= upper_nnz && pos < upper_nnz + diag_nnz) {
+            value = diag[pos - upper_nnz];
+        }
+        if (pos >= upper_nnz + diag_nnz) {
+            value = interface[pos - upper_nnz - diag_nnz];
+        }
+        dense[i] = scale * value;
+    }
+}
+
+void non_symmetric_update_w_interface(const label total_nnz,
+                                      const label diag_nnz,
+                                      const label upper_nnz,
+                                      const label *permute, const scalar scale,
+                                      const scalar *diag, const scalar *upper,
+                                      const scalar *lower,
+                                      const scalar *interface, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        scalar value;
+        if (pos < upper_nnz) {
+            value = upper[pos];
+        }
+        if (pos >= upper_nnz && pos < 2 * upper_nnz) {
+            value = lower[pos - upper_nnz];
+        }
+        if (pos >= 2 * upper_nnz && pos < 2 * upper_nnz + diag_nnz) {
+            value = diag[pos - 2 * upper_nnz];
+        }
+        if (pos >= 2 * upper_nnz + diag_nnz) {
+            value = interface[pos - 2 * upper_nnz - diag_nnz];
+        }
+        dense[i] = scale * value;
+    }
+}
+
+
+void non_symmetric_update(const label total_nnz, const label upper_nnz,
+                          const label *permute, const scalar scale,
+                          const scalar *diag, const scalar *upper,
+                          const scalar *lower, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        if (pos < upper_nnz) {
+            dense[i] = scale * upper[pos];
+            continue;
+        }
+        if (pos >= upper_nnz && pos < 2 * upper_nnz) {
+            dense[i] = scale * lower[pos - upper_nnz];
+            continue;
+        }
+        dense[i] = scale * diag[pos - 2 * upper_nnz];
+    }
+}
+
+
+void init_local_sparsity(const label nrows, const label upper_nnz,
+                         const bool is_symmetric, const label *upper,
+                         const label *lower, label *rows, label *cols,
+                         label *permute)
+{
+    // for OpenFOAMs addressing see
+    // https://openfoamwiki.net/index.php/OpenFOAM_guide/Matrices_in_OpenFOAM
+    // Note that the face order in the wiki seems to be wrong. Entries are
+    // stored such that upper rows are monotonic ascending
+    // upper - rows of lower triangular matrix
+    // lower - columns of lower triangular matrix
+    label after_neighbours = (is_symmetric) ? upper_nnz : 2 * upper_nnz;
+
+    // first pass order elements row wise
+    // scan through all faces
+    std::vector<std::tuple<label, label, label>> tmp_upper;
+    tmp_upper.reserve(upper_nnz);
+    for (label faceI = 0; faceI < upper_nnz; faceI++) {
+        const label col = upper[faceI];
+        const label row = lower[faceI];
+        tmp_upper.emplace_back(row, col, faceI);
+    }
+
+    std::sort(tmp_upper.begin(), tmp_upper.end(),
+              [&](const auto &a, const auto &b) {
+                  auto [row_a, col_a, faceI_a] = a;
+                  auto [row_b, col_b, faceI_b] = b;
+                  return std::tie(row_a, col_a) < std::tie(row_b, col_b);
+              });
+
+    std::vector<std::tuple<label, label, label>> tmp_lower;
+    tmp_lower.reserve(upper_nnz);
+    for (label faceI = 0; faceI < upper_nnz; faceI++) {
+        const label col = lower[faceI];
+        const label row = upper[faceI];
+        tmp_lower.emplace_back(row, col, faceI);
+    }
+
+    std::sort(tmp_lower.begin(), tmp_lower.end(),
+              [&](const auto &a, const auto &b) {
+                  auto [row_a, col_a, faceI_a] = a;
+                  auto [row_b, col_b, faceI_b] = b;
+                  return std::tie(row_a, col_a) < std::tie(row_b, col_b);
+              });
+
+    // now we have tmp_upper and tmp_lower in row order
+
+    label element_ctr = 0;
+    label upper_ctr = 0;
+    label lower_ctr = 0;
+    label lower_size = tmp_lower.size();
+    label upper_size = tmp_upper.size();
+    auto [row_lower, col_lower, faceI_lower] = tmp_lower[0];
+    auto [row_upper, col_upper, faceI_upper] = tmp_upper[0];
+    for (label row = 0; row < nrows; row++) {
+        // check if we have any lower elements to insert
+        while (row_lower == row) {
+            rows[element_ctr] = row_lower;
+            cols[element_ctr] = col_lower;
+            permute[element_ctr] =
+                (is_symmetric) ? faceI_lower : upper_nnz + faceI_lower;
+            element_ctr++;
+            lower_ctr++;
+            if (lower_ctr >= lower_size) {
+                break;
+            }
+            auto [tmp_row_lower, tmp_col_lower, tmp_faceI_lower] =
+                tmp_lower[lower_ctr];
+            row_lower = tmp_row_lower;
+            col_lower = tmp_col_lower;
+            faceI_lower = tmp_faceI_lower;
+        }
+
+        // add diagonal elements
+        rows[element_ctr] = row;
+        cols[element_ctr] = row;
+        permute[element_ctr] = after_neighbours + row;
+        element_ctr++;
+
+        // check if we have any upper elements to insert
+        while (row_upper == row) {
+            rows[element_ctr] = row_upper;
+            cols[element_ctr] = col_upper;
+            permute[element_ctr] = faceI_upper;
+            element_ctr++;
+            upper_ctr++;
+            if (upper_ctr >= upper_size) {
+                break;
+            }
+            auto [tmp_row_upper, tmp_col_upper, tmp_faceI_upper] =
+                tmp_upper[upper_ctr];
+            row_upper = tmp_row_upper;
+            col_upper = tmp_col_upper;
+            faceI_upper = tmp_faceI_upper;
+        }
+    }
+}
+
+}  // namespace Foam

From cb4efb0762e88496193f8b2caed66175abebeca3 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 13 Mar 2024 13:34:36 +0100
Subject: [PATCH 08/90] pass ctr arguments

---
 DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index c09a609b8..85acca92e 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -139,25 +139,29 @@ struct MatrixInitFunctor {
 
     std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
         std::shared_ptr<gko::Executor> exec,
-        std::shared_ptr<gko::experimental::mpi::communicator> comm) const
+        std::shared_ptr<gko::experimental::mpi::communicator> comm,
+        std::vector<gko::span> interfaces) const
     {
         if (matrix_format_ == "Csr") {
             return dist_mtx::create(
                 exec, *comm.get(),
                 gko::with_matrix_type<
-                    CombinationMatrix<scalar, label, gko::matrix::Csr>>());
+                    CombinationMatrix<scalar, label, gko::matrix::Csr>>(
+                    exec, interfaces));
         }
         if (matrix_format_ == "Ell") {
             return dist_mtx::create(
                 exec, *comm.get(),
                 gko::with_matrix_type <
-                    CombinationMatrix<scalar, label, gko::matrix::Ell>());
+                    CombinationMatrix<scalar, label, gko::matrix::Ell>(
+                        exec, interfaces));
         }
         if (matrix_format_ == "Coo") {
             return dist_mtx::create(
                 exec, *comm.get(),
                 gko::with_matrix_type <
-                    CombinationMatrix<scalar, label, gko::matrix::Coo>());
+                    CombinationMatrix<scalar, label, gko::matrix::Coo>(
+                        exec, interfaces));
         }
         // if (matrix_format_ == "Hybrid") {
         //     return dist_mtx::create(

From 8778b1da515869320e71665b503d45861b9192b0 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 13 Mar 2024 13:46:23 +0100
Subject: [PATCH 09/90] add interface spans

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 37 ++++++++++++++-----
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 85acca92e..56823be73 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -140,28 +140,40 @@ struct MatrixInitFunctor {
     std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
         std::shared_ptr<gko::Executor> exec,
         std::shared_ptr<gko::experimental::mpi::communicator> comm,
-        std::vector<gko::span> interfaces) const
+        std::vector<gko::span> local_interfaces,
+        std::vector<gko::span> non_local_interfaces, ) const
     {
         if (matrix_format_ == "Csr") {
             return dist_mtx::create(
                 exec, *comm.get(),
                 gko::with_matrix_type<
                     CombinationMatrix<scalar, label, gko::matrix::Csr>>(
-                    exec, interfaces));
+                    exec, local_interfaces),
+                gko::with_matrix_type<
+                    CombinationMatrix<scalar, label, gko::matrix::Csr>>(
+                    exec, non_local_interfaces));
         }
         if (matrix_format_ == "Ell") {
             return dist_mtx::create(
                 exec, *comm.get(),
                 gko::with_matrix_type <
-                    CombinationMatrix<scalar, label, gko::matrix::Ell>(
-                        exec, interfaces));
+                    gko::with_matrix_type<
+                        CombinationMatrix<scalar, label, gko::matrix::Ell>>(
+                        exec, local_interfaces),
+                gko::with_matrix_type<
+                    CombinationMatrix<scalar, label, gko::matrix::Ell>>(
+                    exec, non_local_interfaces));
         }
         if (matrix_format_ == "Coo") {
             return dist_mtx::create(
                 exec, *comm.get(),
                 gko::with_matrix_type <
-                    CombinationMatrix<scalar, label, gko::matrix::Coo>(
-                        exec, interfaces));
+                    gko::with_matrix_type<
+                        CombinationMatrix<scalar, label, gko::matrix::Coo>>(
+                        exec, local_interfaces),
+                gko::with_matrix_type<
+                    CombinationMatrix<scalar, label, gko::matrix::Coo>>(
+                    exec, non_local_interfaces));
         }
         // if (matrix_format_ == "Hybrid") {
         //     return dist_mtx::create(
@@ -193,14 +205,20 @@ struct MatrixInitFunctor {
         device_matrix_data A_data{exec, gko::dim<2>(num_rows, num_rows),
                                   *rows.get(), *cols.get(), *coeffs.get()};
 
+        auto num_non_local_elems = non_local_cols->get_size();
         device_matrix_data non_local_A_data{
-            exec, gko::dim<2>(num_rows, non_local_cols->get_size()),
+            exec, gko::dim<2>(num_rows, num_non_local_elems),
             *non_local_rows.get(), *non_local_cols.get(),
             *non_local_coeffs.get()};
 
+        std::vector<gko::span> local_interfaces{gko::span(0, num_rows)};
+        std::vector<gko::span> non_local_interfaces{
+            gko::span(0, num_non_local_elems)};
+
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
-            exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm());
+            exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm(),
+            local_interfaces, non_local_interfaces);
 
         auto sparse_comm =
             gko::experimental::distributed::sparse_communicator::create(
@@ -215,7 +233,8 @@ struct MatrixInitFunctor {
         dist_A->read_distributed(local_A_data, non_local_A_data, sparse_comm);
 
         auto device_mat = generate_dist_mtx_with_inner_type(
-            exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
+            exec_.get_device_exec(), exec_.get_gko_mpi_device_comm(),
+            local_interfaces, non_local_interfaces);
         dist_A->move_to(device_mat.get());
         return device_mat;
     }

From 664573f9b85d522ead9ed612621624b5f847bb33 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 14 Mar 2024 17:38:04 +0100
Subject: [PATCH 10/90] add combination implementation

---
 CMakePresets.json                             |  45 +++++-
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 128 +++++++++--------
 .../GkoCombinationMatrix.H                    | 134 ++++++++++++++----
 common/common.C                               |   4 +
 lduLduBase/lduLduBase.H                       |   1 +
 5 files changed, 220 insertions(+), 92 deletions(-)

diff --git a/CMakePresets.json b/CMakePresets.json
index b62aed332..d856ce282 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 6,
   "cmakeMinimumRequired": {
     "major": 3,
     "minor": 20,
@@ -17,6 +17,31 @@
         "CMAKE_INSTALL_PREFIX": "$env{FOAM_USER_LIBBIN}/.."
       }
     },
+    {
+      "name": "debug",
+      "displayName": "Default Config",
+      "description": "Default build using Ninja generator",
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/build/Debug",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_INSTALL_PREFIX": "$env{FOAM_USER_LIBBIN}/.."
+      }
+    },
+    {
+      "name": "ninja-cpuonly-debug",
+      "inherits": "debug",
+      "displayName": "No GPU, Ninja",
+      "description": "Release build without GPU support using Ninja Multi-Config generator",
+      "generator": "Ninja Multi-Config",
+      "binaryDir": "${sourceDir}/build/DebugCPUOnly",
+      "cacheVariables": {
+        "OGL_ALLOW_REFERENCE_ONLY": {
+          "type": "BOOL",
+          "value": "ON"
+        }
+      }
+    },
     {
       "name": "ninja-cpuonly-release",
       "inherits": "default",
@@ -99,6 +124,20 @@
       "configurePreset": "ninja-sycl-release",
       "configuration": "Release"
     }
-  ]
-  
+  ],
+    "workflowPresets": [
+    {
+      "name": "ninja-cpuonly-release",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "ninja-cpuonly-release"
+        },
+        {
+          "type": "build",
+          "name": "ninja-cpuonly-release"
+        }
+      ]
+    }
+    ]
 }
diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 56823be73..20280cfe0 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -9,7 +9,7 @@
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "DevicePersistent/Partition/Partition.H"
 
-#include "MatrixWrapper/GkoCombinationMatrix/GkoCombinatinoMatrix.H"
+#include "MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H"
 
 #include "fvCFD.H"
 
@@ -140,40 +140,43 @@ struct MatrixInitFunctor {
     std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
         std::shared_ptr<gko::Executor> exec,
         std::shared_ptr<gko::experimental::mpi::communicator> comm,
-        std::vector<gko::span> local_interfaces,
-        std::vector<gko::span> non_local_interfaces, ) const
+        gko::dim<2> local_dim, std::vector<gko::span> local_interfaces,
+        gko::dim<2> non_local_dim,
+        std::vector<gko::span> non_local_interfaces) const
     {
-        if (matrix_format_ == "Csr") {
-            return dist_mtx::create(
-                exec, *comm.get(),
-                gko::with_matrix_type<
-                    CombinationMatrix<scalar, label, gko::matrix::Csr>>(
-                    exec, local_interfaces),
-                gko::with_matrix_type<
-                    CombinationMatrix<scalar, label, gko::matrix::Csr>>(
-                    exec, non_local_interfaces));
-        }
-        if (matrix_format_ == "Ell") {
-            return dist_mtx::create(
-                exec, *comm.get(),
-                gko::with_matrix_type <
-                    gko::with_matrix_type<
-                        CombinationMatrix<scalar, label, gko::matrix::Ell>>(
-                        exec, local_interfaces),
-                gko::with_matrix_type<
-                    CombinationMatrix<scalar, label, gko::matrix::Ell>>(
-                    exec, non_local_interfaces));
-        }
+        // if (matrix_format_ == "Csr") {
+        //     using Csr = gko::matrix::Csr<scalar, label>;
+        //     return dist_mtx::create(
+        //         exec, *comm.get(),
+        //         gko::with_matrix_type<
+        //             CombinationMatrix<scalar, label, Csr>>(
+        //             exec, local_dim, local_interfaces),
+        //         gko::with_matrix_type<
+        //             CombinationMatrix<scalar, label, Csr>>(
+        //             exec, non_local_dim, non_local_interfaces));
+        // }
+        // if (matrix_format_ == "Ell") {
+        //     using Ell = gko::matrix::Ell<scalar, label>;
+        //     return dist_mtx::create(
+        //         exec, *comm.get(),
+        //         gko::with_matrix_type <
+        //             gko::with_matrix_type<
+        //                 CombinationMatrix<scalar, label, Ell>>(
+        //                 exec, local_dim, local_interfaces),
+        //         gko::with_matrix_type<
+        //             CombinationMatrix<scalar, label, Ell>>(
+        //             exec, non_local_dim, non_local_interfaces));
+        // }
         if (matrix_format_ == "Coo") {
-            return dist_mtx::create(
-                exec, *comm.get(),
-                gko::with_matrix_type <
-                    gko::with_matrix_type<
-                        CombinationMatrix<scalar, label, gko::matrix::Coo>>(
-                        exec, local_interfaces),
-                gko::with_matrix_type<
-                    CombinationMatrix<scalar, label, gko::matrix::Coo>>(
-                    exec, non_local_interfaces));
+            using Coo = gko::matrix::Coo<scalar, label>;
+
+            auto local_template = CombinationMatrix<scalar, label>::create(
+                exec, local_dim, local_interfaces);
+            auto non_local_template = CombinationMatrix<scalar, label>::create(
+                exec, non_local_dim, non_local_interfaces);
+
+            return dist_mtx::create(exec, *comm.get(), local_template,
+                                    non_local_template);
         }
         // if (matrix_format_ == "Hybrid") {
         //     return dist_mtx::create(
@@ -202,13 +205,15 @@ struct MatrixInitFunctor {
         auto exec = exec_.get_ref_exec();
 
         auto num_rows = partition_.get_local_size();
-        device_matrix_data A_data{exec, gko::dim<2>(num_rows, num_rows),
-                                  *rows.get(), *cols.get(), *coeffs.get()};
+        gko::dim<2> local_dim{num_rows, num_rows};
+
+        gko::device_matrix_data<scalar, label> local_A_data{
+            exec, local_dim, *rows.get(), *cols.get(), *coeffs.get()};
 
         auto num_non_local_elems = non_local_cols->get_size();
-        device_matrix_data non_local_A_data{
-            exec, gko::dim<2>(num_rows, num_non_local_elems),
-            *non_local_rows.get(), *non_local_cols.get(),
+        gko::dim<2> non_local_dim{num_rows, num_non_local_elems};
+        gko::device_matrix_data<scalar, label> non_local_A_data{
+            exec, non_local_dim, *non_local_rows.get(), *non_local_cols.get(),
             *non_local_coeffs.get()};
 
         std::vector<gko::span> local_interfaces{gko::span(0, num_rows)};
@@ -217,25 +222,26 @@ struct MatrixInitFunctor {
 
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
-            exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm(),
-            local_interfaces, non_local_interfaces);
+            exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm(), local_dim,
+            local_interfaces, non_local_dim, non_local_interfaces);
 
         auto sparse_comm =
             gko::experimental::distributed::sparse_communicator::create(
                 *comm.get(), partition_.get_localized_partition());
 
-        auto recv_sizes = sparse_comm->get_recv_sizes();
-        auto recv_offs = sparse_comm->get_recv_offsets();
-        auto send_sizes = sparse_comm->get_send_sizes();
-        auto send_offs = sparse_comm->get_send_offsets();
-
-        // TODO this needs a std::vector version
         dist_A->read_distributed(local_A_data, non_local_A_data, sparse_comm);
 
+        auto local_combination = gko::as<CombinationMatrix<scalar,label>>(dist_A->get_local_matrix().get())->get_combination();
+        std::cout << "local_combination.size " << local_combination->get_operators().size() << "\n";
+
         auto device_mat = generate_dist_mtx_with_inner_type(
-            exec_.get_device_exec(), exec_.get_gko_mpi_device_comm(),
-            local_interfaces, non_local_interfaces);
+            exec_.get_device_exec(), exec_.get_gko_mpi_device_comm(), local_dim,
+            local_interfaces, non_local_dim, non_local_interfaces);
         dist_A->move_to(device_mat.get());
+
+        auto local_device_combination = gko::as<CombinationMatrix<scalar,label>>(device_mat->get_local_matrix().get())->get_combination();
+        std::cout << "device_local_combination.size " << local_device_combination->get_operators().size() << "\n";
+
         return device_mat;
     }
 };
@@ -302,21 +308,27 @@ public:
      */
     void write() const
     {
+    auto dist_matrix = gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(this->get().get());
+    auto local_interfaces = gko::as<CombinationMatrix<scalar, label>>(dist_matrix->get_local_matrix().get())
+            ->get_combination()
+            ->get_operators();
+
+    for (int i = 0; i < local_interfaces.size(); i++) {
         export_mtx(
             field_name_,
-            gko::as<
-                gko::experimental::distributed::Matrix<scalar, label, label>>(
-                get().get())
-                ->get_local_matrix(),
-            "local", db_, matrix_format_);
+            local_interfaces[i],
+            "local_" + std::to_string(i) , db_, matrix_format_);
+    }
 
+    auto non_local_interfaces = gko::as<CombinationMatrix<scalar, label>>(dist_matrix->get_non_local_matrix().get())
+            ->get_combination()
+            ->get_operators();
+    for (int i = 0; i < non_local_interfaces.size(); i++) {
         export_mtx(
             field_name_,
-            gko::as<
-                gko::experimental::distributed::Matrix<scalar, label, label>>(
-                get().get())
-                ->get_non_local_matrix(),
-            "non_local", db_, matrix_format_);
+            non_local_interfaces[i],
+            "non_local_" + std::to_string(i), db_, matrix_format_);
+    }
     }
 };
 
diff --git a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
index 53facf2f9..7f380baf5 100644
--- a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
+++ b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
@@ -4,52 +4,122 @@
 // The CombinationMatrix class is a wrapper around Ginkgos combination linear
 // operator. It provides an interface for creating linear combinations of a
 // giveen inner matrix type and a read implementation
-template <typename ValueType, typename IndexType, typename InnerMatrixType>
+// template <typename ValueType, typename IndexType, typename gko::matrix::Coo>
+template <typename ValueType, typename IndexType>
 class CombinationMatrix
-    : public gko::EnableLinOp<CombinationMatrix<ValueType>>,
-      public gko::EnableCreateMethod<CombinationMatrix<ValueType>>,
+    : public gko::EnableLinOp<CombinationMatrix<ValueType, IndexType>>,
+      public gko::EnableCreateMethod<CombinationMatrix<ValueType, IndexType>>,
       public gko::ReadableFromMatrixData<ValueType, IndexType> {
+    friend class gko::EnableCreateMethod<
+        CombinationMatrix<ValueType, IndexType>>;
+    friend class gko::EnablePolymorphicObject<
+        CombinationMatrix<ValueType, IndexType>, gko::LinOp>;
+
 public:
-    // This constructor will be called by the create method.
-    // Here we initialize, the interface_ranges to create the
-    // the linear combination during the read call
-    CombinationMatrix(std::shared_ptr<const gko::Executor> exec,
-                      std::vector<gko::span> interface_ranges)
-        : gko::EnableLinOp<CombinationMatrix>(exec, gko::dim<2>{size}),
-          interface_ranges_(interface_ranges),
-          comb_(gko::share(gko::Combination<ValueType>::create(exec, size)))
-    {}
+    using gko::EnableLinOp<CombinationMatrix<ValueType, IndexType>>::convert_to;
+    using gko::EnableLinOp<CombinationMatrix<ValueType, IndexType>>::move_to;
 
-protected:
     using vec = gko::matrix::Dense<ValueType>;
     using coef_type = gko::array<ValueType>;
+    using device_matrix_data = gko::device_matrix_data<scalar, label>;
+    using matrix_data = gko::matrix_data<scalar, label>;
+
+    void read(const matrix_data &data) override {}
 
     void read(const device_matrix_data &data) override
     {
         auto exec = this->get_executor();
         for (auto &interface : interface_ranges_) {
-            auto op gko::share(InnerMatrixType::create(this->exec))
+            auto length = interface.length();
+            // NOTE Dont hardcode matrix format
+            auto op = gko::share(gko::matrix::Coo<scalar, label>::create(exec));
 
-                auto sub_array_view = [interface, exec](auto array_data) {
-                    make_const_array_view(exec, interface.length(),
-                                          array_data + interface.begin())
-                };
+            auto offset = interface.begin;
 
-            auto row_idxs = sub_array_view(data.get_const_row_idxs());
-            auto col_idxs = sub_array_view(data.get_const_col_idxs());
-            auto vals = sub_array_view(data.get_const_values());
+            device_matrix_data in(
+                exec, comb_->get_size(),
+                gko::array<IndexType>(
+                    make_const_array_view(exec, length,
+                                          data.get_const_row_idxs() + offset)
+                        .copy_to_array()),
+                gko::array<IndexType>(
+                    make_const_array_view(exec, length,
+                                          data.get_const_col_idxs() + offset)
+                        .copy_to_array()),
+                gko::array<ValueType>(
+                    make_const_array_view(exec, length,
+                                          data.get_const_values() + offset)
+                        .copy_to_array()));
 
-            device_matrix_data in(exec, this->get_size(), row_idxs, col_idxs,
-                                  vals);
+            op->read(std::move(in));
 
-            op->read(std::move(interface_view))
+            this->comb_->add_operators(
+                gko::initialize<gko::matrix::Dense<ValueType>>({1}, exec), op);
+        }
+        std::cout << "operators.size" << this->comb_->get_operators().size() << "\n";
+        std::cout << "operator[0]" << this->comb_->get_operators()[0] << "\n";
+        // 
+        // NOTE find a better place
+        this->set_size(comb_->get_size());
+    }
 
-                combination->add_operators(
-                    gko::initialize<gko::matrix::Dense<ValueType>>({1}, exec),
-                    in);
+    std::shared_ptr<gko::Combination<ValueType>> get_combination() const
+    {
+        return this->comb_;
+    }
+
+
+    /**
+     * Copy-assigns a CombinationMatrix matrix. Preserves executor, copies
+     * everything else.
+     */
+    CombinationMatrix &operator=(const CombinationMatrix &other)
+    {
+        if (&other != this) {
+            gko::EnableLinOp<CombinationMatrix>::operator=(other);
+            interface_ranges_ = std::vector<gko::span>{};
+            for (auto i_range : other.interface_ranges_) {
+                interface_ranges_.emplace_back(i_range.begin, i_range.end);
+            }
+            comb_ = other.comb_;
         }
+        return *this;
     }
 
+    /**
+     * Move-assigns a CombinationMatrix matrix. Preserves executor, moves the
+     * data and leaves the moved-from object in an empty state (0x0 LinOp with
+     * unchanged executor and strategy, no nonzeros and valid row pointers).
+     */
+    CombinationMatrix &operator=(CombinationMatrix &&other)
+    {
+        if (&other != this) {
+            gko::EnableLinOp<CombinationMatrix>::operator=(std::move(other));
+            interface_ranges_ = std::move(other.interface_ranges_);
+            comb_ = std::move(other.comb_);
+            // restore other invariant
+            // other.row_ptrs_.resize_and_reset(1);
+        }
+        return *this;
+    }
+
+
+protected:
+    CombinationMatrix(std::shared_ptr<const gko::Executor> exec)
+        : gko::EnableLinOp<CombinationMatrix>(exec)
+    {}
+
+    // This constructor will be called by the create method.
+    // Here we initialize, the interface_ranges to create the
+    // the linear combination during the read call
+    // TODO dont allow copying
+    CombinationMatrix(std::shared_ptr<const gko::Executor> exec,
+                      gko::dim<2> size, std::vector<gko::span> interface_ranges)
+        : gko::EnableLinOp<CombinationMatrix>(exec),
+          interface_ranges_(interface_ranges),
+          comb_(gko::share(gko::Combination<ValueType>::create(exec, size)))
+    {}
+
     // Here we implement the application of the linear operator, x = A * b.
     // apply_impl will be called by the apply method, after the arguments have
     // been moved to the correct executor and the operators checked for
@@ -58,8 +128,9 @@ protected:
     // For simplicity, we assume that there is always only one right hand side
     // and the stride of consecutive elements in the vectors is 1 (both of these
     // are always true in this example).
-    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override {
-        this->comb_->apply_impl(b,x);
+    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
+    {
+        this->comb_->apply(b, x);
     }
 
     // There is also a version of the apply function which does the operation
@@ -69,10 +140,11 @@ protected:
     void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
                     const gko::LinOp *beta, gko::LinOp *x) const override
     {
-        this->comb_->apply_impl(alpha,b, beta, x);
+        this->comb_->apply(alpha, b, beta, x);
     }
 
+
 private:
     std::vector<gko::span> interface_ranges_;
-    std::shared_ptr<gko::Combination> comb_;
+    std::shared_ptr<gko::Combination<ValueType>> comb_;
 };
diff --git a/common/common.C b/common/common.C
index c2924d74d..61b355e0b 100644
--- a/common/common.C
+++ b/common/common.C
@@ -10,6 +10,8 @@
 #include <filesystem>
 #include "common.H"
 
+#include "MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H"
+
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
 namespace Foam {
@@ -42,10 +44,12 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
 {
     std::string folder{db.time().timePath()};
     std::filesystem::create_directories(folder);
+
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
     std::cout << "exporting " << fn << std::endl;
     std::ofstream stream{fn};
     stream << std::setprecision(15);
+
     if (matrixFormat == "Coo") {
         gko::write(stream, (const gko::matrix::Coo<scalar> *)A.get());
     }
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 2a2b9a3a2..71a9dba01 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -272,6 +272,7 @@ public:
         TIME_WITH_FIELDNAME(verbose_, generate_solver, this->fieldName(),
                             auto solver = solver_gen->generate(dist_A_v);)
         LOG_1(verbose_, "solve solver done")
+        std::cout << "!!! call solver apply\n";
         TIME_WITH_FIELDNAME(verbose_, solve, this->fieldName(),
                             solver->apply(dist_b_v, dist_x_v);)
 

From 86ce03038d352183407a2883750691871af8d140 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 14:30:20 +0100
Subject: [PATCH 11/90] wip clang build asan

---
 CMakeLists.txt                                |  10 +-
 CMakePresets.json                             |   5 +
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 365 ++++++++++--------
 .../GkoCombinationMatrix.H                    |  41 +-
 MatrixWrapper/HostMatrix/HostMatrix.C         |  33 +-
 MatrixWrapper/HostMatrix/HostMatrix.H         |  15 +
 Preconditioner/Preconditioner.H               |  15 +-
 common/common.C                               |  40 +-
 common/common.H                               |   7 +-
 lduLduBase/lduLduBase.H                       |  13 +-
 10 files changed, 329 insertions(+), 215 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f213ab068..e78a9c251 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,8 +104,9 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
       "${CMAKE_CXX_FLAGS_DEBUG} \
       --coverage\
       -fprofile-arcs \
-      -ggdb3")
-  # * fsanitize = address \ - ftest - coverage \
+      -ggdb3 \
+      -fsanitize=address,undefined \
+      ")
 
   set(CMAKE_CXX_FLAGS_RELEASE
       "${CMAKE_CXX_FLAGS_RELEASE} \
@@ -186,6 +187,8 @@ target_sources(
   # LduMatrix / GKOACG / GKOACG.H
 )
 
+target_link_options(OGL PUBLIC "-fsanitize=address")
+
 target_include_directories(
   OGL SYSTEM
   PUBLIC $ENV{FOAM_SRC}/finiteVolume/lnInclude
@@ -197,6 +200,8 @@ target_link_libraries(
   PRIVATE $ENV{FOAM_LIBBIN}/libOpenFOAM.so $ENV{FOAM_LIBBIN}/libfiniteVolume.so
           $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so Ginkgo::ginkgo stdc++fs)
 
+
+
 if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)
 endif()
@@ -206,6 +211,7 @@ if(${GINKGO_BUILD_CUDA})
   target_link_libraries(OGL PUBLIC CUDA::nvToolsExt)
 endif()
 
+
 if(OGL_DATA_VALIDATION)
   target_compile_definitions(OGL PRIVATE DATA_VALIDATION=1)
 endif()
diff --git a/CMakePresets.json b/CMakePresets.json
index d856ce282..a0b5ef4f4 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -109,6 +109,11 @@
       "configurePreset": "ninja-cpuonly-release",
       "configuration": "Release"
     },
+    {
+      "name": "ninja-cpuonly-debug",
+      "configurePreset": "ninja-cpuonly-release",
+      "configuration": "Release"
+    },
     {
       "name": "ninja-cuda-release",
       "configurePreset": "ninja-cuda-release",
diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 20280cfe0..f567823ed 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -16,6 +16,40 @@
 namespace Foam {
 
 
+template <typename InnerMatrixType>
+void perform_update(
+    const std::shared_ptr<gko::Executor> &exec,
+    std::shared_ptr<CombinationMatrix<scalar, label, InnerMatrixType>>
+        combination_matrix,
+    const PersistentSparsityPattern &sparsity,
+    std::shared_ptr<gko::array<scalar>> coeffs)
+{
+    int interface_ctr{0};
+    for (auto &interface :
+         combination_matrix->get_combination()->get_operators()) {
+        auto device_value_ptr =
+            gko::as<InnerMatrixType>(interface)->get_const_values();
+
+        // create a view into data on the device which needs an
+        // update
+        auto num_elems =
+            gko::as<InnerMatrixType>(interface)->get_num_stored_elements();
+        auto device_value_view = val_array::view(
+            exec, num_elems, const_cast<scalar *>(device_value_ptr));
+
+        // create a view into updated values and copy over to
+        // persistent data
+        auto span = sparsity.interface_spans_[interface_ctr];
+        auto update_view = val_array::view(exec, span.length(),
+                                           coeffs->get_data() + span.begin);
+
+        std::cout << "perform update " << span.length() << " " << span.begin
+                  << " " << num_elems << " " << interface->get_size() << "\n";
+        interface_ctr++;
+        device_value_view = update_view;
+    }
+};
+
 struct MatrixInitFunctor {
     using dist_mtx =
         gko::experimental::distributed::Matrix<scalar, label, label>;
@@ -28,15 +62,11 @@ struct MatrixInitFunctor {
 
     const PersistentPartition &partition_;
 
-    const PersistentArray<label> &col_idxs_;
-
-    const PersistentArray<label> &row_idxs_;
+    const PersistentSparsityPattern &local_sparsity_;
 
     const PersistentArray<scalar> &coeffs_;
 
-    const PersistentArray<label> &non_local_col_idxs_;
-
-    const PersistentArray<label> &non_local_row_idxs_;
+    const PersistentSparsityPattern &non_local_sparsity_;
 
     const PersistentArray<scalar> &non_local_coeffs_;
 
@@ -50,22 +80,18 @@ struct MatrixInitFunctor {
 
     MatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
                       const PersistentPartition &partition,
-                      const PersistentArray<label> &col_idxs,
-                      const PersistentArray<label> &row_idxs,
+                      const PersistentSparsityPattern &local_sparsity,
                       const PersistentArray<scalar> &coeffs,
-                      const PersistentArray<label> &non_local_col_idxs,
-                      const PersistentArray<label> &non_local_row_idxs,
+                      const PersistentSparsityPattern &non_local_sparsity,
                       const PersistentArray<scalar> &non_local_coeffs,
                       const word matrix_format, const bool regenerate,
                       const label verbose, const word field_name)
         : db_(db),
           exec_(exec),
           partition_(partition),
-          col_idxs_(col_idxs),
-          row_idxs_(row_idxs),
+          local_sparsity_(local_sparsity),
           coeffs_(coeffs),
-          non_local_col_idxs_(non_local_col_idxs),
-          non_local_row_idxs_(non_local_row_idxs),
+          non_local_sparsity_(non_local_sparsity),
           non_local_coeffs_(non_local_coeffs),
           matrix_format_(matrix_format),
           regenerate_(regenerate),
@@ -81,99 +107,114 @@ struct MatrixInitFunctor {
             persistent_device_matrix->copy_from(reinit.get());
             return;
         }
-        auto coeffs = coeffs_.get_array();
-        auto non_local_coeffs = non_local_coeffs_.get_array();
-
-        auto exec = exec_.get_ref_exec();
-        auto device_exec = exec_.get_device_exec();
-
-        scalar *value_ptr{};
-        scalar *non_local_value_ptr{};
 
-        std::shared_ptr<const gko::LinOp> local_matrix =
-            persistent_device_matrix->get_local_matrix();
-        std::shared_ptr<const gko::LinOp> non_local_matrix =
-            persistent_device_matrix->get_non_local_matrix();
-        if (matrix_format_ == "Csr") {
-            gko::matrix::Csr<scalar, label> *coo_ptr =
-                (gko::matrix::Csr<scalar, label> *)local_matrix.get();
-            value_ptr = coo_ptr->get_values();
-
-            gko::matrix::Csr<scalar, label> *non_local_coo_ptr =
-                (gko::matrix::Csr<scalar, label> *)non_local_matrix.get();
-            non_local_value_ptr = non_local_coo_ptr->get_values();
-        }
-        if (matrix_format_ == "Ell") {
-            gko::matrix::Ell<scalar, label> *coo_ptr =
-                (gko::matrix::Ell<scalar, label> *)local_matrix.get();
-            value_ptr = coo_ptr->get_values();
-
-            gko::matrix::Ell<scalar, label> *non_local_coo_ptr =
-                (gko::matrix::Ell<scalar, label> *)non_local_matrix.get();
-            non_local_value_ptr = non_local_coo_ptr->get_values();
-        }
+        // The local and non-local matrix are a CombinationMatrix and we need to
+        // update the interfaces iteratively
+        std::shared_ptr<gko::LinOp> local_combination_matrix{
+            const_cast<gko::LinOp *>(
+                persistent_device_matrix->get_local_matrix().get())};
+        std::shared_ptr<gko::LinOp> non_local_combination_matrix{
+            const_cast<gko::LinOp *>(
+                persistent_device_matrix->get_non_local_matrix().get())};
         if (matrix_format_ == "Coo") {
-            gko::matrix::Coo<scalar, label> *coo_ptr =
-                (gko::matrix::Coo<scalar, label> *)local_matrix.get();
-            value_ptr = coo_ptr->get_values();
-
-            gko::matrix::Coo<scalar, label> *non_local_coo_ptr =
-                (gko::matrix::Coo<scalar, label> *)non_local_matrix.get();
-            non_local_value_ptr = non_local_coo_ptr->get_values();
+            using Coo = gko::matrix::Coo<scalar, label>;
+            perform_update<Coo>(exec_.get_device_exec(),
+                                gko::as<CombinationMatrix<scalar, label, Coo>>(
+                                    local_combination_matrix),
+                                local_sparsity_, coeffs_.get_array());
+            perform_update<Coo>(exec_.get_device_exec(),
+                                gko::as<CombinationMatrix<scalar, label, Coo>>(
+                                    non_local_combination_matrix),
+                                non_local_sparsity_,
+                                non_local_coeffs_.get_array());
+            std::cout << " local_combination_matrix "
+                      << local_combination_matrix->get_size() << "\n";
+            std::cout << " non_local_combination_matrix "
+                      << non_local_combination_matrix->get_size() << "\n";
+            std::cout << " CsrMatrixWrapper distributed_matrix "
+                      << persistent_device_matrix.get()->get_size() << "\n";
+            std::cout << " CsrMatrixWrapper distributed_matrix get_local "
+                      << persistent_device_matrix.get()
+                             ->get_local_matrix()
+                             ->get_size()
+                      << "\n";
+            std::cout << " CsrMatrixWrapper distributed_matrix get non_local "
+                      << persistent_device_matrix.get()
+                             ->get_non_local_matrix()
+                             ->get_size()
+                      << "\n";
         }
-
-        gko::array<scalar> device_values{device_exec, *coeffs.get()};
-        gko::array<scalar> device_non_local_values{device_exec,
-                                                   *non_local_coeffs.get()};
-
-        auto value_view =
-            val_array::view(device_exec, device_values.get_size(), value_ptr);
-
-        auto non_local_view =
-            val_array::view(device_exec, device_non_local_values.get_size(),
-                            non_local_value_ptr);
-
-        value_view = device_values;
-        non_local_view = device_non_local_values;
-    }
-
-    std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
-        std::shared_ptr<gko::Executor> exec,
-        std::shared_ptr<gko::experimental::mpi::communicator> comm,
-        gko::dim<2> local_dim, std::vector<gko::span> local_interfaces,
-        gko::dim<2> non_local_dim,
-        std::vector<gko::span> non_local_interfaces) const
-    {
         // if (matrix_format_ == "Csr") {
         //     using Csr = gko::matrix::Csr<scalar, label>;
-        //     return dist_mtx::create(
-        //         exec, *comm.get(),
-        //         gko::with_matrix_type<
-        //             CombinationMatrix<scalar, label, Csr>>(
-        //             exec, local_dim, local_interfaces),
-        //         gko::with_matrix_type<
-        //             CombinationMatrix<scalar, label, Csr>>(
-        //             exec, non_local_dim, non_local_interfaces));
+        //     perform_update<Csr>(exec_.get_device_exec(),
+        //                         gko::as<CombinationMatrix<scalar, label,
+        //                         Csr>>(
+        //                             local_combination_matrix),
+        //                         local_sparsity_, coeffs_.get_array());
+        //     perform_update<Csr>(exec_.get_device_exec(),
+        //                         gko::as<CombinationMatrix<scalar, label,
+        //                         Csr>>(
+        //                             non_local_combination_matrix),
+        //                         non_local_sparsity_,
+        //                         non_local_coeffs_.get_array());
         // }
         // if (matrix_format_ == "Ell") {
         //     using Ell = gko::matrix::Ell<scalar, label>;
-        //     return dist_mtx::create(
-        //         exec, *comm.get(),
-        //         gko::with_matrix_type <
-        //             gko::with_matrix_type<
-        //                 CombinationMatrix<scalar, label, Ell>>(
-        //                 exec, local_dim, local_interfaces),
-        //         gko::with_matrix_type<
-        //             CombinationMatrix<scalar, label, Ell>>(
-        //             exec, non_local_dim, non_local_interfaces));
+        //     perform_update<Ell>(exec_.get_device_exec(),
+        //                         gko::as<CombinationMatrix<scalar, label,
+        //                         Ell>>(
+        //                             local_combination_matrix),
+        //                         local_sparsity_, coeffs_.get_array());
+        //     perform_update<Ell>(exec_.get_device_exec(),
+        //                         gko::as<CombinationMatrix<scalar, label,
+        //                         Ell>>(
+        //                             non_local_combination_matrix),
+        //                         non_local_sparsity_,
+        //                         non_local_coeffs_.get_array());
         // }
+    }
+
+    std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
+        std::shared_ptr<gko::Executor> exec,
+        std::shared_ptr<gko::experimental::mpi::communicator> comm) const
+    {
+        auto local_dim = local_sparsity_.dim_;
+        auto non_local_dim = non_local_sparsity_.dim_;
+        auto local_interfaces = local_sparsity_.interface_spans_;
+        auto non_local_interfaces = non_local_sparsity_.interface_spans_;
+
+        if (matrix_format_ == "Ell") {
+            using Ell = gko::matrix::Ell<scalar, label>;
+
+            auto local_template = CombinationMatrix<scalar, label, Ell>::create(
+                exec, local_dim, local_interfaces);
+            auto non_local_template =
+                CombinationMatrix<scalar, label, Ell>::create(
+                    exec, non_local_dim, non_local_interfaces);
+
+            return dist_mtx::create(exec, *comm.get(), local_template,
+                                    non_local_template);
+        }
+        if (matrix_format_ == "Csr") {
+            using Csr = gko::matrix::Csr<scalar, label>;
+
+            auto local_template = CombinationMatrix<scalar, label, Csr>::create(
+                exec, local_dim, local_interfaces);
+            auto non_local_template =
+                CombinationMatrix<scalar, label, Csr>::create(
+                    exec, non_local_dim, non_local_interfaces);
+
+            return dist_mtx::create(exec, *comm.get(), local_template,
+                                    non_local_template);
+        }
         if (matrix_format_ == "Coo") {
             using Coo = gko::matrix::Coo<scalar, label>;
 
-            auto local_template = CombinationMatrix<scalar, label>::create(
+            auto local_template = CombinationMatrix<scalar, label, Coo>::create(
                 exec, local_dim, local_interfaces);
-            auto non_local_template = CombinationMatrix<scalar, label>::create(
-                exec, non_local_dim, non_local_interfaces);
+            auto non_local_template =
+                CombinationMatrix<scalar, label, Coo>::create(
+                    exec, non_local_dim, non_local_interfaces);
 
             return dist_mtx::create(exec, *comm.get(), local_template,
                                     non_local_template);
@@ -194,55 +235,35 @@ struct MatrixInitFunctor {
                  std::to_string(local_size)};
         LOG_1(verbose_, msg)
 
-        auto coeffs = coeffs_.get_array();
-        auto cols = col_idxs_.get_array();
-        auto rows = row_idxs_.get_array();
-
-        auto non_local_coeffs = non_local_coeffs_.get_array();
-        auto non_local_cols = non_local_col_idxs_.get_array();
-        auto non_local_rows = non_local_row_idxs_.get_array();
-
         auto exec = exec_.get_ref_exec();
 
-        auto num_rows = partition_.get_local_size();
-        gko::dim<2> local_dim{num_rows, num_rows};
-
+        auto coeffs = coeffs_.get_array();
+        auto cols = local_sparsity_.col_idxs_.get_array();
+        auto rows = local_sparsity_.row_idxs_.get_array();
         gko::device_matrix_data<scalar, label> local_A_data{
-            exec, local_dim, *rows.get(), *cols.get(), *coeffs.get()};
+            exec, local_sparsity_.dim_, *rows.get(), *cols.get(),
+            *coeffs.get()};
 
-        auto num_non_local_elems = non_local_cols->get_size();
-        gko::dim<2> non_local_dim{num_rows, num_non_local_elems};
+        auto non_local_coeffs = non_local_coeffs_.get_array();
+        auto non_local_cols = non_local_sparsity_.col_idxs_.get_array();
+        auto non_local_rows = non_local_sparsity_.row_idxs_.get_array();
         gko::device_matrix_data<scalar, label> non_local_A_data{
-            exec, non_local_dim, *non_local_rows.get(), *non_local_cols.get(),
-            *non_local_coeffs.get()};
-
-        std::vector<gko::span> local_interfaces{gko::span(0, num_rows)};
-        std::vector<gko::span> non_local_interfaces{
-            gko::span(0, num_non_local_elems)};
+            exec, non_local_sparsity_.dim_, *non_local_rows.get(),
+            *non_local_cols.get(), *non_local_coeffs.get()};
 
         auto comm = exec_.get_gko_mpi_host_comm();
-        auto dist_A = generate_dist_mtx_with_inner_type(
-            exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm(), local_dim,
-            local_interfaces, non_local_dim, non_local_interfaces);
-
         auto sparse_comm =
             gko::experimental::distributed::sparse_communicator::create(
                 *comm.get(), partition_.get_localized_partition());
 
+        auto dist_A = generate_dist_mtx_with_inner_type(
+            exec_.get_device_exec(), exec_.get_gko_mpi_host_comm());
         dist_A->read_distributed(local_A_data, non_local_A_data, sparse_comm);
 
-        auto local_combination = gko::as<CombinationMatrix<scalar,label>>(dist_A->get_local_matrix().get())->get_combination();
-        std::cout << "local_combination.size " << local_combination->get_operators().size() << "\n";
-
-        auto device_mat = generate_dist_mtx_with_inner_type(
-            exec_.get_device_exec(), exec_.get_gko_mpi_device_comm(), local_dim,
-            local_interfaces, non_local_dim, non_local_interfaces);
-        dist_A->move_to(device_mat.get());
-
-        auto local_device_combination = gko::as<CombinationMatrix<scalar,label>>(device_mat->get_local_matrix().get())->get_combination();
-        std::cout << "device_local_combination.size " << local_device_combination->get_operators().size() << "\n";
-
-        return device_mat;
+        // auto device_mat = generate_dist_mtx_with_inner_type(
+        //     exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
+        // dist_A->move_to(device_mat.get());
+        return dist_A;
     }
 };
 
@@ -269,11 +290,9 @@ private:
 
 public:
     MatrixWrapper(const objectRegistry &db, const PersistentExecutor &exec,
-                  const PersistentArray<label> &col_idxs,
-                  const PersistentArray<label> &row_idxs,
+                  const PersistentSparsityPattern &local_sparsity,
                   const PersistentArray<scalar> &coeffs,
-                  const PersistentArray<label> &non_local_col_idxs,
-                  const PersistentArray<label> &non_local_row_idxs,
+                  const PersistentSparsityPattern &non_local_sparsity,
                   const PersistentArray<scalar> &non_local_coeffs,
                   const PersistentPartition &partition,
                   const dictionary &controlDict, const word sys_matrix_name,
@@ -287,9 +306,8 @@ public:
           gkomatrix_{
               sys_matrix_name + "_matrix", db,
               MatrixInitFunctor(
-                  db, exec, partition, col_idxs, row_idxs, coeffs,
-                  non_local_col_idxs, non_local_row_idxs, non_local_coeffs,
-                  matrix_format_,
+                  db, exec, partition, local_sparsity, coeffs,
+                  non_local_sparsity, non_local_coeffs, matrix_format_,
                   controlDict.lookupOrDefault<Switch>("regenerate", false),
                   verbose_, sys_matrix_name),
               controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
@@ -308,27 +326,60 @@ public:
      */
     void write() const
     {
-    auto dist_matrix = gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(this->get().get());
-    auto local_interfaces = gko::as<CombinationMatrix<scalar, label>>(dist_matrix->get_local_matrix().get())
-            ->get_combination()
-            ->get_operators();
-
-    for (int i = 0; i < local_interfaces.size(); i++) {
-        export_mtx(
-            field_name_,
-            local_interfaces[i],
-            "local_" + std::to_string(i) , db_, matrix_format_);
-    }
+        auto dist_matrix = gko::as<
+            gko::experimental::distributed::Matrix<scalar, label, label>>(
+            this->get());
 
-    auto non_local_interfaces = gko::as<CombinationMatrix<scalar, label>>(dist_matrix->get_non_local_matrix().get())
-            ->get_combination()
-            ->get_operators();
-    for (int i = 0; i < non_local_interfaces.size(); i++) {
-        export_mtx(
-            field_name_,
-            non_local_interfaces[i],
-            "non_local_" + std::to_string(i), db_, matrix_format_);
-    }
+        if (matrix_format_ == "Coo") {
+            using Coo = gko::matrix::Coo<scalar, label>;
+            std::vector<std::shared_ptr<const gko::LinOp>> local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Coo>>(
+                    dist_matrix->get_local_matrix())
+                    ->get_combination()
+                    ->get_operators();
+            auto non_local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Coo>>(
+                    dist_matrix->get_non_local_matrix())
+                    ->get_combination()
+                    ->get_operators();
+            export_mtx<Coo>(word(field_name_ + "_local"), local_interfaces, db_);
+            export_mtx<Coo>(field_name_ + "_non_local", non_local_interfaces,
+                            db_);
+        }
+
+        if (matrix_format_ == "Csr") {
+            using Csr = gko::matrix::Csr<scalar, label>;
+            auto local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Csr>>(
+                    dist_matrix->get_local_matrix().get())
+                    ->get_combination()
+                    ->get_operators();
+            auto non_local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Csr>>(
+                    dist_matrix->get_non_local_matrix().get())
+                    ->get_combination()
+                    ->get_operators();
+            export_mtx<Csr>(word(field_name_ + "_local"), local_interfaces, db_);
+            export_mtx<Csr>(field_name_ + "_non_local", non_local_interfaces,
+                            db_);
+        }
+
+        if (matrix_format_ == "Ell") {
+            using Ell = gko::matrix::Ell<scalar, label>;
+            auto local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Ell>>(
+                    dist_matrix->get_local_matrix().get())
+                    ->get_combination()
+                    ->get_operators();
+            auto non_local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Ell>>(
+                    dist_matrix->get_non_local_matrix().get())
+                    ->get_combination()
+                    ->get_operators();
+            export_mtx<Ell>(field_name_ + "_local", local_interfaces, db_);
+            export_mtx<Ell>(field_name_ + "_non_local", non_local_interfaces,
+                            db_);
+        }
     }
 };
 
diff --git a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
index 7f380baf5..e4c8f6b95 100644
--- a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
+++ b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
@@ -5,19 +5,23 @@
 // operator. It provides an interface for creating linear combinations of a
 // giveen inner matrix type and a read implementation
 // template <typename ValueType, typename IndexType, typename gko::matrix::Coo>
-template <typename ValueType, typename IndexType>
+template <typename ValueType, typename IndexType, typename InnerMatrixType>
 class CombinationMatrix
-    : public gko::EnableLinOp<CombinationMatrix<ValueType, IndexType>>,
-      public gko::EnableCreateMethod<CombinationMatrix<ValueType, IndexType>>,
+    : public gko::EnableLinOp<
+          CombinationMatrix<ValueType, IndexType, InnerMatrixType>>,
+      public gko::EnableCreateMethod<
+          CombinationMatrix<ValueType, IndexType, InnerMatrixType>>,
       public gko::ReadableFromMatrixData<ValueType, IndexType> {
     friend class gko::EnableCreateMethod<
-        CombinationMatrix<ValueType, IndexType>>;
+        CombinationMatrix<ValueType, IndexType, InnerMatrixType>>;
     friend class gko::EnablePolymorphicObject<
-        CombinationMatrix<ValueType, IndexType>, gko::LinOp>;
+        CombinationMatrix<ValueType, IndexType, InnerMatrixType>, gko::LinOp>;
 
 public:
-    using gko::EnableLinOp<CombinationMatrix<ValueType, IndexType>>::convert_to;
-    using gko::EnableLinOp<CombinationMatrix<ValueType, IndexType>>::move_to;
+    using gko::EnableLinOp<
+        CombinationMatrix<ValueType, IndexType, InnerMatrixType>>::convert_to;
+    using gko::EnableLinOp<
+        CombinationMatrix<ValueType, IndexType, InnerMatrixType>>::move_to;
 
     using vec = gko::matrix::Dense<ValueType>;
     using coef_type = gko::array<ValueType>;
@@ -29,36 +33,39 @@ public:
     void read(const device_matrix_data &data) override
     {
         auto exec = this->get_executor();
+        auto data_exec = data.get_executor();
+
         for (auto &interface : interface_ranges_) {
             auto length = interface.length();
-            // NOTE Dont hardcode matrix format
-            auto op = gko::share(gko::matrix::Coo<scalar, label>::create(exec));
-
             auto offset = interface.begin;
+            auto op = gko::share(InnerMatrixType::create(exec));
 
             device_matrix_data in(
                 exec, comb_->get_size(),
                 gko::array<IndexType>(
-                    make_const_array_view(exec, length,
+                    exec,
+                    make_const_array_view(data_exec, length,
                                           data.get_const_row_idxs() + offset)
                         .copy_to_array()),
                 gko::array<IndexType>(
-                    make_const_array_view(exec, length,
+                    exec,
+                    make_const_array_view(data_exec, length,
                                           data.get_const_col_idxs() + offset)
                         .copy_to_array()),
                 gko::array<ValueType>(
-                    make_const_array_view(exec, length,
+                    exec,
+                    make_const_array_view(data_exec, length,
                                           data.get_const_values() + offset)
                         .copy_to_array()));
 
             op->read(std::move(in));
+            std::cout << "!!! gkocombinationmatrix length " << length
+                      << " offset " << offset << " op->get_size() "
+                      << op->get_size() << "\n";
 
             this->comb_->add_operators(
                 gko::initialize<gko::matrix::Dense<ValueType>>({1}, exec), op);
         }
-        std::cout << "operators.size" << this->comb_->get_operators().size() << "\n";
-        std::cout << "operator[0]" << this->comb_->get_operators()[0] << "\n";
-        // 
         // NOTE find a better place
         this->set_size(comb_->get_size());
     }
@@ -78,7 +85,7 @@ public:
         if (&other != this) {
             gko::EnableLinOp<CombinationMatrix>::operator=(other);
             interface_ranges_ = std::vector<gko::span>{};
-            for (auto i_range : other.interface_ranges_) {
+            for (auto &i_range : other.interface_ranges_) {
                 interface_ranges_.emplace_back(i_range.begin, i_range.end);
             }
             comb_ = other.comb_;
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
index 383d1739d..585728b6d 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -274,12 +274,13 @@ CommunicationPattern
 HostMatrixWrapper<MatrixType>::create_communication_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
+    using comm_size_type = CommunicationPattern::comm_size_type;
     // temp map, mapping from neighbour rank interface cells
     std::map<label, std::vector<label>> interface_cell_map{};
 
     // iterate all interfaces, count number of neighbour procs
     // and store rows to send to neighbour procs
-    label n_procs = 0;
+    gko::size_type n_procs = 0;
     interface_iterator<processorFvPatch>(
         interfaces, [&](label, const label, const processorFvPatch &patch,
                         const lduInterfaceField *iface) {
@@ -298,8 +299,6 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
             }
         });
 
-    using comm_size_type = CommunicationPattern::comm_size_type;
-
     // create index_sets
     std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
     for (auto [proc, interface_cells] : interface_cell_map) {
@@ -311,10 +310,8 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     }
 
     // convert to gko::array
-    gko::array<comm_size_type> target_ids{exec_.get_ref_exec(),
-                                          static_cast<comm_size_type>(n_procs)};
-    gko::array<comm_size_type> target_sizes{
-        exec_.get_ref_exec(), static_cast<comm_size_type>(n_procs)};
+    gko::array<comm_size_type> target_ids{exec_.get_ref_exec(), n_procs};
+    gko::array<comm_size_type> target_sizes{exec_.get_ref_exec(), n_procs};
 
     label iter = 0;
     for (const auto &[proc, interface_cells] : interface_cell_map) {
@@ -484,6 +481,7 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
         permute[element_ctr] = interface_idx;
         element_ctr += 1;
     }
+
 }
 
 template <class MatrixType>
@@ -561,7 +559,7 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
             auto [interface_idx, interface_row, interface_col] = interface;
 
             // copy from existing matrix coefficients
-            while ([&]() {
+            while ([&](auto interface_row, auto interface_col) {
                 // check for length
                 if (current_idx_ctr == local_matrix_nnz_) {
                     return false;
@@ -579,7 +577,7 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
                     return false;
                 }
                 return true;
-            }()) {
+            }(interface_row, interface_col)) {
                 // insert coeffs
                 rows[total_ctr] = rows_copy[current_idx_ctr];
                 cols[total_ctr] = cols_copy[current_idx_ctr];
@@ -624,6 +622,7 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     // TODO this does not work for Ell
     auto permute = local_sparsity_.ldu_mapping_.get_data();
 
+
     if (reorder_on_copy_) {
         auto dense = local_coeffs_.get_data();
         if (local_interface_nnz_) {
@@ -634,6 +633,7 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
                     local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
                     permute, scaling_, diag.data(), upper.data(),
                     couple_coeffs.data(), dense);
+
             } else {
                 non_symmetric_update_w_interface(
                     local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
@@ -721,6 +721,14 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
 
         dense_vec->copy_from(row_collection);
     }
+
+    std::cout << "!!! set dim_ " << nrows_ << " " << local_matrix_nnz_ << "\n";
+    // NOTE interface_spans and dim are not persistent so we need to
+    // recreate this for every solver call
+    // TODO this needs a propper implementation once we know how to handle interfaces
+    local_sparsity_.interface_spans_.emplace_back(0, local_matrix_nnz_);
+    local_sparsity_.dim_ = gko::dim<2>{nrows_, nrows_};
+
 }
 
 
@@ -748,6 +756,13 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     auto i_device_view = gko::array<scalar>::view(
         ref_exec, non_local_matrix_nnz_, non_local_coeffs_.get_data());
     i_device_view = tmp_contiguous_iface;
+
+    std::cout << "!!! set dim_ " << nrows_ << " " << non_local_matrix_nnz_ << "\n";
+    // NOTE interface_spans and dim are not persistent so we need to
+    // We have now the full non_local matrix and need to add the start and end
+    // to the PersistentSparsityPattern data structure
+    non_local_sparsity_.interface_spans_.emplace_back(0, non_local_matrix_nnz_);
+    non_local_sparsity_.dim_ = gko::dim<2>{nrows_, non_local_matrix_nnz_};
 }
 
 template HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.H b/MatrixWrapper/HostMatrix/HostMatrix.H
index b0016e311..26b3a598a 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.H
+++ b/MatrixWrapper/HostMatrix/HostMatrix.H
@@ -40,12 +40,14 @@ namespace Foam {
 
 
 struct PersistentSparsityPattern {
+
     PersistentSparsityPattern(
       const word &fieldName,
       const objectRegistry &db,
       const ExecutorHandler &exec,
       const label size,
       const label verbose):
+    size_(size),
     col_idxs_{
         fieldName + "_cols",
         db,
@@ -76,6 +78,9 @@ struct PersistentSparsityPattern {
     }
     {}
 
+    // number of nnz entries
+    const label size_;
+
     mutable PersistentArray<label> col_idxs_;
 
     mutable PersistentArray<label> row_idxs_;
@@ -83,6 +88,12 @@ struct PersistentSparsityPattern {
     // ldu_mapping[sorted_(csr)_position] =
     // unsorted_(consecutive_ldu_)_position
     mutable PersistentArray<label> ldu_mapping_;
+
+    // dimensions of the resulting matrix
+    mutable gko::dim<2> dim_;
+
+    // A vector of spans indicating begin and end of each interface
+    mutable std::vector<gko::span> interface_spans_;
 };
 
 struct CommunicationPattern {
@@ -432,6 +443,8 @@ public:
 
     const ExecutorHandler &get_exec_handler() const { return exec_; }
 
+    PersistentSparsityPattern &get_local_sparsity_pattern() const { return local_sparsity_; };
+
     PersistentArray<scalar> &get_local_coeffs() const { return local_coeffs_; };
 
     PersistentArray<label> &get_local_col_idxs() const
@@ -444,6 +457,8 @@ public:
         return local_sparsity_.row_idxs_;
     };
 
+    PersistentSparsityPattern &get_non_local_sparsity_pattern() const { return non_local_sparsity_; };
+
     PersistentArray<scalar> &get_non_local_coeffs() const
     {
         return non_local_coeffs_;
diff --git a/Preconditioner/Preconditioner.H b/Preconditioner/Preconditioner.H
index 6433b6012..88e046adf 100644
--- a/Preconditioner/Preconditioner.H
+++ b/Preconditioner/Preconditioner.H
@@ -90,8 +90,9 @@ public:
 
         if (name == "BJ") {
             // TODO for non constant system matrix reuse block pointers
-            label max_block_size(
-                controls.lookupOrDefault("maxBlockSize", label(1)));
+            const unsigned int max_block_size(
+                controls.lookupOrDefault("maxBlockSize", label(1))
+                    );
 
             word msg = "Generate preconditioner " + name + " MaxBlockSize " +
                        std::to_string(max_block_size);
@@ -148,9 +149,9 @@ public:
             auto trisolve_factory =
                 ir::build()
                     .with_solver(
-                        bj::build().with_max_block_size(1).on(device_exec))
+                        bj::build().with_max_block_size(1u).on(device_exec))
                     .with_criteria(
-                        gko::stop::Iteration::build().with_max_iters(5).on(
+                        gko::stop::Iteration::build().with_max_iters(5u).on(
                             device_exec))
                     .on(device_exec);
 
@@ -294,7 +295,7 @@ public:
             //                                      .on(device_exec))
             //                   .on(device_exec));
 
-            const label coarse_solver_iters(
+            const unsigned int coarse_solver_iters(
                 controls.lookupOrDefault("coarseSolverIters", label(4)));
             auto coarsest_gen = gko::share(
                 cg::build()
@@ -311,8 +312,8 @@ public:
             if (cycle_name == "f") cycle = gko::solver::multigrid::cycle::f;
 
             // Create multigrid factory
-            label max_levels(controls.lookupOrDefault("maxLevels", label(9)));
-            label min_coarse_rows(
+            const unsigned int max_levels(controls.lookupOrDefault("maxLevels", label(9)));
+            const unsigned int  min_coarse_rows(
                 controls.lookupOrDefault("minCoarseRows", label(10)));
             bool zeroGuess(controls.lookupOrDefault<Switch>("zeroGuess", true));
 
diff --git a/common/common.C b/common/common.C
index 61b355e0b..78e63b846 100644
--- a/common/common.C
+++ b/common/common.C
@@ -38,29 +38,39 @@ void export_vec(const word fieldName, const gko::matrix::Dense<scalar> *x,
     export_x(fn, x);
 }
 
-void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
-                const word local, const objectRegistry &db,
-                const word matrixFormat)
+template<typename Mtx>
+void export_mtx(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>&  As,
+                const objectRegistry &db
+                )
 {
     std::string folder{db.time().timePath()};
     std::filesystem::create_directories(folder);
 
-    std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
-    std::cout << "exporting " << fn << std::endl;
-    std::ofstream stream{fn};
-    stream << std::setprecision(15);
+    for (int i = 0; i < As.size(); i++) {
+        std::string fn{folder + "/" + fieldName + "_A_" + std::to_string(i) + ".mtx"};
+        std::cout << "exporting " << fn << std::endl;
+        std::ofstream stream{fn};
+        stream << std::setprecision(15);
 
-    if (matrixFormat == "Coo") {
-        gko::write(stream, (const gko::matrix::Coo<scalar> *)A.get());
-    }
-    if (matrixFormat == "Csr") {
-        gko::write(stream, (const gko::matrix::Csr<scalar> *)A.get());
-    }
-    if (matrixFormat == "Ell") {
-        gko::write(stream, (const gko::matrix::Ell<scalar> *)A.get());
+        gko::write(stream, gko::as<Mtx>(As[i]).get());
     }
 }
 
+template 
+void export_mtx<gko::matrix::Coo<scalar, label>>(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>&  As,
+                const objectRegistry &db
+                );
+
+template 
+void export_mtx<gko::matrix::Csr<scalar, label>>(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>&  As,
+                const objectRegistry &db
+                );
+template 
+void export_mtx<gko::matrix::Ell<scalar, label>>(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>&  As,
+                const objectRegistry &db
+                );
+
+
 void export_system(const word fieldName, const gko::matrix::Csr<scalar> *A,
                    const gko::matrix::Dense<scalar> *x,
                    const gko::matrix::Dense<scalar> *b, const word time)
diff --git a/common/common.H b/common/common.H
index ba4540719..62e6ebefd 100644
--- a/common/common.H
+++ b/common/common.H
@@ -103,9 +103,10 @@ void export_system(const word fieldName, const gko::matrix::Csr<scalar> *A,
 void export_vec(const word fieldName, const gko::matrix::Dense<scalar> *x,
                 const objectRegistry &db);
 
-void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
-                const word local, const objectRegistry &db,
-                const word matrixFormat);
+template<typename Mtx>
+void export_mtx(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>& As,
+                const objectRegistry &db
+                );
 
 void set_solve_prev_iters(const word sys_matrix_name, const objectRegistry &db,
                           label prev_solve_iters, const bool is_final);
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 71a9dba01..2c775c5dd 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -203,11 +203,9 @@ public:
         // TODO this is now the local matrix part for dist_A
         MatrixWrapper dist_A{db_,
                              this->get_exec_handler(),  // get exec
-                             this->get_local_col_idxs(),
-                             this->get_local_row_idxs(),
+                             this->get_local_sparsity_pattern(),
                              this->get_local_coeffs(),
-                             this->get_non_local_col_idxs(),
-                             this->get_non_local_row_idxs(),
+                             this->get_non_local_sparsity_pattern(),
                              this->get_non_local_coeffs(),
                              partition,
                              solver_controls_,
@@ -239,6 +237,9 @@ public:
         auto dist_b_v = dist_b.get_vector();
         auto dist_A_v = dist_A.get();
 
+        std::cout << " ldulduBase distributed_matrix get non_local " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_non_local_matrix()->get_size() << "\n";
+
+        std::cout << " ldulduBase distributed_matrix get local " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_local_matrix()->get_size() << "\n";
         scalar scaling =
             solver_controls_.lookupOrDefault<scalar>("scaling", 1.0);
         if (scaling != 1) {
@@ -251,6 +252,7 @@ public:
                 dist_b.get_vector()->scale(dense_scaling.get());)
         }
 
+        std::cout << " ldulduBase distributed_matrix get local 1 " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_local_matrix()->get_size() << "\n";
         TIME_WITH_FIELDNAME(
             verbose_, init_precond, this->fieldName(),
             auto precond = this->init_preconditioner(
@@ -265,6 +267,7 @@ public:
 
 
         LOG_1(verbose_, "create solver")
+        std::cout << " ldulduBase distributed_matrix get local 2 " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_local_matrix()->get_size() << "\n";
         auto solver_gen = this->create_dist_solver(
             this->get_exec_handler().get_device_exec(), dist_A_v, dist_x_v,
             dist_b_v, verbose_, dist_A.get_export(), precond);
@@ -272,7 +275,7 @@ public:
         TIME_WITH_FIELDNAME(verbose_, generate_solver, this->fieldName(),
                             auto solver = solver_gen->generate(dist_A_v);)
         LOG_1(verbose_, "solve solver done")
-        std::cout << "!!! call solver apply\n";
+        std::cout << " ldulduBase distributed_matrix get local 3 " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_local_matrix()->get_size() << "\n";
         TIME_WITH_FIELDNAME(verbose_, solve, this->fieldName(),
                             solver->apply(dist_b_v, dist_x_v);)
 

From 1b32e3e9b02091022d9899aebba3460890a0bb08 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 16:49:43 +0100
Subject: [PATCH 12/90] enhance workflow

---
 .github/workflows/build-foam.yml   | 17 +++---------
 .github/workflows/test_matrix.json | 18 +++++++++++--
 CMakePresets.json                  | 43 +++++++++++++++++-------------
 3 files changed, 43 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index 30585b861..16df67434 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -83,25 +83,14 @@ jobs:
       run: |
         source /root/OpenFOAM/${{matrix.OF.path}}/etc/bashrc
         mkdir -p ${{matrix.OF.foam_user_libbin}}
-        mkdir -p build
-        cd build
-        [ -d "/github/home/$GINKGO_CHECKOUT_VERSION" ] && cp -rp /github/home/$GINKGO_CHECKOUT_VERSION third_party
-        cmake -GNinja -DOGL_ALLOW_REFERENCE_ONLY=On -DOGL_BUILD_UNITTEST=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_INSTALL_PREFIX=${{matrix.OF.foam_user_libbin}} ..
+        # [ -d "/github/home/$GINKGO_CHECKOUT_VERSION" ] && cp -rp /github/home/$GINKGO_CHECKOUT_VERSION third_party
+        CC=${{matrix.OF.CC }} CXX=${{matrix.OF.CXX}} cmake --preset ninja-cpuonly-release -DOGL_BUILD_UNITTEST=ON 
 
     - name: Build OGL
       working-directory: ${{github.workspace}}/build
       run: |
         source /root/OpenFOAM/${{matrix.OF.path}}/etc/bashrc
-        cmake --build . --config $BUILD_TYPE
-
-    - name: Install OGL
-      working-directory: ${{github.workspace}}
-      run: |
-        source /root/OpenFOAM/${{matrix.OF.path}}/etc/bashrc
-        cmake --install ./build
-        ls $FOAM_USER_LIBBIN
-        ls ${{matrix.OF.foam_user_libbin}}
-
+        cmake --build --preset ninja-cpuonly-release --target install
 
     - name: Check folders
       working-directory: ${{github.workspace}}
diff --git a/.github/workflows/test_matrix.json b/.github/workflows/test_matrix.json
index 838863ca9..2fd67e3a7 100644
--- a/.github/workflows/test_matrix.json
+++ b/.github/workflows/test_matrix.json
@@ -6,7 +6,19 @@
    "path": "openfoam",
    "repo": "https://develop.openfoam.com/Development/openfoam.git",
    "cyclic_case": "periodicPlaneChannel" ,
-   "foam_user_libbin": "/github/home/OpenFOAM/user-v2312/platforms/linux64GccDPInt32Opt/lib/"
+   "foam_user_libbin": "/github/home/OpenFOAM/user-v2312/platforms/linux64GccDPInt32Opt/lib/",
+   "CC": "gcc",
+   "CXX": "g++"
+    },
+  {
+   "version": "v2212",
+   "fork": "ESI",
+   "path": "openfoam",
+   "repo": "https://develop.openfoam.com/Development/openfoam.git",
+   "cyclic_case": "periodicPlaneChannel" ,
+   "foam_user_libbin": "/github/home/OpenFOAM/user-v2312/platforms/linux64GccDPInt32Opt/lib/",
+   "CC": "clang",
+   "CXX": "clang++"
     },
   {
    "version": "10",
@@ -14,7 +26,9 @@
    "path": "OpenFOAM-10",
    "repo": "https://github.com/OpenFOAM/OpenFOAM-10.git",
    "cyclic_case": "channel395" ,
-   "foam_user_libbin": "/github/home/OpenFOAM/user-10/platforms/linux64GccDPInt32Opt/lib"
+   "foam_user_libbin": "/github/home/OpenFOAM/user-10/platforms/linux64GccDPInt32Opt/lib",
+   "CC": "gcc",
+   "CXX": "g++"
     }
 ]
 }
diff --git a/CMakePresets.json b/CMakePresets.json
index a0b5ef4f4..7e5aec5dd 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -8,10 +8,10 @@
   "configurePresets": [
     {
       "name": "default",
+      "hidden": true,
       "displayName": "Default Config",
       "description": "Default build using Ninja generator",
       "generator": "Ninja",
-      "binaryDir": "${sourceDir}/build/Release",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "CMAKE_INSTALL_PREFIX": "$env{FOAM_USER_LIBBIN}/.."
@@ -19,42 +19,47 @@
     },
     {
       "name": "debug",
+      "hidden": true,
       "displayName": "Default Config",
       "description": "Default build using Ninja generator",
       "generator": "Ninja",
-      "binaryDir": "${sourceDir}/build/Debug",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Debug",
         "CMAKE_INSTALL_PREFIX": "$env{FOAM_USER_LIBBIN}/.."
       }
     },
     {
-      "name": "ninja-cpuonly-debug",
-      "inherits": "debug",
-      "displayName": "No GPU, Ninja",
-      "description": "Release build without GPU support using Ninja Multi-Config generator",
-      "generator": "Ninja Multi-Config",
-      "binaryDir": "${sourceDir}/build/DebugCPUOnly",
+      "name": "ninja",
+      "hidden": true,
+      "displayName": "Default Config",
+      "description": "Default build using Ninja generator",
+      "generator": "Ninja"
+    },
+    {
+      "name": "cpuonly",
+      "hidden": true,
+      "displayName": "Default Config",
+      "description": "Default build using Ninja generator",
+      "generator": "Ninja",
       "cacheVariables": {
         "OGL_ALLOW_REFERENCE_ONLY": {
           "type": "BOOL",
           "value": "ON"
         }
-      }
+    },
+    {
+      "name": "ninja-cpuonly-debug",
+      "inherits": ["debug","cpuonly", "ninja"],
+      "displayName": "No GPU, Ninja",
+      "description": "Debug build without GPU support using Ninja Multi-Config generator",
+      "binaryDir": "${sourceDir}/build/DebugCPUOnly"
     },
     {
       "name": "ninja-cpuonly-release",
-      "inherits": "default",
+      "inherits": ["release","cpuonly", "ninja"],
       "displayName": "No GPU, Ninja",
-      "description": "Release build without GPU support using Ninja Multi-Config generator",
-      "generator": "Ninja Multi-Config",
-      "binaryDir": "${sourceDir}/build/ReleaseCPUOnly",
-      "cacheVariables": {
-        "OGL_ALLOW_REFERENCE_ONLY": {
-          "type": "BOOL",
-          "value": "ON"
-        }
-      }
+      "description": "Debug build without GPU support using Ninja Multi-Config generator",
+      "binaryDir": "${sourceDir}/build/ReleaseCPUOnly"
     },
     {
       "name": "ninja-cuda-release",

From a58082db0aac72a8f10318b1b4d57577a1c7c6a1 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 16:51:13 +0100
Subject: [PATCH 13/90] add compiler to workflow name

---
 .github/workflows/build-foam.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index 16df67434..3f5aff8d7 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     container: greole/ofbase
     needs: setup_build_matrix
-    name: build-${{matrix.OF.version}}
+    name: build-${{matrix.OF.version}}-${{matrix.OF.CXX}}
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.setup_build_matrix.outputs.matrix) }}

From f58fb3c6948fcb9a6dd74fc6cf99e731b1a48030 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 17:00:24 +0100
Subject: [PATCH 14/90] fix format

---
 CMakePresets.json | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/CMakePresets.json b/CMakePresets.json
index 7e5aec5dd..302965006 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -46,17 +46,26 @@
           "type": "BOOL",
           "value": "ON"
         }
+      }
     },
     {
       "name": "ninja-cpuonly-debug",
-      "inherits": ["debug","cpuonly", "ninja"],
+      "inherits": [
+        "debug",
+        "cpuonly",
+        "ninja"
+      ],
       "displayName": "No GPU, Ninja",
       "description": "Debug build without GPU support using Ninja Multi-Config generator",
       "binaryDir": "${sourceDir}/build/DebugCPUOnly"
     },
     {
       "name": "ninja-cpuonly-release",
-      "inherits": ["release","cpuonly", "ninja"],
+      "inherits": [
+        "release",
+        "cpuonly",
+        "ninja"
+      ],
       "displayName": "No GPU, Ninja",
       "description": "Debug build without GPU support using Ninja Multi-Config generator",
       "binaryDir": "${sourceDir}/build/ReleaseCPUOnly"
@@ -135,7 +144,7 @@
       "configuration": "Release"
     }
   ],
-    "workflowPresets": [
+  "workflowPresets": [
     {
       "name": "ninja-cpuonly-release",
       "steps": [
@@ -149,5 +158,5 @@
         }
       ]
     }
-    ]
+  ]
 }

From d03980a1789e87eec2e0057e9c95db19f083ebc0 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 17:29:46 +0100
Subject: [PATCH 15/90] change required version

---
 CMakePresets.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakePresets.json b/CMakePresets.json
index 302965006..e5ca1b548 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -1,5 +1,5 @@
 {
-  "version": 6,
+  "version": 3,
   "cmakeMinimumRequired": {
     "major": 3,
     "minor": 20,

From ae27b657d168adc481008f9e66dc8c33bdeb1a58 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 17:51:47 +0100
Subject: [PATCH 16/90] fix required version

---
 CMakePresets.json | 4 ++--
 README.md         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CMakePresets.json b/CMakePresets.json
index e5ca1b548..42de93f5e 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -1,8 +1,8 @@
 {
-  "version": 3,
+  "version": 6,
   "cmakeMinimumRequired": {
     "major": 3,
-    "minor": 20,
+    "minor": 25,
     "patch": 0
   },
   "configurePresets": [
diff --git a/README.md b/README.md
index 7efc5eb51..bfcac463e 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ OGL is a wrapper for [ginkgo](https://github.com/ginkgo-project/ginkgo) solvers
 
 OGL has the following requirements
 
-*   _cmake 3.13+_
+*   _cmake 3.13+_ or 3.25+ for CMakePresets
 *   _OpenFOAM 6+_ or _v2106_
 *   _Ginkgo 1.5.0+_ (It is recommended to install via OGL)
 *   C++17 compliant compiler (gcc or clang)
@@ -50,7 +50,7 @@ Then, compile and install by
 
 ### CMakePresets and Ninja builds
 
-If you have Ninja installed on your system we recommend to use ninja over gnu make for better compilation times. We also provide a list of Cmake presets which can be used a recent version of Cmake (>3.20). To display available presets use: 
+If you have a recent cmake version (3.25+) and  Ninja installed on your system we recommend to use ninja over gnu make for better compilation times. We also provide a list of Cmake presets which can be used a recent version of Cmake (>3.20). To display available presets use: 
 
     cmake --list-preset
     

From 364a990421594bd11c650125d1d16d06b399467a Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 18:27:41 +0100
Subject: [PATCH 17/90] format

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       |  6 +-
 MatrixWrapper/HostMatrix/HostMatrix.C         |  2 +-
 MatrixWrapper/HostMatrix/HostMatrix.H         | 78 ++++++++++---------
 Preconditioner/Preconditioner.H               |  8 +-
 common/common.C                               | 34 ++++----
 common/common.H                               |  8 +-
 lduLduBase/lduLduBase.H                       | 37 +++++++--
 7 files changed, 100 insertions(+), 73 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index f567823ed..f9bff46f4 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -342,7 +342,8 @@ public:
                     dist_matrix->get_non_local_matrix())
                     ->get_combination()
                     ->get_operators();
-            export_mtx<Coo>(word(field_name_ + "_local"), local_interfaces, db_);
+            export_mtx<Coo>(word(field_name_ + "_local"), local_interfaces,
+                            db_);
             export_mtx<Coo>(field_name_ + "_non_local", non_local_interfaces,
                             db_);
         }
@@ -359,7 +360,8 @@ public:
                     dist_matrix->get_non_local_matrix().get())
                     ->get_combination()
                     ->get_operators();
-            export_mtx<Csr>(word(field_name_ + "_local"), local_interfaces, db_);
+            export_mtx<Csr>(word(field_name_ + "_local"), local_interfaces,
+                            db_);
             export_mtx<Csr>(field_name_ + "_non_local", non_local_interfaces,
                             db_);
         }
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
index 585728b6d..c95c64bfd 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -725,7 +725,7 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     std::cout << "!!! set dim_ " << nrows_ << " " << local_matrix_nnz_ << "\n";
     // NOTE interface_spans and dim are not persistent so we need to
     // recreate this for every solver call
-    // TODO this needs a propper implementation once we know how to handle interfaces
+    // TODO this needs a proper implementation once we know how to handle interfaces
     local_sparsity_.interface_spans_.emplace_back(0, local_matrix_nnz_);
     local_sparsity_.dim_ = gko::dim<2>{nrows_, nrows_};
 
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.H b/MatrixWrapper/HostMatrix/HostMatrix.H
index 26b3a598a..46cd1bc46 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.H
+++ b/MatrixWrapper/HostMatrix/HostMatrix.H
@@ -40,42 +40,38 @@ namespace Foam {
 
 
 struct PersistentSparsityPattern {
-
-    PersistentSparsityPattern(
-      const word &fieldName,
-      const objectRegistry &db,
-      const ExecutorHandler &exec,
-      const label size,
-      const label verbose):
-    size_(size),
-    col_idxs_{
-        fieldName + "_cols",
-        db,
-        exec,
-        size,
-        verbose,
-        false,  // For now we assume columns and rows to be constant
-        false   // we don't init on device since the indices are written using the pointer
-                // to the data
-    },
-    row_idxs_{
-        fieldName + "_rows",
-        db,
-        exec,
-        size,
-        verbose,
-        false,  // For now we assume columns and rows to be constant
-        false   // same as for cols don't init on device
-    },
-    ldu_mapping_{
-        fieldName + "_ldu_map",
-        db,
-        exec,
-        size,
-        verbose,
-        false,
-        false // same as for rows and cols dont init on device
-    }
+    PersistentSparsityPattern(const word &fieldName, const objectRegistry &db,
+                              const ExecutorHandler &exec, const label size,
+                              const label verbose)
+        : size_(size),
+          col_idxs_{
+              fieldName + "_cols",
+              db,
+              exec,
+              size,
+              verbose,
+              false,  // For now we assume columns and rows to be constant
+              false   // we don't init on device since the indices are written
+                     // using the pointer to the data
+          },
+          row_idxs_{
+              fieldName + "_rows",
+              db,
+              exec,
+              size,
+              verbose,
+              false,  // For now we assume columns and rows to be constant
+              false   // same as for cols don't init on device
+          },
+          ldu_mapping_{
+              fieldName + "_ldu_map",
+              db,
+              exec,
+              size,
+              verbose,
+              false,
+              false  // same as for rows and cols dont init on device
+          }
     {}
 
     // number of nnz entries
@@ -443,7 +439,10 @@ public:
 
     const ExecutorHandler &get_exec_handler() const { return exec_; }
 
-    PersistentSparsityPattern &get_local_sparsity_pattern() const { return local_sparsity_; };
+    PersistentSparsityPattern &get_local_sparsity_pattern() const
+    {
+        return local_sparsity_;
+    };
 
     PersistentArray<scalar> &get_local_coeffs() const { return local_coeffs_; };
 
@@ -457,7 +456,10 @@ public:
         return local_sparsity_.row_idxs_;
     };
 
-    PersistentSparsityPattern &get_non_local_sparsity_pattern() const { return non_local_sparsity_; };
+    PersistentSparsityPattern &get_non_local_sparsity_pattern() const
+    {
+        return non_local_sparsity_;
+    };
 
     PersistentArray<scalar> &get_non_local_coeffs() const
     {
diff --git a/Preconditioner/Preconditioner.H b/Preconditioner/Preconditioner.H
index 88e046adf..5c3820e1e 100644
--- a/Preconditioner/Preconditioner.H
+++ b/Preconditioner/Preconditioner.H
@@ -91,8 +91,7 @@ public:
         if (name == "BJ") {
             // TODO for non constant system matrix reuse block pointers
             const unsigned int max_block_size(
-                controls.lookupOrDefault("maxBlockSize", label(1))
-                    );
+                controls.lookupOrDefault("maxBlockSize", label(1)));
 
             word msg = "Generate preconditioner " + name + " MaxBlockSize " +
                        std::to_string(max_block_size);
@@ -312,8 +311,9 @@ public:
             if (cycle_name == "f") cycle = gko::solver::multigrid::cycle::f;
 
             // Create multigrid factory
-            const unsigned int max_levels(controls.lookupOrDefault("maxLevels", label(9)));
-            const unsigned int  min_coarse_rows(
+            const unsigned int max_levels(
+                controls.lookupOrDefault("maxLevels", label(9)));
+            const unsigned int min_coarse_rows(
                 controls.lookupOrDefault("minCoarseRows", label(10)));
             bool zeroGuess(controls.lookupOrDefault<Switch>("zeroGuess", true));
 
diff --git a/common/common.C b/common/common.C
index 78e63b846..e0234043d 100644
--- a/common/common.C
+++ b/common/common.C
@@ -38,16 +38,17 @@ void export_vec(const word fieldName, const gko::matrix::Dense<scalar> *x,
     export_x(fn, x);
 }
 
-template<typename Mtx>
-void export_mtx(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>&  As,
-                const objectRegistry &db
-                )
+template <typename Mtx>
+void export_mtx(const word fieldName,
+                std::vector<std::shared_ptr<const gko::LinOp>> &As,
+                const objectRegistry &db)
 {
     std::string folder{db.time().timePath()};
     std::filesystem::create_directories(folder);
 
     for (int i = 0; i < As.size(); i++) {
-        std::string fn{folder + "/" + fieldName + "_A_" + std::to_string(i) + ".mtx"};
+        std::string fn{folder + "/" + fieldName + "_A_" + std::to_string(i) +
+                       ".mtx"};
         std::cout << "exporting " << fn << std::endl;
         std::ofstream stream{fn};
         stream << std::setprecision(15);
@@ -56,19 +57,16 @@ void export_mtx(const word fieldName, std::vector<std::shared_ptr<const gko::Lin
     }
 }
 
-template 
-void export_mtx<gko::matrix::Coo<scalar, label>>(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>&  As,
-                const objectRegistry &db
-                );
-
-template 
-void export_mtx<gko::matrix::Csr<scalar, label>>(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>&  As,
-                const objectRegistry &db
-                );
-template 
-void export_mtx<gko::matrix::Ell<scalar, label>>(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>&  As,
-                const objectRegistry &db
-                );
+template void export_mtx<gko::matrix::Coo<scalar, label>>(
+    const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>> &As,
+    const objectRegistry &db);
+
+template void export_mtx<gko::matrix::Csr<scalar, label>>(
+    const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>> &As,
+    const objectRegistry &db);
+template void export_mtx<gko::matrix::Ell<scalar, label>>(
+    const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>> &As,
+    const objectRegistry &db);
 
 
 void export_system(const word fieldName, const gko::matrix::Csr<scalar> *A,
diff --git a/common/common.H b/common/common.H
index 62e6ebefd..f0cf66a57 100644
--- a/common/common.H
+++ b/common/common.H
@@ -103,10 +103,10 @@ void export_system(const word fieldName, const gko::matrix::Csr<scalar> *A,
 void export_vec(const word fieldName, const gko::matrix::Dense<scalar> *x,
                 const objectRegistry &db);
 
-template<typename Mtx>
-void export_mtx(const word fieldName, std::vector<std::shared_ptr<const gko::LinOp>>& As,
-                const objectRegistry &db
-                );
+template <typename Mtx>
+void export_mtx(const word fieldName,
+                std::vector<std::shared_ptr<const gko::LinOp>> &As,
+                const objectRegistry &db);
 
 void set_solve_prev_iters(const word sys_matrix_name, const objectRegistry &db,
                           label prev_solve_iters, const bool is_final);
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 2c775c5dd..1b442ca6b 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -237,9 +237,19 @@ public:
         auto dist_b_v = dist_b.get_vector();
         auto dist_A_v = dist_A.get();
 
-        std::cout << " ldulduBase distributed_matrix get non_local " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_non_local_matrix()->get_size() << "\n";
-
-        std::cout << " ldulduBase distributed_matrix get local " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_local_matrix()->get_size() << "\n";
+        std::cout << " ldulduBase distributed_matrix get non_local "
+                  << gko::as<gko::experimental::distributed::Matrix<
+                         scalar, label, label>>(dist_A_v)
+                         ->get_non_local_matrix()
+                         ->get_size()
+                  << "\n";
+
+        std::cout << " ldulduBase distributed_matrix get local "
+                  << gko::as<gko::experimental::distributed::Matrix<
+                         scalar, label, label>>(dist_A_v)
+                         ->get_local_matrix()
+                         ->get_size()
+                  << "\n";
         scalar scaling =
             solver_controls_.lookupOrDefault<scalar>("scaling", 1.0);
         if (scaling != 1) {
@@ -252,7 +262,12 @@ public:
                 dist_b.get_vector()->scale(dense_scaling.get());)
         }
 
-        std::cout << " ldulduBase distributed_matrix get local 1 " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_local_matrix()->get_size() << "\n";
+        std::cout << " ldulduBase distributed_matrix get local 1 "
+                  << gko::as<gko::experimental::distributed::Matrix<
+                         scalar, label, label>>(dist_A_v)
+                         ->get_local_matrix()
+                         ->get_size()
+                  << "\n";
         TIME_WITH_FIELDNAME(
             verbose_, init_precond, this->fieldName(),
             auto precond = this->init_preconditioner(
@@ -267,7 +282,12 @@ public:
 
 
         LOG_1(verbose_, "create solver")
-        std::cout << " ldulduBase distributed_matrix get local 2 " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_local_matrix()->get_size() << "\n";
+        std::cout << " ldulduBase distributed_matrix get local 2 "
+                  << gko::as<gko::experimental::distributed::Matrix<
+                         scalar, label, label>>(dist_A_v)
+                         ->get_local_matrix()
+                         ->get_size()
+                  << "\n";
         auto solver_gen = this->create_dist_solver(
             this->get_exec_handler().get_device_exec(), dist_A_v, dist_x_v,
             dist_b_v, verbose_, dist_A.get_export(), precond);
@@ -275,7 +295,12 @@ public:
         TIME_WITH_FIELDNAME(verbose_, generate_solver, this->fieldName(),
                             auto solver = solver_gen->generate(dist_A_v);)
         LOG_1(verbose_, "solve solver done")
-        std::cout << " ldulduBase distributed_matrix get local 3 " << gko::as<gko::experimental::distributed::Matrix<scalar, label, label>>(dist_A_v)->get_local_matrix()->get_size() << "\n";
+        std::cout << " ldulduBase distributed_matrix get local 3 "
+                  << gko::as<gko::experimental::distributed::Matrix<
+                         scalar, label, label>>(dist_A_v)
+                         ->get_local_matrix()
+                         ->get_size()
+                  << "\n";
         TIME_WITH_FIELDNAME(verbose_, solve, this->fieldName(),
                             solver->apply(dist_b_v, dist_x_v);)
 

From 36022304fb735f8e364122a32f56927746282346 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 20:55:46 +0100
Subject: [PATCH 18/90] fixup! Presets

---
 CMakePresets.json | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/CMakePresets.json b/CMakePresets.json
index 42de93f5e..f46b6bc2c 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -50,22 +50,14 @@
     },
     {
       "name": "ninja-cpuonly-debug",
-      "inherits": [
-        "debug",
-        "cpuonly",
-        "ninja"
-      ],
       "displayName": "No GPU, Ninja",
+      "inherits": ["debug", "ninja", "cpuonly"],
       "description": "Debug build without GPU support using Ninja Multi-Config generator",
       "binaryDir": "${sourceDir}/build/DebugCPUOnly"
     },
     {
       "name": "ninja-cpuonly-release",
-      "inherits": [
-        "release",
-        "cpuonly",
-        "ninja"
-      ],
+      "inherits": ["default", "ninja", "cpuonly"],
       "displayName": "No GPU, Ninja",
       "description": "Debug build without GPU support using Ninja Multi-Config generator",
       "binaryDir": "${sourceDir}/build/ReleaseCPUOnly"
@@ -125,7 +117,7 @@
     },
     {
       "name": "ninja-cpuonly-debug",
-      "configurePreset": "ninja-cpuonly-release",
+      "configurePreset": "ninja-cpuonly-debug",
       "configuration": "Release"
     },
     {

From 3c42567e1f3bbde72e4e7cd6db67432f0d89f57d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 21:32:45 +0100
Subject: [PATCH 19/90] fixup! Presets

---
 .github/workflows/build-foam.yml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index 3f5aff8d7..a08914af0 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -76,7 +76,7 @@ jobs:
         path: |
           ${{matrix.OF.foam_user_libbin}}
 
-    - name: Configure CMake
+    - name: Configure and build
       # Use a bash shell so we can use the same syntax for environment variable
       # access regardless of the host operating system
       working-directory: ${{github.workspace}}
@@ -85,13 +85,9 @@ jobs:
         mkdir -p ${{matrix.OF.foam_user_libbin}}
         # [ -d "/github/home/$GINKGO_CHECKOUT_VERSION" ] && cp -rp /github/home/$GINKGO_CHECKOUT_VERSION third_party
         CC=${{matrix.OF.CC }} CXX=${{matrix.OF.CXX}} cmake --preset ninja-cpuonly-release -DOGL_BUILD_UNITTEST=ON 
-
-    - name: Build OGL
-      working-directory: ${{github.workspace}}/build
-      run: |
-        source /root/OpenFOAM/${{matrix.OF.path}}/etc/bashrc
         cmake --build --preset ninja-cpuonly-release --target install
 
+
     - name: Check folders
       working-directory: ${{github.workspace}}
       run: |

From fabe2c11c7b6c5a66a85e42d38a8863563907e28 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 17 Mar 2024 22:04:58 +0100
Subject: [PATCH 20/90] add sanitizer

---
 CMakeLists.txt        |  19 +++++++-
 cmake/Sanitizer.cmake | 103 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 cmake/Sanitizer.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e78a9c251..5da376a95 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,8 +32,15 @@ option(OGL_CUDA_ARCHITECTURES "Which cuda architecture to build for" "native")
 option(GINKGO_BUILD_REFERENCE "Build Ginkgo with reference backend" TRUE)
 option(OGL_ALLOW_REFERENCE_ONLY
        "Enable builds of Ginkgo with reference backend only" FALSE)
+option(OGL_ENABLE_SANITIZE_ADDRESS "Enable address sanitizer" FALSE)
+option(OGL_ENABLE_SANITIZE_LEAK "Enable leak sanitizer" FALSE)
+option(OGL_ENABLE_SANITIZE_UB "Enable undefined behaviour sanitizer" FALSE)
+option(OGL_ENABLE_SANITIZE_THREAD "Enable thread sanitizer" FALSE)
+option(OGL_ENABLE_SANITIZE_MEMORY "Enable memory sanitizer" FALSE)
+
 option(GINKGO_WITH_OGL_EXTENSIONS "Whether ginkgo was build with OGL extension"
        FALSE)
+option(GINKGO_BUILD_REFERENCE "Build Ginkgo with reference backend" TRUE)
 
 option(GINKGO_FORCE_GPU_AWARE_MPI "Build Ginkgo using device aware MPI" TRUE)
 option(GINKGO_BUILD_CUDA "Build Ginkgo with cuda backend" FALSE)
@@ -67,6 +74,8 @@ check_include_file_cxx(cxxabi.h GKO_HAVE_CXXABI_H)
 include(cmake/package_helpers.cmake)
 include(cmake/build_helpers.cmake)
 include(cmake/install_helpers.cmake)
+include(cmake/Sanitizer.cmake)
+
 
 ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.7.0)
 
@@ -187,7 +196,15 @@ target_sources(
   # LduMatrix / GKOACG / GKOACG.H
 )
 
-target_link_options(OGL PUBLIC "-fsanitize=address")
+
+enable_sanitizers(
+    OGL
+    ${OGL_ENABLE_SANITIZE_ADDRESS}
+    ${OGL_ENABLE_SANITIZE_LEAK}
+    ${OGL_ENABLE_SANITIZE_UB}
+    ${OGL_ENABLE_SANITIZE_THREAD}
+    ${OGL_ENABLE_SANITIZE_MEMORY}
+)
 
 target_include_directories(
   OGL SYSTEM
diff --git a/cmake/Sanitizer.cmake b/cmake/Sanitizer.cmake
new file mode 100644
index 000000000..c79772ffa
--- /dev/null
+++ b/cmake/Sanitizer.cmake
@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Unlicense
+# SPDX-FileCopyrightText: 2023 Jason Turner
+# SPDX-FileCopyrightText: 2023 NeoFOAM authors
+##############################################################################
+# This function will enable sanitizers                                       #
+# from here                                                                  #
+# https://github.com/cpp-best-practices/cmake_template                       #
+##############################################################################
+
+function(
+  enable_sanitizers
+  project_name
+  ENABLE_SANITIZE_ADDRESS
+  ENABLE_SANITIZE_LEAK
+  ENABLE_SANITIZE_UNDEFINED_BEHAVIOR
+  ENABLE_SANITIZE_THREAD
+  ENABLE_SANITIZE_MEMORY)
+
+#  message(FATAL_ERROR "add addr ${ENABLE_NEOFOAM_SANITIZE_ADDRESS}")
+
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+    set(SANITIZERS "")
+
+    if(${ENABLE_SANITIZE_ADDRESS})
+      list(APPEND SANITIZERS "address")
+    endif()
+
+    if(${ENABLE_SANITIZE_LEAK})
+      message(FATAL_ERROR "add leak")
+      list(APPEND SANITIZERS "leak")
+    endif()
+
+    if(${ENABLE_SANITIZE_UNDEFINED_BEHAVIOR})
+      list(APPEND SANITIZERS "undefined")
+    endif()
+
+    if(${ENABLE_SANITIZE_THREAD})
+      if("address" IN_LIST SANITIZERS OR "leak" IN_LIST SANITIZERS)
+        message(WARNING "Thread sanitizer does not work with Address and Leak sanitizer enabled")
+      else()
+        list(APPEND SANITIZERS "thread")
+      endif()
+    endif()
+
+    if(${ENABLE_SANITIZE_MEMORY} AND CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+      message(
+        WARNING
+          "Memory sanitizer requires all the code (including libc++) to be MSan-instrumented otherwise it reports false positives"
+      )
+      if("address" IN_LIST SANITIZERS
+         OR "thread" IN_LIST SANITIZERS
+         OR "leak" IN_LIST SANITIZERS)
+ # message(FATAL_ERROR "Memory sanitizer does not work with Address, Thread or Leak sanitizer enabled")
+      else()
+        list(APPEND SANITIZERS "memory")
+      endif()
+    endif()
+  elseif(MSVC)
+    if(${ENABLE_SANITIZE_ADDRESS})
+      list(APPEND SANITIZERS "address")
+    endif()
+    if(${ENABLE_SANITIZE_LEAK}
+       OR ${ENABLE_SANITIZE_UNDEFINED_BEHAVIOR}
+       OR ${ENABLE_SANITIZE_THREAD}
+       OR ${ENABLE_SANITIZE_MEMORY})
+      message(WARNING "MSVC only supports address sanitizer")
+    endif()
+  endif()
+
+  list(
+    JOIN
+    SANITIZERS
+    ","
+    LIST_OF_SANITIZERS)
+
+  if(LIST_OF_SANITIZERS)
+    if(NOT
+       "${LIST_OF_SANITIZERS}"
+       STREQUAL
+       "")
+      if(NOT MSVC)
+        # message(FATAL_ERROR "${project_name} ${LIST_OF_SANITIZERS}")
+        target_compile_options(${project_name} PRIVATE -fsanitize=${LIST_OF_SANITIZERS})
+        target_link_options(${project_name} PRIVATE -fsanitize=${LIST_OF_SANITIZERS})
+      else()
+        string(FIND "$ENV{PATH}" "$ENV{VSINSTALLDIR}" index_of_vs_install_dir)
+        if("${index_of_vs_install_dir}" STREQUAL "-1")
+          message(
+            SEND_ERROR
+              "Using MSVC sanitizers requires setting the MSVC environment before building the project. Please manually open the MSVC command prompt and rebuild the project."
+          )
+        endif()
+        target_compile_options(${project_name} INTERFACE /fsanitize=${LIST_OF_SANITIZERS} /Zi /INCREMENTAL:NO)
+        target_compile_definitions(${project_name} INTERFACE _DISABLE_VECTOR_ANNOTATION _DISABLE_STRING_ANNOTATION)
+        target_link_options(${project_name} INTERFACE /INCREMENTAL:NO)
+      endif()
+    endif()
+  endif()
+
+endfunction()
+
+
+

From 8cbdbfb20bc0534770f6543529f4c9617756920a Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 18 Mar 2024 16:44:14 +0100
Subject: [PATCH 21/90] fix const correctness, formating, pointer issue

---
 BaseWrapper/lduBase/GKOlduBase.H              |  33 ++---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 113 +++++++-----------
 .../GkoCombinationMatrix.H                    |   4 -
 MatrixWrapper/HostMatrix/HostMatrix.C         |   7 +-
 Preconditioner/Preconditioner.H               |   7 +-
 Solver/CG/GKOCG.H                             |   1 -
 StoppingCriterion/StoppingCriterion.H         |  17 +--
 lduLduBase/lduLduBase.H                       |  31 -----
 8 files changed, 74 insertions(+), 139 deletions(-)

diff --git a/BaseWrapper/lduBase/GKOlduBase.H b/BaseWrapper/lduBase/GKOlduBase.H
index 3bb48f573..f6a0f178f 100644
--- a/BaseWrapper/lduBase/GKOlduBase.H
+++ b/BaseWrapper/lduBase/GKOlduBase.H
@@ -20,22 +20,23 @@ namespace Foam {
                            Class GKOlduBaseSolver Declaration
 \*---------------------------------------------------------------------------*/
 
-#define CREATE_SOLVER_METHODS(Solver)                                       \
-    std::unique_ptr<Solver::Factory, std::default_delete<Solver::Factory>>  \
-    create_dist_solver(                                                     \
-        std::shared_ptr<gko::Executor> exec,                                \
-        std::shared_ptr<gko::LinOp> gkomatrix, std::shared_ptr<dist_vec> x, \
-        std::shared_ptr<dist_vec> b, const label verbose,                   \
-        const bool export_res, std::shared_ptr<gko::LinOp> precond) const   \
-    {                                                                       \
-        stoppingCriterionVec_.push_back(                                    \
-            stoppingCriterion_.build_dist_stopping_criterion(               \
-                exec, gkomatrix, x, b, verbose, export_res,                 \
-                get_prev_number_of_iterations(),                            \
-                get_solve_prev_rel_res_cost()));                            \
-                                                                            \
-        if (precond != NULL) return create_precond(exec, precond);          \
-        return create_default(exec);                                        \
+#define CREATE_SOLVER_METHODS(Solver)                                          \
+    std::unique_ptr<Solver::Factory, std::default_delete<Solver::Factory>>     \
+    create_dist_solver(std::shared_ptr<gko::Executor> exec,                    \
+                       std::shared_ptr<const gko::LinOp> gkomatrix,            \
+                       std::shared_ptr<const dist_vec> x,                      \
+                       std::shared_ptr<const dist_vec> b, const label verbose, \
+                       const bool export_res,                                  \
+                       std::shared_ptr<gko::LinOp> precond) const              \
+    {                                                                          \
+        stoppingCriterionVec_.push_back(                                       \
+            stoppingCriterion_.build_dist_stopping_criterion(                  \
+                exec, gkomatrix, x, b, verbose, export_res,                    \
+                get_prev_number_of_iterations(),                               \
+                get_solve_prev_rel_res_cost()));                               \
+                                                                               \
+        if (precond != NULL) return create_precond(exec, precond);             \
+        return create_default(exec);                                           \
     };
 
 
diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index f9bff46f4..ce0bfc89f 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -15,41 +15,36 @@
 
 namespace Foam {
 
-
 template <typename InnerMatrixType>
 void perform_update(
-    const std::shared_ptr<gko::Executor> &exec,
-    std::shared_ptr<CombinationMatrix<scalar, label, InnerMatrixType>>
-        combination_matrix,
+    std::shared_ptr<const gko::Executor> exec,
+    CombinationMatrix<scalar, label, InnerMatrixType> *combination_matrix,
     const PersistentSparsityPattern &sparsity,
-    std::shared_ptr<gko::array<scalar>> coeffs)
+    std::shared_ptr<const gko::array<scalar>> coeffs)
 {
     int interface_ctr{0};
-    for (auto &interface :
+    for (auto interface :
          combination_matrix->get_combination()->get_operators()) {
+        auto span = sparsity.interface_spans_[interface_ctr];
         auto device_value_ptr =
-            gko::as<InnerMatrixType>(interface)->get_const_values();
+            ((InnerMatrixType *)(interface.get()))->get_values();
 
         // create a view into data on the device which needs an
         // update
-        auto num_elems =
-            gko::as<InnerMatrixType>(interface)->get_num_stored_elements();
-        auto device_value_view = val_array::view(
-            exec, num_elems, const_cast<scalar *>(device_value_ptr));
+        auto device_value_view =
+            val_array::view(exec, span.length(), device_value_ptr);
 
         // create a view into updated values and copy over to
         // persistent data
-        auto span = sparsity.interface_spans_[interface_ctr];
-        auto update_view = val_array::view(exec, span.length(),
-                                           coeffs->get_data() + span.begin);
+        auto update_view = val_array::const_view(
+            exec, span.length(), coeffs->get_const_data() + span.begin);
 
-        std::cout << "perform update " << span.length() << " " << span.begin
-                  << " " << num_elems << " " << interface->get_size() << "\n";
         interface_ctr++;
         device_value_view = update_view;
     }
 };
 
+
 struct MatrixInitFunctor {
     using dist_mtx =
         gko::experimental::distributed::Matrix<scalar, label, label>;
@@ -110,68 +105,46 @@ struct MatrixInitFunctor {
 
         // The local and non-local matrix are a CombinationMatrix and we need to
         // update the interfaces iteratively
-        std::shared_ptr<gko::LinOp> local_combination_matrix{
-            const_cast<gko::LinOp *>(
-                persistent_device_matrix->get_local_matrix().get())};
-        std::shared_ptr<gko::LinOp> non_local_combination_matrix{
-            const_cast<gko::LinOp *>(
-                persistent_device_matrix->get_non_local_matrix().get())};
+        std::shared_ptr<const gko::LinOp> local_combination_matrix{
+            persistent_device_matrix->get_local_matrix()};
+        std::shared_ptr<const gko::LinOp> non_local_combination_matrix{
+            persistent_device_matrix->get_non_local_matrix()};
         if (matrix_format_ == "Coo") {
             using Coo = gko::matrix::Coo<scalar, label>;
             perform_update<Coo>(exec_.get_device_exec(),
-                                gko::as<CombinationMatrix<scalar, label, Coo>>(
-                                    local_combination_matrix),
+                                (CombinationMatrix<scalar, label, Coo>
+                                     *)(local_combination_matrix.get()),
                                 local_sparsity_, coeffs_.get_array());
             perform_update<Coo>(exec_.get_device_exec(),
-                                gko::as<CombinationMatrix<scalar, label, Coo>>(
-                                    non_local_combination_matrix),
+                                (CombinationMatrix<scalar, label, Coo>
+                                     *)(non_local_combination_matrix.get()),
+                                non_local_sparsity_,
+                                non_local_coeffs_.get_array());
+        }
+        if (matrix_format_ == "Csr") {
+            using Csr = gko::matrix::Csr<scalar, label>;
+            perform_update<Csr>(exec_.get_device_exec(),
+                                (CombinationMatrix<scalar, label, Csr>
+                                     *)(local_combination_matrix.get()),
+                                local_sparsity_, coeffs_.get_array());
+            perform_update<Csr>(exec_.get_device_exec(),
+                                (CombinationMatrix<scalar, label, Csr>
+                                     *)(non_local_combination_matrix.get()),
+                                non_local_sparsity_,
+                                non_local_coeffs_.get_array());
+        }
+        if (matrix_format_ == "Ell") {
+            using Ell = gko::matrix::Ell<scalar, label>;
+            perform_update<Ell>(exec_.get_device_exec(),
+                                (CombinationMatrix<scalar, label, Ell>
+                                     *)(local_combination_matrix.get()),
+                                local_sparsity_, coeffs_.get_array());
+            perform_update<Ell>(exec_.get_device_exec(),
+                                (CombinationMatrix<scalar, label, Ell>
+                                     *)(non_local_combination_matrix.get()),
                                 non_local_sparsity_,
                                 non_local_coeffs_.get_array());
-            std::cout << " local_combination_matrix "
-                      << local_combination_matrix->get_size() << "\n";
-            std::cout << " non_local_combination_matrix "
-                      << non_local_combination_matrix->get_size() << "\n";
-            std::cout << " CsrMatrixWrapper distributed_matrix "
-                      << persistent_device_matrix.get()->get_size() << "\n";
-            std::cout << " CsrMatrixWrapper distributed_matrix get_local "
-                      << persistent_device_matrix.get()
-                             ->get_local_matrix()
-                             ->get_size()
-                      << "\n";
-            std::cout << " CsrMatrixWrapper distributed_matrix get non_local "
-                      << persistent_device_matrix.get()
-                             ->get_non_local_matrix()
-                             ->get_size()
-                      << "\n";
         }
-        // if (matrix_format_ == "Csr") {
-        //     using Csr = gko::matrix::Csr<scalar, label>;
-        //     perform_update<Csr>(exec_.get_device_exec(),
-        //                         gko::as<CombinationMatrix<scalar, label,
-        //                         Csr>>(
-        //                             local_combination_matrix),
-        //                         local_sparsity_, coeffs_.get_array());
-        //     perform_update<Csr>(exec_.get_device_exec(),
-        //                         gko::as<CombinationMatrix<scalar, label,
-        //                         Csr>>(
-        //                             non_local_combination_matrix),
-        //                         non_local_sparsity_,
-        //                         non_local_coeffs_.get_array());
-        // }
-        // if (matrix_format_ == "Ell") {
-        //     using Ell = gko::matrix::Ell<scalar, label>;
-        //     perform_update<Ell>(exec_.get_device_exec(),
-        //                         gko::as<CombinationMatrix<scalar, label,
-        //                         Ell>>(
-        //                             local_combination_matrix),
-        //                         local_sparsity_, coeffs_.get_array());
-        //     perform_update<Ell>(exec_.get_device_exec(),
-        //                         gko::as<CombinationMatrix<scalar, label,
-        //                         Ell>>(
-        //                             non_local_combination_matrix),
-        //                         non_local_sparsity_,
-        //                         non_local_coeffs_.get_array());
-        // }
     }
 
     std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
diff --git a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
index e4c8f6b95..276af09d2 100644
--- a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
+++ b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
@@ -59,10 +59,6 @@ public:
                         .copy_to_array()));
 
             op->read(std::move(in));
-            std::cout << "!!! gkocombinationmatrix length " << length
-                      << " offset " << offset << " op->get_size() "
-                      << op->get_size() << "\n";
-
             this->comb_->add_operators(
                 gko::initialize<gko::matrix::Dense<ValueType>>({1}, exec), op);
         }
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
index c95c64bfd..f1b56ff2d 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -481,7 +481,6 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
         permute[element_ctr] = interface_idx;
         element_ctr += 1;
     }
-
 }
 
 template <class MatrixType>
@@ -722,13 +721,12 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
         dense_vec->copy_from(row_collection);
     }
 
-    std::cout << "!!! set dim_ " << nrows_ << " " << local_matrix_nnz_ << "\n";
     // NOTE interface_spans and dim are not persistent so we need to
     // recreate this for every solver call
-    // TODO this needs a proper implementation once we know how to handle interfaces
+    // TODO this needs a proper implementation once we know how to handle
+    // interfaces
     local_sparsity_.interface_spans_.emplace_back(0, local_matrix_nnz_);
     local_sparsity_.dim_ = gko::dim<2>{nrows_, nrows_};
-
 }
 
 
@@ -757,7 +755,6 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
         ref_exec, non_local_matrix_nnz_, non_local_coeffs_.get_data());
     i_device_view = tmp_contiguous_iface;
 
-    std::cout << "!!! set dim_ " << nrows_ << " " << non_local_matrix_nnz_ << "\n";
     // NOTE interface_spans and dim are not persistent so we need to
     // We have now the full non_local matrix and need to add the start and end
     // to the PersistentSparsityPattern data structure
diff --git a/Preconditioner/Preconditioner.H b/Preconditioner/Preconditioner.H
index 5c3820e1e..350382d4f 100644
--- a/Preconditioner/Preconditioner.H
+++ b/Preconditioner/Preconditioner.H
@@ -11,7 +11,6 @@
 
 namespace Foam {
 class Preconditioner {
-    using mtx = gko::matrix::Csr<scalar>;
     using bj = gko::preconditioner::Jacobi<>;
     using ic = gko::preconditioner::Ic<>;
     using ir = gko::solver::Ir<scalar>;
@@ -46,7 +45,7 @@ public:
 
     template <typename PrecondFactory>
     std::shared_ptr<gko::LinOp> wrap_schwarz(
-        std::shared_ptr<gko::LinOp> gkomatrix,
+        std::shared_ptr<const gko::LinOp> gkomatrix,
         std::shared_ptr<gko::Executor> device_exec,
         std::unique_ptr<PrecondFactory> precond) const
     {
@@ -65,7 +64,7 @@ public:
 
     template <typename PrecondFactory, typename Factorization>
     std::shared_ptr<gko::LinOp> wrap_schwarz(
-        std::shared_ptr<gko::LinOp> gkomatrix,
+        std::shared_ptr<const gko::LinOp> gkomatrix,
         std::shared_ptr<gko::Executor> device_exec,
         std::unique_ptr<PrecondFactory> precond,
         std::shared_ptr<Factorization> factorization) const
@@ -82,7 +81,7 @@ public:
 
     std::shared_ptr<gko::LinOp> init_preconditioner_impl(
         const word name, const dictionary &controls,
-        std::shared_ptr<gko::LinOp> gkomatrix,
+        std::shared_ptr<const gko::LinOp> gkomatrix,
         std::shared_ptr<gko::Executor> device_exec) const
     {
         bool skip_sorting =
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 71ed83510..b4a8858b7 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -12,7 +12,6 @@ namespace Foam {
 
 class GKOCGFactory {
 private:
-    using mtx = gko::matrix::Csr<scalar>;
     using vec = gko::matrix::Dense<scalar>;
     using cg = gko::solver::Cg<scalar>;
 
diff --git a/StoppingCriterion/StoppingCriterion.H b/StoppingCriterion/StoppingCriterion.H
index 2db8e8449..394673d40 100644
--- a/StoppingCriterion/StoppingCriterion.H
+++ b/StoppingCriterion/StoppingCriterion.H
@@ -15,7 +15,6 @@ namespace Foam {
 
 class StoppingCriterion {
     using vec = gko::matrix::Dense<scalar>;
-    using mtx = gko::matrix::Csr<scalar>;
     using val_array = gko::array<scalar>;
     using idx_array = gko::array<label>;
 
@@ -64,11 +63,12 @@ class StoppingCriterion {
 
             bool GKO_FACTORY_PARAMETER(export_res, false);
 
-            std::shared_ptr<gko::LinOp> GKO_FACTORY_PARAMETER(gkomatrix, {});
+            std::shared_ptr<const gko::LinOp> GKO_FACTORY_PARAMETER(gkomatrix,
+                                                                    {});
 
-            std::shared_ptr<dist_vec> GKO_FACTORY_PARAMETER(x, {});
+            std::shared_ptr<const dist_vec> GKO_FACTORY_PARAMETER(x, {});
 
-            std::shared_ptr<dist_vec> GKO_FACTORY_PARAMETER(b, {});
+            std::shared_ptr<const dist_vec> GKO_FACTORY_PARAMETER(b, {});
         };
 
         GKO_ENABLE_CRITERION_FACTORY(OpenFOAMDistStoppingCriterion, parameters,
@@ -190,10 +190,11 @@ public:
 
     std::shared_ptr<const gko::stop::CriterionFactory>
     build_dist_stopping_criterion(std::shared_ptr<gko::Executor> device_exec,
-                                  std::shared_ptr<gko::LinOp> gkomatrix,
-                                  std::shared_ptr<dist_vec> x,
-                                  std::shared_ptr<dist_vec> b, label verbose,
-                                  bool export_res, label prev_solve_iters,
+                                  std::shared_ptr<const gko::LinOp> gkomatrix,
+                                  std::shared_ptr<const dist_vec> x,
+                                  std::shared_ptr<const dist_vec> b,
+                                  label verbose, bool export_res,
+                                  label prev_solve_iters,
                                   scalar prev_rel_cost) const
     {
         label minIter = minIter_;
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 1b442ca6b..73d801cc3 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -237,19 +237,6 @@ public:
         auto dist_b_v = dist_b.get_vector();
         auto dist_A_v = dist_A.get();
 
-        std::cout << " ldulduBase distributed_matrix get non_local "
-                  << gko::as<gko::experimental::distributed::Matrix<
-                         scalar, label, label>>(dist_A_v)
-                         ->get_non_local_matrix()
-                         ->get_size()
-                  << "\n";
-
-        std::cout << " ldulduBase distributed_matrix get local "
-                  << gko::as<gko::experimental::distributed::Matrix<
-                         scalar, label, label>>(dist_A_v)
-                         ->get_local_matrix()
-                         ->get_size()
-                  << "\n";
         scalar scaling =
             solver_controls_.lookupOrDefault<scalar>("scaling", 1.0);
         if (scaling != 1) {
@@ -262,12 +249,6 @@ public:
                 dist_b.get_vector()->scale(dense_scaling.get());)
         }
 
-        std::cout << " ldulduBase distributed_matrix get local 1 "
-                  << gko::as<gko::experimental::distributed::Matrix<
-                         scalar, label, label>>(dist_A_v)
-                         ->get_local_matrix()
-                         ->get_size()
-                  << "\n";
         TIME_WITH_FIELDNAME(
             verbose_, init_precond, this->fieldName(),
             auto precond = this->init_preconditioner(
@@ -282,12 +263,6 @@ public:
 
 
         LOG_1(verbose_, "create solver")
-        std::cout << " ldulduBase distributed_matrix get local 2 "
-                  << gko::as<gko::experimental::distributed::Matrix<
-                         scalar, label, label>>(dist_A_v)
-                         ->get_local_matrix()
-                         ->get_size()
-                  << "\n";
         auto solver_gen = this->create_dist_solver(
             this->get_exec_handler().get_device_exec(), dist_A_v, dist_x_v,
             dist_b_v, verbose_, dist_A.get_export(), precond);
@@ -295,12 +270,6 @@ public:
         TIME_WITH_FIELDNAME(verbose_, generate_solver, this->fieldName(),
                             auto solver = solver_gen->generate(dist_A_v);)
         LOG_1(verbose_, "solve solver done")
-        std::cout << " ldulduBase distributed_matrix get local 3 "
-                  << gko::as<gko::experimental::distributed::Matrix<
-                         scalar, label, label>>(dist_A_v)
-                         ->get_local_matrix()
-                         ->get_size()
-                  << "\n";
         TIME_WITH_FIELDNAME(verbose_, solve, this->fieldName(),
                             solver->apply(dist_b_v, dist_x_v);)
 

From f12634bdafc41f04d2e0e9798ba55f8c0c4b3e00 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 3 Apr 2024 16:24:56 +0200
Subject: [PATCH 22/90] remove commented code

---
 DevicePersistent/Base/Base.H | 1 -
 1 file changed, 1 deletion(-)

diff --git a/DevicePersistent/Base/Base.H b/DevicePersistent/Base/Base.H
index 200310866..8a5897a5f 100644
--- a/DevicePersistent/Base/Base.H
+++ b/DevicePersistent/Base/Base.H
@@ -72,7 +72,6 @@ private:
 
 public:
     PersistentBase(const word name, const objectRegistry &db,
-                   // const ExecutorHandler &exec,
                    const InitFunctor f, const bool update, const label verbose)
         : name_(name),
           db_(db),

From a5bc4d515ba2533df700e23a7c3381d1a02d491b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 4 Apr 2024 11:22:21 +0200
Subject: [PATCH 23/90] add interface spans

---
 MatrixWrapper/HostMatrix/HostMatrix.C | 127 +++++++++-----------------
 1 file changed, 44 insertions(+), 83 deletions(-)

diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
index f1b56ff2d..de3b934e0 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -463,6 +463,9 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     auto cols = non_local_sparsity_.col_idxs_.get_data();
     auto permute = non_local_sparsity_.ldu_mapping_.get_data();
 
+    // TODO if we treat all interfaces separately we don't need to sort
+    // anymore
+    //
     // Sorting of the interfaces is still needed since we can we have
     // multiple interfaces using the same rows/send_idxs
     // if the resulting non_local_matrix is not in row major order
@@ -509,98 +512,39 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
 
     // Scan through given rows and insert row and column indices into array
     //
-    //  position after all local offdiagonal elements, needed for
-    //  permutation matrix
+    // position after all local offdiagonal elements, needed for
+    // permutation matrix
     //
-    //  TODO in order to simplify when local interfaces exists set
-    //  local_sparsity to size of nrows_w_interfaces, if interfaces exist
-    //  local_sparsity is only valid till nrows_
+    // TODO in order to simplify when local interfaces exists set
+    // local_sparsity to size of nrows_w_interfaces, if interfaces exist
+    // local_sparsity is only valid till nrows_
     label after_neighbours = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
     init_local_sparsity(nrows_, upper_nnz_, is_symmetric, upper, lower, rows,
                         cols, permute);
 
     // if no local interfaces are present we are done here
-    // otherwise we need to add local interfaces in order
+    // otherwise we need to add local interfaces to local_sparsity in order
+    // of the interfaces to end of the col and row idx arrays. This will produce
+    // idx = [d_1, u_1, l_2, d_2, u_2, ... d_n, i_11, i_12, .., i_nn] where
+    // i_j,k j=interface index and k cell index on the interface
     if (local_interface_nnz_) {
+        // NOTE currently, this copies the interface indizes first to a vector
+        // of tuples before inserting it into the persistent arrays. We could
+        // remove the unnessary copy via the vector of tuples and
+        // let collect_local_interface_indices_impl write directly to rows, cols, permute etc
         auto local_interfaces = collect_local_interface_indices(interfaces);
-        // sort local interfaces to be in row major order
-        // and keep original indices
-        std::sort(local_interfaces.begin(), local_interfaces.end(),
-                  [&](const auto &a, const auto &b) {
-                      auto [interface_idx_a, row_a, col_a] = a;
-                      auto [interface_idx_b, row_b, col_b] = b;
-                      return std::tie(row_a, col_a) < std::tie(row_b, col_b);
-                  });
-
-        // copy current rows and columns to tmp array
-        // TODO this has full length but is only valid till nrows_
-        auto rows_copy_a = gko::array<label>(
-            *local_sparsity_.row_idxs_.get_persistent_object().get());
-        auto cols_copy_a = gko::array<label>(
-            *local_sparsity_.col_idxs_.get_persistent_object().get());
-        auto permute_copy_a = gko::array<label>(
-            *local_sparsity_.ldu_mapping_.get_persistent_object().get());
-
-        auto rows_copy = rows_copy_a.get_data();
-        auto cols_copy = cols_copy_a.get_data();
-        auto permute_copy = permute_copy_a.get_data();
-
-        auto rows = local_sparsity_.row_idxs_.get_data();
-        auto cols = local_sparsity_.col_idxs_.get_data();
-        auto permute = local_sparsity_.ldu_mapping_.get_data();
-
-        label current_idx_ctr = 0;
-        label total_ctr = 0;
-        // iterate local interfaces i
-        // insert all coeffs for row < interface[i].row
-        // find local_interface with lowes row, column
-        for (auto const &interface : local_interfaces) {
-            auto [interface_idx, interface_row, interface_col] = interface;
-
-            // copy from existing matrix coefficients
-            while ([&](auto interface_row, auto interface_col) {
-                // check for length
-                if (current_idx_ctr == local_matrix_nnz_) {
-                    return false;
-                }
-
-                if (rows_copy[current_idx_ctr] > interface_row) {
-                    return false;
-                }
-
-                // the copy rows are or equal
-                // in that case we need to check if the
-                // copy columns are lower
-                if (rows_copy[current_idx_ctr] == interface_row &&
-                    cols_copy[current_idx_ctr] > interface_col) {
-                    return false;
-                }
-                return true;
-            }(interface_row, interface_col)) {
-                // insert coeffs
-                rows[total_ctr] = rows_copy[current_idx_ctr];
-                cols[total_ctr] = cols_copy[current_idx_ctr];
-                permute[total_ctr] = permute_copy[current_idx_ctr];
-                current_idx_ctr++;
-                total_ctr++;
-            }
-            rows[total_ctr] = interface_row;
-            cols[total_ctr] = interface_col;
-            // store the original position of the contiguous interfaces
-            permute[total_ctr] = after_neighbours + nrows_ + interface_idx;
-            total_ctr++;
-        }
 
-        // post insert if local interfaces were consumed but stuff remains
-        // in rows_copy and cols_copy
-        for (label i = total_ctr; i < local_matrix_w_interfaces_nnz_; i++) {
-            rows[i] = rows_copy[current_idx_ctr];
-            cols[i] = cols_copy[current_idx_ctr];
-            permute[i] = permute_copy[current_idx_ctr];
-            current_idx_ctr++;
+        label local_interface_ctr {0};
+        for (label i=local_matrix_nnz_; i<local_matrix_w_interfaces_nnz_ ;++i) {
+          auto [interface_idx, interface_row, interface_col] = local_interfaces[local_interface_ctr];
+          rows[i] = interface_row;
+          cols[i] = interface_col;
+          // if interfaces are treated separately we don't need to permute
+          // interface values
+          permute[i] = i;
+          local_interface_ctr++;
         }
     }
-
     LOG_1(verbose_, "done init host matrix")
 }
 
@@ -621,7 +565,6 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     // TODO this does not work for Ell
     auto permute = local_sparsity_.ldu_mapping_.get_data();
 
-
     if (reorder_on_copy_) {
         auto dense = local_coeffs_.get_data();
         if (local_interface_nnz_) {
@@ -725,8 +668,26 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     // recreate this for every solver call
     // TODO this needs a proper implementation once we know how to handle
     // interfaces
-    local_sparsity_.interface_spans_.emplace_back(0, local_matrix_nnz_);
     local_sparsity_.dim_ = gko::dim<2>{nrows_, nrows_};
+
+    local_sparsity_.interface_spans_.emplace_back(0, local_matrix_nnz_);
+    if (local_interface_nnz_) {
+        auto local_interfaces = collect_local_interface_indices(interfaces);
+
+        label local_interface_ctr {0};
+        label prev_interface_ctr {0};
+        label end = {0};
+        label start = local_matrix_nnz_;
+        for (auto [interface_idx, interface_row, interface_col] : local_interfaces){
+          // A new interface started
+          if (interface_idx > prev_interface_ctr) {
+            end = start + i;
+            local_sparsity_.interface_spans_.emplace_back(start, end);
+            start = end;
+            prev_interface_ctr = interface_idx;
+          }
+        }
+    }
 }
 
 

From 5b7bd4f8f168f43c1f2dcf03ec4c3a956991e9e4 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 4 Apr 2024 12:25:21 +0200
Subject: [PATCH 24/90] fixup! brokin idx

---
 MatrixWrapper/HostMatrix/HostMatrix.C | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
index de3b934e0..5b275f901 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -681,11 +681,12 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
         for (auto [interface_idx, interface_row, interface_col] : local_interfaces){
           // A new interface started
           if (interface_idx > prev_interface_ctr) {
-            end = start + i;
+            end = start + local_interface_ctr;
             local_sparsity_.interface_spans_.emplace_back(start, end);
             start = end;
             prev_interface_ctr = interface_idx;
           }
+          local_interface_ctr++;
         }
     }
 }

From 7eea220a1d7d69f1ba1447014294ab2507dd0aca Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 4 Apr 2024 12:42:39 +0200
Subject: [PATCH 25/90] add implementation for non local interfaces

---
 MatrixWrapper/HostMatrix/HostMatrix.C | 41 +++++++++++++++------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
index 5b275f901..624b01063 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -463,25 +463,13 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     auto cols = non_local_sparsity_.col_idxs_.get_data();
     auto permute = non_local_sparsity_.ldu_mapping_.get_data();
 
-    // TODO if we treat all interfaces separately we don't need to sort
-    // anymore
-    //
-    // Sorting of the interfaces is still needed since we can we have
-    // multiple interfaces using the same rows/send_idxs
-    // if the resulting non_local_matrix is not in row major order
-    // we might get non converging solvers on GPU devices
-    std::sort(non_local_row_indices.begin(), non_local_row_indices.end(),
-              [&](const auto &a, const auto &b) {
-                  auto [interface_idx_a, row_a] = a;
-                  auto [interface_idx_b, row_b] = b;
-                  return row_a < row_b;
-              });
-
     label element_ctr = 0;
+    // TODO currently we set permute eventhough this is not required
+    // anymore, remove permute from non_local_interfaces
     for (auto [interface_idx, row] : non_local_row_indices) {
         rows[element_ctr] = row;
         cols[element_ctr] = interface_idx;
-        permute[element_ctr] = interface_idx;
+        permute[element_ctr] = element_ctr;
         element_ctr += 1;
     }
 }
@@ -676,8 +664,8 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
 
         label local_interface_ctr {0};
         label prev_interface_ctr {0};
-        label end = {0};
-        label start = local_matrix_nnz_;
+        label end {0};
+        label start { local_matrix_nnz_ };
         for (auto [interface_idx, interface_row, interface_col] : local_interfaces){
           // A new interface started
           if (interface_idx > prev_interface_ctr) {
@@ -720,8 +708,25 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     // NOTE interface_spans and dim are not persistent so we need to
     // We have now the full non_local matrix and need to add the start and end
     // to the PersistentSparsityPattern data structure
-    non_local_sparsity_.interface_spans_.emplace_back(0, non_local_matrix_nnz_);
     non_local_sparsity_.dim_ = gko::dim<2>{nrows_, non_local_matrix_nnz_};
+
+    auto non_local_interfaces = collect_cells_on_non_local_interface(interfaces);
+
+    // TODO this can be merged with algorithm in update_local_matrix_data
+    // refactor to separate function
+    label interface_ctr {0};
+    label prev_interface_ctr {0};
+    label end {0};
+    label start {0};
+    for (auto [interface_idx, interface_row] : non_local_interfaces){
+        if (interface_idx > prev_interface_ctr) {
+          end = start + interface_ctr;
+          non_local_sparsity_.interface_spans_.emplace_back(start, end);
+          start = end;
+          prev_interface_ctr = interface_idx;
+        }
+        interface_ctr++;
+    }
 }
 
 template HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(

From 560ae32de82e223638d34e1153e836232c2a0e57 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 4 Apr 2024 16:10:46 +0200
Subject: [PATCH 26/90] add missing ginkgo dependency

---
 unitTests/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 5d8c7b28b..57d05a909 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -33,6 +33,7 @@ target_link_libraries(
          $ENV{FOAM_LIBBIN}/libfiniteVolume.so
          $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so
          OGL
+         Ginkgo::ginkgo
          # MPI::MPI_CXX
          ${CMAKE_DL_LIBS})
 

From d025ea48600c94711c3efade7b5d6f518fd561b9 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 9 Apr 2024 20:56:07 +0200
Subject: [PATCH 27/90] remove sanitizer flags in favour of seperate cmake
 files

---
 CMakeLists.txt | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5da376a95..be91b8a7b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -94,8 +94,7 @@ if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
 
   set(CMAKE_CXX_FLAGS_DEBUG
       "${CMAKE_CXX_FLAGS_DEBUG} \
-    -fno-omit-frame-pointer \
-    -fsanitize=address")
+    -fno-omit-frame-pointer")
 
   set(CMAKE_CXX_FLAGS_RELEASE
       "${CMAKE_CXX_FLAGS_RELEASE} \
@@ -114,7 +113,6 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
       --coverage\
       -fprofile-arcs \
       -ggdb3 \
-      -fsanitize=address,undefined \
       ")
 
   set(CMAKE_CXX_FLAGS_RELEASE

From 569694c6bdcf1100b6131f47141f0003ab4ae05d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 9 Apr 2024 20:57:57 +0200
Subject: [PATCH 28/90] add non_local_interfaces

---
 MatrixWrapper/HostMatrix/HostMatrix.C | 115 +++++++++++++++-----------
 MatrixWrapper/HostMatrix/HostMatrix.H |   5 +-
 2 files changed, 68 insertions(+), 52 deletions(-)

diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
index 624b01063..cabfeaa58 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -231,6 +231,7 @@ template <class Sel, class Func>
 void interface_iterator(const lduInterfaceFieldPtrsList &interfaces, Func func)
 {
     label element_ctr = 0;
+    label interface_ctr = 0;
     for (label i = 0; i < interfaces.size(); i++) {
         if (interface_getter(interfaces, i) == nullptr) {
             continue;
@@ -241,7 +242,8 @@ void interface_iterator(const lduInterfaceFieldPtrsList &interfaces, Func func)
 
         if (isA<Sel>(iface->interface())) {
             const Sel &patch = refCast<const Sel>(iface->interface());
-            func(element_ctr, interface_size, patch, iface);
+            func(element_ctr, interface_ctr, interface_size, patch, iface);
+            interface_ctr++;
         }
     }
 }
@@ -282,7 +284,7 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     // and store rows to send to neighbour procs
     gko::size_type n_procs = 0;
     interface_iterator<processorFvPatch>(
-        interfaces, [&](label, const label, const processorFvPatch &patch,
+        interfaces, [&](label, label, label, const processorFvPatch &patch,
                         const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
             const label neighbProcNo = patch.neighbProcNo();
@@ -428,25 +430,26 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
 }
 
 template <class MatrixType>
-std::vector<std::tuple<label, label>>
+std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_cells_on_non_local_interface(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // vector of neighbour cell idx connected to interface
-    std::vector<std::tuple<label, label>> non_local_idxs{};
+    std::vector<std::tuple<label, label, label>> non_local_idxs{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
     label interface_ctr = 0;
-    interface_iterator<processorFvPatch>(
-        interfaces,
-        [&](label, const label interface_size, const processorLduInterface &,
-            const lduInterfaceField *iface) {
-            const auto &face_cells{iface->interface().faceCells()};
-            for (label cellI = 0; cellI < interface_size; cellI++) {
-                auto local_row = face_cells[cellI];
-                non_local_idxs.push_back({interface_ctr, local_row});
-                interface_ctr += 1;
-            }
-        });
+    interface_iterator<
+        processorFvPatch>(interfaces, [&](label, label interface_id,
+                                          label interface_size,
+                                          const processorLduInterface &,
+                                          const lduInterfaceField *iface) {
+        const auto &face_cells{iface->interface().faceCells()};
+        for (label cellI = 0; cellI < interface_size; cellI++) {
+            auto local_row = face_cells[cellI];
+            non_local_idxs.push_back({interface_id, interface_ctr, local_row});
+            interface_ctr += 1;
+        }
+    });
 
     word msg = "done collecting neighbouring processor cell id";
     LOG_2(verbose_, msg)
@@ -466,7 +469,7 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     label element_ctr = 0;
     // TODO currently we set permute eventhough this is not required
     // anymore, remove permute from non_local_interfaces
-    for (auto [interface_idx, row] : non_local_row_indices) {
+    for (auto [interface_id, interface_idx, row] : non_local_row_indices) {
         rows[element_ctr] = row;
         cols[element_ctr] = interface_idx;
         permute[element_ctr] = element_ctr;
@@ -519,18 +522,21 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
         // NOTE currently, this copies the interface indizes first to a vector
         // of tuples before inserting it into the persistent arrays. We could
         // remove the unnessary copy via the vector of tuples and
-        // let collect_local_interface_indices_impl write directly to rows, cols, permute etc
+        // let collect_local_interface_indices_impl write directly to rows,
+        // cols, permute etc
         auto local_interfaces = collect_local_interface_indices(interfaces);
 
-        label local_interface_ctr {0};
-        for (label i=local_matrix_nnz_; i<local_matrix_w_interfaces_nnz_ ;++i) {
-          auto [interface_idx, interface_row, interface_col] = local_interfaces[local_interface_ctr];
-          rows[i] = interface_row;
-          cols[i] = interface_col;
-          // if interfaces are treated separately we don't need to permute
-          // interface values
-          permute[i] = i;
-          local_interface_ctr++;
+        label local_interface_ctr{0};
+        for (label i = local_matrix_nnz_; i < local_matrix_w_interfaces_nnz_;
+             ++i) {
+            auto [interface_idx, interface_row, interface_col] =
+                local_interfaces[local_interface_ctr];
+            rows[i] = interface_row;
+            cols[i] = interface_col;
+            // if interfaces are treated separately we don't need to permute
+            // interface values
+            permute[i] = i;
+            local_interface_ctr++;
         }
     }
     LOG_1(verbose_, "done init host matrix")
@@ -662,19 +668,22 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     if (local_interface_nnz_) {
         auto local_interfaces = collect_local_interface_indices(interfaces);
 
-        label local_interface_ctr {0};
-        label prev_interface_ctr {0};
-        label end {0};
-        label start { local_matrix_nnz_ };
-        for (auto [interface_idx, interface_row, interface_col] : local_interfaces){
-          // A new interface started
-          if (interface_idx > prev_interface_ctr) {
-            end = start + local_interface_ctr;
-            local_sparsity_.interface_spans_.emplace_back(start, end);
-            start = end;
-            prev_interface_ctr = interface_idx;
-          }
-          local_interface_ctr++;
+        label local_interface_ctr{0};
+        label prev_interface_ctr{0};
+        label end{0};
+        label start{local_matrix_nnz_};
+        for (auto [interface_idx, interface_row, interface_col] :
+             local_interfaces) {
+            // check if a new interface has started or final interface has been
+            // reache
+            if (interface_idx > prev_interface_ctr ||
+                local_interface_ctr + 1 == local_interfaces.size()) {
+                end = start + local_interface_ctr;
+                local_sparsity_.interface_spans_.emplace_back(start, end);
+                start = end;
+                prev_interface_ctr = interface_idx;
+            }
+            local_interface_ctr++;
         }
     }
 }
@@ -710,20 +719,26 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     // to the PersistentSparsityPattern data structure
     non_local_sparsity_.dim_ = gko::dim<2>{nrows_, non_local_matrix_nnz_};
 
-    auto non_local_interfaces = collect_cells_on_non_local_interface(interfaces);
+    auto non_local_interfaces =
+        collect_cells_on_non_local_interface(interfaces);
 
     // TODO this can be merged with algorithm in update_local_matrix_data
     // refactor to separate function
-    label interface_ctr {0};
-    label prev_interface_ctr {0};
-    label end {0};
-    label start {0};
-    for (auto [interface_idx, interface_row] : non_local_interfaces){
-        if (interface_idx > prev_interface_ctr) {
-          end = start + interface_ctr;
-          non_local_sparsity_.interface_spans_.emplace_back(start, end);
-          start = end;
-          prev_interface_ctr = interface_idx;
+    label interface_ctr{0};
+    label prev_interface_ctr{0};
+    label end{0};
+    label start{0};
+    for (auto [interface_idx, interface_cell_ctr, interface_row] :
+         non_local_interfaces) {
+        std::cout << "updating interface span " << interface_idx << "\n";
+        // check if a new interface has started or final interface has been
+        // reache
+        if (interface_idx > prev_interface_ctr ||
+            interface_ctr + 1 == non_local_interfaces.size()) {
+            end = start + interface_ctr;
+            non_local_sparsity_.interface_spans_.emplace_back(start, end);
+            start = end;
+            prev_interface_ctr = interface_idx;
         }
         interface_ctr++;
     }
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.H b/MatrixWrapper/HostMatrix/HostMatrix.H
index 46cd1bc46..2a8fad2ca 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.H
+++ b/MatrixWrapper/HostMatrix/HostMatrix.H
@@ -52,7 +52,7 @@ struct PersistentSparsityPattern {
               verbose,
               false,  // For now we assume columns and rows to be constant
               false   // we don't init on device since the indices are written
-                     // using the pointer to the data
+                      // using the pointer to the data
           },
           row_idxs_{
               fieldName + "_rows",
@@ -362,7 +362,8 @@ private:
     ** ret = [(1,2,1),(2,20, 2), (3, 20, 2) ...]
     **         i0   i1,   i...
     */
-    std::vector<std::tuple<label, label>> collect_cells_on_non_local_interface(
+    std::vector<std::tuple<label, label, label>>
+    collect_cells_on_non_local_interface(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
     /** Based on OpenFOAMs ldu matrix format this function computes two

From aa1b44520be8264bc29476e46f7bec19f6660b58 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 10 Apr 2024 16:55:04 +0200
Subject: [PATCH 29/90] fix span bounds

---
 MatrixWrapper/HostMatrix/HostMatrix.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
index cabfeaa58..408d13ff1 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -735,7 +735,7 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
         // reache
         if (interface_idx > prev_interface_ctr ||
             interface_ctr + 1 == non_local_interfaces.size()) {
-            end = start + interface_ctr;
+            end = start + interface_ctr + 1;
             non_local_sparsity_.interface_spans_.emplace_back(start, end);
             start = end;
             prev_interface_ctr = interface_idx;

From 05340a46498d6c87aadea38bff55285641980c66 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 11 Apr 2024 10:30:53 +0200
Subject: [PATCH 30/90] fix multi interface case

---
 .../GkoCombinationMatrix/GkoCombinationMatrix.H |  1 +
 MatrixWrapper/HostMatrix/HostMatrix.C           | 17 +++++++++--------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
index 276af09d2..4214c9d06 100644
--- a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
+++ b/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
@@ -38,6 +38,7 @@ public:
         for (auto &interface : interface_ranges_) {
             auto length = interface.length();
             auto offset = interface.begin;
+            std::cout << " read interface " << length << " " << offset << "\n";
             auto op = gko::share(InnerMatrixType::create(exec));
 
             device_matrix_data in(
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/HostMatrix/HostMatrix.C
index 408d13ff1..fac414169 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/HostMatrix/HostMatrix.C
@@ -668,20 +668,22 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     if (local_interface_nnz_) {
         auto local_interfaces = collect_local_interface_indices(interfaces);
 
+        // counter how many interface elements are found on current interface
         label local_interface_ctr{0};
-        label prev_interface_ctr{0};
+        label prev_interface_idx{0};
         label end{0};
         label start{local_matrix_nnz_};
         for (auto [interface_idx, interface_row, interface_col] :
              local_interfaces) {
             // check if a new interface has started or final interface has been
             // reache
-            if (interface_idx > prev_interface_ctr ||
+            if (interface_idx > prev_interface_idx ||
                 local_interface_ctr + 1 == local_interfaces.size()) {
                 end = start + local_interface_ctr;
+                local_interface_ctr = 0;
                 local_sparsity_.interface_spans_.emplace_back(start, end);
                 start = end;
-                prev_interface_ctr = interface_idx;
+                prev_interface_idx = interface_idx;
             }
             local_interface_ctr++;
         }
@@ -697,7 +699,6 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     auto ref_exec = exec_.get_ref_exec();
     auto interface_coeffs =
         collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
-    auto permute = non_local_sparsity_.ldu_mapping_.get_data();
 
     // copy interfaces
     auto tmp_contiguous_iface =
@@ -706,7 +707,7 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
 
     for (label element_ctr = 0; element_ctr < non_local_matrix_nnz_;
          element_ctr++) {
-        contiguous_iface[element_ctr] = interface_coeffs[permute[element_ctr]];
+        contiguous_iface[element_ctr] = interface_coeffs[element_ctr];
     }
 
     // copy to persistent
@@ -730,12 +731,12 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     label start{0};
     for (auto [interface_idx, interface_cell_ctr, interface_row] :
          non_local_interfaces) {
-        std::cout << "updating interface span " << interface_idx << "\n";
         // check if a new interface has started or final interface has been
         // reache
         if (interface_idx > prev_interface_ctr ||
-            interface_ctr + 1 == non_local_interfaces.size()) {
-            end = start + interface_ctr + 1;
+            interface_ctr == non_local_interfaces.size() - 1) {
+            end = start + interface_ctr;
+            interface_ctr = 0;
             non_local_sparsity_.interface_spans_.emplace_back(start, end);
             start = end;
             prev_interface_ctr = interface_idx;

From f2e3a24a13d25d38004aa2463cd0f11278471a35 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 12 Apr 2024 09:47:09 +0200
Subject: [PATCH 31/90] WIP DistMatrixWrapper

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 81 ++++++++++---------
 lduLduBase/lduLduBase.H                       |  4 +-
 2 files changed, 47 insertions(+), 38 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index ce0bfc89f..23ba900cf 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -10,6 +10,7 @@
 #include "DevicePersistent/Partition/Partition.H"
 
 #include "MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H"
+#include "MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H"
 
 #include "fvCFD.H"
 
@@ -42,12 +43,11 @@ void perform_update(
         interface_ctr++;
         device_value_view = update_view;
     }
-};
+}
 
 
 struct MatrixInitFunctor {
-    using dist_mtx =
-        gko::experimental::distributed::Matrix<scalar, label, label>;
+    using dist_mtx = RepartDistMatrix<scalar, label, label>;
 
     using val_array = gko::array<scalar>;
 
@@ -159,39 +159,37 @@ struct MatrixInitFunctor {
         if (matrix_format_ == "Ell") {
             using Ell = gko::matrix::Ell<scalar, label>;
 
-            auto local_template = CombinationMatrix<scalar, label, Ell>::create(
-                exec, local_dim, local_interfaces);
-            auto non_local_template =
+            return gko::share(dist_mtx::create(
+                exec, comm,
+                CombinationMatrix<scalar, label, Ell>::create(exec, local_dim,
+                                                              local_interfaces),
                 CombinationMatrix<scalar, label, Ell>::create(
-                    exec, non_local_dim, non_local_interfaces);
-
-            return dist_mtx::create(exec, *comm.get(), local_template,
-                                    non_local_template);
-        }
-        if (matrix_format_ == "Csr") {
-            using Csr = gko::matrix::Csr<scalar, label>;
-
-            auto local_template = CombinationMatrix<scalar, label, Csr>::create(
-                exec, local_dim, local_interfaces);
-            auto non_local_template =
-                CombinationMatrix<scalar, label, Csr>::create(
-                    exec, non_local_dim, non_local_interfaces);
-
-            return dist_mtx::create(exec, *comm.get(), local_template,
-                                    non_local_template);
-        }
-        if (matrix_format_ == "Coo") {
-            using Coo = gko::matrix::Coo<scalar, label>;
-
-            auto local_template = CombinationMatrix<scalar, label, Coo>::create(
-                exec, local_dim, local_interfaces);
-            auto non_local_template =
-                CombinationMatrix<scalar, label, Coo>::create(
-                    exec, non_local_dim, non_local_interfaces);
-
-            return dist_mtx::create(exec, *comm.get(), local_template,
-                                    non_local_template);
+                    exec, non_local_dim, non_local_interfaces)));
         }
+        // if (matrix_format_ == "Csr") {
+        //     using Csr = gko::matrix::Csr<scalar, label>;
+        //
+        //     auto local_template = CombinationMatrix<scalar, label, Csr>::create(
+        //         exec, local_dim, local_interfaces);
+        //     auto non_local_template =
+        //         CombinationMatrix<scalar, label, Csr>::create(
+        //             exec, non_local_dim, non_local_interfaces);
+        //
+        //     return dist_mtx::create(exec, comm, local_template,
+        //                             non_local_template);
+        // }
+        // if (matrix_format_ == "Coo") {
+        //     using Coo = gko::matrix::Coo<scalar, label>;
+        //
+        //     auto local_template = CombinationMatrix<scalar, label, Coo>::create(
+        //         exec, local_dim, local_interfaces);
+        //     auto non_local_template =
+        //         CombinationMatrix<scalar, label, Coo>::create(
+        //             exec, non_local_dim, non_local_interfaces);
+        //
+        //     return dist_mtx::create(exec, comm, local_template,
+        //                             non_local_template);
+        // }
         // if (matrix_format_ == "Hybrid") {
         //     return dist_mtx::create(
         //         *comm.get(), gko::with_matrix_type<gko::matrix::Hybrid>());
@@ -225,12 +223,24 @@ struct MatrixInitFunctor {
             *non_local_cols.get(), *non_local_coeffs.get()};
 
         auto comm = exec_.get_gko_mpi_host_comm();
+
+        // TODO here we would need a source and target comm / partition to allow
+        // repartitioning
         auto sparse_comm =
             gko::experimental::distributed::sparse_communicator::create(
                 *comm.get(), partition_.get_localized_partition());
 
         auto dist_A = generate_dist_mtx_with_inner_type(
             exec_.get_device_exec(), exec_.get_gko_mpi_host_comm());
+
+        // TODO here we would need a source and target comm / partition to allow
+        // repartitioning However repartitioning is also specific to the
+        // combination matrix since it moves data between interfaces thus it
+        // would need a function that does repartition(local, non_local) we
+        // could do that by implementing a wrapper for the distributed matrix
+        // type GKORepartitionedDistributedMatrix after the first read it
+        // constructs everything needed for repartitioning update would than be
+        // handled via repeat_create()
         dist_A->read_distributed(local_A_data, non_local_A_data, sparse_comm);
 
         // auto device_mat = generate_dist_mtx_with_inner_type(
@@ -243,8 +253,7 @@ struct MatrixInitFunctor {
 
 class MatrixWrapper {
 private:
-    using dist_mtx =
-        gko::experimental::distributed::Matrix<scalar, label, label>;
+    using dist_mtx = RepartDistMatrix<scalar, label, label>;
 
     const objectRegistry &db_;
 
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 73d801cc3..508cefde3 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -121,8 +121,8 @@ class lduLduBase : public OGL_Info,
                    public Preconditioner {
 private:
     using dist_vec = gko::experimental::distributed::Vector<scalar>;
-    using dist_mtx =
-        gko::experimental::distributed::Matrix<scalar, label, label>;
+    using dist_mtx = RepartDistMatrix<scalar, label, label>;
+
     using mtx = gko::matrix::Csr<scalar>;
     using coo_mtx = gko::matrix::Coo<scalar>;
     using csr_mtx = gko::matrix::Csr<scalar>;

From 8a4b72ed2158df86cf12dca5ae44f419c2c348a4 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 12 Apr 2024 14:39:20 +0200
Subject: [PATCH 32/90] add basic RepartDistMatrix implementation

---
 .../RepartDistMatrix/RepartDistMatrix.H       | 132 ++++++++++++++++++
 1 file changed, 132 insertions(+)
 create mode 100644 MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H

diff --git a/MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H b/MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H
new file mode 100644
index 000000000..b62ec4f06
--- /dev/null
+++ b/MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H
@@ -0,0 +1,132 @@
+#include <ginkgo/ginkgo.hpp>
+
+
+// The RepartDistMatrix class is a wrapper around Ginkgos distributed Matrix class
+// 
+// It adds functionality for repeated read and repartitioning operatitions. As a constraint
+// it is required that the inner matrix types of the distributed matrix are of RepartDistMatrix type.
+template <typename ValueType, typename LocalIndexType, typename GlobalIndexType>
+class RepartDistMatrix
+    : public gko::EnableLinOp<
+          RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>,
+      public gko::EnableCreateMethod<
+          RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>> {
+    friend class gko::EnableCreateMethod<
+        RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>;
+    friend class gko::EnablePolymorphicObject<
+        RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>, gko::LinOp>;
+
+    using dist_mtx =
+        gko::experimental::distributed::Matrix<ValueType, LocalIndexType, GlobalIndexType>;
+
+public:
+    using gko::EnableLinOp<
+        RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>::convert_to;
+    using gko::EnableLinOp<
+        RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>::move_to;
+
+    using vec = gko::matrix::Dense<ValueType>;
+    using coef_type = gko::array<ValueType>;
+    using device_matrix_data = gko::device_matrix_data<scalar, label>;
+    using matrix_data = gko::matrix_data<scalar, label>;
+    using sparse_communicator = gko::experimental::distributed::sparse_communicator;
+
+
+    void read_distributed(
+        const device_matrix_data& local_data,
+        const device_matrix_data& non_local_data,
+        std::shared_ptr<const sparse_communicator> sparse_comm)
+    {
+        this->dist_mtx_->read_distributed(local_data, non_local_data, sparse_comm);
+    }
+
+    std::shared_ptr<const gko::LinOp> get_non_local_matrix() {
+        this->dist_mtx_->get_local_matrix();
+    }
+
+    std::shared_ptr<const gko::LinOp> get_local_matrix() {
+        this->dist_mtx_->get_non_local_matrix();
+    }
+
+    // Function with the exact same interface as create but reuses intermediate computations 
+    static std::unique_ptr<RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType> > create(
+        std::shared_ptr<gko::Executor> exec,
+        std::shared_ptr<gko::experimental::mpi::communicator> comm,
+        std::shared_ptr<const gko::LinOp> local_template,
+        std::shared_ptr<const gko::LinOp> non_local_template
+        ); 
+
+    // Function with the exact same interface as create but reuses intermediate computations 
+    void update(
+        std::shared_ptr<gko::Executor> exec,
+        std::shared_ptr<gko::experimental::mpi::communicator> comm,
+        std::shared_ptr<gko::LinOp> local_template,
+        std::shared_ptr<gko::LinOp> non_local_template
+            ) {}
+
+    /**
+     * Copy-assigns a CombinationMatrix matrix. Preserves executor, copies
+     * everything else.
+     */
+    RepartDistMatrix &operator=(const RepartDistMatrix &other)
+    {
+        if (&other != this) {
+            *dist_mtx_.get() = *other.dist_mtx_.get();
+        }
+        return *this;
+    }
+
+    /**
+     * Move-assigns a CombinationMatrix matrix. Preserves executor, moves the
+     * data and leaves the moved-from object in an empty state (0x0 LinOp with
+     * unchanged executor and strategy, no nonzeros and valid row pointers).
+     */
+    RepartDistMatrix &operator=(RepartDistMatrix &&other)
+    {
+        if (&other != this) {
+            gko::EnableLinOp<RepartDistMatrix>::operator=(std::move(other));
+            dist_mtx_ = std::move(other.dist_mtx_);
+        }
+        return *this;
+    }
+
+protected:
+    RepartDistMatrix(std::shared_ptr<const gko::Executor> exec)
+        : gko::EnableLinOp<RepartDistMatrix>(exec)
+    {}
+
+    RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
+        std::shared_ptr<const sparse_communicator> sparse_comm,
+        std::unique_ptr<const gko::LinOp> local_template,
+        std::unique_ptr<const gko::LinOp> non_local_template
+                      )
+        : gko::EnableLinOp<RepartDistMatrix>(exec),
+          dist_mtx_(dist_mtx::create(exec, sparse_comm, local_template, non_local_template))
+    {}
+
+    // Here we implement the application of the linear operator, x = A * b.
+    // apply_impl will be called by the apply method, after the arguments have
+    // been moved to the correct executor and the operators checked for
+    // conforming sizes.
+    //
+    // For simplicity, we assume that there is always only one right hand side
+    // and the stride of consecutive elements in the vectors is 1 (both of these
+    // are always true in this example).
+    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
+    {
+        this->dist_mtx_->apply(b, x);
+    }
+
+    // There is also a version of the apply function which does the operation
+    // x = alpha * A * b + beta * x. This function is commonly used and can
+    // often be better optimized than implementing it using x = A * b. However,
+    // for simplicity, we will implement it exactly like that in this example.
+    void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
+                    const gko::LinOp *beta, gko::LinOp *x) const override
+    {
+        this->dist_mtx_->apply(alpha, b, beta, x);
+    }
+
+private:
+    std::unique_ptr<dist_mtx> dist_mtx_;
+};

From 5e1c769a8b9f200663eb7793c285ce2cd365dd5e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 15 Apr 2024 09:44:10 +0200
Subject: [PATCH 33/90] Fixup issuse with RepartDistMatrix implementation

---
 DevicePersistent/Base/Base.C                  |  6 +-
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 57 ++++++++-----------
 .../RepartDistMatrix/RepartDistMatrix.H       | 20 +++----
 3 files changed, 37 insertions(+), 46 deletions(-)

diff --git a/DevicePersistent/Base/Base.C b/DevicePersistent/Base/Base.C
index 66660a4d4..5deecede0 100644
--- a/DevicePersistent/Base/Base.C
+++ b/DevicePersistent/Base/Base.C
@@ -4,6 +4,8 @@
 
 #include "Base.H"
 
+#include "../../MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H"
+
 namespace Foam {
 
 // to store the std::shared_ptr<T> in the IO registry the type needs to be
@@ -30,7 +32,7 @@ typedef gko::experimental::distributed::localized_partition<label> Partition;
 defineTemplateTypeNameWithName(DevicePersistentBase<Partition>,
                                "PersistentPartition");
 
-typedef gko::experimental::distributed::Matrix<scalar, label, label> GkoMatrix;
-defineTemplateTypeNameWithName(DevicePersistentBase<GkoMatrix>,
+typedef RepartDistMatrix<scalar, label, label> RepartDistMatrix;
+defineTemplateTypeNameWithName(DevicePersistentBase<RepartDistMatrix>,
                                "PersistentMatrix");
 }  // namespace Foam
diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 23ba900cf..ce8898cff 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -157,46 +157,39 @@ struct MatrixInitFunctor {
         auto non_local_interfaces = non_local_sparsity_.interface_spans_;
 
         if (matrix_format_ == "Ell") {
-            using Ell = gko::matrix::Ell<scalar, label>;
+            using mtx_type = gko::matrix::Ell<scalar, label>;
+
+            return gko::share(dist_mtx::create(
+                exec, comm,
+                CombinationMatrix<scalar, label, mtx_type>::create(exec, local_dim,
+                                                              local_interfaces),
+                CombinationMatrix<scalar, label, mtx_type>::create(
+                    exec, non_local_dim, non_local_interfaces)));
+        }
+        if (matrix_format_ == "Csr") {
+            using mtx_type = gko::matrix::Csr<scalar, label>;
+
+            return gko::share(dist_mtx::create(
+                exec, comm,
+                CombinationMatrix<scalar, label, mtx_type>::create(exec, local_dim,
+                                                              local_interfaces),
+                CombinationMatrix<scalar, label, mtx_type>::create(
+                    exec, non_local_dim, non_local_interfaces)));
+        }
+        if (matrix_format_ == "Coo") {
+            using mtx_type = gko::matrix::Coo<scalar, label>;
 
             return gko::share(dist_mtx::create(
                 exec, comm,
-                CombinationMatrix<scalar, label, Ell>::create(exec, local_dim,
+                CombinationMatrix<scalar, label, mtx_type>::create(exec, local_dim,
                                                               local_interfaces),
-                CombinationMatrix<scalar, label, Ell>::create(
+                CombinationMatrix<scalar, label, mtx_type>::create(
                     exec, non_local_dim, non_local_interfaces)));
         }
-        // if (matrix_format_ == "Csr") {
-        //     using Csr = gko::matrix::Csr<scalar, label>;
-        //
-        //     auto local_template = CombinationMatrix<scalar, label, Csr>::create(
-        //         exec, local_dim, local_interfaces);
-        //     auto non_local_template =
-        //         CombinationMatrix<scalar, label, Csr>::create(
-        //             exec, non_local_dim, non_local_interfaces);
-        //
-        //     return dist_mtx::create(exec, comm, local_template,
-        //                             non_local_template);
-        // }
-        // if (matrix_format_ == "Coo") {
-        //     using Coo = gko::matrix::Coo<scalar, label>;
-        //
-        //     auto local_template = CombinationMatrix<scalar, label, Coo>::create(
-        //         exec, local_dim, local_interfaces);
-        //     auto non_local_template =
-        //         CombinationMatrix<scalar, label, Coo>::create(
-        //             exec, non_local_dim, non_local_interfaces);
-        //
-        //     return dist_mtx::create(exec, comm, local_template,
-        //                             non_local_template);
-        // }
-        // if (matrix_format_ == "Hybrid") {
-        //     return dist_mtx::create(
-        //         *comm.get(), gko::with_matrix_type<gko::matrix::Hybrid>());
-        // }
 
         FatalErrorInFunction << "Matrix format " << matrix_format_
-                             << " not supported " << abort(FatalError);
+                             << " not supported. Supported formats are: Ell, Csr, and Coo."
+                             << abort(FatalError);
     }
 
     std::shared_ptr<dist_mtx> init() const
diff --git a/MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H b/MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H
index b62ec4f06..05f54187d 100644
--- a/MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H
+++ b/MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H
@@ -29,6 +29,7 @@ public:
     using coef_type = gko::array<ValueType>;
     using device_matrix_data = gko::device_matrix_data<scalar, label>;
     using matrix_data = gko::matrix_data<scalar, label>;
+    using communicator = gko::experimental::mpi::communicator;
     using sparse_communicator = gko::experimental::distributed::sparse_communicator;
 
 
@@ -38,24 +39,17 @@ public:
         std::shared_ptr<const sparse_communicator> sparse_comm)
     {
         this->dist_mtx_->read_distributed(local_data, non_local_data, sparse_comm);
+        this->set_size(dist_mtx_->get_size());
     }
 
-    std::shared_ptr<const gko::LinOp> get_non_local_matrix() {
+    std::shared_ptr<const gko::LinOp> get_local_matrix() {
         this->dist_mtx_->get_local_matrix();
     }
 
-    std::shared_ptr<const gko::LinOp> get_local_matrix() {
+    std::shared_ptr<const gko::LinOp> get_non_local_matrix() {
         this->dist_mtx_->get_non_local_matrix();
     }
 
-    // Function with the exact same interface as create but reuses intermediate computations 
-    static std::unique_ptr<RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType> > create(
-        std::shared_ptr<gko::Executor> exec,
-        std::shared_ptr<gko::experimental::mpi::communicator> comm,
-        std::shared_ptr<const gko::LinOp> local_template,
-        std::shared_ptr<const gko::LinOp> non_local_template
-        ); 
-
     // Function with the exact same interface as create but reuses intermediate computations 
     void update(
         std::shared_ptr<gko::Executor> exec,
@@ -72,6 +66,7 @@ public:
     {
         if (&other != this) {
             *dist_mtx_.get() = *other.dist_mtx_.get();
+            this->set_size(dist_mtx_->get_size());
         }
         return *this;
     }
@@ -86,6 +81,7 @@ public:
         if (&other != this) {
             gko::EnableLinOp<RepartDistMatrix>::operator=(std::move(other));
             dist_mtx_ = std::move(other.dist_mtx_);
+            this->set_size(dist_mtx_->get_size());
         }
         return *this;
     }
@@ -96,12 +92,12 @@ protected:
     {}
 
     RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
-        std::shared_ptr<const sparse_communicator> sparse_comm,
+        std::shared_ptr<const communicator> comm,
         std::unique_ptr<const gko::LinOp> local_template,
         std::unique_ptr<const gko::LinOp> non_local_template
                       )
         : gko::EnableLinOp<RepartDistMatrix>(exec),
-          dist_mtx_(dist_mtx::create(exec, sparse_comm, local_template, non_local_template))
+          dist_mtx_(dist_mtx::create(exec, *comm.get(), local_template, non_local_template))
     {}
 
     // Here we implement the application of the linear operator, x = A * b.

From e88012995201ff7f796916cdd6c64fc030c7eb20 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 15 Apr 2024 11:38:35 +0200
Subject: [PATCH 34/90] Rename files

---
 CMakeLists.txt                                                | 2 +-
 DevicePersistent/Base/Base.C                                  | 2 +-
 DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H          | 4 ++--
 .../GkoCombinationMatrix.H => Combination/Combination.H}      | 0
 .../RepartDistMatrix.H => Distributed/Distributed.H}          | 0
 common/common.C                                               | 2 +-
 6 files changed, 5 insertions(+), 5 deletions(-)
 rename MatrixWrapper/{GkoCombinationMatrix/GkoCombinationMatrix.H => Combination/Combination.H} (100%)
 rename MatrixWrapper/{RepartDistMatrix/RepartDistMatrix.H => Distributed/Distributed.H} (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index be91b8a7b..84e3d4100 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -175,7 +175,7 @@ target_sources(
          StoppingCriterion/StoppingCriterion.H
          lduLduBase/lduLduBase.H
          MatrixWrapper/HostMatrix/HostMatrix.H
-         MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
+         MatrixWrapper/Combination/Combination.H
          DevicePersistent/Base/Base.H
          DevicePersistent/Partition/Partition.H
          DevicePersistent/Array/Array.H
diff --git a/DevicePersistent/Base/Base.C b/DevicePersistent/Base/Base.C
index 5deecede0..78052dffa 100644
--- a/DevicePersistent/Base/Base.C
+++ b/DevicePersistent/Base/Base.C
@@ -4,7 +4,7 @@
 
 #include "Base.H"
 
-#include "../../MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H"
+#include "../../MatrixWrapper/Distributed/Distributed.H"
 
 namespace Foam {
 
diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index ce8898cff..3aacb051d 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -9,8 +9,8 @@
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "DevicePersistent/Partition/Partition.H"
 
-#include "MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H"
-#include "MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H"
+#include "MatrixWrapper/Combination/Combination.H"
+#include "MatrixWrapper/Distributed/Distributed.H"
 
 #include "fvCFD.H"
 
diff --git a/MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H b/MatrixWrapper/Combination/Combination.H
similarity index 100%
rename from MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H
rename to MatrixWrapper/Combination/Combination.H
diff --git a/MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H b/MatrixWrapper/Distributed/Distributed.H
similarity index 100%
rename from MatrixWrapper/RepartDistMatrix/RepartDistMatrix.H
rename to MatrixWrapper/Distributed/Distributed.H
diff --git a/common/common.C b/common/common.C
index e0234043d..9f7ec3650 100644
--- a/common/common.C
+++ b/common/common.C
@@ -10,7 +10,7 @@
 #include <filesystem>
 #include "common.H"
 
-#include "MatrixWrapper/GkoCombinationMatrix/GkoCombinationMatrix.H"
+#include "MatrixWrapper/Combination/Combination.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 

From 4af6f25e7ad8c9397220d57bcbab3bd4c2011f2c Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 15 Apr 2024 11:41:51 +0200
Subject: [PATCH 35/90] fixup! path

---
 DevicePersistent/Base/Base.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DevicePersistent/Base/Base.C b/DevicePersistent/Base/Base.C
index 78052dffa..1332b88c6 100644
--- a/DevicePersistent/Base/Base.C
+++ b/DevicePersistent/Base/Base.C
@@ -4,7 +4,7 @@
 
 #include "Base.H"
 
-#include "../../MatrixWrapper/Distributed/Distributed.H"
+#include "MatrixWrapper/Distributed/Distributed.H"
 
 namespace Foam {
 

From ffa80372828947e7eed5e6ef9cbd05e7c5712868 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 15 Apr 2024 21:54:38 +0200
Subject: [PATCH 36/90] WIP refactor CSRMatrix

---
 CMakeLists.txt                                |  67 ++++---
 .../LDUMatrixAdapter.C}                       |   0
 .../LDUMatrixAdapter/LDUMatrixAdapter.H       | 138 ++++++++++++++
 .../{HostMatrix => LDUMatrix}/HostMatrix.C    | 178 +++---------------
 .../{HostMatrix => LDUMatrix}/HostMatrix.H    | 117 +++++-------
 .../HostMatrixFreeFunctions.C                 |   0
 lduLduBase/lduLduBase.H                       |  52 +++--
 7 files changed, 283 insertions(+), 269 deletions(-)
 rename DevicePersistent/{CsrMatrixWrapper/CsrMatrixWrapper.C => LDUMatrixAdapter/LDUMatrixAdapter.C} (100%)
 create mode 100644 DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H
 rename MatrixWrapper/{HostMatrix => LDUMatrix}/HostMatrix.C (78%)
 rename MatrixWrapper/{HostMatrix => LDUMatrix}/HostMatrix.H (85%)
 rename MatrixWrapper/{HostMatrix => LDUMatrix}/HostMatrixFreeFunctions.C (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 84e3d4100..71c0bf903 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,10 +48,18 @@ option(GINKGO_BUILD_HIP "Build Ginkgo with hip backend" FALSE)
 option(GINKGO_BUILD_SYCL "Build Ginkgo with sycl backend" FALSE)
 option(GINKGO_BUILD_OMP "Build Ginkgo with omp backend" FALSE)
 
-set(GINKGO_BUILD_MPI ON CACHE INTERNAL "")
-set(GINKGO_BUILD_TESTS OFF CACHE INTERNAL "")
-set(GINKGO_BUILD_EXAMPLES OFF CACHE INTERNAL "")
-set(GINKGO_BUILD_BENCHMARKS OFF CACHE INTERNAL "")
+set(GINKGO_BUILD_MPI
+    ON
+    CACHE INTERNAL "")
+set(GINKGO_BUILD_TESTS
+    OFF
+    CACHE INTERNAL "")
+set(GINKGO_BUILD_EXAMPLES
+    OFF
+    CACHE INTERNAL "")
+set(GINKGO_BUILD_BENCHMARKS
+    OFF
+    CACHE INTERNAL "")
 
 if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO))
   if((NOT GINKGO_BUILD_CUDA)
@@ -76,12 +84,10 @@ include(cmake/build_helpers.cmake)
 include(cmake/install_helpers.cmake)
 include(cmake/Sanitizer.cmake)
 
-
 ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.7.0)
 
 add_subdirectory(third_party)
 
-
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0 -ggdb")
 set(CMAKE_CXX_FLAGS_RELEASE
     "${CMAKE_CXX_FLAGS_RELEASE} \
@@ -92,8 +98,7 @@ if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
   message("Compiling with clang")
   set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-  set(CMAKE_CXX_FLAGS_DEBUG
-      "${CMAKE_CXX_FLAGS_DEBUG} \
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} \
     -fno-omit-frame-pointer")
 
   set(CMAKE_CXX_FLAGS_RELEASE
@@ -160,12 +165,12 @@ target_sources(
           DevicePersistent/Vector/Vector.C
           DevicePersistent/DeviceIdGuard/DeviceIdGuard.C
           DevicePersistent/ExecutorHandler/ExecutorHandler.C
-          DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.C
+          DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.C
           Preconditioner/Preconditioner.C
           BaseWrapper/lduBase/GKOlduBase.C
           BaseWrapper/CoupledLduBase/GKOCoupledLduBase.C
-          MatrixWrapper/HostMatrix/HostMatrix.C
-          MatrixWrapper/HostMatrix/HostMatrixFreeFunctions.C
+          MatrixWrapper/LDUMatrix/HostMatrix.C
+          MatrixWrapper/LDUMatrix/HostMatrixFreeFunctions.C
           Solver/CG/GKOCG.C
           Solver/BiCGStab/GKOBiCGStab.C
           # Solver / IR / GKOIR.C Solver / Multigrid / GKOMultigrid.C
@@ -174,7 +179,7 @@ target_sources(
   PUBLIC common/common.H
          StoppingCriterion/StoppingCriterion.H
          lduLduBase/lduLduBase.H
-         MatrixWrapper/HostMatrix/HostMatrix.H
+         MatrixWrapper/LDUMatrix/HostMatrix.H
          MatrixWrapper/Combination/Combination.H
          DevicePersistent/Base/Base.H
          DevicePersistent/Partition/Partition.H
@@ -182,7 +187,7 @@ target_sources(
          DevicePersistent/Vector/Vector.H
          DevicePersistent/ExecutorHandler/ExecutorHandler.H
          DevicePersistent/DeviceIdGuard/DeviceIdGuard.H
-         DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+         DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H
          Preconditioner/Preconditioner.H
          BaseWrapper/lduBase/GKOlduBase.H
          BaseWrapper/CoupledLduBase/GKOCoupledLduBase.H
@@ -194,15 +199,10 @@ target_sources(
   # LduMatrix / GKOACG / GKOACG.H
 )
 
-
 enable_sanitizers(
-    OGL
-    ${OGL_ENABLE_SANITIZE_ADDRESS}
-    ${OGL_ENABLE_SANITIZE_LEAK}
-    ${OGL_ENABLE_SANITIZE_UB}
-    ${OGL_ENABLE_SANITIZE_THREAD}
-    ${OGL_ENABLE_SANITIZE_MEMORY}
-)
+  OGL ${OGL_ENABLE_SANITIZE_ADDRESS} ${OGL_ENABLE_SANITIZE_LEAK}
+  ${OGL_ENABLE_SANITIZE_UB} ${OGL_ENABLE_SANITIZE_THREAD}
+  ${OGL_ENABLE_SANITIZE_MEMORY})
 
 target_include_directories(
   OGL SYSTEM
@@ -213,9 +213,8 @@ target_include_directories(
 target_link_libraries(
   OGL
   PRIVATE $ENV{FOAM_LIBBIN}/libOpenFOAM.so $ENV{FOAM_LIBBIN}/libfiniteVolume.so
-          $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so Ginkgo::ginkgo stdc++fs)
-
-
+          $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so Ginkgo::ginkgo
+          stdc++fs)
 
 if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)
@@ -226,7 +225,6 @@ if(${GINKGO_BUILD_CUDA})
   target_link_libraries(OGL PUBLIC CUDA::nvToolsExt)
 endif()
 
-
 if(OGL_DATA_VALIDATION)
   target_compile_definitions(OGL PRIVATE DATA_VALIDATION=1)
 endif()
@@ -248,12 +246,13 @@ add_custom_target(
   COMMAND find . -name CMakeLists.txt -exec cmake-format -i {} +
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
 
-install(TARGETS OGL
-    ginkgo
-    ginkgo_device
-    ginkgo_hip
-    ginkgo_cuda
-    ginkgo_omp
-    ginkgo_dpcpp
-    ginkgo_reference
-    DESTINATION $ENV{FOAM_USER_LIBBIN})
+install(
+  TARGETS OGL
+          ginkgo
+          ginkgo_device
+          ginkgo_hip
+          ginkgo_cuda
+          ginkgo_omp
+          ginkgo_dpcpp
+          ginkgo_reference
+  DESTINATION $ENV{FOAM_USER_LIBBIN})
diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.C b/DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.C
similarity index 100%
rename from DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.C
rename to DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.C
diff --git a/DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H b/DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H
new file mode 100644
index 000000000..008a4354b
--- /dev/null
+++ b/DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H
@@ -0,0 +1,138 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::GKOCG
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    GKOCG.C
+
+\*---------------------------------------------------------------------------*/
+#ifndef OGL_IOGKOMatrixHandler_INCLUDED_H
+#define OGL_IOGKOMatrixHandler_INCLUDED_H
+
+
+#include "DevicePersistent/Array/Array.H"
+#include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
+#include "DevicePersistent/Partition/Partition.H"
+
+#include "MatrixWrapper/Combination/Combination.H"
+#include "MatrixWrapper/Distributed/Distributed.H"
+
+#include "fvCFD.H"
+
+namespace Foam {
+
+struct lduMatrixInitFunctor {
+
+    const objectRegistry &db_;
+
+    const ExecutorHandler &exec_;
+
+    const lduMatrix & matrix_;
+
+    const FieldField<Field, scalar> &interfaceBouCoeffs_;
+
+    const FieldField<Field, scalar> &interfaceIntCoeffs_;
+
+    const lduInterfaceFieldPtrsList &interfaces_;
+
+    MatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
+                      const lduMatrix & matrix,
+                      const word field_name, const label verbose,
+                      const FieldField<Field, scalar> &interfaceBouCoeffs,
+                      const FieldField<Field, scalar> &interfaceIntCoeffs,
+                      const lduInterfaceFieldPtrsList &interfaces)
+        : db_(db),
+          exec_(exec),
+          matrix_(matrix),
+          field_name_(field_name) verbose_(verbose),
+          interfaceBouCoeffs_(interfaceBouCoeffs),
+        interfaceIntCoeffs_(interfaceIntCoeffs),
+        interfaces_(interfaces)
+    {}
+
+    void update(std::shared_ptr<LduMatrix> persistent_device_matrix) const
+    {
+        persistent_device_matrix.update(exec_, interfaceBouCoeffs_,
+                                        interfaceIntCoeffs_, interfaces_);
+    }
+
+    std::shared_ptr<LduMatrix> init() const
+    {
+        return LduMatrix::create(
+          exec_,
+          matrix_.diag().size(),
+          matrix_.lduAddr().upperAddr().size(),
+          matrix_.symmetric(),
+          matrix_.diag().begin(),
+          matrix_.upper().begin(),
+          matrix_.lower().begin(),
+          matrix_.lduAddr().upperAddr().begin(),
+          matrix_.lduAddr().lowerAddr().begin(),
+          interfaceBouCoeffs_,
+          interfaceIntCoeffs_,
+          interfaces_);
+    }
+};
+
+
+/* This Class makes the Host LDUMatrix device persistent. It basically only
+ * provides a functor to call LduMatrix::create and LduMatrix::update
+ */
+class MatrixWrapper {
+private:
+    const objectRegistry &db_;
+
+    const label verbose_;
+
+    const word field_name_;
+
+    mutable PersistentBase<LDUMatrix, MatrixInitFunctor> host_matrix_;
+
+
+public:
+    MatrixWrapper(const objectRegistry &db, const PersistentExecutor &exec,
+                  const dictionary &controlDict, const word sys_matrix_name,
+                  const label verbose,
+                  const FieldField<Field, scalar> &interfaceBouCoeffs,
+                  const FieldField<Field, scalar> &interfaceIntCoeffs,
+                  const lduInterfaceFieldPtrsList &interfaces)
+        : db_(db),
+          verbose_(verbose),
+          export_(controlDict.lookupOrDefault<Switch>("export", false)),
+          field_name_(sys_matrix_name),
+          host_matrix_{
+              sys_matrix_name + "_host_matrix", db,
+              MatrixInitFunctor(
+                  db, exec, verbose_, sys_matrix_name + "_host_matrix",
+                  interfaceBouCoeffs, interfaceIntCoeffs, interfaces),
+              controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
+              verbose_}
+    {}
+
+    std::shared_ptr<gko::LinOp> get() const
+    {
+        return gkomatrix_.get_persistent_object();
+    }
+};
+
+}  // namespace Foam
+
+#endif
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
similarity index 78%
rename from MatrixWrapper/HostMatrix/HostMatrix.C
rename to MatrixWrapper/LDUMatrix/HostMatrix.C
index fac414169..123aff0bd 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -33,150 +33,40 @@ SourceFiles
 
 namespace Foam {
 
-template <class MatrixType>
+
 HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
-    const objectRegistry &db, const MatrixType &matrix,
-    // coeffs for cells on boundaries
+    label nrows, label upper_nnz, bool symmetric, const scalar *diag,
+    const scalar *upper, const scalar *lower, const label *lowerAddr,
+    const label *upperAddr, const objectRegistry &db,
     const FieldField<Field, scalar> &interfaceBouCoeffs,
-    // coeffs for internal cells
     const FieldField<Field, scalar> &interfaceIntCoeffs,
-    // pointers to interfaces can be used to access concrete
-    // functions such as transferring indices, patch neighbours etc
     const lduInterfaceFieldPtrsList &interfaces,
-    const dictionary &solverControls, const word &fieldName)
-    : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
-                         interfaceIntCoeffs, interfaces, solverControls),
-      exec_{db, solverControls, fieldName},
-      device_id_guard_{db, fieldName, exec_.get_device_exec()},
-      verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
-      reorder_on_copy_(
-          solverControls.lookupOrDefault<Switch>("reorderOnHost", false)),
-      scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
-      nrows_(matrix.diag().size()),
-      local_interface_nnz_(count_interface_nnz(interfaces, false)),
-      upper_nnz_(matrix.lduAddr().upperAddr().size()),
-      non_diag_nnz_(2 * upper_nnz_),
-      local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
-      local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
-      local_sparsity_{
-          fieldName + "_local",           db,       exec_,
-          local_matrix_w_interfaces_nnz_, verbose_,
-      },
-      local_coeffs_{
-          fieldName + "_local_coeffs",
-          db,
-          exec_,
-          local_matrix_w_interfaces_nnz_,
-          verbose_,
-          true,              // needs to be updated
-          !reorder_on_copy_  // whether to init on device
-                             // if reorderOnHost is selected, the array needs
-                             // to be initialized on the host
-      },
-      non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
-      communication_pattern_(create_communication_pattern(interfaces)),
-      non_local_sparsity_{
-          fieldName + "_non_local", db, exec_, non_local_matrix_nnz_, verbose_,
-      },
-      non_local_coeffs_{
-          fieldName + "_non_local_coeffs",
-          db,
-          exec_,
-          non_local_matrix_nnz_,
-          verbose_,
-          true,  // needs to be updated
-          false  // leave it on host once it is turned into a distributed
-                 // matrix it will be put on the device
-      },
-      permutation_matrix_name_{"PermutationMatrix"},
-      permutation_stored_{
-          db.template foundObject<regIOobject>(permutation_matrix_name_)},
-      P_{(permutation_stored_)
-             ? db.template lookupObjectRef<DevicePersistentBase<gko::LinOp>>(
-                     permutation_matrix_name_)
-                   .get_ptr()
-             : nullptr}
-{
-    if (!local_sparsity_.col_idxs_.get_stored() ||
-        local_sparsity_.col_idxs_.get_update()) {
-        TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
-                            this->fieldName(),
-                            init_local_sparsity_pattern(interfaces);)
-        TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
-                            this->fieldName(),
-                            init_non_local_sparsity_pattern(interfaces);)
-    }
-    if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
-        TIME_WITH_FIELDNAME(
-            verbose_, update_local_matrix_data, this->fieldName(),
-            update_local_matrix_data(interfaces, interfaceBouCoeffs);)
-        TIME_WITH_FIELDNAME(
-            verbose_, update_non_local_matrix_data, this->fieldName(),
-            update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
-    }
-}
-
-
-template <class MatrixType>
-HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
-    const objectRegistry &db, const MatrixType &matrix,
-    const dictionary &solverControls, const word &fieldName)
-    : MatrixType::solver(fieldName, matrix, solverControls),
-      exec_{db, solverControls, fieldName},
+    const dictionary &solverControls, const word &fieldName, label verbose)
+    : exec_{db, solverControls, fieldName},
       device_id_guard_{db, fieldName, exec_.get_device_exec()},
-      verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      verbose_(verbose),
+      fieldName_(fieldName),
       reorder_on_copy_(
           solverControls.lookupOrDefault<Switch>("reorderOnHost", true)),
       scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
-      nrows_(matrix.diag().size()),
+      nrows_(nrows),
       local_interface_nnz_(0),
-      upper_nnz_(matrix.lduAddr().upperAddr().size()),
+      upper_nnz_(upper_nnz),
       non_diag_nnz_(2 * upper_nnz_),
       local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
       local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
-      local_sparsity_{
-          fieldName + "_cols", db, exec_, local_matrix_nnz_, verbose_,
-      },
-      local_coeffs_{
-          fieldName + "_coeffs",
-          db,
-          exec_,
-          local_matrix_nnz_,
-          verbose_,
-          true,              // needs to be updated
-          !reorder_on_copy_  // whether to init on device
-                             // if reorderOnHost is selected, the array needs
-                             // to be initialized on the host
-      },
-      non_local_matrix_nnz_(),
-      // proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
-      non_local_sparsity_{
-          fieldName + "_non_local", db, exec_, non_local_matrix_nnz_, verbose_,
-      },
-      non_local_coeffs_{
-          fieldName + "_non_local_coeffs",
-          db,
-          exec_,
-          non_local_matrix_nnz_,
-          verbose_,
-          true,  // needs to be updated
-          false  // leave it on host once it is turned into a distributed
-                 // matrix it will be put on the device
-      },
-      permutation_matrix_name_{"PermutationMatrix"},
-      permutation_stored_{
-          db.template foundObject<regIOobject>(permutation_matrix_name_)},
-      P_{(permutation_stored_)
-             ? db.template lookupObjectRef<DevicePersistentBase<gko::LinOp>>(
-                     permutation_matrix_name_)
-                   .get_ptr()
-             : nullptr}
+      local_sparsity_{exec_.get_host_exec(), local_matrix_nnz_},
+      non_local_matrix_nnz_{count_interface_nnz(interfaces, true)},
+      non_local_sparsity_{exec_.get_host_exec(), non_local_matrix_nnz_}
 {
-    FatalErrorInFunction << "This constructor is currently not implemented"
-                         << abort(FatalError);
+    TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
+                        fieldName_,
+                        init_local_sparsity_pattern(interfaces);)
+    TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
+                        fieldName_,
+                        init_non_local_sparsity_pattern(interfaces);)
 }
 
-template <class MatrixType>
 label HostMatrixWrapper<MatrixType>::count_interface_nnz(
     const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
 {
@@ -198,7 +88,6 @@ label HostMatrixWrapper<MatrixType>::count_interface_nnz(
     return ctr;
 }
 
-template <class MatrixType>
 std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs, const bool local) const
@@ -271,7 +160,6 @@ void neg_interface_iterator(const lduInterfaceFieldPtrsList &interfaces,
 }
 
 
-template <class MatrixType>
 CommunicationPattern
 HostMatrixWrapper<MatrixType>::create_communication_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
@@ -402,7 +290,6 @@ void collect_local_interface_indices_impl_cyclicACMIFvPatch(
 }
 #endif
 
-template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
     const lduInterfaceFieldPtrsList &interfaces) const
@@ -429,7 +316,6 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
     return local_interface_idxs;
 }
 
-template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_cells_on_non_local_interface(
     const lduInterfaceFieldPtrsList &interfaces) const
@@ -456,8 +342,7 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_non_local_interface(
     return non_local_idxs;
 }
 
-template <class MatrixType>
-void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
+void HostMatrixWrapper::init_non_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     auto non_local_row_indices =
@@ -477,12 +362,10 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     }
 }
 
-template <class MatrixType>
-void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
+void HostMatrixWrapper::init_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     LOG_1(verbose_, "start init host sparsity pattern")
-    bool is_symmetric{this->matrix().symmetric()};
 
     auto lower_local = idx_array::view(
         exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
@@ -509,8 +392,8 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     // TODO in order to simplify when local interfaces exists set
     // local_sparsity to size of nrows_w_interfaces, if interfaces exist
     // local_sparsity is only valid till nrows_
-    label after_neighbours = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
-    init_local_sparsity(nrows_, upper_nnz_, is_symmetric, upper, lower, rows,
+    label after_neighbours = (symmetric_) ? upper_nnz_ : 2 * upper_nnz_;
+    init_local_sparsity(nrows_, upper_nnz_, symmetric_, upper, lower, rows,
                         cols, permute);
 
     // if no local interfaces are present we are done here
@@ -539,12 +422,11 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
             local_interface_ctr++;
         }
     }
-    LOG_1(verbose_, "done init host matrix")
+    LOG_1(verbose_, "done init host sparsity pattern")
 }
 
 
-template <class MatrixType>
-void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
+void HostMatrixWrapper::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs) const
 {
@@ -554,7 +436,6 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     auto diag = this->matrix().diag();
 
     label diag_nnz = diag.size();
-    bool is_symmetric{this->matrix().symmetric()};
 
     // TODO this does not work for Ell
     auto permute = local_sparsity_.ldu_mapping_.get_data();
@@ -564,7 +445,7 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
         if (local_interface_nnz_) {
             auto couple_coeffs =
                 collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
-            if (is_symmetric) {
+            if (symmetric_) {
                 symmetric_update_w_interface(
                     local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
                     permute, scaling_, diag.data(), upper.data(),
@@ -577,7 +458,7 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
                     couple_coeffs.data(), dense);
             }
         } else {
-            if (is_symmetric) {
+            if (symmetric_) {
                 symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
                                  scaling_, diag.data(), upper.data(), dense);
             } else {
@@ -691,8 +572,7 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
 }
 
 
-template <class MatrixType>
-void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
+void HostMatrixWrapper::update_non_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs) const
 {
@@ -745,7 +625,7 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     }
 }
 
-template HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(
+template HostMatrixWrapper::HostMatrixWrapper(
     const objectRegistry &db, const lduMatrix &matrix,
     // coeffs for cells on boundaries
     const FieldField<Field, scalar> &interfaceBouCoeffs,
diff --git a/MatrixWrapper/HostMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
similarity index 85%
rename from MatrixWrapper/HostMatrix/HostMatrix.H
rename to MatrixWrapper/LDUMatrix/HostMatrix.H
index 2a8fad2ca..6738857d0 100644
--- a/MatrixWrapper/HostMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -33,62 +33,46 @@ SourceFiles
 #include "DevicePersistent/Array/Array.H"
 #include "DevicePersistent/DeviceIdGuard/DeviceIdGuard.H"
 #include "DevicePersistent/IOGlobalIndex/gkoGlobalIndex.H"
+#include "MatrixWrapper/Combination/Combination.H"
 #include "common/common.H"
 
 
 namespace Foam {
 
 
-struct PersistentSparsityPattern {
-    PersistentSparsityPattern(const word &fieldName, const objectRegistry &db,
-                              const ExecutorHandler &exec, const label size,
-                              const label verbose)
+struct SparsityPattern {
+    SparsityPattern(const ExecutorHandler &exec, const label size)
         : size_(size),
           col_idxs_{
-              fieldName + "_cols",
-              db,
-              exec,
-              size,
-              verbose,
-              false,  // For now we assume columns and rows to be constant
-              false   // we don't init on device since the indices are written
-                      // using the pointer to the data
+              exec.get_host_executor(),
+              size
           },
           row_idxs_{
-              fieldName + "_rows",
-              db,
-              exec,
-              size,
-              verbose,
-              false,  // For now we assume columns and rows to be constant
-              false   // same as for cols don't init on device
+              exec.get_host_executor(),
+              size
           },
           ldu_mapping_{
-              fieldName + "_ldu_map",
-              db,
-              exec,
-              size,
-              verbose,
-              false,
-              false  // same as for rows and cols dont init on device
+              exec.get_host_executor(),
+              size
           }
     {}
 
     // number of nnz entries
     const label size_;
 
-    mutable PersistentArray<label> col_idxs_;
+    mutable gko::Array<label> col_idxs_;
 
-    mutable PersistentArray<label> row_idxs_;
+    mutable gko::Array<label> row_idxs_;
 
     // ldu_mapping[sorted_(csr)_position] =
     // unsorted_(consecutive_ldu_)_position
-    mutable PersistentArray<label> ldu_mapping_;
+    mutable gko::Array<label> ldu_mapping_;
 
     // dimensions of the resulting matrix
     mutable gko::dim<2> dim_;
 
     // A vector of spans indicating begin and end of each interface
+    // this is used to keep all col and row idx consecutive in memory
     mutable std::vector<gko::span> interface_spans_;
 };
 
@@ -243,12 +227,14 @@ void non_symmetric_update_w_interface(
  * format into Ginkgo array data structures
  *
  * It mainly generates:
- * - sparsity pattern/device_matrix_data of the local_matrix and
- * - non_local_matrix (with global columns)
+ * - sparsity pattern of the local_matrix and
+ *   non_local_matrix (with global columns)
  * - CommunicationPattern
+ * - implements an interfaces so that
+ *   RepartDistMatrix can read_distributed(A_host)
  * */
-template <class MatrixType>
-class HostMatrixWrapper : public MatrixType::solver {
+class HostMatrixWrapper
+{
 private:
     using vec = gko::matrix::Dense<scalar>;
     using idx_array = gko::array<label>;
@@ -259,30 +245,33 @@ private:
 
     const label verbose_;
 
+    const word fieldName_;
+
     // Whether the matrix coefficients should be reordered
     // during copy or on device
-    // NOTE Not implemented
     const bool reorder_on_copy_;
 
     // multiply the complete system by this factor, ie sAx=sb
-    // NOTE this could be also achieved by just fliping the sign
+    // NOTE this needed to avoid negative diagonal matrix entries, but
+    // this could be also achieved by just fliping the sign
     const scalar scaling_;
 
     // number of local matrix rows
     const label nrows_;
 
+    // number of local upper elements
+    // ie coefficients which column_idx < nrows_
+    const label upper_nnz_;
+
+    const bool symmetric_;
+
     // number of local elements on interfaces
     // ie number of interfaces which column_idx is < nrows_
     // and have to be sorted into local matrix
     const label local_interface_nnz_;
 
-    // number of local upper elements
-    // ie coefficients which column_idx < nrows_
-    const label upper_nnz_;
-
     // total number of local upper and lower elements
     // ie 2*upper_nnz_ since the sparsity pattern is symmetric
-    // TODO should this include the local_interface_nnz_ ?
     const label non_diag_nnz_;
 
     // nnz of local matrix wo local interfaces
@@ -291,27 +280,14 @@ private:
     // nnzs of local matrix including local interfaces
     const label local_matrix_w_interfaces_nnz_;
 
-    mutable PersistentSparsityPattern local_sparsity_;
-
-    // matrix coefficients
-    mutable PersistentArray<scalar> local_coeffs_;
+    mutable SparsityPattern local_sparsity_pattern_;
 
     // non-local indices
     const label non_local_matrix_nnz_;
 
-    const CommunicationPattern communication_pattern_;
-
     mutable PersistentSparsityPattern non_local_sparsity_;
 
-    mutable PersistentArray<scalar> non_local_coeffs_;
-
-    // FIXME TODO unused
-    const word permutation_matrix_name_;
-
-    const bool permutation_stored_;
-
-    mutable std::shared_ptr<gko::LinOp> P_;
-
+    const CommunicationPattern communication_pattern_;
 
     /* Iterates all interfaces and collects the coefficients into a vector
     **
@@ -402,28 +378,29 @@ private:
         const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
 public:
-    // segregated wrapper constructor
-    // TODO - compute local_interface_nnzs these are matrix coefficient
-    //      that are on an interface but not on a different rank
-    //      - then a new array/vector is needed to store local_interfaces
-    //      - if local interfaces are present init_local_sparsity_pattern
-    //      needs to be adapted
-    //      - local_sparsity_.ldu_mapping needs a new section for interfaces
+
+    // segregated matrix wrapper constructor
     HostMatrixWrapper(
-        const objectRegistry &db, const MatrixType &matrix,
-        // coeffs for cells on boundaries
+        label nrows,
+        label upper_nnz,
+        bool symmetric,
+        const scalar* diag,
+        const scalar* upper,
+        const scalar* lower,
+        const label* lowerAddr,
+        const label* upperAddr,
+        const objectRegistry &db,
         const FieldField<Field, scalar> &interfaceBouCoeffs,
-        // coeffs for internal cells
         const FieldField<Field, scalar> &interfaceIntCoeffs,
-        // pointers to interfaces can be used to access concrete
-        // functions such as transferring indices, patch neighbours etc
         const lduInterfaceFieldPtrsList &interfaces,
-        const dictionary &solverControls, const word &fieldName);
+        const dictionary &solverControls,
+        const word &fieldName,
+        label verbose);
 
-    // TODO reimplement this at some point
     // coupled wrapper constructor
     HostMatrixWrapper(const objectRegistry &db, const MatrixType &matrix,
-                      const dictionary &solverControls, const word &fieldName);
+                      const dictionary &solverControls, const word &fieldName)
+        {OGL_NOT_IMPLEMENTED};
 
 
     // getter
diff --git a/MatrixWrapper/HostMatrix/HostMatrixFreeFunctions.C b/MatrixWrapper/LDUMatrix/HostMatrixFreeFunctions.C
similarity index 100%
rename from MatrixWrapper/HostMatrix/HostMatrixFreeFunctions.C
rename to MatrixWrapper/LDUMatrix/HostMatrixFreeFunctions.C
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 508cefde3..68688622f 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -117,7 +117,6 @@ public:
 template <class MatrixType, class SolverFactory>
 class lduLduBase : public OGL_Info,
                    public SolverFactory,
-                   public HostMatrixWrapper<MatrixType>,
                    public Preconditioner {
 private:
     using dist_vec = gko::experimental::distributed::Vector<scalar>;
@@ -132,10 +131,19 @@ private:
 
     const objectRegistry &db_;
 
+    const MatrixType& matrix_;
+
+    const FieldField<Field, scalar> &interfaceBouCoeffs_,
+
+    const FieldField<Field, scalar> &interfaceIntCoeffs_,
+
+    const lduInterfaceFieldPtrsList &interfaces_,
+
     const dictionary &solver_controls_;
 
     const label verbose_;
 
+
 public:
     // constructor for segregated solver
     lduLduBase(const word &fieldName, const lduMatrix &matrix,
@@ -145,16 +153,13 @@ public:
                const dictionary &solverControls)
         : OGL_Info(fieldName, solverControls, matrix.mesh().thisDb()),
           SolverFactory{solverControls, matrix.mesh().thisDb(), fieldName},
-          HostMatrixWrapper<MatrixType>{matrix.mesh().thisDb(),
-                                        matrix,
-                                        interfaceBouCoeffs,
-                                        interfaceIntCoeffs,
-                                        interfaces,
-                                        solverControls,
-                                        fieldName},
           Preconditioner(fieldName, matrix.mesh().thisDb(), solverControls,
                          this->get_verbose()),
           db_(matrix.mesh().thisDb()),
+          matrix_(matrix),
+          interfaceBouCoeffs_(interfaceBouCoeffs),
+          interfaceIntCoeffs_(interfaceIntCoeffs),
+          interfaces_(interfaces),
           solver_controls_(solverControls),
           verbose_(solverControls.lookupOrDefault<label>("verbose", 0))
     {}
@@ -168,11 +173,15 @@ public:
                const dictionary &solverControls)
         : OGL_Info(fieldName, solverControls, matrix.mesh().thisDb()),
           SolverFactory{solverControls, matrix.mesh().thisDb(), fieldName},
-          HostMatrixWrapper<MatrixType>(matrix.mesh().thisDb(), matrix,
-                                        solverControls, fieldName),
+          // HostMatrixWrapper<MatrixType>(matrix.mesh().thisDb(), matrix,
+          //                               solverControls, fieldName),
           Preconditioner(fieldName, matrix.mesh().thisDb(), solverControls,
                          this->get_verbose()),
           db_(matrix.mesh().thisDb()),
+          matrix_(matrix),
+          interfaceBouCoeffs_(),
+          interfaceIntCoeffs_(),
+          interfaces_(),
           solver_controls_(solverControls),
           verbose_(solverControls.lookupOrDefault<label>("verbose", 0))
     {}
@@ -192,6 +201,19 @@ public:
     {
         auto ref_exec = this->get_exec_handler().get_ref_exec();
 
+        // using a thin wrapper around OpenFOAMs LDU matrix class
+        PersistentLDUMatrix host_A{
+          db_,
+          this->get_exec_handler(),
+          solver_controls_
+          this->fieldName(),
+          this->get_verbose(),
+          matrix_,
+          interfaceBouCoeffs_,
+          interfaceIntCoeffs_,
+          interfaces_,
+       };
+
         PersistentPartition partition{
             db_,
             this->get_exec_handler(),
@@ -200,13 +222,10 @@ public:
             psi.size(),  // number of elements
             this->get_communication_pattern()};
 
-        // TODO this is now the local matrix part for dist_A
+        // create a distributed matrix from the host matrix
+        // here also repartitioning is performed if required
         MatrixWrapper dist_A{db_,
-                             this->get_exec_handler(),  // get exec
-                             this->get_local_sparsity_pattern(),
-                             this->get_local_coeffs(),
-                             this->get_non_local_sparsity_pattern(),
-                             this->get_non_local_coeffs(),
+                            host_A,
                              partition,
                              solver_controls_,
                              this->fieldName(),
@@ -233,6 +252,7 @@ public:
             solver_controls_.lookupOrDefault<Switch>("updateInitGuess", false),
             false  // whether data for init is on device
         };
+
         auto dist_x_v = dist_x.get_vector();
         auto dist_b_v = dist_b.get_vector();
         auto dist_A_v = dist_A.get();

From f8644946fef8ccf23be7372797adfb2255267da7 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 16 Apr 2024 16:13:52 +0200
Subject: [PATCH 37/90] WIP refactor CSRMatrix

---
 CMakeLists.txt                                |   4 +-
 .../DistributedMatrixAdapter.C}               |   3 +-
 .../DistributedMatrixAdapter.H                | 136 ++++++++++++++++
 .../ExecutorHandler/ExecutorHandler.H         |   2 -
 .../HostMatrixAdapter/HostMatrixAdapter.C     |  32 ++++
 .../HostMatrixAdapter/HostMatrixAdapter.H     | 147 ++++++++++++++++++
 .../LDUMatrixAdapter/LDUMatrixAdapter.H       | 138 ----------------
 DevicePersistent/Partition/Partition.H        |   2 +-
 MatrixWrapper/Combination/Combination.H       |   3 +-
 .../CommunicationPattern.H                    |  42 +++++
 MatrixWrapper/Distributed/Distributed.H       |  15 ++
 MatrixWrapper/LDUMatrix/HostMatrix.C          | 134 ++++++++--------
 MatrixWrapper/LDUMatrix/HostMatrix.H          | 138 ++++++++--------
 lduLduBase/lduLduBase.H                       | 109 ++++++-------
 14 files changed, 563 insertions(+), 342 deletions(-)
 rename DevicePersistent/{LDUMatrixAdapter/LDUMatrixAdapter.C => DistributedMatrixAdapter/DistributedMatrixAdapter.C} (85%)
 create mode 100644 DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
 create mode 100644 DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.C
 create mode 100644 DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
 delete mode 100644 DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H
 create mode 100644 MatrixWrapper/CommunicationPattern/CommunicationPattern.H

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 71c0bf903..17ce88027 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -165,7 +165,7 @@ target_sources(
           DevicePersistent/Vector/Vector.C
           DevicePersistent/DeviceIdGuard/DeviceIdGuard.C
           DevicePersistent/ExecutorHandler/ExecutorHandler.C
-          DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.C
+          DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.C
           Preconditioner/Preconditioner.C
           BaseWrapper/lduBase/GKOlduBase.C
           BaseWrapper/CoupledLduBase/GKOCoupledLduBase.C
@@ -187,7 +187,7 @@ target_sources(
          DevicePersistent/Vector/Vector.H
          DevicePersistent/ExecutorHandler/ExecutorHandler.H
          DevicePersistent/DeviceIdGuard/DeviceIdGuard.H
-         DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H
+         DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
          Preconditioner/Preconditioner.H
          BaseWrapper/lduBase/GKOlduBase.H
          BaseWrapper/CoupledLduBase/GKOCoupledLduBase.H
diff --git a/DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.C b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.C
similarity index 85%
rename from DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.C
rename to DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.C
index 940b457be..7ca685d33 100644
--- a/DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.C
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.C
@@ -3,7 +3,8 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include <ginkgo/ginkgo.hpp>
-#include "CsrMatrixWrapper.H"
+
+#include "HostMatrixAdapter.H"
 #include "common/common.H"
 
 namespace Foam {}  // namespace Foam
diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
new file mode 100644
index 000000000..23c5c8a82
--- /dev/null
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -0,0 +1,136 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::GKOCG
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    GKOCG.C
+
+\*---------------------------------------------------------------------------*/
+#pragma once
+
+#include "DevicePersistent/Array/Array.H"
+#include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
+#include "DevicePersistent/Partition/Partition.H"
+#include "MatrixWrapper/Distributed/Distributed.H"
+
+#include "fvCFD.H"
+
+namespace Foam {
+
+struct DistributedMatrixInitFunctor {
+    const objectRegistry &db_;
+
+    const ExecutorHandler &exec_;
+
+    const word field_name_;
+
+    const label verbose_;
+
+    const dictionary& solver_controls_;
+
+    const PersistentHostMatrixAdapter& host_A_;
+
+    const PersistentPartition& partition_;
+
+    DistributedMatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
+                         const label verbose,
+                         const word field_name,
+                         const PersistentHostMatrixAdapter& host_A,
+                         const PersistentPartition& partition,
+                         const dictionary& solver_controls
+                         )
+        : db_(db),
+          exec_(exec),
+          field_name_(field_name),
+          verbose_(verbose),
+          solver_controls_(solver_controls),
+          partition_(partition),
+          host_A_(host_A)
+    {}
+
+    void update(
+        std::shared_ptr<RepartDistMatrix<scalar, label, label>> persistent_device_matrix) const
+    {
+        // persistent_device_matrix->update(
+        //     matrix_.diag().begin(), matrix_.upper().begin(),
+        //     matrix_.lower().begin(), interfaceBouCoeffs_, interfaceIntCoeffs_,
+        //     interfaces_);
+    }
+
+    std::shared_ptr<RepartDistMatrix<scalar, label, label>> init() const
+    {
+        // return RepartDistMatrix<scalar, label, label>::create(
+        //     exec_, host_A_);
+    }
+};
+
+
+/* This Class makes the Host LDUMatrix device persistent. It basically only
+ * provides a functor to call HostMatrixWrapper::create and LduMatrix::update
+ */
+class PersistentDistributedMatrix {
+private:
+    const objectRegistry &db_;
+
+    const label verbose_;
+
+    const word field_name_;
+
+    mutable PersistentBase<RepartDistMatrix<scalar, label, label>, DistributedMatrixInitFunctor> host_matrix_;
+
+
+public:
+    PersistentDistributedMatrix(
+                  const objectRegistry &db,
+                  const ExecutorHandler &exec,
+                  const word field_name,
+                  const PersistentHostMatrixAdapter& host_A,
+                  const PersistentPartition& partition,
+                  const dictionary &controlDict,
+                  const label verbose
+                  )
+        : db_(db),
+          verbose_(verbose),
+          field_name_(field_name),
+          host_matrix_{
+              field_name + "_distributed_matrix", db,
+              DistributedMatrixInitFunctor(db, exec, verbose_,
+                                   field_name + "_distributed_matrix",
+                                   host_A, partition,
+                                   controlDict),
+              controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
+              verbose_}
+    {}
+
+    std::shared_ptr<RepartDistMatrix<scalar, label, label>> get() const
+    {
+        return host_matrix_.get_persistent_object();
+    }
+
+    void write() 
+    {
+        this->get()->write();
+    }
+
+    bool get_export();
+};
+
+}  // namespace Foam
diff --git a/DevicePersistent/ExecutorHandler/ExecutorHandler.H b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
index 6434df777..dd53fa2f2 100644
--- a/DevicePersistent/ExecutorHandler/ExecutorHandler.H
+++ b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
@@ -169,7 +169,5 @@ public:
     }
 };
 
-using PersistentExecutor = ExecutorHandler;
-
 }  // namespace Foam
 // namespace Foam
diff --git a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.C b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.C
new file mode 100644
index 000000000..dc9270932
--- /dev/null
+++ b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.C
@@ -0,0 +1,32 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::GKOCG
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    GKOCG.C
+
+\*---------------------------------------------------------------------------*/
+#include <ginkgo/ginkgo.hpp>
+
+#include "HostMatrixAdapter.H"
+#include "common/common.H"
+
+namespace Foam {}  // namespace Foam
diff --git a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
new file mode 100644
index 000000000..595b9bd95
--- /dev/null
+++ b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
@@ -0,0 +1,147 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::GKOCG
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    GKOCG.C
+
+\*---------------------------------------------------------------------------*/
+#pragma once
+
+#include "DevicePersistent/Array/Array.H"
+#include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
+#include "DevicePersistent/Partition/Partition.H"
+#include "MatrixWrapper/Combination/Combination.H"
+#include "MatrixWrapper/LDUMatrix/HostMatrix.H"
+
+#include "fvCFD.H"
+
+namespace Foam {
+
+struct lduMatrixInitFunctor {
+    const objectRegistry &db_;
+
+    const ExecutorHandler &exec_;
+
+    const word field_name_;
+
+    const label verbose_;
+
+    const lduMatrix &matrix_;
+
+    const FieldField<Field, scalar> &interfaceBouCoeffs_;
+
+    const FieldField<Field, scalar> &interfaceIntCoeffs_;
+
+    const lduInterfaceFieldPtrsList &interfaces_;
+
+    const dictionary &solver_controls_;
+
+    lduMatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
+                         const label verbose,
+                         const word field_name,
+                         const lduMatrix &matrix,
+                         const FieldField<Field, scalar> &interfaceBouCoeffs,
+                         const FieldField<Field, scalar> &interfaceIntCoeffs,
+                         const lduInterfaceFieldPtrsList &interfaces,
+                         const dictionary &solver_controls)
+        : db_(db),
+          exec_(exec),
+          matrix_(matrix),
+          field_name_(field_name),
+          verbose_(verbose),
+          interfaceBouCoeffs_(interfaceBouCoeffs),
+          interfaceIntCoeffs_(interfaceIntCoeffs),
+          interfaces_(interfaces),
+          solver_controls_(solver_controls)
+    {}
+
+    void update(
+        std::shared_ptr<HostMatrixWrapper> persistent_device_matrix) const
+    {
+        persistent_device_matrix->update(
+            matrix_.diag().begin(), matrix_.upper().begin(),
+            matrix_.lower().begin(), interfaceBouCoeffs_, interfaceIntCoeffs_,
+            interfaces_);
+    }
+
+    std::shared_ptr<HostMatrixWrapper> init() const
+    {
+        return HostMatrixWrapper::create(
+            exec_, db_, matrix_.diag().size(), matrix_.upper().size(),
+            matrix_.symmetric(), matrix_.diag().begin(),
+            matrix_.upper().begin(), matrix_.lower().begin(),
+            matrix_.lduAddr(), interfaceBouCoeffs_,
+            interfaceIntCoeffs_, interfaces_, solver_controls_, field_name_,
+            verbose_);
+    }
+};
+
+
+/* This Class makes the Host LDUMatrix device persistent. It basically only
+ * provides a functor to call HostMatrixWrapper::create and LduMatrix::update
+ */
+class PersistentHostMatrixAdapter {
+private:
+    const objectRegistry &db_;
+
+    const label verbose_;
+
+    const word field_name_;
+
+    mutable PersistentBase<HostMatrixWrapper, lduMatrixInitFunctor> host_matrix_;
+
+
+public:
+    template<typename matrixType>
+    PersistentHostMatrixAdapter(const objectRegistry &db, const ExecutorHandler &exec,
+                  const dictionary &controlDict, const word field_name,
+                  const label verbose,
+                  const matrixType& matrix,
+                  const FieldField<Field, scalar> &interfaceBouCoeffs,
+                  const FieldField<Field, scalar> &interfaceIntCoeffs,
+                  const lduInterfaceFieldPtrsList &interfaces)
+        : db_(db),
+          verbose_(verbose),
+          field_name_(field_name),
+          host_matrix_{
+              field_name_ + "_host_matrix", db,
+              lduMatrixInitFunctor(db, exec, verbose_,
+                                   field_name + "_host_matrix",
+                                   matrix,
+                                   interfaceBouCoeffs, interfaceIntCoeffs,
+                                   interfaces, controlDict),
+              controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
+              verbose_}
+    {}
+
+    std::shared_ptr<HostMatrixWrapper> get() const
+    {
+        return host_matrix_.get_persistent_object();
+    }
+
+    const CommunicationPattern& get_communication_pattern() const
+    {
+        return this->get()->get_communication_pattern();
+    }
+};
+
+}  // namespace Foam
diff --git a/DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H b/DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H
deleted file mode 100644
index 008a4354b..000000000
--- a/DevicePersistent/LDUMatrixAdapter/LDUMatrixAdapter.H
+++ /dev/null
@@ -1,138 +0,0 @@
-/*---------------------------------------------------------------------------*\
-License
-    This file is part of OGL.
-
-    OGL is free software: you can redistribute it and/or modify it
-    under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    OGL is distributed in the hope that it will be useful, but WITHOUT
-    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-    for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
-
-Class
-    Foam::GKOCG
-
-Author: Gregor Olenik <go@hpsim.de>
-
-SourceFiles
-    GKOCG.C
-
-\*---------------------------------------------------------------------------*/
-#ifndef OGL_IOGKOMatrixHandler_INCLUDED_H
-#define OGL_IOGKOMatrixHandler_INCLUDED_H
-
-
-#include "DevicePersistent/Array/Array.H"
-#include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
-#include "DevicePersistent/Partition/Partition.H"
-
-#include "MatrixWrapper/Combination/Combination.H"
-#include "MatrixWrapper/Distributed/Distributed.H"
-
-#include "fvCFD.H"
-
-namespace Foam {
-
-struct lduMatrixInitFunctor {
-
-    const objectRegistry &db_;
-
-    const ExecutorHandler &exec_;
-
-    const lduMatrix & matrix_;
-
-    const FieldField<Field, scalar> &interfaceBouCoeffs_;
-
-    const FieldField<Field, scalar> &interfaceIntCoeffs_;
-
-    const lduInterfaceFieldPtrsList &interfaces_;
-
-    MatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
-                      const lduMatrix & matrix,
-                      const word field_name, const label verbose,
-                      const FieldField<Field, scalar> &interfaceBouCoeffs,
-                      const FieldField<Field, scalar> &interfaceIntCoeffs,
-                      const lduInterfaceFieldPtrsList &interfaces)
-        : db_(db),
-          exec_(exec),
-          matrix_(matrix),
-          field_name_(field_name) verbose_(verbose),
-          interfaceBouCoeffs_(interfaceBouCoeffs),
-        interfaceIntCoeffs_(interfaceIntCoeffs),
-        interfaces_(interfaces)
-    {}
-
-    void update(std::shared_ptr<LduMatrix> persistent_device_matrix) const
-    {
-        persistent_device_matrix.update(exec_, interfaceBouCoeffs_,
-                                        interfaceIntCoeffs_, interfaces_);
-    }
-
-    std::shared_ptr<LduMatrix> init() const
-    {
-        return LduMatrix::create(
-          exec_,
-          matrix_.diag().size(),
-          matrix_.lduAddr().upperAddr().size(),
-          matrix_.symmetric(),
-          matrix_.diag().begin(),
-          matrix_.upper().begin(),
-          matrix_.lower().begin(),
-          matrix_.lduAddr().upperAddr().begin(),
-          matrix_.lduAddr().lowerAddr().begin(),
-          interfaceBouCoeffs_,
-          interfaceIntCoeffs_,
-          interfaces_);
-    }
-};
-
-
-/* This Class makes the Host LDUMatrix device persistent. It basically only
- * provides a functor to call LduMatrix::create and LduMatrix::update
- */
-class MatrixWrapper {
-private:
-    const objectRegistry &db_;
-
-    const label verbose_;
-
-    const word field_name_;
-
-    mutable PersistentBase<LDUMatrix, MatrixInitFunctor> host_matrix_;
-
-
-public:
-    MatrixWrapper(const objectRegistry &db, const PersistentExecutor &exec,
-                  const dictionary &controlDict, const word sys_matrix_name,
-                  const label verbose,
-                  const FieldField<Field, scalar> &interfaceBouCoeffs,
-                  const FieldField<Field, scalar> &interfaceIntCoeffs,
-                  const lduInterfaceFieldPtrsList &interfaces)
-        : db_(db),
-          verbose_(verbose),
-          export_(controlDict.lookupOrDefault<Switch>("export", false)),
-          field_name_(sys_matrix_name),
-          host_matrix_{
-              sys_matrix_name + "_host_matrix", db,
-              MatrixInitFunctor(
-                  db, exec, verbose_, sys_matrix_name + "_host_matrix",
-                  interfaceBouCoeffs, interfaceIntCoeffs, interfaces),
-              controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
-              verbose_}
-    {}
-
-    std::shared_ptr<gko::LinOp> get() const
-    {
-        return gkomatrix_.get_persistent_object();
-    }
-};
-
-}  // namespace Foam
-
-#endif
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 7d57da673..671d8a162 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -10,7 +10,7 @@
 
 #include "DevicePersistent/Base/Base.H"
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
-#include "MatrixWrapper/HostMatrix/HostMatrix.H"
+#include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
 #include "common/common.H"
 
 namespace Foam {
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index 4214c9d06..2d30d2078 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -1,5 +1,6 @@
-#include <ginkgo/ginkgo.hpp>
+#pragma once
 
+#include <ginkgo/ginkgo.hpp>
 
 // The CombinationMatrix class is a wrapper around Ginkgos combination linear
 // operator. It provides an interface for creating linear combinations of a
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
new file mode 100644
index 000000000..2c369d7de
--- /dev/null
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -0,0 +1,42 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::IOSortingIdxHandler
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    CommunicationPattern.H
+
+\*---------------------------------------------------------------------------*/
+
+#pragma once
+
+struct CommunicationPattern {
+    using comm_size_type = int;
+    // an array storing to which rank to communicate
+    gko::array<comm_size_type> target_ids;
+
+    // an array storing how many elements to communicate
+    // to the corresponding target_id
+    gko::array<comm_size_type> target_sizes;
+
+    // send_idx stores the index_set ie which cells
+    // are owned by the interface and the corresponding target rank
+    std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
+};
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 05f54187d..958d5008e 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -86,6 +86,20 @@ public:
         return *this;
     }
 
+    static std::shared_ptr<RepartDistMatrix> create(
+        std::shared_ptr<const gko::Executor> exec,
+        std::shared_ptr<const communicator> comm
+        );
+
+    void update(
+        std::shared_ptr<const gko::Executor> exec,
+        std::shared_ptr<const communicator> comm,
+        std::unique_ptr<const gko::LinOp> local_template,
+        std::unique_ptr<const gko::LinOp> non_local_template
+        );
+
+    void write();
+
 protected:
     RepartDistMatrix(std::shared_ptr<const gko::Executor> exec)
         : gko::EnableLinOp<RepartDistMatrix>(exec)
@@ -123,6 +137,7 @@ protected:
         this->dist_mtx_->apply(alpha, b, beta, x);
     }
 
+
 private:
     std::unique_ptr<dist_mtx> dist_mtx_;
 };
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index 123aff0bd..d949385f4 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -34,10 +34,11 @@ SourceFiles
 namespace Foam {
 
 
-HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
+HostMatrixWrapper::HostMatrixWrapper(
     label nrows, label upper_nnz, bool symmetric, const scalar *diag,
-    const scalar *upper, const scalar *lower, const label *lowerAddr,
-    const label *upperAddr, const objectRegistry &db,
+    const scalar *upper, const scalar *lower,
+    const lduAddressing& addr,
+    const objectRegistry &db,
     const FieldField<Field, scalar> &interfaceBouCoeffs,
     const FieldField<Field, scalar> &interfaceIntCoeffs,
     const lduInterfaceFieldPtrsList &interfaces,
@@ -45,29 +46,34 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
     : exec_{db, solverControls, fieldName},
       device_id_guard_{db, fieldName, exec_.get_device_exec()},
       verbose_(verbose),
-      fieldName_(fieldName),
+      field_name_(fieldName),
       reorder_on_copy_(
           solverControls.lookupOrDefault<Switch>("reorderOnHost", true)),
+      addr_(addr),
+      diag_(diag),
+      upper_(upper),
+      lower_(lower),
       scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
       nrows_(nrows),
       local_interface_nnz_(0),
       upper_nnz_(upper_nnz),
+      symmetric_(symmetric),
       non_diag_nnz_(2 * upper_nnz_),
       local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
       local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
-      local_sparsity_{exec_.get_host_exec(), local_matrix_nnz_},
+      local_sparsity_pattern_{exec_, local_matrix_nnz_},
       non_local_matrix_nnz_{count_interface_nnz(interfaces, true)},
-      non_local_sparsity_{exec_.get_host_exec(), non_local_matrix_nnz_}
+      non_local_sparsity_pattern_{exec_, non_local_matrix_nnz_}
 {
     TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
-                        fieldName_,
+                        field_name_,
                         init_local_sparsity_pattern(interfaces);)
     TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
-                        fieldName_,
+                        field_name_,
                         init_non_local_sparsity_pattern(interfaces);)
 }
 
-label HostMatrixWrapper<MatrixType>::count_interface_nnz(
+label HostMatrixWrapper::count_interface_nnz(
     const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
 {
     label ctr{0};
@@ -88,7 +94,7 @@ label HostMatrixWrapper<MatrixType>::count_interface_nnz(
     return ctr;
 }
 
-std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
+std::vector<scalar> HostMatrixWrapper::collect_interface_coeffs(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs, const bool local) const
 {
@@ -161,7 +167,7 @@ void neg_interface_iterator(const lduInterfaceFieldPtrsList &interfaces,
 
 
 CommunicationPattern
-HostMatrixWrapper<MatrixType>::create_communication_pattern(
+HostMatrixWrapper::create_communication_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     using comm_size_type = CommunicationPattern::comm_size_type;
@@ -291,13 +297,12 @@ void collect_local_interface_indices_impl_cyclicACMIFvPatch(
 #endif
 
 std::vector<std::tuple<label, label, label>>
-HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
+HostMatrixWrapper::collect_local_interface_indices(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     std::vector<std::tuple<label, label, label>> local_interface_idxs{};
     local_interface_idxs.reserve(local_interface_nnz_);
 
-    const auto &addr = this->matrix().lduAddr();
     neg_interface_iterator<processorFvPatch>(
         interfaces, [&](label &element_ctr, const label interface_size,
                         const lduInterfaceField *iface) {
@@ -305,19 +310,19 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
             // cyclicAMIFvPatch or cyclicACMIFvPatch and collect local interface
             // indices
             collect_local_interface_indices_impl<cyclicFvPatch>(
-                element_ctr, iface, addr, local_interface_idxs);
+                element_ctr, iface, addr_, local_interface_idxs);
             collect_local_interface_indices_impl_cyclicAMIFvPatch(
-                element_ctr, iface, addr, local_interface_idxs);
+                element_ctr, iface, addr_, local_interface_idxs);
 #ifdef WITH_ESI_VERSION
             collect_local_interface_indices_impl_cyclicACMIFvPatch(
-                element_ctr, iface, addr, local_interface_idxs);
+                element_ctr, iface, addr_, local_interface_idxs);
 #endif
         });
     return local_interface_idxs;
 }
 
 std::vector<std::tuple<label, label, label>>
-HostMatrixWrapper<MatrixType>::collect_cells_on_non_local_interface(
+HostMatrixWrapper::collect_cells_on_non_local_interface(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // vector of neighbour cell idx connected to interface
@@ -347,9 +352,9 @@ void HostMatrixWrapper::init_non_local_sparsity_pattern(
 {
     auto non_local_row_indices =
         collect_cells_on_non_local_interface(interfaces);
-    auto rows = non_local_sparsity_.row_idxs_.get_data();
-    auto cols = non_local_sparsity_.col_idxs_.get_data();
-    auto permute = non_local_sparsity_.ldu_mapping_.get_data();
+    auto rows = non_local_sparsity_pattern_.row_idxs_.get_data();
+    auto cols = non_local_sparsity_pattern_.col_idxs_.get_data();
+    auto permute = non_local_sparsity_pattern_.ldu_mapping_.get_data();
 
     label element_ctr = 0;
     // TODO currently we set permute eventhough this is not required
@@ -369,20 +374,20 @@ void HostMatrixWrapper::init_local_sparsity_pattern(
 
     auto lower_local = idx_array::view(
         exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
-        const_cast<label *>(&this->matrix().lduAddr().lowerAddr()[0]));
+        const_cast<label *>(addr_.lowerAddr().begin()));
 
     auto upper_local = idx_array::view(
         exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
-        const_cast<label *>(&this->matrix().lduAddr().upperAddr()[0]));
+        const_cast<label *>(addr_.upperAddr().begin()));
 
     // row of upper, col of lower
     const auto lower = lower_local.get_const_data();
     // col of upper, row of lower
     const auto upper = upper_local.get_const_data();
 
-    auto rows = local_sparsity_.row_idxs_.get_data();
-    auto cols = local_sparsity_.col_idxs_.get_data();
-    const auto permute = local_sparsity_.ldu_mapping_.get_data();
+    auto rows = local_sparsity_pattern_.row_idxs_.get_data();
+    auto cols = local_sparsity_pattern_.col_idxs_.get_data();
+    const auto permute = local_sparsity_pattern_.ldu_mapping_.get_data();
 
     // Scan through given rows and insert row and column indices into array
     //
@@ -428,43 +433,40 @@ void HostMatrixWrapper::init_local_sparsity_pattern(
 
 void HostMatrixWrapper::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
-    const FieldField<Field, scalar> &interfaceBouCoeffs) const
+    const FieldField<Field, scalar> &interfaceBouCoeffs,
+    gko::array<scalar>& local_coeffs
+    ) const
 {
     auto ref_exec = exec_.get_ref_exec();
-    auto upper = this->matrix().upper();
-    auto lower = this->matrix().lower();
-    auto diag = this->matrix().diag();
-
-    label diag_nnz = diag.size();
 
     // TODO this does not work for Ell
-    auto permute = local_sparsity_.ldu_mapping_.get_data();
+    auto permute = local_sparsity_pattern_.ldu_mapping_.get_data();
 
     if (reorder_on_copy_) {
-        auto dense = local_coeffs_.get_data();
+        auto dense = local_coeffs.get_data();
         if (local_interface_nnz_) {
             auto couple_coeffs =
                 collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
             if (symmetric_) {
                 symmetric_update_w_interface(
-                    local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
-                    permute, scaling_, diag.data(), upper.data(),
+                    local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_,
+                    permute, scaling_, diag_, upper_,
                     couple_coeffs.data(), dense);
 
             } else {
                 non_symmetric_update_w_interface(
-                    local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
-                    permute, scaling_, diag.data(), upper.data(), lower.data(),
+                    local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_,
+                    permute, scaling_, diag_, upper_, lower_,
                     couple_coeffs.data(), dense);
             }
         } else {
             if (symmetric_) {
                 symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-                                 scaling_, diag.data(), upper.data(), dense);
+                                 scaling_, diag_, upper_, dense);
             } else {
                 non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-                                     scaling_, diag.data(), upper.data(),
-                                     lower.data(), dense);
+                                     scaling_, diag_, upper_,
+                                     lower_, dense);
             }
         }
     } else {
@@ -478,30 +480,27 @@ void HostMatrixWrapper::update_local_matrix_data(
             (reorder_on_copy_) ? ref_exec : exec_.get_device_exec();
 
         // copy upper
-        auto upper = this->matrix().upper();
         auto u_host_view =
-            gko::array<scalar>::view(ref_exec, upper_nnz_, &upper[0]);
+            gko::array<scalar>::const_view(ref_exec, upper_nnz_, upper_);
         auto u_device_view = gko::array<scalar>::view(target_exec, upper_nnz_,
-                                                      local_coeffs_.get_data());
+                                                      local_coeffs.get_data());
         u_device_view = u_host_view;
 
         // for the non-symmetric case copy lower to the device
-        if (!is_symmetric) {
-            auto lower = this->matrix().lower();
+        if (!symmetric_) {
             auto l_host_view =
-                gko::array<scalar>::view(ref_exec, upper_nnz_, &lower[0]);
+                gko::array<scalar>::const_view(ref_exec, upper_nnz_, lower_);
             auto l_device_view = gko::array<scalar>::view(
-                target_exec, upper_nnz_, &local_coeffs_.get_data()[upper_nnz_]);
+                target_exec, upper_nnz_, &local_coeffs.get_data()[upper_nnz_]);
             l_device_view = l_host_view;
         }
 
         // copy diag
-        auto diag = this->matrix().diag();
         auto diag_host_view =
-            gko::array<scalar>::view(ref_exec, nrows_, &diag[0]);
-        const label diag_start = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
+            gko::array<scalar>::const_view(ref_exec, nrows_, diag_);
+        const label diag_start = (symmetric_) ? upper_nnz_ : 2 * upper_nnz_;
         auto diag_device_view = gko::array<scalar>::view(
-            target_exec, nrows_, &local_coeffs_.get_data()[diag_start]);
+            target_exec, nrows_, &local_coeffs.get_data()[diag_start]);
         diag_device_view = diag_host_view;
 
         // copy local interfaces
@@ -513,7 +512,7 @@ void HostMatrixWrapper::update_local_matrix_data(
                 ref_exec, local_interface_nnz_, couple_coeffs.data());
             auto interface_device_view = gko::array<scalar>::view(
                 target_exec, local_interface_nnz_,
-                &local_coeffs_.get_data()[interface_start]);
+                &local_coeffs.get_data()[interface_start]);
             interface_device_view = interface_host_view;
         }
 
@@ -529,11 +528,11 @@ void HostMatrixWrapper::update_local_matrix_data(
                         1},
             gko::array<scalar>::view(target_exec,
                                      local_matrix_w_interfaces_nnz_,
-                                     local_coeffs_.get_data()),
+                                     local_coeffs.get_data()),
             1);
 
         // TODO this needs to copy ldu_mapping to the device
-        dense_vec->row_gather(local_sparsity_.ldu_mapping_.get_array().get(),
+        dense_vec->row_gather(&local_sparsity_pattern_.ldu_mapping_,
                               row_collection.get());
 
         dense_vec->copy_from(row_collection);
@@ -543,9 +542,9 @@ void HostMatrixWrapper::update_local_matrix_data(
     // recreate this for every solver call
     // TODO this needs a proper implementation once we know how to handle
     // interfaces
-    local_sparsity_.dim_ = gko::dim<2>{nrows_, nrows_};
+    local_sparsity_pattern_.dim_ = gko::dim<2>{nrows_, nrows_};
 
-    local_sparsity_.interface_spans_.emplace_back(0, local_matrix_nnz_);
+    local_sparsity_pattern_.interface_spans_.emplace_back(0, local_matrix_nnz_);
     if (local_interface_nnz_) {
         auto local_interfaces = collect_local_interface_indices(interfaces);
 
@@ -562,7 +561,7 @@ void HostMatrixWrapper::update_local_matrix_data(
                 local_interface_ctr + 1 == local_interfaces.size()) {
                 end = start + local_interface_ctr;
                 local_interface_ctr = 0;
-                local_sparsity_.interface_spans_.emplace_back(start, end);
+                local_sparsity_pattern_.interface_spans_.emplace_back(start, end);
                 start = end;
                 prev_interface_idx = interface_idx;
             }
@@ -574,7 +573,9 @@ void HostMatrixWrapper::update_local_matrix_data(
 
 void HostMatrixWrapper::update_non_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
-    const FieldField<Field, scalar> &interfaceBouCoeffs) const
+    const FieldField<Field, scalar> &interfaceBouCoeffs,
+    gko::array<scalar>& non_local_coeffs
+    ) const
 {
     auto ref_exec = exec_.get_ref_exec();
     auto interface_coeffs =
@@ -592,13 +593,13 @@ void HostMatrixWrapper::update_non_local_matrix_data(
 
     // copy to persistent
     auto i_device_view = gko::array<scalar>::view(
-        ref_exec, non_local_matrix_nnz_, non_local_coeffs_.get_data());
+        ref_exec, non_local_matrix_nnz_, non_local_coeffs.get_data());
     i_device_view = tmp_contiguous_iface;
 
     // NOTE interface_spans and dim are not persistent so we need to
     // We have now the full non_local matrix and need to add the start and end
     // to the PersistentSparsityPattern data structure
-    non_local_sparsity_.dim_ = gko::dim<2>{nrows_, non_local_matrix_nnz_};
+    non_local_sparsity_pattern_.dim_ = gko::dim<2>{nrows_, non_local_matrix_nnz_};
 
     auto non_local_interfaces =
         collect_cells_on_non_local_interface(interfaces);
@@ -617,7 +618,7 @@ void HostMatrixWrapper::update_non_local_matrix_data(
             interface_ctr == non_local_interfaces.size() - 1) {
             end = start + interface_ctr;
             interface_ctr = 0;
-            non_local_sparsity_.interface_spans_.emplace_back(start, end);
+            non_local_sparsity_pattern_.interface_spans_.emplace_back(start, end);
             start = end;
             prev_interface_ctr = interface_idx;
         }
@@ -625,15 +626,4 @@ void HostMatrixWrapper::update_non_local_matrix_data(
     }
 }
 
-template HostMatrixWrapper::HostMatrixWrapper(
-    const objectRegistry &db, const lduMatrix &matrix,
-    // coeffs for cells on boundaries
-    const FieldField<Field, scalar> &interfaceBouCoeffs,
-    // coeffs for internal cells
-    const FieldField<Field, scalar> &interfaceIntCoeffs,
-    // pointers to interfaces can be used to access concrete
-    // functions such as transferring indices, patch neighbours etc
-    const lduInterfaceFieldPtrsList &interfaces,
-    const dictionary &solverControls, const word &fieldName);
-
 }  // namespace Foam
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 6738857d0..995cf016f 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -34,6 +34,7 @@ SourceFiles
 #include "DevicePersistent/DeviceIdGuard/DeviceIdGuard.H"
 #include "DevicePersistent/IOGlobalIndex/gkoGlobalIndex.H"
 #include "MatrixWrapper/Combination/Combination.H"
+#include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
 #include "common/common.H"
 
 
@@ -43,18 +44,9 @@ namespace Foam {
 struct SparsityPattern {
     SparsityPattern(const ExecutorHandler &exec, const label size)
         : size_(size),
-          col_idxs_{
-              exec.get_host_executor(),
-              size
-          },
-          row_idxs_{
-              exec.get_host_executor(),
-              size
-          },
-          ldu_mapping_{
-              exec.get_host_executor(),
-              size
-          }
+          col_idxs_{exec.get_ref_exec(), size},
+          row_idxs_{exec.get_ref_exec(), size},
+          ldu_mapping_{exec.get_ref_exec(), size}
     {}
 
     // number of nnz entries
@@ -76,21 +68,6 @@ struct SparsityPattern {
     mutable std::vector<gko::span> interface_spans_;
 };
 
-struct CommunicationPattern {
-    using comm_size_type = int;
-    // an array storing to which rank to communicate
-    gko::array<comm_size_type> target_ids;
-
-    // an array storing how many elements to communicate
-    // to the corresponding target_id
-    gko::array<comm_size_type> target_sizes;
-
-    // send_idx stores the index_set ie which cells
-    // are owned by the interface and the corresponding target rank
-    std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
-};
-
-
 // Free functions
 const lduInterfaceField *interface_getter(
     const lduInterfaceFieldPtrsList &interfaces, const label i);
@@ -233,8 +210,7 @@ void non_symmetric_update_w_interface(
  * - implements an interfaces so that
  *   RepartDistMatrix can read_distributed(A_host)
  * */
-class HostMatrixWrapper
-{
+class HostMatrixWrapper {
 private:
     using vec = gko::matrix::Dense<scalar>;
     using idx_array = gko::array<label>;
@@ -245,12 +221,20 @@ private:
 
     const label verbose_;
 
-    const word fieldName_;
+    const word field_name_;
 
     // Whether the matrix coefficients should be reordered
     // during copy or on device
     const bool reorder_on_copy_;
 
+    const lduAddressing& addr_;
+
+    const scalar* diag_;
+
+    const scalar* upper_;
+
+    const scalar* lower_;
+
     // multiply the complete system by this factor, ie sAx=sb
     // NOTE this needed to avoid negative diagonal matrix entries, but
     // this could be also achieved by just fliping the sign
@@ -285,7 +269,7 @@ private:
     // non-local indices
     const label non_local_matrix_nnz_;
 
-    mutable PersistentSparsityPattern non_local_sparsity_;
+    mutable SparsityPattern non_local_sparsity_pattern_;
 
     const CommunicationPattern communication_pattern_;
 
@@ -368,44 +352,59 @@ private:
      **/
     void update_local_matrix_data(
         const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        gko::array<scalar>& local_coeffs
+        ) const;
 
     /** Copies the interface matrix coefficients to non_local_coeffs without
     ** changing or reinstantiating the sparsity pattern.
     **/
     void update_non_local_matrix_data(
         const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        gko::array<scalar>& local_coeffs
+        ) const;
 
 public:
-
     // segregated matrix wrapper constructor
-    HostMatrixWrapper(
-        label nrows,
-        label upper_nnz,
-        bool symmetric,
-        const scalar* diag,
-        const scalar* upper,
-        const scalar* lower,
-        const label* lowerAddr,
-        const label* upperAddr,
-        const objectRegistry &db,
-        const FieldField<Field, scalar> &interfaceBouCoeffs,
-        const FieldField<Field, scalar> &interfaceIntCoeffs,
-        const lduInterfaceFieldPtrsList &interfaces,
-        const dictionary &solverControls,
-        const word &fieldName,
-        label verbose);
+    HostMatrixWrapper(label nrows, label upper_nnz, bool symmetric,
+                      const scalar *diag, const scalar *upper,
+                      const scalar *lower,
+                      const lduAddressing& addr,
+                      const objectRegistry &db,
+                      const FieldField<Field, scalar> &interfaceBouCoeffs,
+                      const FieldField<Field, scalar> &interfaceIntCoeffs,
+                      const lduInterfaceFieldPtrsList &interfaces,
+                      const dictionary &solverControls, const word &fieldName,
+                      label verbose);
 
     // coupled wrapper constructor
-    HostMatrixWrapper(const objectRegistry &db, const MatrixType &matrix,
-                      const dictionary &solverControls, const word &fieldName)
-        {OGL_NOT_IMPLEMENTED};
+    // HostMatrixWrapper(const objectRegistry &db, const MatrixType &matrix,
+    //                   const dictionary &solverControls, const word
+    //                   &fieldName)
+    //     {OGL_NOT_IMPLEMENTED};
 
 
     // getter
 
 public:
+    static std::shared_ptr<HostMatrixWrapper> create(
+        const ExecutorHandler &exec, 
+        const objectRegistry &db,
+        label nrows, label upper_nnz, bool symmetric,
+        const scalar *diag, const scalar *upper, const scalar *lower,
+        const lduAddressing& addr,
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        const FieldField<Field, scalar> &interfaceIntCoeffs,
+        const lduInterfaceFieldPtrsList &interfaces,
+        const dictionary &solverControls, const word &fieldName, label verbose);
+
+    void update(const scalar *diag, const scalar *upper, const scalar *lower,
+                const FieldField<Field, scalar> &interfaceBouCoeffs,
+                const FieldField<Field, scalar> &interfaceIntCoeffs,
+                const lduInterfaceFieldPtrsList &interfaces);
+
+
     bool get_verbose() const { return verbose_; }
 
     const CommunicationPattern &get_communication_pattern() const
@@ -417,41 +416,38 @@ public:
 
     const ExecutorHandler &get_exec_handler() const { return exec_; }
 
-    PersistentSparsityPattern &get_local_sparsity_pattern() const
-    {
-        return local_sparsity_;
-    };
-
-    PersistentArray<scalar> &get_local_coeffs() const { return local_coeffs_; };
+    gko::Array<scalar> &get_local_coeffs() const {};
 
-    PersistentArray<label> &get_local_col_idxs() const
+    gko::Array<label> &get_local_col_idxs() const
     {
-        return local_sparsity_.col_idxs_;
+        return local_sparsity_pattern_.col_idxs_;
     };
 
-    PersistentArray<label> &get_local_row_idxs() const
+    gko::Array<label> &get_local_row_idxs() const
     {
-        return local_sparsity_.row_idxs_;
+        return local_sparsity_pattern_.row_idxs_;
     };
 
-    PersistentSparsityPattern &get_non_local_sparsity_pattern() const
+    SparsityPattern &get_local_sparsity_pattern() const
     {
-        return non_local_sparsity_;
+        return local_sparsity_pattern_;
     };
 
-    PersistentArray<scalar> &get_non_local_coeffs() const
+    SparsityPattern &get_non_local_sparsity_pattern() const
     {
-        return non_local_coeffs_;
+        return non_local_sparsity_pattern_;
     };
 
-    PersistentArray<label> &get_non_local_col_idxs() const
+    gko::Array<scalar> &get_non_local_coeffs() const {};
+
+    gko::Array<label> &get_non_local_col_idxs() const
     {
-        return non_local_sparsity_.col_idxs_;
+        return non_local_sparsity_pattern_.col_idxs_;
     };
 
-    PersistentArray<label> &get_non_local_row_idxs() const
+    gko::Array<label> &get_non_local_row_idxs() const
     {
-        return non_local_sparsity_.row_idxs_;
+        return non_local_sparsity_pattern_.row_idxs_;
     };
 };
 
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 68688622f..ba3b8d175 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -4,9 +4,9 @@
 
 #pragma once
 
-#include "DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H"
+#include "DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H"
+#include "DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H"
 #include "DevicePersistent/Vector/Vector.H"
-#include "MatrixWrapper/HostMatrix/HostMatrix.H"
 #include "Preconditioner/Preconditioner.H"
 #include "common/common.H"
 
@@ -116,11 +116,11 @@ public:
  */
 template <class MatrixType, class SolverFactory>
 class lduLduBase : public OGL_Info,
+                   public MatrixType::solver,
                    public SolverFactory,
                    public Preconditioner {
 private:
     using dist_vec = gko::experimental::distributed::Vector<scalar>;
-    using dist_mtx = RepartDistMatrix<scalar, label, label>;
 
     using mtx = gko::matrix::Csr<scalar>;
     using coo_mtx = gko::matrix::Coo<scalar>;
@@ -131,18 +131,17 @@ private:
 
     const objectRegistry &db_;
 
-    const MatrixType& matrix_;
-
-    const FieldField<Field, scalar> &interfaceBouCoeffs_,
-
-    const FieldField<Field, scalar> &interfaceIntCoeffs_,
-
-    const lduInterfaceFieldPtrsList &interfaces_,
+    const MatrixType &matrix_;
 
     const dictionary &solver_controls_;
 
+    const word & field_name_;
+
     const label verbose_;
 
+    const ExecutorHandler& exec_handler_;
+
+    PersistentHostMatrixAdapter host_A_;
 
 public:
     // constructor for segregated solver
@@ -152,16 +151,31 @@ public:
                const lduInterfaceFieldPtrsList &interfaces,
                const dictionary &solverControls)
         : OGL_Info(fieldName, solverControls, matrix.mesh().thisDb()),
+          MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
+                         interfaceIntCoeffs, interfaces, solverControls),
           SolverFactory{solverControls, matrix.mesh().thisDb(), fieldName},
           Preconditioner(fieldName, matrix.mesh().thisDb(), solverControls,
-                         this->get_verbose()),
+                solverControls.lookupOrDefault<label>("verbose", 0)     
+                  ),
           db_(matrix.mesh().thisDb()),
           matrix_(matrix),
-          interfaceBouCoeffs_(interfaceBouCoeffs),
-          interfaceIntCoeffs_(interfaceIntCoeffs),
-          interfaces_(interfaces),
           solver_controls_(solverControls),
-          verbose_(solverControls.lookupOrDefault<label>("verbose", 0))
+          field_name_(fieldName),
+          verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+          exec_handler_{
+              db_, solver_controls_, fieldName
+          },
+          host_A_{
+              db_,
+              exec_handler_,
+              solver_controls_,
+              fieldName,
+              verbose_,
+              matrix_,
+              interfaceBouCoeffs,
+              interfaceIntCoeffs,
+              interfaces,
+          }
     {}
 
 
@@ -176,12 +190,9 @@ public:
           // HostMatrixWrapper<MatrixType>(matrix.mesh().thisDb(), matrix,
           //                               solverControls, fieldName),
           Preconditioner(fieldName, matrix.mesh().thisDb(), solverControls,
-                         this->get_verbose()),
+                         verbose_),
           db_(matrix.mesh().thisDb()),
           matrix_(matrix),
-          interfaceBouCoeffs_(),
-          interfaceIntCoeffs_(),
-          interfaces_(),
           solver_controls_(solverControls),
           verbose_(solverControls.lookupOrDefault<label>("verbose", 0))
     {}
@@ -199,43 +210,33 @@ public:
                                            const scalarField &source,
                                            solverPerformance &solverPerf) const
     {
-        auto ref_exec = this->get_exec_handler().get_ref_exec();
+        auto ref_exec = exec_handler_.get_ref_exec();
 
         // using a thin wrapper around OpenFOAMs LDU matrix class
-        PersistentLDUMatrix host_A{
-          db_,
-          this->get_exec_handler(),
-          solver_controls_
-          this->fieldName(),
-          this->get_verbose(),
-          matrix_,
-          interfaceBouCoeffs_,
-          interfaceIntCoeffs_,
-          interfaces_,
-       };
 
         PersistentPartition partition{
             db_,
-            this->get_exec_handler(),
+            exec_handler_,
             verbose_,
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
             psi.size(),  // number of elements
-            this->get_communication_pattern()};
+            host_A_.get_communication_pattern()};
 
         // create a distributed matrix from the host matrix
         // here also repartitioning is performed if required
-        MatrixWrapper dist_A{db_,
-                            host_A,
+        PersistentDistributedMatrix dist_A{db_,
+                             exec_handler_,
+                             field_name_,
+                             host_A_,
                              partition,
                              solver_controls_,
-                             this->fieldName(),
-                             this->get_verbose()};
+                             verbose_};
 
         PersistentVector<scalar> dist_b{
             &source[0],
-            this->fieldName() + "_rhs",
+            field_name_ + "_rhs",
             db_,
-            this->get_exec_handler(),
+            exec_handler_,
             partition,
             verbose_,
             solver_controls_.lookupOrDefault<Switch>("updateRHS", true),
@@ -244,9 +245,9 @@ public:
 
         PersistentVector<scalar> dist_x{
             &psi[0],
-            this->fieldName() + "_solution",
+            field_name_ + "_solution",
             db_,
-            this->get_exec_handler(),
+            exec_handler_,
             partition,
             verbose_,
             solver_controls_.lookupOrDefault<Switch>("updateInitGuess", false),
@@ -265,14 +266,14 @@ public:
                     {scaling}, ref_exec));
 
             TIME_WITH_FIELDNAME(
-                verbose_, scale_RHS, this->fieldName(),
+                verbose_, scale_RHS, field_name_,
                 dist_b.get_vector()->scale(dense_scaling.get());)
         }
 
         TIME_WITH_FIELDNAME(
-            verbose_, init_precond, this->fieldName(),
+            verbose_, init_precond, field_name_,
             auto precond = this->init_preconditioner(
-                dist_A_v, this->get_exec_handler().get_device_exec());)
+                dist_A_v, exec_handler_.get_device_exec());)
 
         bool debug(solver_controls_.lookupOrDefault<Switch>("debug", false));
         if (debug && db_.time().writeTime()) {
@@ -284,16 +285,16 @@ public:
 
         LOG_1(verbose_, "create solver")
         auto solver_gen = this->create_dist_solver(
-            this->get_exec_handler().get_device_exec(), dist_A_v, dist_x_v,
+            exec_handler_.get_device_exec(), dist_A_v, dist_x_v,
             dist_b_v, verbose_, dist_A.get_export(), precond);
 
-        TIME_WITH_FIELDNAME(verbose_, generate_solver, this->fieldName(),
+        TIME_WITH_FIELDNAME(verbose_, generate_solver, field_name_,
                             auto solver = solver_gen->generate(dist_A_v);)
         LOG_1(verbose_, "solve solver done")
-        TIME_WITH_FIELDNAME(verbose_, solve, this->fieldName(),
+        TIME_WITH_FIELDNAME(verbose_, solve, field_name_,
                             solver->apply(dist_b_v, dist_x_v);)
 
-        TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
+        TIME_WITH_FIELDNAME(verbose_, copy_x_back, field_name_,
                             dist_x.copy_back();)
         auto bandwidth_copy_back = sizeof(scalar) * partition.get_local_size() /
                                    delta_t_copy_x_back / 1000.0;
@@ -306,8 +307,8 @@ public:
         auto time_per_iter =
             delta_t_solve / max(this->get_number_of_iterations(), 1);
         scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
-        this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
-            this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1, 0);
+        exec_handler_.get_gko_mpi_host_comm()->broadcast(
+            exec_handler_.get_ref_exec(), &prev_rel_res_cost, 1, 0);
         this->set_prev_rel_res_cost(prev_rel_res_cost);
         auto time_per_iter_and_dof =
             time_per_iter * 1000.0 / partition.get_total_size();
@@ -331,14 +332,14 @@ public:
     {
         // --- Setup class containing solver performance data
         solverPerformance solverPerf(
-            lduMatrix::preconditioner::getName(this->controlDict_) +
-                this->get_exec_handler().get_exec_name() + typeName,
-            this->fieldName());
+            lduMatrix::preconditioner::getName(solver_controls_) +
+                exec_handler_.get_exec_name() + typeName,
+            field_name_);
 
         // Solve system
         if (Pstream::parRun()) {
             TIME_WITH_FIELDNAME(
-                verbose_, solve_multi_gpu, this->fieldName(),
+                verbose_, solve_multi_gpu, field_name_,
                 auto res = solve_multi_gpu_impl(psi, source, solverPerf);)
             return res;
         } else {

From d1d53b7f7d3b95cab64e8a079888aabfab22a2a8 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 16 Apr 2024 16:47:18 +0200
Subject: [PATCH 38/90] WIP refactor CSRMatrix

---
 .../DistributedMatrixAdapter.H                | 10 ++++------
 DevicePersistent/Partition/Partition.H        |  2 +-
 MatrixWrapper/Distributed/Distributed.H       | 20 ++++++-------------
 3 files changed, 11 insertions(+), 21 deletions(-)

diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 23c5c8a82..771605db2 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -69,16 +69,14 @@ struct DistributedMatrixInitFunctor {
     void update(
         std::shared_ptr<RepartDistMatrix<scalar, label, label>> persistent_device_matrix) const
     {
-        // persistent_device_matrix->update(
-        //     matrix_.diag().begin(), matrix_.upper().begin(),
-        //     matrix_.lower().begin(), interfaceBouCoeffs_, interfaceIntCoeffs_,
-        //     interfaces_);
+        persistent_device_matrix->update(
+            host_A_.get());
     }
 
     std::shared_ptr<RepartDistMatrix<scalar, label, label>> init() const
     {
-        // return RepartDistMatrix<scalar, label, label>::create(
-        //     exec_, host_A_);
+        return RepartDistMatrix<scalar, label, label>::create(
+            host_A_.get(), partition_.get());
     }
 };
 
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 671d8a162..a51dc37f8 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -121,7 +121,7 @@ public:
         global_elements_ = local_elements;
     }
 
-    std::shared_ptr<part_type> get_localized_partition() const
+    std::shared_ptr<part_type> get() const
     {
         return this->get_persistent_object();
     }
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 958d5008e..3ae6951bf 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -1,5 +1,6 @@
 #include <ginkgo/ginkgo.hpp>
 
+#include  "MatrixWrapper/LDUMatrix/HostMatrix.H"
 
 // The RepartDistMatrix class is a wrapper around Ginkgos distributed Matrix class
 // 
@@ -18,6 +19,8 @@ class RepartDistMatrix
 
     using dist_mtx =
         gko::experimental::distributed::Matrix<ValueType, LocalIndexType, GlobalIndexType>;
+    using part_type =
+        gko::experimental::distributed::localized_partition<label>;
 
 public:
     using gko::EnableLinOp<
@@ -50,14 +53,6 @@ public:
         this->dist_mtx_->get_non_local_matrix();
     }
 
-    // Function with the exact same interface as create but reuses intermediate computations 
-    void update(
-        std::shared_ptr<gko::Executor> exec,
-        std::shared_ptr<gko::experimental::mpi::communicator> comm,
-        std::shared_ptr<gko::LinOp> local_template,
-        std::shared_ptr<gko::LinOp> non_local_template
-            ) {}
-
     /**
      * Copy-assigns a CombinationMatrix matrix. Preserves executor, copies
      * everything else.
@@ -87,15 +82,12 @@ public:
     }
 
     static std::shared_ptr<RepartDistMatrix> create(
-        std::shared_ptr<const gko::Executor> exec,
-        std::shared_ptr<const communicator> comm
+        std::shared_ptr<const HostMatrixWrapper> host_A,
+        std::shared_ptr<const part_type> partition
         );
 
     void update(
-        std::shared_ptr<const gko::Executor> exec,
-        std::shared_ptr<const communicator> comm,
-        std::unique_ptr<const gko::LinOp> local_template,
-        std::unique_ptr<const gko::LinOp> non_local_template
+        std::shared_ptr<const HostMatrixWrapper> host_A
         );
 
     void write();

From 70c43bf0f656552e238640a55eb909ff45b4a804 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 16 Apr 2024 21:03:25 +0200
Subject: [PATCH 39/90] WIP refactor CSRMatrix

---
 .../DistributedMatrixAdapter.H                | 60 +++++++++----------
 1 file changed, 27 insertions(+), 33 deletions(-)

diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 771605db2..03d83b7d9 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -44,19 +44,18 @@ struct DistributedMatrixInitFunctor {
 
     const label verbose_;
 
-    const dictionary& solver_controls_;
+    const dictionary &solver_controls_;
 
-    const PersistentHostMatrixAdapter& host_A_;
+    const PersistentHostMatrixAdapter &host_A_;
 
-    const PersistentPartition& partition_;
+    const PersistentPartition &partition_;
 
-    DistributedMatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
-                         const label verbose,
-                         const word field_name,
-                         const PersistentHostMatrixAdapter& host_A,
-                         const PersistentPartition& partition,
-                         const dictionary& solver_controls
-                         )
+    DistributedMatrixInitFunctor(const objectRegistry &db,
+                                 const ExecutorHandler &exec,
+                                 const label verbose, const word field_name,
+                                 const PersistentHostMatrixAdapter &host_A,
+                                 const PersistentPartition &partition,
+                                 const dictionary &solver_controls)
         : db_(db),
           exec_(exec),
           field_name_(field_name),
@@ -66,17 +65,16 @@ struct DistributedMatrixInitFunctor {
           host_A_(host_A)
     {}
 
-    void update(
-        std::shared_ptr<RepartDistMatrix<scalar, label, label>> persistent_device_matrix) const
+    void update(std::shared_ptr<RepartDistMatrix<scalar, label, label>>
+                    persistent_device_matrix) const
     {
-        persistent_device_matrix->update(
-            host_A_.get());
+        persistent_device_matrix->update(host_A_.get());
     }
 
     std::shared_ptr<RepartDistMatrix<scalar, label, label>> init() const
     {
-        return RepartDistMatrix<scalar, label, label>::create(
-            host_A_.get(), partition_.get());
+        return RepartDistMatrix<scalar, label, label>::create(host_A_.get(),
+                                                              partition_.get());
     }
 };
 
@@ -92,28 +90,27 @@ private:
 
     const word field_name_;
 
-    mutable PersistentBase<RepartDistMatrix<scalar, label, label>, DistributedMatrixInitFunctor> host_matrix_;
+    mutable PersistentBase<RepartDistMatrix<scalar, label, label>,
+                           DistributedMatrixInitFunctor>
+        host_matrix_;
 
 
 public:
-    PersistentDistributedMatrix(
-                  const objectRegistry &db,
-                  const ExecutorHandler &exec,
-                  const word field_name,
-                  const PersistentHostMatrixAdapter& host_A,
-                  const PersistentPartition& partition,
-                  const dictionary &controlDict,
-                  const label verbose
-                  )
+    PersistentDistributedMatrix(const objectRegistry &db,
+                                const ExecutorHandler &exec,
+                                const word field_name,
+                                const PersistentHostMatrixAdapter &host_A,
+                                const PersistentPartition &partition,
+                                const dictionary &controlDict,
+                                const label verbose)
         : db_(db),
           verbose_(verbose),
           field_name_(field_name),
           host_matrix_{
               field_name + "_distributed_matrix", db,
               DistributedMatrixInitFunctor(db, exec, verbose_,
-                                   field_name + "_distributed_matrix",
-                                   host_A, partition,
-                                   controlDict),
+                                           field_name + "_distributed_matrix",
+                                           host_A, partition, controlDict),
               controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
               verbose_}
     {}
@@ -123,10 +120,7 @@ public:
         return host_matrix_.get_persistent_object();
     }
 
-    void write() 
-    {
-        this->get()->write();
-    }
+    void write() { this->get()->write(); }
 
     bool get_export();
 };

From 4fa2fe666f640fb53a22a4b37f082956b613c72c Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 17 Apr 2024 22:25:02 +0200
Subject: [PATCH 40/90] WIP refactor CSRMatrix

---
 DevicePersistent/Base/Base.C                  |   3 +
 .../DistributedMatrixAdapter.H                |  59 +--
 .../ExecutorHandler/ExecutorHandler.H         |   2 +-
 .../HostMatrixAdapter/HostMatrixAdapter.H     |   5 -
 DevicePersistent/Partition/Partition.H        |   4 +-
 DevicePersistent/Vector/Vector.H              |  44 +-
 MatrixWrapper/Distributed/Distributed.H       | 152 ++++--
 MatrixWrapper/LDUMatrix/HostMatrix.C          | 438 +++++++++---------
 MatrixWrapper/LDUMatrix/HostMatrix.H          | 159 +++----
 lduLduBase/lduLduBase.H                       |  40 +-
 10 files changed, 476 insertions(+), 430 deletions(-)

diff --git a/DevicePersistent/Base/Base.C b/DevicePersistent/Base/Base.C
index 1332b88c6..27fdb3361 100644
--- a/DevicePersistent/Base/Base.C
+++ b/DevicePersistent/Base/Base.C
@@ -35,4 +35,7 @@ defineTemplateTypeNameWithName(DevicePersistentBase<Partition>,
 typedef RepartDistMatrix<scalar, label, label> RepartDistMatrix;
 defineTemplateTypeNameWithName(DevicePersistentBase<RepartDistMatrix>,
                                "PersistentMatrix");
+
+defineTemplateTypeNameWithName(DevicePersistentBase<HostMatrixWrapper>,
+                               "PersistentHostMatrixWrapper");
 }  // namespace Foam
diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 03d83b7d9..411b3c2e7 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -44,37 +44,35 @@ struct DistributedMatrixInitFunctor {
 
     const label verbose_;
 
-    const dictionary &solver_controls_;
+    const dictionary& solver_controls_;
 
-    const PersistentHostMatrixAdapter &host_A_;
+    const PersistentHostMatrixAdapter& host_A_;
 
-    const PersistentPartition &partition_;
-
-    DistributedMatrixInitFunctor(const objectRegistry &db,
-                                 const ExecutorHandler &exec,
-                                 const label verbose, const word field_name,
-                                 const PersistentHostMatrixAdapter &host_A,
-                                 const PersistentPartition &partition,
-                                 const dictionary &solver_controls)
+    DistributedMatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
+                         const label verbose,
+                         const word field_name,
+                         const PersistentHostMatrixAdapter& host_A,
+                         const dictionary& solver_controls
+                         )
         : db_(db),
           exec_(exec),
           field_name_(field_name),
           verbose_(verbose),
           solver_controls_(solver_controls),
-          partition_(partition),
           host_A_(host_A)
     {}
 
-    void update(std::shared_ptr<RepartDistMatrix<scalar, label, label>>
-                    persistent_device_matrix) const
+    void update(
+        std::shared_ptr<RepartDistMatrix<scalar, label, label>> persistent_device_matrix) const
     {
-        persistent_device_matrix->update(host_A_.get());
+        // persistent_device_matrix->update();
     }
 
     std::shared_ptr<RepartDistMatrix<scalar, label, label>> init() const
     {
-        return RepartDistMatrix<scalar, label, label>::create(host_A_.get(),
-                                                              partition_.get());
+        return RepartDistMatrix<scalar, label, label>::create(
+              solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
+              host_A_.get());
     }
 };
 
@@ -90,27 +88,27 @@ private:
 
     const word field_name_;
 
-    mutable PersistentBase<RepartDistMatrix<scalar, label, label>,
-                           DistributedMatrixInitFunctor>
-        host_matrix_;
+    mutable PersistentBase<RepartDistMatrix<scalar, label, label>, DistributedMatrixInitFunctor> host_matrix_;
 
 
 public:
-    PersistentDistributedMatrix(const objectRegistry &db,
-                                const ExecutorHandler &exec,
-                                const word field_name,
-                                const PersistentHostMatrixAdapter &host_A,
-                                const PersistentPartition &partition,
-                                const dictionary &controlDict,
-                                const label verbose)
+    PersistentDistributedMatrix(
+                  const objectRegistry &db,
+                  const ExecutorHandler &exec,
+                  const word field_name,
+                  const PersistentHostMatrixAdapter& host_A,
+                  const dictionary &controlDict,
+                  const label verbose
+                  )
         : db_(db),
           verbose_(verbose),
           field_name_(field_name),
           host_matrix_{
               field_name + "_distributed_matrix", db,
               DistributedMatrixInitFunctor(db, exec, verbose_,
-                                           field_name + "_distributed_matrix",
-                                           host_A, partition, controlDict),
+                                   field_name + "_distributed_matrix",
+                                   host_A, 
+                                   controlDict),
               controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
               verbose_}
     {}
@@ -120,7 +118,10 @@ public:
         return host_matrix_.get_persistent_object();
     }
 
-    void write() { this->get()->write(); }
+    void write() 
+    {
+        this->get()->write();
+    }
 
     bool get_export();
 };
diff --git a/DevicePersistent/ExecutorHandler/ExecutorHandler.H b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
index dd53fa2f2..e7a8fe7d0 100644
--- a/DevicePersistent/ExecutorHandler/ExecutorHandler.H
+++ b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
@@ -163,7 +163,7 @@ public:
     }
 
     std::shared_ptr<gko::experimental::mpi::communicator>
-    get_gko_mpi_device_comm() const
+    get_communicator() const
     {
         return this->device_comm_;
     }
diff --git a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
index 595b9bd95..265e78e47 100644
--- a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
+++ b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
@@ -137,11 +137,6 @@ public:
     {
         return host_matrix_.get_persistent_object();
     }
-
-    const CommunicationPattern& get_communication_pattern() const
-    {
-        return this->get()->get_communication_pattern();
-    }
 };
 
 }  // namespace Foam
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index a51dc37f8..49f6c87a3 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -57,7 +57,6 @@ struct PartitionInitFunctor {
     std::shared_ptr<part_type> init() const
     {
         auto exec = exec_.get_ref_exec();
-        auto comm = exec_.get_gko_mpi_host_comm();
 
         word msg{"initialising partition of size " +
                  std::to_string(local_size_)};
@@ -66,7 +65,7 @@ struct PartitionInitFunctor {
         auto host_partition = part_type::build_from_blocked_recv(
             exec, local_size_, send_idxs_, target_ids_, target_sizes_);
 
-        if (ranks_per_gpu_ == 1) return host_partition;
+        return host_partition;
     }
 };
 
@@ -93,6 +92,7 @@ class PersistentPartition
 public:
     using part_type =
         gko::experimental::distributed::localized_partition<label>;
+
     /* PersistentPartition constructor using existing memory
      *
      * @param objectRegistry reference to registry for storage
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 9a3b23308..706f08447 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -25,9 +25,6 @@ struct VectorInitFunctor {
 
     const ExecutorHandler &exec_;
 
-    // representing the partitioning that OF uses
-    const PersistentPartition &partition_;
-
     const label verbose_;
 
     const bool on_device_;
@@ -35,16 +32,19 @@ struct VectorInitFunctor {
     // Memory from which array will be initialised
     const T *other_;
 
+    const label size_;
+
 
     VectorInitFunctor(const ExecutorHandler &exec, const word name,
-                      const PersistentPartition &partition, const T *other,
+                      const T *other,
+                      const label size,
                       const label verbose, const bool on_device = false)
-        : exec_(exec),
-          name_(name),
-          partition_(partition),
+        : name_(name),
+          exec_(exec),
           on_device_(on_device),
           verbose_(verbose),
-          other_(other)
+          other_(other),
+          size_(size)
     {}
 
 
@@ -52,7 +52,7 @@ struct VectorInitFunctor {
     void update(std::shared_ptr<gko::experimental::distributed::Vector<T>>
                     persistent_vector) const
     {
-        const auto local_size = partition_.get_local_size();
+        const auto local_size = size_;
         word msg{"updating array " + name_ + " of size " +
                  std::to_string(local_size)};
         LOG_1(verbose_, msg)
@@ -66,7 +66,7 @@ struct VectorInitFunctor {
             (on_device_) ? exec_.get_device_exec() : exec_.get_ref_exec();
         auto comm = exec_.get_gko_mpi_host_comm();
 
-        const auto local_size = partition_.get_local_size();
+        const auto local_size = size_;
         word msg{"initialising vector " + name_ + " of size " +
                  std::to_string(local_size)};
         msg += (on_device_) ? " on device" : " on host";
@@ -75,11 +75,9 @@ struct VectorInitFunctor {
         auto host_view = gko::array<T>::view(exec_.get_ref_exec(), local_size,
                                              const_cast<T *>(other_));
 
-        if (partition_.get_ranks_per_gpu() == 1) {
-            return gko::share(dist_vec::create(
-                exec, *exec_.get_gko_mpi_device_comm().get(),
+        return gko::share(dist_vec::create(
+                exec, *exec_.get_communicator().get(),
                 vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)));
-        }
     }
 };
 
@@ -89,6 +87,7 @@ class PersistentVector
     : public PersistentBase<gko::experimental::distributed::Vector<T>,
                             VectorInitFunctor<T>> {
     using dist_vec = gko::experimental::distributed::Vector<scalar>;
+    using part_type = gko::experimental::distributed::Partition<label, label>;
     using vec = gko::matrix::Dense<scalar>;
 
     const objectRegistry &db_;
@@ -97,7 +96,7 @@ class PersistentVector
 
     const T *memory_;
 
-    const PersistentPartition partition_;
+    const label size_;
 
     const ExecutorHandler &exec_;
 
@@ -121,30 +120,30 @@ public:
      * device or host
      * @param ranks_per_gpu
      */
-    PersistentVector(const T *memory, const word name, const objectRegistry &db,
+    PersistentVector(const T *memory, const label size, const word name, const objectRegistry &db,
                      const ExecutorHandler &exec,
-                     const PersistentPartition partition, const label verbose,
+                     const label verbose,
                      const bool update, const bool init_on_device)
         : PersistentBase<gko::experimental::distributed::Vector<T>,
                          VectorInitFunctor<T>>(
               name, db,
-              VectorInitFunctor<T>(exec, name, partition, memory, verbose,
+              VectorInitFunctor<T>(exec, name, memory, size, verbose,
                                    init_on_device),
               update, verbose),
           db_(db),
           name_(name),
           memory_(memory),
-          partition_(partition),
+          size_(size),
           exec_(exec),
           update_(update)
     {}
 
     /** Copies the content of the distributed vector back to the original source
      **/
-    void copy_back()
+    void copy_back(std::shared_ptr<const part_type> partition)
     {
-        const auto local_size = partition_.get_local_size();
-        auto global_size = partition_.get_total_size();
+        const label local_size = size_;
+        auto global_size = partition->get_size();
 
         auto comm = exec_.get_gko_mpi_host_comm();
         auto host_device_vector =
@@ -176,7 +175,6 @@ public:
     }
 
     // getter and setter
-
     bool get_update() const { return update_; }
 
     T *get_data() const { return this->get_persistent_object()->get_data(); }
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 3ae6951bf..e7322614c 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -1,11 +1,13 @@
 #include <ginkgo/ginkgo.hpp>
 
-#include  "MatrixWrapper/LDUMatrix/HostMatrix.H"
-
-// The RepartDistMatrix class is a wrapper around Ginkgos distributed Matrix class
-// 
-// It adds functionality for repeated read and repartitioning operatitions. As a constraint
-// it is required that the inner matrix types of the distributed matrix are of RepartDistMatrix type.
+#include "MatrixWrapper/LDUMatrix/HostMatrix.H"
+
+// The RepartDistMatrix class is a wrapper around Ginkgos distributed Matrix
+// class
+//
+// It adds functionality for repeated read and repartitioning operatitions. As a
+// constraint it is required that the inner matrix types of the distributed
+// matrix are of RepartDistMatrix type.
 template <typename ValueType, typename LocalIndexType, typename GlobalIndexType>
 class RepartDistMatrix
     : public gko::EnableLinOp<
@@ -15,16 +17,22 @@ class RepartDistMatrix
     friend class gko::EnableCreateMethod<
         RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>;
     friend class gko::EnablePolymorphicObject<
-        RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>, gko::LinOp>;
+        RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>,
+        gko::LinOp>;
 
     using dist_mtx =
-        gko::experimental::distributed::Matrix<ValueType, LocalIndexType, GlobalIndexType>;
+        gko::experimental::distributed::Matrix<ValueType, LocalIndexType,
+                                               GlobalIndexType>;
+
     using part_type =
+        gko::experimental::distributed::Partition<label, label>;
+
+    using localized_partition =
         gko::experimental::distributed::localized_partition<label>;
 
 public:
-    using gko::EnableLinOp<
-        RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>::convert_to;
+    using gko::EnableLinOp<RepartDistMatrix<ValueType, LocalIndexType,
+                                            GlobalIndexType>>::convert_to;
     using gko::EnableLinOp<
         RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>::move_to;
 
@@ -33,23 +41,27 @@ public:
     using device_matrix_data = gko::device_matrix_data<scalar, label>;
     using matrix_data = gko::matrix_data<scalar, label>;
     using communicator = gko::experimental::mpi::communicator;
-    using sparse_communicator = gko::experimental::distributed::sparse_communicator;
+    using sparse_communicator =
+        gko::experimental::distributed::sparse_communicator;
 
 
     void read_distributed(
-        const device_matrix_data& local_data,
-        const device_matrix_data& non_local_data,
+        const device_matrix_data &local_data,
+        const device_matrix_data &non_local_data,
         std::shared_ptr<const sparse_communicator> sparse_comm)
     {
-        this->dist_mtx_->read_distributed(local_data, non_local_data, sparse_comm);
+        this->dist_mtx_->read_distributed(local_data, non_local_data,
+                                          sparse_comm);
         this->set_size(dist_mtx_->get_size());
     }
 
-    std::shared_ptr<const gko::LinOp> get_local_matrix() {
+    std::shared_ptr<const gko::LinOp> get_local_matrix()
+    {
         this->dist_mtx_->get_local_matrix();
     }
 
-    std::shared_ptr<const gko::LinOp> get_non_local_matrix() {
+    std::shared_ptr<const gko::LinOp> get_non_local_matrix()
+    {
         this->dist_mtx_->get_non_local_matrix();
     }
 
@@ -81,14 +93,95 @@ public:
         return *this;
     }
 
-    static std::shared_ptr<RepartDistMatrix> create(
-        std::shared_ptr<const HostMatrixWrapper> host_A,
-        std::shared_ptr<const part_type> partition
-        );
+    std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
+        word matrix_format, std::shared_ptr<gko::Executor> exec,
+        std::shared_ptr<gko::experimental::mpi::communicator> comm) const
+    {
+        if (matrix_format == "Csr") {
+            return dist_mtx::create(exec, *comm.get(),
+                                    gko::with_matrix_type<gko::matrix::Csr>());
+        }
+        if (matrix_format == "Ell") {
+            return dist_mtx::create(exec, *comm.get(),
+                                    gko::with_matrix_type<gko::matrix::Ell>());
+        }
+        if (matrix_format == "Coo") {
+            return dist_mtx::create(exec, *comm.get(),
+                                    gko::with_matrix_type<gko::matrix::Coo>());
+        }
+
+        if (matrix_format == "Ldu") {
+        }
+
+        FatalErrorInFunction
+            << "Matrix format " << matrix_format
+            << " not supported! Supported formats: Csr, Ell, Coo"
+            << abort(FatalError);
+    }
 
-    void update(
-        std::shared_ptr<const HostMatrixWrapper> host_A
-        );
+    std::shared_ptr<part_type> get_partition() {return partition_;}
+
+    static std::shared_ptr<RepartDistMatrix> create(
+        word matrix_format, std::shared_ptr<const HostMatrixWrapper> host_A )
+    {
+        using part_type =
+            gko::experimental::distributed::localized_partition<label>;
+
+        auto exec_handler = host_A.get()->get_exec_handler();
+        auto exec = exec_handler.get_device_exec();
+
+        // repartition things here first by creating device_matrix_data on
+        // device with correct size on each rank
+        //
+        // NOTE Start by copying first and repartition later
+        auto local_sparsity_pattern = host_A->compute_local_sparsity(exec);
+        auto non_local_sparsity_pattern = host_A->compute_non_local_sparsity(exec);
+
+        // create original communicator pattern
+        auto src_comm_pattern = host_A.get()->create_communication_pattern();
+        // auto dst_comm_pattern = repartition_comm_pattern(src_comm_pattern);
+
+        std::cout << "!!!!! END\n";
+
+        // // if accumulate_ranks
+        // // 1. move to shared buffer
+        // // 2. switch interfaces to local_data
+        // // 3. modify communication pattern
+        // auto [local_sparsity_pattern_r, non_local_sparsity_pattern_r] =
+        //     repartition_indices(ranks_per_gpu, source_partition,
+        //                         local_sparsity_pattern,
+        //                         non_local_sparsity_pattern);
+        //
+        // auto [local_coeffs, non_local_coeffs] = host_A->compute_coeffs();
+        // auto [local_coeff_r, non_local_coeff_r] =
+        //     repartition_coeffs(ranks_per_gpu, source_partition,
+        //                        local_device_data, non_local_device_data);
+        //
+        //
+        // partition = part_type::build_from_blocked_recv(
+        //     exec, local_size_, comm_pattern.send_idxs,
+        //     comm_pattern.target_ids, comm_pattern.target_sizes);
+        //
+        // auto sparse_comm =
+        //     gko::experimental::distributed::sparse_communicator::create(
+        //         *comm.get(), partition_.get_localized_partition());
+        //
+        // auto dist_A = generate_dist_mtx_with_inner_type(
+        //     exec, host_A->get_communicator(), );
+        //
+        // // calls read of inner matrix type
+        // dist_A->read_distributed(local_device_data, non_local_device_data,
+        //                          sparse_comm);
+        //
+        // return dist_A;
+    };
+
+    void update(std::shared_ptr<const HostMatrixWrapper> host_A ){
+        // On update dist_mtx already exists
+        // and only local and non local coeffs need to be updated
+        //
+        // repartition_coeffs(local_device_data, non_local_device_data)
+    };
 
     void write();
 
@@ -98,12 +191,12 @@ protected:
     {}
 
     RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
-        std::shared_ptr<const communicator> comm,
-        std::unique_ptr<const gko::LinOp> local_template,
-        std::unique_ptr<const gko::LinOp> non_local_template
-                      )
+                     std::shared_ptr<const communicator> comm,
+                     std::unique_ptr<const gko::LinOp> local_template,
+                     std::unique_ptr<const gko::LinOp> non_local_template)
         : gko::EnableLinOp<RepartDistMatrix>(exec),
-          dist_mtx_(dist_mtx::create(exec, *comm.get(), local_template, non_local_template))
+          dist_mtx_(dist_mtx::create(exec, *comm.get(), local_template,
+                                     non_local_template))
     {}
 
     // Here we implement the application of the linear operator, x = A * b.
@@ -132,4 +225,7 @@ protected:
 
 private:
     std::unique_ptr<dist_mtx> dist_mtx_;
+
+    std::shared_ptr<part_type> partition_;
+
 };
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index d949385f4..dc467b5f3 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -35,15 +35,15 @@ namespace Foam {
 
 
 HostMatrixWrapper::HostMatrixWrapper(
-    label nrows, label upper_nnz, bool symmetric, const scalar *diag,
-    const scalar *upper, const scalar *lower,
-    const lduAddressing& addr,
-    const objectRegistry &db,
+    const ExecutorHandler &exec, const objectRegistry &db, label nrows,
+    label upper_nnz, bool symmetric, const scalar *diag, const scalar *upper,
+    const scalar *lower, const lduAddressing &addr,
     const FieldField<Field, scalar> &interfaceBouCoeffs,
     const FieldField<Field, scalar> &interfaceIntCoeffs,
     const lduInterfaceFieldPtrsList &interfaces,
-    const dictionary &solverControls, const word &fieldName, label verbose)
-    : exec_{db, solverControls, fieldName},
+    const dictionary &solverControls, const word &fieldName, label verbose
+    )
+    : exec_{exec},
       device_id_guard_{db, fieldName, exec_.get_device_exec()},
       verbose_(verbose),
       field_name_(fieldName),
@@ -55,24 +55,18 @@ HostMatrixWrapper::HostMatrixWrapper(
       lower_(lower),
       scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
       nrows_(nrows),
-      local_interface_nnz_(0),
+      local_interface_nnz_{count_interface_nnz(interfaces, false)},
       upper_nnz_(upper_nnz),
       symmetric_(symmetric),
       non_diag_nnz_(2 * upper_nnz_),
       local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
       local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
-      local_sparsity_pattern_{exec_, local_matrix_nnz_},
-      non_local_matrix_nnz_{count_interface_nnz(interfaces, true)},
-      non_local_sparsity_pattern_{exec_, non_local_matrix_nnz_}
+      interfaces_(interfaces),
+      non_local_matrix_nnz_{count_interface_nnz(interfaces, true)}
 {
-    TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
-                        field_name_,
-                        init_local_sparsity_pattern(interfaces);)
-    TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
-                        field_name_,
-                        init_non_local_sparsity_pattern(interfaces);)
 }
 
+
 label HostMatrixWrapper::count_interface_nnz(
     const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
 {
@@ -166,10 +160,9 @@ void neg_interface_iterator(const lduInterfaceFieldPtrsList &interfaces,
 }
 
 
-CommunicationPattern
-HostMatrixWrapper::create_communication_pattern(
-    const lduInterfaceFieldPtrsList &interfaces) const
+CommunicationPattern HostMatrixWrapper::create_communication_pattern() const
 {
+
     using comm_size_type = CommunicationPattern::comm_size_type;
     // temp map, mapping from neighbour rank interface cells
     std::map<label, std::vector<label>> interface_cell_map{};
@@ -178,7 +171,7 @@ HostMatrixWrapper::create_communication_pattern(
     // and store rows to send to neighbour procs
     gko::size_type n_procs = 0;
     interface_iterator<processorFvPatch>(
-        interfaces, [&](label, label, label, const processorFvPatch &patch,
+        interfaces_, [&](label, label, label, const processorFvPatch &patch,
                         const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
             const label neighbProcNo = patch.neighbProcNo();
@@ -347,14 +340,14 @@ HostMatrixWrapper::collect_cells_on_non_local_interface(
     return non_local_idxs;
 }
 
-void HostMatrixWrapper::init_non_local_sparsity_pattern(
-    const lduInterfaceFieldPtrsList &interfaces) const
+SparsityPattern HostMatrixWrapper::compute_non_local_sparsity(std::shared_ptr<const gko::Executor> exec) const
 {
+    SparsityPattern sparsity {exec->get_master(), non_local_matrix_nnz_};
     auto non_local_row_indices =
-        collect_cells_on_non_local_interface(interfaces);
-    auto rows = non_local_sparsity_pattern_.row_idxs_.get_data();
-    auto cols = non_local_sparsity_pattern_.col_idxs_.get_data();
-    auto permute = non_local_sparsity_pattern_.ldu_mapping_.get_data();
+        collect_cells_on_non_local_interface(interfaces_);
+    auto rows = sparsity.row_idxs.get_data();
+    auto cols = sparsity.col_idxs.get_data();
+    auto permute = sparsity.ldu_mapping.get_data();
 
     label element_ctr = 0;
     // TODO currently we set permute eventhough this is not required
@@ -365,29 +358,32 @@ void HostMatrixWrapper::init_non_local_sparsity_pattern(
         permute[element_ctr] = element_ctr;
         element_ctr += 1;
     }
+
+    return sparsity;
 }
 
-void HostMatrixWrapper::init_local_sparsity_pattern(
-    const lduInterfaceFieldPtrsList &interfaces) const
-{
+SparsityPattern HostMatrixWrapper::compute_local_sparsity(std::shared_ptr<const gko::Executor> exec) const {
     LOG_1(verbose_, "start init host sparsity pattern")
 
-    auto lower_local = idx_array::view(
-        exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
-        const_cast<label *>(addr_.lowerAddr().begin()));
+    SparsityPattern sparsity {exec->get_master(), local_matrix_w_interfaces_nnz_};
 
-    auto upper_local = idx_array::view(
-        exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
-        const_cast<label *>(addr_.upperAddr().begin()));
+    auto lower_local =
+        idx_array::view(exec, upper_nnz_,
+                        const_cast<label *>(addr_.lowerAddr().begin()));
+
+    // TODO const_view ?
+    auto upper_local =
+        idx_array::view(exec, upper_nnz_,
+                        const_cast<label *>(addr_.upperAddr().begin()));
 
     // row of upper, col of lower
     const auto lower = lower_local.get_const_data();
     // col of upper, row of lower
     const auto upper = upper_local.get_const_data();
 
-    auto rows = local_sparsity_pattern_.row_idxs_.get_data();
-    auto cols = local_sparsity_pattern_.col_idxs_.get_data();
-    const auto permute = local_sparsity_pattern_.ldu_mapping_.get_data();
+    auto rows = sparsity.row_idxs.get_data();
+    auto cols = sparsity.col_idxs.get_data();
+    const auto permute = sparsity.ldu_mapping.get_data();
 
     // Scan through given rows and insert row and column indices into array
     //
@@ -412,7 +408,7 @@ void HostMatrixWrapper::init_local_sparsity_pattern(
         // remove the unnessary copy via the vector of tuples and
         // let collect_local_interface_indices_impl write directly to rows,
         // cols, permute etc
-        auto local_interfaces = collect_local_interface_indices(interfaces);
+        auto local_interfaces = collect_local_interface_indices(interfaces_);
 
         label local_interface_ctr{0};
         for (label i = local_matrix_nnz_; i < local_matrix_w_interfaces_nnz_;
@@ -428,202 +424,202 @@ void HostMatrixWrapper::init_local_sparsity_pattern(
         }
     }
     LOG_1(verbose_, "done init host sparsity pattern")
+    return sparsity;
 }
 
 
 void HostMatrixWrapper::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs,
-    gko::array<scalar>& local_coeffs
-    ) const
+    gko::array<scalar> &local_coeffs) const
 {
-    auto ref_exec = exec_.get_ref_exec();
-
-    // TODO this does not work for Ell
-    auto permute = local_sparsity_pattern_.ldu_mapping_.get_data();
-
-    if (reorder_on_copy_) {
-        auto dense = local_coeffs.get_data();
-        if (local_interface_nnz_) {
-            auto couple_coeffs =
-                collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
-            if (symmetric_) {
-                symmetric_update_w_interface(
-                    local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_,
-                    permute, scaling_, diag_, upper_,
-                    couple_coeffs.data(), dense);
-
-            } else {
-                non_symmetric_update_w_interface(
-                    local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_,
-                    permute, scaling_, diag_, upper_, lower_,
-                    couple_coeffs.data(), dense);
-            }
-        } else {
-            if (symmetric_) {
-                symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-                                 scaling_, diag_, upper_, dense);
-            } else {
-                non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-                                     scaling_, diag_, upper_,
-                                     lower_, dense);
-            }
-        }
-    } else {
-        // TODO
-        // - this should be moved to separate function to avoid making this
-        // too long
-
-        using dim_type = gko::dim<2>::dimension_type;
-
-        auto target_exec =
-            (reorder_on_copy_) ? ref_exec : exec_.get_device_exec();
-
-        // copy upper
-        auto u_host_view =
-            gko::array<scalar>::const_view(ref_exec, upper_nnz_, upper_);
-        auto u_device_view = gko::array<scalar>::view(target_exec, upper_nnz_,
-                                                      local_coeffs.get_data());
-        u_device_view = u_host_view;
-
-        // for the non-symmetric case copy lower to the device
-        if (!symmetric_) {
-            auto l_host_view =
-                gko::array<scalar>::const_view(ref_exec, upper_nnz_, lower_);
-            auto l_device_view = gko::array<scalar>::view(
-                target_exec, upper_nnz_, &local_coeffs.get_data()[upper_nnz_]);
-            l_device_view = l_host_view;
-        }
-
-        // copy diag
-        auto diag_host_view =
-            gko::array<scalar>::const_view(ref_exec, nrows_, diag_);
-        const label diag_start = (symmetric_) ? upper_nnz_ : 2 * upper_nnz_;
-        auto diag_device_view = gko::array<scalar>::view(
-            target_exec, nrows_, &local_coeffs.get_data()[diag_start]);
-        diag_device_view = diag_host_view;
-
-        // copy local interfaces
-        if (local_interface_nnz_) {
-            const label interface_start = diag_start + nrows_;
-            auto couple_coeffs =
-                collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
-            auto interface_host_view = gko::array<scalar>::view(
-                ref_exec, local_interface_nnz_, couple_coeffs.data());
-            auto interface_device_view = gko::array<scalar>::view(
-                target_exec, local_interface_nnz_,
-                &local_coeffs.get_data()[interface_start]);
-            interface_device_view = interface_host_view;
-        }
-
-        // permute and update local_coeffs
-        auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
-            target_exec,
-            gko::dim<2>{static_cast<dim_type>(local_matrix_w_interfaces_nnz_),
-                        1}));
-
-        auto dense_vec = vec::create(
-            target_exec,
-            gko::dim<2>{static_cast<dim_type>(local_matrix_w_interfaces_nnz_),
-                        1},
-            gko::array<scalar>::view(target_exec,
-                                     local_matrix_w_interfaces_nnz_,
-                                     local_coeffs.get_data()),
-            1);
-
-        // TODO this needs to copy ldu_mapping to the device
-        dense_vec->row_gather(&local_sparsity_pattern_.ldu_mapping_,
-                              row_collection.get());
-
-        dense_vec->copy_from(row_collection);
-    }
-
-    // NOTE interface_spans and dim are not persistent so we need to
-    // recreate this for every solver call
-    // TODO this needs a proper implementation once we know how to handle
-    // interfaces
-    local_sparsity_pattern_.dim_ = gko::dim<2>{nrows_, nrows_};
-
-    local_sparsity_pattern_.interface_spans_.emplace_back(0, local_matrix_nnz_);
-    if (local_interface_nnz_) {
-        auto local_interfaces = collect_local_interface_indices(interfaces);
-
-        // counter how many interface elements are found on current interface
-        label local_interface_ctr{0};
-        label prev_interface_idx{0};
-        label end{0};
-        label start{local_matrix_nnz_};
-        for (auto [interface_idx, interface_row, interface_col] :
-             local_interfaces) {
-            // check if a new interface has started or final interface has been
-            // reache
-            if (interface_idx > prev_interface_idx ||
-                local_interface_ctr + 1 == local_interfaces.size()) {
-                end = start + local_interface_ctr;
-                local_interface_ctr = 0;
-                local_sparsity_pattern_.interface_spans_.emplace_back(start, end);
-                start = end;
-                prev_interface_idx = interface_idx;
-            }
-            local_interface_ctr++;
-        }
-    }
+    // auto ref_exec = exec_.get_ref_exec();
+    //
+    // // TODO this does not work for Ell
+    // auto permute = local_sparsity_pattern_.ldu_mapping_.get_data();
+    //
+    // if (reorder_on_copy_) {
+    //     auto dense = local_coeffs.get_data();
+    //     if (local_interface_nnz_) {
+    //         auto couple_coeffs =
+    //             collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
+    //         if (symmetric_) {
+    //             symmetric_update_w_interface(
+    //                 local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_, permute,
+    //                 scaling_, diag_, upper_, couple_coeffs.data(), dense);
+    //
+    //         } else {
+    //             non_symmetric_update_w_interface(
+    //                 local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_, permute,
+    //                 scaling_, diag_, upper_, lower_, couple_coeffs.data(),
+    //                 dense);
+    //         }
+    //     } else {
+    //         if (symmetric_) {
+    //             symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+    //                              scaling_, diag_, upper_, dense);
+    //         } else {
+    //             non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+    //                                  scaling_, diag_, upper_, lower_, dense);
+    //         }
+    //     }
+    // } else {
+    //     // TODO
+    //     // - this should be moved to separate function to avoid making this
+    //     // too long
+    //
+    //     using dim_type = gko::dim<2>::dimension_type;
+    //
+    //     auto target_exec =
+    //         (reorder_on_copy_) ? ref_exec : exec_.get_device_exec();
+    //
+    //     // copy upper
+    //     auto u_host_view =
+    //         gko::array<scalar>::const_view(ref_exec, upper_nnz_, upper_);
+    //     auto u_device_view = gko::array<scalar>::view(target_exec, upper_nnz_,
+    //                                                   local_coeffs.get_data());
+    //     u_device_view = u_host_view;
+    //
+    //     // for the non-symmetric case copy lower to the device
+    //     if (!symmetric_) {
+    //         auto l_host_view =
+    //             gko::array<scalar>::const_view(ref_exec, upper_nnz_, lower_);
+    //         auto l_device_view = gko::array<scalar>::view(
+    //             target_exec, upper_nnz_, &local_coeffs.get_data()[upper_nnz_]);
+    //         l_device_view = l_host_view;
+    //     }
+    //
+    //     // copy diag
+    //     auto diag_host_view =
+    //         gko::array<scalar>::const_view(ref_exec, nrows_, diag_);
+    //     const label diag_start = (symmetric_) ? upper_nnz_ : 2 * upper_nnz_;
+    //     auto diag_device_view = gko::array<scalar>::view(
+    //         target_exec, nrows_, &local_coeffs.get_data()[diag_start]);
+    //     diag_device_view = diag_host_view;
+    //
+    //     // copy local interfaces
+    //     if (local_interface_nnz_) {
+    //         const label interface_start = diag_start + nrows_;
+    //         auto couple_coeffs =
+    //             collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
+    //         auto interface_host_view = gko::array<scalar>::view(
+    //             ref_exec, local_interface_nnz_, couple_coeffs.data());
+    //         auto interface_device_view = gko::array<scalar>::view(
+    //             target_exec, local_interface_nnz_,
+    //             &local_coeffs.get_data()[interface_start]);
+    //         interface_device_view = interface_host_view;
+    //     }
+    //
+    //     // permute and update local_coeffs
+    //     auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
+    //         target_exec,
+    //         gko::dim<2>{static_cast<dim_type>(local_matrix_w_interfaces_nnz_),
+    //                     1}));
+    //
+    //     auto dense_vec = vec::create(
+    //         target_exec,
+    //         gko::dim<2>{static_cast<dim_type>(local_matrix_w_interfaces_nnz_),
+    //                     1},
+    //         gko::array<scalar>::view(target_exec,
+    //                                  local_matrix_w_interfaces_nnz_,
+    //                                  local_coeffs.get_data()),
+    //         1);
+    //
+    //     // TODO this needs to copy ldu_mapping to the device
+    //     dense_vec->row_gather(&local_sparsity_pattern_.ldu_mapping_,
+    //                           row_collection.get());
+    //
+    //     dense_vec->copy_from(row_collection);
+    // }
+    //
+    // // NOTE interface_spans and dim are not persistent so we need to
+    // // recreate this for every solver call
+    // // TODO this needs a proper implementation once we know how to handle
+    // // interfaces
+    // local_sparsity_pattern_.dim_ = gko::dim<2>{nrows_, nrows_};
+    //
+    // local_sparsity_pattern_.interface_spans_.emplace_back(0, local_matrix_nnz_);
+    // if (local_interface_nnz_) {
+    //     auto local_interfaces = collect_local_interface_indices(interfaces);
+    //
+    //     // counter how many interface elements are found on current interface
+    //     label local_interface_ctr{0};
+    //     label prev_interface_idx{0};
+    //     label end{0};
+    //     label start{local_matrix_nnz_};
+    //     for (auto [interface_idx, interface_row, interface_col] :
+    //          local_interfaces) {
+    //         // check if a new interface has started or final interface has been
+    //         // reache
+    //         if (interface_idx > prev_interface_idx ||
+    //             local_interface_ctr + 1 == local_interfaces.size()) {
+    //             end = start + local_interface_ctr;
+    //             local_interface_ctr = 0;
+    //             local_sparsity_pattern_.interface_spans_.emplace_back(start,
+    //                                                                   end);
+    //             start = end;
+    //             prev_interface_idx = interface_idx;
+    //         }
+    //         local_interface_ctr++;
+    //     }
+    // }
 }
 
 
 void HostMatrixWrapper::update_non_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs,
-    gko::array<scalar>& non_local_coeffs
-    ) const
+    gko::array<scalar> &non_local_coeffs) const
 {
-    auto ref_exec = exec_.get_ref_exec();
-    auto interface_coeffs =
-        collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
-
-    // copy interfaces
-    auto tmp_contiguous_iface =
-        gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
-    auto contiguous_iface = tmp_contiguous_iface.get_data();
-
-    for (label element_ctr = 0; element_ctr < non_local_matrix_nnz_;
-         element_ctr++) {
-        contiguous_iface[element_ctr] = interface_coeffs[element_ctr];
-    }
-
-    // copy to persistent
-    auto i_device_view = gko::array<scalar>::view(
-        ref_exec, non_local_matrix_nnz_, non_local_coeffs.get_data());
-    i_device_view = tmp_contiguous_iface;
-
-    // NOTE interface_spans and dim are not persistent so we need to
-    // We have now the full non_local matrix and need to add the start and end
-    // to the PersistentSparsityPattern data structure
-    non_local_sparsity_pattern_.dim_ = gko::dim<2>{nrows_, non_local_matrix_nnz_};
-
-    auto non_local_interfaces =
-        collect_cells_on_non_local_interface(interfaces);
-
-    // TODO this can be merged with algorithm in update_local_matrix_data
-    // refactor to separate function
-    label interface_ctr{0};
-    label prev_interface_ctr{0};
-    label end{0};
-    label start{0};
-    for (auto [interface_idx, interface_cell_ctr, interface_row] :
-         non_local_interfaces) {
-        // check if a new interface has started or final interface has been
-        // reache
-        if (interface_idx > prev_interface_ctr ||
-            interface_ctr == non_local_interfaces.size() - 1) {
-            end = start + interface_ctr;
-            interface_ctr = 0;
-            non_local_sparsity_pattern_.interface_spans_.emplace_back(start, end);
-            start = end;
-            prev_interface_ctr = interface_idx;
-        }
-        interface_ctr++;
-    }
+    // auto ref_exec = exec_.get_ref_exec();
+    // auto interface_coeffs =
+    //     collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
+    //
+    // // copy interfaces
+    // auto tmp_contiguous_iface =
+    //     gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
+    // auto contiguous_iface = tmp_contiguous_iface.get_data();
+    //
+    // for (label element_ctr = 0; element_ctr < non_local_matrix_nnz_;
+    //      element_ctr++) {
+    //     contiguous_iface[element_ctr] = interface_coeffs[element_ctr];
+    // }
+    //
+    // // copy to persistent
+    // auto i_device_view = gko::array<scalar>::view(
+    //     ref_exec, non_local_matrix_nnz_, non_local_coeffs.get_data());
+    // i_device_view = tmp_contiguous_iface;
+    //
+    // // NOTE interface_spans and dim are not persistent so we need to
+    // // We have now the full non_local matrix and need to add the start and end
+    // // to the PersistentSparsityPattern data structure
+    // non_local_sparsity_pattern_.dim_ =
+    //     gko::dim<2>{nrows_, non_local_matrix_nnz_};
+    //
+    // auto non_local_interfaces =
+    //     collect_cells_on_non_local_interface(interfaces);
+    //
+    // // TODO this can be merged with algorithm in update_local_matrix_data
+    // // refactor to separate function
+    // label interface_ctr{0};
+    // label prev_interface_ctr{0};
+    // label end{0};
+    // label start{0};
+    // for (auto [interface_idx, interface_cell_ctr, interface_row] :
+    //      non_local_interfaces) {
+    //     // check if a new interface has started or final interface has been
+    //     // reache
+    //     if (interface_idx > prev_interface_ctr ||
+    //         interface_ctr == non_local_interfaces.size() - 1) {
+    //         end = start + interface_ctr;
+    //         interface_ctr = 0;
+    //         non_local_sparsity_pattern_.interface_spans_.emplace_back(start,
+    //                                                                   end);
+    //         start = end;
+    //         prev_interface_ctr = interface_idx;
+    //     }
+    //     interface_ctr++;
+    // }
 }
 
 }  // namespace Foam
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 995cf016f..40bf46f47 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -42,30 +42,34 @@ namespace Foam {
 
 
 struct SparsityPattern {
-    SparsityPattern(const ExecutorHandler &exec, const label size)
+    SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size)
         : size_(size),
-          col_idxs_{exec.get_ref_exec(), size},
-          row_idxs_{exec.get_ref_exec(), size},
-          ldu_mapping_{exec.get_ref_exec(), size}
+          col_idxs{exec, size},
+          row_idxs{exec, size},
+          ldu_mapping{exec, size}
     {}
 
+    // TODO rename the public members and remove trailing _
     // number of nnz entries
     const label size_;
 
-    mutable gko::Array<label> col_idxs_;
+    mutable gko::Array<label> col_idxs;
 
-    mutable gko::Array<label> row_idxs_;
+    mutable gko::Array<label> row_idxs;
 
     // ldu_mapping[sorted_(csr)_position] =
     // unsorted_(consecutive_ldu_)_position
-    mutable gko::Array<label> ldu_mapping_;
+    mutable gko::Array<label> ldu_mapping;
 
     // dimensions of the resulting matrix
-    mutable gko::dim<2> dim_;
+    mutable gko::dim<2> dim;
 
     // A vector of spans indicating begin and end of each interface
     // this is used to keep all col and row idx consecutive in memory
-    mutable std::vector<gko::span> interface_spans_;
+    mutable std::vector<gko::span> interface_spans;
+
+    // keep track of original rank of each interface or matrix block
+    mutable std::vector<label> rank;
 };
 
 // Free functions
@@ -227,13 +231,13 @@ private:
     // during copy or on device
     const bool reorder_on_copy_;
 
-    const lduAddressing& addr_;
+    const lduAddressing &addr_;
 
-    const scalar* diag_;
+    const scalar *diag_;
 
-    const scalar* upper_;
+    const scalar *upper_;
 
-    const scalar* lower_;
+    const scalar *lower_;
 
     // multiply the complete system by this factor, ie sAx=sb
     // NOTE this needed to avoid negative diagonal matrix entries, but
@@ -264,12 +268,13 @@ private:
     // nnzs of local matrix including local interfaces
     const label local_matrix_w_interfaces_nnz_;
 
-    mutable SparsityPattern local_sparsity_pattern_;
+    // TODO remove
+    // mutable SparsityPattern local_sparsity_pattern_;
 
     // non-local indices
     const label non_local_matrix_nnz_;
 
-    mutable SparsityPattern non_local_sparsity_pattern_;
+    const lduInterfaceFieldPtrsList &interfaces_;
 
     const CommunicationPattern communication_pattern_;
 
@@ -303,16 +308,6 @@ private:
         const lduInterfaceFieldPtrsList &interfaces_) const;
 
 
-    /** Iterates all interfaces and counts the number of unique neighbour
-     ** processors and number of interfaces in total for this processor
-     ** and collects all interface cells of this rank.
-     **
-     ** @param interfaces The list of interfaces for the search
-     ** @return the CommunicationPattern
-     */
-    CommunicationPattern create_communication_pattern(
-        const lduInterfaceFieldPtrsList &interfaces) const;
-
     /** Iterates all interfaces and collect the corresponding cell id (row)
     ** and a unique counter
     **
@@ -326,22 +321,6 @@ private:
     collect_cells_on_non_local_interface(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
-    /** Based on OpenFOAMs ldu matrix format this function computes two
-     ** consecutive index arrays (local_sparsisty_.row_idxs and col_idxs) in row
-     ** major ordering and the permutation index (local_sparsity_.ldu_mapping),
-     ** which are required to create to a ginkgo matrix
-     **/
-    void init_local_sparsity_pattern(
-        const lduInterfaceFieldPtrsList &interfaces) const;
-
-    /** Based on OpenFOAMs interfaces this function computes two
-     ** consecutive index arrays (non_local_sparsisty_.row_idxs and col_idxs) in
-     ** row *major ordering and the permutation index
-     ** (non_local_sparsity_.ldu_mapping), which are required to create a ginkgo
-     ** matrix
-     **/
-    void init_non_local_sparsity_pattern(
-        const lduInterfaceFieldPtrsList &interfaces) const;
 
     // updater
 
@@ -353,8 +332,7 @@ private:
     void update_local_matrix_data(
         const lduInterfaceFieldPtrsList &interfaces,
         const FieldField<Field, scalar> &interfaceBouCoeffs,
-        gko::array<scalar>& local_coeffs
-        ) const;
+        gko::array<scalar> &local_coeffs) const;
 
     /** Copies the interface matrix coefficients to non_local_coeffs without
     ** changing or reinstantiating the sparsity pattern.
@@ -362,55 +340,71 @@ private:
     void update_non_local_matrix_data(
         const lduInterfaceFieldPtrsList &interfaces,
         const FieldField<Field, scalar> &interfaceBouCoeffs,
-        gko::array<scalar>& local_coeffs
-        ) const;
+        gko::array<scalar> &local_coeffs) const;
 
 public:
     // segregated matrix wrapper constructor
-    HostMatrixWrapper(label nrows, label upper_nnz, bool symmetric,
+    HostMatrixWrapper(const ExecutorHandler &exec, const objectRegistry &db,
+                      label nrows, label upper_nnz, bool symmetric,
                       const scalar *diag, const scalar *upper,
-                      const scalar *lower,
-                      const lduAddressing& addr,
-                      const objectRegistry &db,
+                      const scalar *lower, const lduAddressing &addr,
                       const FieldField<Field, scalar> &interfaceBouCoeffs,
                       const FieldField<Field, scalar> &interfaceIntCoeffs,
                       const lduInterfaceFieldPtrsList &interfaces,
                       const dictionary &solverControls, const word &fieldName,
                       label verbose);
 
-    // coupled wrapper constructor
-    // HostMatrixWrapper(const objectRegistry &db, const MatrixType &matrix,
-    //                   const dictionary &solverControls, const word
-    //                   &fieldName)
-    //     {OGL_NOT_IMPLEMENTED};
-
-
     // getter
 
 public:
     static std::shared_ptr<HostMatrixWrapper> create(
-        const ExecutorHandler &exec, 
-        const objectRegistry &db,
-        label nrows, label upper_nnz, bool symmetric,
-        const scalar *diag, const scalar *upper, const scalar *lower,
-        const lduAddressing& addr,
+        const ExecutorHandler &exec, const objectRegistry &db, label nrows,
+        label upper_nnz, bool symmetric, const scalar *diag,
+        const scalar *upper, const scalar *lower, const lduAddressing &addr,
         const FieldField<Field, scalar> &interfaceBouCoeffs,
         const FieldField<Field, scalar> &interfaceIntCoeffs,
         const lduInterfaceFieldPtrsList &interfaces,
-        const dictionary &solverControls, const word &fieldName, label verbose);
+        const dictionary &solverControls, const word &fieldName, label verbose)
+    {
+        return std::make_shared<HostMatrixWrapper>(
+            exec, db, nrows, upper_nnz, symmetric, diag, upper, lower, addr,
+            interfaceBouCoeffs, interfaceIntCoeffs, interfaces, solverControls,
+            fieldName, verbose);
+    };
 
     void update(const scalar *diag, const scalar *upper, const scalar *lower,
                 const FieldField<Field, scalar> &interfaceBouCoeffs,
                 const FieldField<Field, scalar> &interfaceIntCoeffs,
                 const lduInterfaceFieldPtrsList &interfaces);
 
+    /** Based on OpenFOAMs ldu matrix format this function computes two
+     ** consecutive index arrays (local_sparsisty_.row_idxs and col_idxs) in row
+     ** major ordering and the permutation index (local_sparsity_.ldu_mapping),
+     ** which are required to create to a ginkgo matrix
+     **/
+    SparsityPattern compute_local_sparsity(
+            std::shared_ptr<const gko::Executor> exec) const;
 
-    bool get_verbose() const { return verbose_; }
+    /** Based on OpenFOAMs interfaces this function computes two
+     ** consecutive index arrays (non_local_sparsisty_.row_idxs and col_idxs) in
+     ** row *major ordering and the permutation index
+     ** (non_local_sparsity_.ldu_mapping), which are required to create a ginkgo
+     ** matrix
+     **/
+    SparsityPattern compute_non_local_sparsity(
+            std::shared_ptr<const gko::Executor> exec) const;
 
-    const CommunicationPattern &get_communication_pattern() const
-    {
-        return communication_pattern_;
-    };
+    /** Iterates all interfaces and counts the number of unique neighbour
+     ** processors and number of interfaces in total for this processor
+     ** and collects all interface cells of this rank.
+     **
+     ** @param interfaces The list of interfaces for the search
+     ** @return the CommunicationPattern
+     */
+    CommunicationPattern create_communication_pattern() const;
+
+
+    bool get_verbose() const { return verbose_; }
 
     label get_local_nrows() const { return nrows_; }
 
@@ -418,37 +412,6 @@ public:
 
     gko::Array<scalar> &get_local_coeffs() const {};
 
-    gko::Array<label> &get_local_col_idxs() const
-    {
-        return local_sparsity_pattern_.col_idxs_;
-    };
-
-    gko::Array<label> &get_local_row_idxs() const
-    {
-        return local_sparsity_pattern_.row_idxs_;
-    };
-
-    SparsityPattern &get_local_sparsity_pattern() const
-    {
-        return local_sparsity_pattern_;
-    };
-
-    SparsityPattern &get_non_local_sparsity_pattern() const
-    {
-        return non_local_sparsity_pattern_;
-    };
-
-    gko::Array<scalar> &get_non_local_coeffs() const {};
-
-    gko::Array<label> &get_non_local_col_idxs() const
-    {
-        return non_local_sparsity_pattern_.col_idxs_;
-    };
-
-    gko::Array<label> &get_non_local_row_idxs() const
-    {
-        return non_local_sparsity_pattern_.row_idxs_;
-    };
 };
 
 
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index ba3b8d175..ebfcf9b00 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -141,6 +141,7 @@ private:
 
     const ExecutorHandler& exec_handler_;
 
+    // A thin wrapper around OpenFOAMs LDU matrix class
     PersistentHostMatrixAdapter host_A_;
 
 public:
@@ -174,7 +175,7 @@ public:
               matrix_,
               interfaceBouCoeffs,
               interfaceIntCoeffs,
-              interfaces,
+              interfaces
           }
     {}
 
@@ -210,45 +211,38 @@ public:
                                            const scalarField &source,
                                            solverPerformance &solverPerf) const
     {
-        auto ref_exec = exec_handler_.get_ref_exec();
-
-        // using a thin wrapper around OpenFOAMs LDU matrix class
-
-        PersistentPartition partition{
-            db_,
-            exec_handler_,
-            verbose_,
-            solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
-            psi.size(),  // number of elements
-            host_A_.get_communication_pattern()};
+        auto exec_handler = host_A_.get()->get_exec_handler();
+        auto ref_exec = exec_handler.get_ref_exec();
 
         // create a distributed matrix from the host matrix
         // here also repartitioning is performed if required
+        std::cout << "Start persistent distributed matrix\n";
         PersistentDistributedMatrix dist_A{db_,
-                             exec_handler_,
+                             exec_handler,
                              field_name_,
                              host_A_,
-                             partition,
                              solver_controls_,
                              verbose_};
 
+        auto partition = dist_A.get()->get_partition();
+
         PersistentVector<scalar> dist_b{
-            &source[0],
+            source.begin(),
+            source.size(),
             field_name_ + "_rhs",
             db_,
-            exec_handler_,
-            partition,
+            exec_handler,
             verbose_,
             solver_controls_.lookupOrDefault<Switch>("updateRHS", true),
             false  // whether data for init is on device
         };
 
         PersistentVector<scalar> dist_x{
-            &psi[0],
+            psi.begin(),
+            psi.size(),
             field_name_ + "_solution",
             db_,
-            exec_handler_,
-            partition,
+            exec_handler,
             verbose_,
             solver_controls_.lookupOrDefault<Switch>("updateInitGuess", false),
             false  // whether data for init is on device
@@ -295,8 +289,8 @@ public:
                             solver->apply(dist_b_v, dist_x_v);)
 
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, field_name_,
-                            dist_x.copy_back();)
-        auto bandwidth_copy_back = sizeof(scalar) * partition.get_local_size() /
+                            dist_x.copy_back(partition);)
+        auto bandwidth_copy_back = sizeof(scalar) * psi.size() /
                                    delta_t_copy_x_back / 1000.0;
 
         solverPerf.initialResidual() = this->get_init_res_norm();
@@ -311,7 +305,7 @@ public:
             exec_handler_.get_ref_exec(), &prev_rel_res_cost, 1, 0);
         this->set_prev_rel_res_cost(prev_rel_res_cost);
         auto time_per_iter_and_dof =
-            time_per_iter * 1000.0 / partition.get_total_size();
+            time_per_iter * 1000.0 / partition->get_size();
         word msg =
             "\nStatistics:\n\tTime per iteration: " +
             std::to_string(time_per_iter) +

From 59a89a5f41f1fb02adc1b4f3a13a61ecdcdf5a99 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 18 Apr 2024 21:26:07 +0200
Subject: [PATCH 41/90] WIP refactor CSRMatrix

---
 .../DistributedMatrixAdapter.H                |  10 +-
 MatrixWrapper/Distributed/Distributed.H       | 257 +++++++++++------
 MatrixWrapper/LDUMatrix/HostMatrix.C          | 258 +++++++++---------
 MatrixWrapper/LDUMatrix/HostMatrix.H          |  37 ++-
 Solver/CG/GKOCG.H                             |   3 -
 lduLduBase/lduLduBase.H                       |  18 +-
 6 files changed, 329 insertions(+), 254 deletions(-)

diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 411b3c2e7..249a9d7f9 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -118,12 +118,10 @@ public:
         return host_matrix_.get_persistent_object();
     }
 
-    void write() 
-    {
-        this->get()->write();
-    }
-
-    bool get_export();
+    // void write() 
+    // {
+    //     this->get()->write();
+    // }
 };
 
 }  // namespace Foam
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index e7322614c..bc5089bd3 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -2,6 +2,57 @@
 
 #include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 
+
+template <typename DistMtxType>
+std::unique_ptr<DistMtxType> generate_dist_mtx_with_inner_type(
+    word matrix_format, std::shared_ptr<gko::Executor> exec,
+    std::shared_ptr<gko::experimental::distributed::sparse_communicator> comm,
+    const SparsityPattern &local_sparsity,
+    const SparsityPattern &non_local_sparsity)
+{
+    auto local_dim = local_sparsity.dim;
+    auto non_local_dim = non_local_sparsity.dim;
+    auto local_interfaces = local_sparsity.interface_spans;
+    auto non_local_interfaces = non_local_sparsity.interface_spans;
+
+    if (matrix_format == "Ell") {
+        using mtx_type = gko::matrix::Ell<scalar, label>;
+
+        return DistMtxType::create(
+            exec, comm,
+            CombinationMatrix<scalar, label, mtx_type>::create(
+                exec, local_dim, local_interfaces),
+            CombinationMatrix<scalar, label, mtx_type>::create(
+                exec, non_local_dim, non_local_interfaces));
+    }
+    if (matrix_format == "Csr") {
+        using mtx_type = gko::matrix::Csr<scalar, label>;
+
+        return DistMtxType::create(
+            exec, comm,
+            CombinationMatrix<scalar, label, mtx_type>::create(
+                exec, local_dim, local_interfaces),
+            CombinationMatrix<scalar, label, mtx_type>::create(
+                exec, non_local_dim, non_local_interfaces));
+    }
+    if (matrix_format == "Coo") {
+        using mtx_type = gko::matrix::Coo<scalar, label>;
+
+        return DistMtxType::create(
+            exec, comm,
+            CombinationMatrix<scalar, label, mtx_type>::create(
+                exec, local_dim, local_interfaces),
+            CombinationMatrix<scalar, label, mtx_type>::create(
+                exec, non_local_dim, non_local_interfaces));
+    }
+
+    FatalErrorInFunction << "Matrix format " << matrix_format
+                         << " not supported! Supported formats: Csr, Ell, Coo"
+                         << abort(FatalError);
+    return {};
+}
+
+
 // The RepartDistMatrix class is a wrapper around Ginkgos distributed Matrix
 // class
 //
@@ -24,8 +75,7 @@ class RepartDistMatrix
         gko::experimental::distributed::Matrix<ValueType, LocalIndexType,
                                                GlobalIndexType>;
 
-    using part_type =
-        gko::experimental::distributed::Partition<label, label>;
+    using part_type = gko::experimental::distributed::Partition<label, label>;
 
     using localized_partition =
         gko::experimental::distributed::localized_partition<label>;
@@ -44,17 +94,6 @@ public:
     using sparse_communicator =
         gko::experimental::distributed::sparse_communicator;
 
-
-    void read_distributed(
-        const device_matrix_data &local_data,
-        const device_matrix_data &non_local_data,
-        std::shared_ptr<const sparse_communicator> sparse_comm)
-    {
-        this->dist_mtx_->read_distributed(local_data, non_local_data,
-                                          sparse_comm);
-        this->set_size(dist_mtx_->get_size());
-    }
-
     std::shared_ptr<const gko::LinOp> get_local_matrix()
     {
         this->dist_mtx_->get_local_matrix();
@@ -71,11 +110,8 @@ public:
      */
     RepartDistMatrix &operator=(const RepartDistMatrix &other)
     {
-        if (&other != this) {
-            *dist_mtx_.get() = *other.dist_mtx_.get();
-            this->set_size(dist_mtx_->get_size());
-        }
-        return *this;
+        FatalErrorInFunction
+            << "Copying the RepartDistMatrix is disallowed for performance reasons" << abort(FatalError);
     }
 
     /**
@@ -87,42 +123,17 @@ public:
     {
         if (&other != this) {
             gko::EnableLinOp<RepartDistMatrix>::operator=(std::move(other));
-            dist_mtx_ = std::move(other.dist_mtx_);
-            this->set_size(dist_mtx_->get_size());
+            this->dist_mtx_ = std::move(other.dist_mtx_);
+            this->set_size(this->dist_mtx_->get_size());
         }
         return *this;
     }
 
-    std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
-        word matrix_format, std::shared_ptr<gko::Executor> exec,
-        std::shared_ptr<gko::experimental::mpi::communicator> comm) const
-    {
-        if (matrix_format == "Csr") {
-            return dist_mtx::create(exec, *comm.get(),
-                                    gko::with_matrix_type<gko::matrix::Csr>());
-        }
-        if (matrix_format == "Ell") {
-            return dist_mtx::create(exec, *comm.get(),
-                                    gko::with_matrix_type<gko::matrix::Ell>());
-        }
-        if (matrix_format == "Coo") {
-            return dist_mtx::create(exec, *comm.get(),
-                                    gko::with_matrix_type<gko::matrix::Coo>());
-        }
-
-        if (matrix_format == "Ldu") {
-        }
-
-        FatalErrorInFunction
-            << "Matrix format " << matrix_format
-            << " not supported! Supported formats: Csr, Ell, Coo"
-            << abort(FatalError);
-    }
 
-    std::shared_ptr<part_type> get_partition() {return partition_;}
+    std::shared_ptr<part_type> get_partition() { return this->partition_; }
 
     static std::shared_ptr<RepartDistMatrix> create(
-        word matrix_format, std::shared_ptr<const HostMatrixWrapper> host_A )
+        word matrix_format, std::shared_ptr<const HostMatrixWrapper> host_A)
     {
         using part_type =
             gko::experimental::distributed::localized_partition<label>;
@@ -134,15 +145,13 @@ public:
         // device with correct size on each rank
         //
         // NOTE Start by copying first and repartition later
-        auto local_sparsity_pattern = host_A->compute_local_sparsity(exec);
-        auto non_local_sparsity_pattern = host_A->compute_non_local_sparsity(exec);
+        auto local_sparsity = host_A->compute_local_sparsity(exec);
+        auto non_local_sparsity = host_A->compute_non_local_sparsity(exec);
 
         // create original communicator pattern
         auto src_comm_pattern = host_A.get()->create_communication_pattern();
         // auto dst_comm_pattern = repartition_comm_pattern(src_comm_pattern);
 
-        std::cout << "!!!!! END\n";
-
         // // if accumulate_ranks
         // // 1. move to shared buffer
         // // 2. switch interfaces to local_data
@@ -152,53 +161,139 @@ public:
         //                         local_sparsity_pattern,
         //                         non_local_sparsity_pattern);
         //
-        // auto [local_coeffs, non_local_coeffs] = host_A->compute_coeffs();
         // auto [local_coeff_r, non_local_coeff_r] =
         //     repartition_coeffs(ranks_per_gpu, source_partition,
         //                        local_device_data, non_local_device_data);
         //
         //
-        // partition = part_type::build_from_blocked_recv(
-        //     exec, local_size_, comm_pattern.send_idxs,
-        //     comm_pattern.target_ids, comm_pattern.target_sizes);
-        //
-        // auto sparse_comm =
-        //     gko::experimental::distributed::sparse_communicator::create(
-        //         *comm.get(), partition_.get_localized_partition());
-        //
-        // auto dist_A = generate_dist_mtx_with_inner_type(
-        //     exec, host_A->get_communicator(), );
-        //
-        // // calls read of inner matrix type
-        // dist_A->read_distributed(local_device_data, non_local_device_data,
-        //                          sparse_comm);
-        //
-        // return dist_A;
+        auto comm_pattern = src_comm_pattern;
+        const label local_nnz = local_sparsity.size_;
+        const label non_local_nnz = non_local_sparsity.size_;
+
+        auto localized_partition = part_type::build_from_blocked_recv(
+            exec, local_sparsity.dim[0], comm_pattern.send_idxs, comm_pattern.target_ids,
+            comm_pattern.target_sizes);
+
+        auto comm = *exec_handler.get_communicator().get();
+
+        auto sparse_comm =
+            gko::experimental::distributed::sparse_communicator::create(
+                comm, localized_partition);
+
+        auto dist_A = generate_dist_mtx_with_inner_type<dist_mtx>(
+            matrix_format, exec, sparse_comm, local_sparsity,
+            non_local_sparsity);
+
+        auto local_coeffs = gko::array<scalar>(exec, local_nnz);
+        auto non_local_coeffs = gko::array<scalar>(exec, non_local_nnz);
+
+        host_A.get()->compute_local_coeffs(local_sparsity, local_coeffs);
+        host_A.get()->compute_non_local_coeffs(non_local_sparsity, non_local_coeffs);
+
+        dist_A->read_distributed(
+            device_matrix_data(exec,
+                local_sparsity.dim,
+                local_sparsity.row_idxs,
+                local_sparsity.col_idxs,
+                local_coeffs),
+            device_matrix_data(exec,
+                non_local_sparsity.dim,
+                non_local_sparsity.row_idxs,
+                non_local_sparsity.col_idxs,
+                non_local_coeffs),
+                sparse_comm);
+
+        return std::make_shared<RepartDistMatrix>(exec, dist_A->get_size(), std::move(dist_A));
     };
 
-    void update(std::shared_ptr<const HostMatrixWrapper> host_A ){
+    void update(std::shared_ptr<const HostMatrixWrapper> host_A){
         // On update dist_mtx already exists
         // and only local and non local coeffs need to be updated
         //
         // repartition_coeffs(local_device_data, non_local_device_data)
     };
 
-    void write();
+    RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
+                     gko::dim<2> size,
+                     std::unique_ptr<dist_mtx> dist_mtx)
+        : gko::EnableLinOp<RepartDistMatrix>(exec),
+          dist_mtx_(std::move(dist_mtx))
+    {
+        this->set_size(size);
+    }
+
+    void write(const word field_name_, const objectRegistry &db_
+            ) const {
+        auto dist_matrix = gko::as<
+            gko::experimental::distributed::Matrix<scalar, label, label>>(
+            this->dist_mtx_.get());
+        word matrix_format_ { "Coo" };
+
+        if (matrix_format_ == "Coo") {
+            using Coo = gko::matrix::Coo<scalar, label>;
+            std::vector<std::shared_ptr<const gko::LinOp>> local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Coo>>(
+                    dist_matrix->get_local_matrix())
+                    ->get_combination()
+                    ->get_operators();
+            auto non_local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Coo>>(
+                    dist_matrix->get_non_local_matrix())
+                    ->get_combination()
+                    ->get_operators();
+            export_mtx<Coo>(word(field_name_ + "_local"), local_interfaces,
+                            db_);
+            export_mtx<Coo>(field_name_ + "_non_local", non_local_interfaces,
+                            db_);
+        }
+
+        if (matrix_format_ == "Csr") {
+            using Csr = gko::matrix::Csr<scalar, label>;
+            auto local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Csr>>(
+                    dist_matrix->get_local_matrix().get())
+                    ->get_combination()
+                    ->get_operators();
+            auto non_local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Csr>>(
+                    dist_matrix->get_non_local_matrix().get())
+                    ->get_combination()
+                    ->get_operators();
+            export_mtx<Csr>(word(field_name_ + "_local"), local_interfaces,
+                            db_);
+            export_mtx<Csr>(field_name_ + "_non_local", non_local_interfaces,
+                            db_);
+        }
+
+        if (matrix_format_ == "Ell") {
+            using Ell = gko::matrix::Ell<scalar, label>;
+            auto local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Ell>>(
+                    dist_matrix->get_local_matrix().get())
+                    ->get_combination()
+                    ->get_operators();
+            auto non_local_interfaces =
+                gko::as<CombinationMatrix<scalar, label, Ell>>(
+                    dist_matrix->get_non_local_matrix().get())
+                    ->get_combination()
+                    ->get_operators();
+            export_mtx<Ell>(field_name_ + "_local", local_interfaces, db_);
+            export_mtx<Ell>(field_name_ + "_non_local", non_local_interfaces,
+                            db_);
+        }
+    }
+
+    void solve(const gko::LinOp *b, gko::LinOp *x) const
+    {
+        std::cout << "call solve\n";
+        this->dist_mtx_->apply(b, x);
+    }
 
 protected:
     RepartDistMatrix(std::shared_ptr<const gko::Executor> exec)
         : gko::EnableLinOp<RepartDistMatrix>(exec)
     {}
 
-    RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
-                     std::shared_ptr<const communicator> comm,
-                     std::unique_ptr<const gko::LinOp> local_template,
-                     std::unique_ptr<const gko::LinOp> non_local_template)
-        : gko::EnableLinOp<RepartDistMatrix>(exec),
-          dist_mtx_(dist_mtx::create(exec, *comm.get(), local_template,
-                                     non_local_template))
-    {}
-
     // Here we implement the application of the linear operator, x = A * b.
     // apply_impl will be called by the apply method, after the arguments have
     // been moved to the correct executor and the operators checked for
@@ -209,9 +304,11 @@ protected:
     // are always true in this example).
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
+        std::cout << "call apply\n";
         this->dist_mtx_->apply(b, x);
     }
 
+
     // There is also a version of the apply function which does the operation
     // x = alpha * A * b + beta * x. This function is commonly used and can
     // often be better optimized than implementing it using x = A * b. However,
@@ -219,6 +316,7 @@ protected:
     void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
                     const gko::LinOp *beta, gko::LinOp *x) const override
     {
+        std::cout << "call apply 2\n";
         this->dist_mtx_->apply(alpha, b, beta, x);
     }
 
@@ -227,5 +325,4 @@ private:
     std::unique_ptr<dist_mtx> dist_mtx_;
 
     std::shared_ptr<part_type> partition_;
-
 };
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index dc467b5f3..8a7337bb7 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -41,8 +41,7 @@ HostMatrixWrapper::HostMatrixWrapper(
     const FieldField<Field, scalar> &interfaceBouCoeffs,
     const FieldField<Field, scalar> &interfaceIntCoeffs,
     const lduInterfaceFieldPtrsList &interfaces,
-    const dictionary &solverControls, const word &fieldName, label verbose
-    )
+    const dictionary &solverControls, const word &fieldName, label verbose)
     : exec_{exec},
       device_id_guard_{db, fieldName, exec_.get_device_exec()},
       verbose_(verbose),
@@ -62,9 +61,9 @@ HostMatrixWrapper::HostMatrixWrapper(
       local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
       local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
       interfaces_(interfaces),
+      interfaceBouCoeffs_(interfaceBouCoeffs),
       non_local_matrix_nnz_{count_interface_nnz(interfaces, true)}
-{
-}
+{}
 
 
 label HostMatrixWrapper::count_interface_nnz(
@@ -162,7 +161,6 @@ void neg_interface_iterator(const lduInterfaceFieldPtrsList &interfaces,
 
 CommunicationPattern HostMatrixWrapper::create_communication_pattern() const
 {
-
     using comm_size_type = CommunicationPattern::comm_size_type;
     // temp map, mapping from neighbour rank interface cells
     std::map<label, std::vector<label>> interface_cell_map{};
@@ -172,7 +170,7 @@ CommunicationPattern HostMatrixWrapper::create_communication_pattern() const
     gko::size_type n_procs = 0;
     interface_iterator<processorFvPatch>(
         interfaces_, [&](label, label, label, const processorFvPatch &patch,
-                        const lduInterfaceField *iface) {
+                         const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
             const label neighbProcNo = patch.neighbProcNo();
 
@@ -340,9 +338,12 @@ HostMatrixWrapper::collect_cells_on_non_local_interface(
     return non_local_idxs;
 }
 
-SparsityPattern HostMatrixWrapper::compute_non_local_sparsity(std::shared_ptr<const gko::Executor> exec) const
+SparsityPattern HostMatrixWrapper::compute_non_local_sparsity(
+    std::shared_ptr<const gko::Executor> exec) const
 {
-    SparsityPattern sparsity {exec->get_master(), non_local_matrix_nnz_};
+    SparsityPattern sparsity{exec->get_master(), non_local_matrix_nnz_};
+    sparsity.dim = gko::dim<2>{nrows_, non_local_matrix_nnz_};
+
     auto non_local_row_indices =
         collect_cells_on_non_local_interface(interfaces_);
     auto rows = sparsity.row_idxs.get_data();
@@ -359,22 +360,45 @@ SparsityPattern HostMatrixWrapper::compute_non_local_sparsity(std::shared_ptr<co
         element_ctr += 1;
     }
 
+    // TODO this can be merged with algorithm in update_local_matrix_data
+    auto non_local_interfaces =
+        collect_cells_on_non_local_interface(interfaces_);
+    label interface_ctr{0};
+    label prev_interface_ctr{0};
+    label end{0};
+    label start{0};
+    for (auto [interface_idx, interface_cell_ctr, interface_row] :
+         non_local_interfaces) {
+        // check if a new interface has started or final interface has been
+        // reache
+        if (interface_idx > prev_interface_ctr ||
+            interface_ctr == non_local_interfaces.size() - 1) {
+            end = start + interface_ctr;
+            interface_ctr = 0;
+            sparsity.interface_spans.emplace_back(start, end);
+            start = end;
+            prev_interface_ctr = interface_idx;
+        }
+        interface_ctr++;
+    }
+
     return sparsity;
 }
 
-SparsityPattern HostMatrixWrapper::compute_local_sparsity(std::shared_ptr<const gko::Executor> exec) const {
+SparsityPattern HostMatrixWrapper::compute_local_sparsity(
+    std::shared_ptr<const gko::Executor> exec) const
+{
     LOG_1(verbose_, "start init host sparsity pattern")
 
-    SparsityPattern sparsity {exec->get_master(), local_matrix_w_interfaces_nnz_};
+    SparsityPattern sparsity{exec->get_master(),
+                             local_matrix_w_interfaces_nnz_};
 
-    auto lower_local =
-        idx_array::view(exec, upper_nnz_,
-                        const_cast<label *>(addr_.lowerAddr().begin()));
+    auto lower_local = idx_array::view(
+        exec, upper_nnz_, const_cast<label *>(addr_.lowerAddr().begin()));
 
     // TODO const_view ?
-    auto upper_local =
-        idx_array::view(exec, upper_nnz_,
-                        const_cast<label *>(addr_.upperAddr().begin()));
+    auto upper_local = idx_array::view(
+        exec, upper_nnz_, const_cast<label *>(addr_.upperAddr().begin()));
 
     // row of upper, col of lower
     const auto lower = lower_local.get_const_data();
@@ -397,6 +421,9 @@ SparsityPattern HostMatrixWrapper::compute_local_sparsity(std::shared_ptr<const
     init_local_sparsity(nrows_, upper_nnz_, symmetric_, upper, lower, rows,
                         cols, permute);
 
+    sparsity.dim = gko::dim<2>{nrows_, nrows_};
+    sparsity.interface_spans.emplace_back(0, local_matrix_nnz_);
+
     // if no local interfaces are present we are done here
     // otherwise we need to add local interfaces to local_sparsity in order
     // of the interfaces to end of the col and row idx arrays. This will produce
@@ -422,47 +449,66 @@ SparsityPattern HostMatrixWrapper::compute_local_sparsity(std::shared_ptr<const
             permute[i] = i;
             local_interface_ctr++;
         }
+
+        // TODO merge with above
+        local_interface_ctr = 0;
+        label prev_interface_idx{0};
+        label end{0};
+        label start{local_matrix_nnz_};
+        for (auto [interface_idx, interface_row, interface_col] :
+             local_interfaces) {
+            // check if a new interface has started or final interface has been
+            // reache
+            if (interface_idx > prev_interface_idx ||
+                local_interface_ctr + 1 == local_interfaces.size()) {
+                end = start + local_interface_ctr;
+                local_interface_ctr = 0;
+                sparsity.interface_spans.emplace_back(start, end);
+                start = end;
+                prev_interface_idx = interface_idx;
+            }
+            local_interface_ctr++;
+        }
     }
+
     LOG_1(verbose_, "done init host sparsity pattern")
     return sparsity;
 }
 
 
-void HostMatrixWrapper::update_local_matrix_data(
-    const lduInterfaceFieldPtrsList &interfaces,
-    const FieldField<Field, scalar> &interfaceBouCoeffs,
-    gko::array<scalar> &local_coeffs) const
+void HostMatrixWrapper::compute_local_coeffs(
+    const SparsityPattern &sparsity, gko::array<scalar> &local_coeffs) const
 {
-    // auto ref_exec = exec_.get_ref_exec();
-    //
+    auto ref_exec = exec_.get_ref_exec();
+    auto permute = sparsity.ldu_mapping.get_data();
+
     // // TODO this does not work for Ell
-    // auto permute = local_sparsity_pattern_.ldu_mapping_.get_data();
-    //
-    // if (reorder_on_copy_) {
-    //     auto dense = local_coeffs.get_data();
-    //     if (local_interface_nnz_) {
-    //         auto couple_coeffs =
-    //             collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
-    //         if (symmetric_) {
-    //             symmetric_update_w_interface(
-    //                 local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_, permute,
-    //                 scaling_, diag_, upper_, couple_coeffs.data(), dense);
-    //
-    //         } else {
-    //             non_symmetric_update_w_interface(
-    //                 local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_, permute,
-    //                 scaling_, diag_, upper_, lower_, couple_coeffs.data(),
-    //                 dense);
-    //         }
-    //     } else {
-    //         if (symmetric_) {
-    //             symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-    //                              scaling_, diag_, upper_, dense);
-    //         } else {
-    //             non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-    //                                  scaling_, diag_, upper_, lower_, dense);
-    //         }
-    //     }
+    if (reorder_on_copy_) {
+        auto dense = local_coeffs.get_data();
+        if (local_interface_nnz_) {
+            auto couple_coeffs = collect_interface_coeffs(
+                interfaces_, interfaceBouCoeffs_, true);
+            if (symmetric_) {
+                symmetric_update_w_interface(
+                    local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_, permute,
+                    scaling_, diag_, upper_, couple_coeffs.data(), dense);
+
+            } else {
+                non_symmetric_update_w_interface(
+                    local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_, permute,
+                    scaling_, diag_, upper_, lower_, couple_coeffs.data(),
+                    dense);
+            }
+        } else {
+            if (symmetric_) {
+                symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+                                 scaling_, diag_, upper_, dense);
+            } else {
+                non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+                                     scaling_, diag_, upper_, lower_, dense);
+            }
+        }
+    }
     // } else {
     //     // TODO
     //     // - this should be moved to separate function to avoid making this
@@ -476,7 +522,8 @@ void HostMatrixWrapper::update_local_matrix_data(
     //     // copy upper
     //     auto u_host_view =
     //         gko::array<scalar>::const_view(ref_exec, upper_nnz_, upper_);
-    //     auto u_device_view = gko::array<scalar>::view(target_exec, upper_nnz_,
+    //     auto u_device_view = gko::array<scalar>::view(target_exec,
+    //     upper_nnz_,
     //                                                   local_coeffs.get_data());
     //     u_device_view = u_host_view;
     //
@@ -485,7 +532,8 @@ void HostMatrixWrapper::update_local_matrix_data(
     //         auto l_host_view =
     //             gko::array<scalar>::const_view(ref_exec, upper_nnz_, lower_);
     //         auto l_device_view = gko::array<scalar>::view(
-    //             target_exec, upper_nnz_, &local_coeffs.get_data()[upper_nnz_]);
+    //             target_exec, upper_nnz_,
+    //             &local_coeffs.get_data()[upper_nnz_]);
     //         l_device_view = l_host_view;
     //     }
     //
@@ -501,7 +549,8 @@ void HostMatrixWrapper::update_local_matrix_data(
     //     if (local_interface_nnz_) {
     //         const label interface_start = diag_start + nrows_;
     //         auto couple_coeffs =
-    //             collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
+    //             collect_interface_coeffs(interfaces, interfaceBouCoeffs,
+    //             true);
     //         auto interface_host_view = gko::array<scalar>::view(
     //             ref_exec, local_interface_nnz_, couple_coeffs.data());
     //         auto interface_device_view = gko::array<scalar>::view(
@@ -532,94 +581,31 @@ void HostMatrixWrapper::update_local_matrix_data(
     //     dense_vec->copy_from(row_collection);
     // }
     //
-    // // NOTE interface_spans and dim are not persistent so we need to
-    // // recreate this for every solver call
-    // // TODO this needs a proper implementation once we know how to handle
-    // // interfaces
-    // local_sparsity_pattern_.dim_ = gko::dim<2>{nrows_, nrows_};
-    //
-    // local_sparsity_pattern_.interface_spans_.emplace_back(0, local_matrix_nnz_);
-    // if (local_interface_nnz_) {
-    //     auto local_interfaces = collect_local_interface_indices(interfaces);
-    //
-    //     // counter how many interface elements are found on current interface
-    //     label local_interface_ctr{0};
-    //     label prev_interface_idx{0};
-    //     label end{0};
-    //     label start{local_matrix_nnz_};
-    //     for (auto [interface_idx, interface_row, interface_col] :
-    //          local_interfaces) {
-    //         // check if a new interface has started or final interface has been
-    //         // reache
-    //         if (interface_idx > prev_interface_idx ||
-    //             local_interface_ctr + 1 == local_interfaces.size()) {
-    //             end = start + local_interface_ctr;
-    //             local_interface_ctr = 0;
-    //             local_sparsity_pattern_.interface_spans_.emplace_back(start,
-    //                                                                   end);
-    //             start = end;
-    //             prev_interface_idx = interface_idx;
-    //         }
-    //         local_interface_ctr++;
-    //     }
-    // }
 }
 
 
-void HostMatrixWrapper::update_non_local_matrix_data(
-    const lduInterfaceFieldPtrsList &interfaces,
-    const FieldField<Field, scalar> &interfaceBouCoeffs,
-    gko::array<scalar> &non_local_coeffs) const
+void HostMatrixWrapper::compute_non_local_coeffs(
+    const SparsityPattern &, gko::array<scalar> &coeffs) const
 {
-    // auto ref_exec = exec_.get_ref_exec();
-    // auto interface_coeffs =
-    //     collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
-    //
-    // // copy interfaces
-    // auto tmp_contiguous_iface =
-    //     gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
-    // auto contiguous_iface = tmp_contiguous_iface.get_data();
-    //
-    // for (label element_ctr = 0; element_ctr < non_local_matrix_nnz_;
-    //      element_ctr++) {
-    //     contiguous_iface[element_ctr] = interface_coeffs[element_ctr];
-    // }
-    //
-    // // copy to persistent
-    // auto i_device_view = gko::array<scalar>::view(
-    //     ref_exec, non_local_matrix_nnz_, non_local_coeffs.get_data());
-    // i_device_view = tmp_contiguous_iface;
-    //
-    // // NOTE interface_spans and dim are not persistent so we need to
-    // // We have now the full non_local matrix and need to add the start and end
-    // // to the PersistentSparsityPattern data structure
-    // non_local_sparsity_pattern_.dim_ =
-    //     gko::dim<2>{nrows_, non_local_matrix_nnz_};
-    //
-    // auto non_local_interfaces =
-    //     collect_cells_on_non_local_interface(interfaces);
-    //
-    // // TODO this can be merged with algorithm in update_local_matrix_data
-    // // refactor to separate function
-    // label interface_ctr{0};
-    // label prev_interface_ctr{0};
-    // label end{0};
-    // label start{0};
-    // for (auto [interface_idx, interface_cell_ctr, interface_row] :
-    //      non_local_interfaces) {
-    //     // check if a new interface has started or final interface has been
-    //     // reache
-    //     if (interface_idx > prev_interface_ctr ||
-    //         interface_ctr == non_local_interfaces.size() - 1) {
-    //         end = start + interface_ctr;
-    //         interface_ctr = 0;
-    //         non_local_sparsity_pattern_.interface_spans_.emplace_back(start,
-    //                                                                   end);
-    //         start = end;
-    //         prev_interface_ctr = interface_idx;
-    //     }
-    //     interface_ctr++;
-    // }
+    auto ref_exec = exec_.get_ref_exec();
+    auto interface_coeffs =
+        collect_interface_coeffs(interfaces_, interfaceBouCoeffs_, false);
+
+    // copy interfaces
+    auto tmp_contiguous_iface =
+        gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
+    auto contiguous_iface = tmp_contiguous_iface.get_data();
+
+    for (label element_ctr = 0; element_ctr < non_local_matrix_nnz_;
+         element_ctr++) {
+        contiguous_iface[element_ctr] = interface_coeffs[element_ctr];
+    }
+
+    // copy to persistent
+    auto i_device_view = gko::array<scalar>::view(
+        ref_exec, non_local_matrix_nnz_, coeffs.get_data());
+
+    i_device_view = tmp_contiguous_iface;
 }
 
 }  // namespace Foam
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 40bf46f47..c4956855f 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -276,6 +276,8 @@ private:
 
     const lduInterfaceFieldPtrsList &interfaces_;
 
+    const FieldField<Field, scalar> &interfaceBouCoeffs_;
+
     const CommunicationPattern communication_pattern_;
 
     /* Iterates all interfaces and collects the coefficients into a vector
@@ -324,23 +326,7 @@ private:
 
     // updater
 
-    /** Copies the LDU matrix coefficients to local_coeffs without changing or
-     ** reinstantiating the sparsity pattern.
-     ** This uses the local_sparsity_.ldu_mapping to permute the data already
-     ** on the host or device to be in row major order.
-     **/
-    void update_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs,
-        gko::array<scalar> &local_coeffs) const;
 
-    /** Copies the interface matrix coefficients to non_local_coeffs without
-    ** changing or reinstantiating the sparsity pattern.
-    **/
-    void update_non_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs,
-        gko::array<scalar> &local_coeffs) const;
 
 public:
     // segregated matrix wrapper constructor
@@ -383,7 +369,21 @@ public:
      ** which are required to create to a ginkgo matrix
      **/
     SparsityPattern compute_local_sparsity(
-            std::shared_ptr<const gko::Executor> exec) const;
+        std::shared_ptr<const gko::Executor> exec) const;
+
+    /** Copies the LDU matrix coefficients to local_coeffs without changing or
+     ** reinstantiating the sparsity pattern.
+     ** This uses the local_sparsity_.ldu_mapping to permute the data already
+     ** on the host or device to be in row major order.
+     **/
+    void compute_local_coeffs(const SparsityPattern &sparsity,
+                              gko::array<scalar> &target_coeffs) const;
+
+    /** Copies the interface matrix coefficients to non_local_coeffs without
+    ** changing or reinstantiating the sparsity pattern.
+    **/
+    void compute_non_local_coeffs(const SparsityPattern &sparsity,
+                                  gko::array<scalar> &target_coeffs) const;
 
     /** Based on OpenFOAMs interfaces this function computes two
      ** consecutive index arrays (non_local_sparsisty_.row_idxs and col_idxs) in
@@ -392,7 +392,7 @@ public:
      ** matrix
      **/
     SparsityPattern compute_non_local_sparsity(
-            std::shared_ptr<const gko::Executor> exec) const;
+        std::shared_ptr<const gko::Executor> exec) const;
 
     /** Iterates all interfaces and counts the number of unique neighbour
      ** processors and number of interfaces in total for this processor
@@ -411,7 +411,6 @@ public:
     const ExecutorHandler &get_exec_handler() const { return exec_; }
 
     gko::Array<scalar> &get_local_coeffs() const {};
-
 };
 
 
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index b4a8858b7..646480a4c 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -14,10 +14,7 @@ class GKOCGFactory {
 private:
     using vec = gko::matrix::Dense<scalar>;
     using cg = gko::solver::Cg<scalar>;
-
     using dist_vec = gko::experimental::distributed::Vector<scalar>;
-    using dist_mtx =
-        gko::experimental::distributed::Matrix<scalar, label, label>;
 
     const dictionary &solverControls_;
 
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index ebfcf9b00..311d17d18 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -216,7 +216,6 @@ public:
 
         // create a distributed matrix from the host matrix
         // here also repartitioning is performed if required
-        std::cout << "Start persistent distributed matrix\n";
         PersistentDistributedMatrix dist_A{db_,
                              exec_handler,
                              field_name_,
@@ -224,8 +223,6 @@ public:
                              solver_controls_,
                              verbose_};
 
-        auto partition = dist_A.get()->get_partition();
-
         PersistentVector<scalar> dist_b{
             source.begin(),
             source.size(),
@@ -269,25 +266,26 @@ public:
             auto precond = this->init_preconditioner(
                 dist_A_v, exec_handler_.get_device_exec());)
 
-        bool debug(solver_controls_.lookupOrDefault<Switch>("debug", false));
-        if (debug && db_.time().writeTime()) {
+        bool write(solver_controls_.lookupOrDefault<Switch>("export", false));
+        if (write && db_.time().writeTime()) {
             LOG_0(verbose_, "Export system")
-            dist_A.write();
-            dist_b.write();
+            dist_A.get()->write(field_name_, db_);
+            //dist_b.write(field_name_, db_);
         }
 
 
         LOG_1(verbose_, "create solver")
         auto solver_gen = this->create_dist_solver(
-            exec_handler_.get_device_exec(), dist_A_v, dist_x_v,
-            dist_b_v, verbose_, dist_A.get_export(), precond);
+            exec_handler.get_device_exec(), dist_A_v, dist_x_v,
+            dist_b_v, verbose_, write, precond);
 
         TIME_WITH_FIELDNAME(verbose_, generate_solver, field_name_,
                             auto solver = solver_gen->generate(dist_A_v);)
-        LOG_1(verbose_, "solve solver done")
+        LOG_1(verbose_, "generate solver done")
         TIME_WITH_FIELDNAME(verbose_, solve, field_name_,
                             solver->apply(dist_b_v, dist_x_v);)
 
+        auto partition = dist_A.get()->get_partition();
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, field_name_,
                             dist_x.copy_back(partition);)
         auto bandwidth_copy_back = sizeof(scalar) * psi.size() /

From 5408ec3c3a8aaf6594413ca601cd9146e64b8803 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 22 Apr 2024 11:51:35 +0200
Subject: [PATCH 42/90] WIP combination adoptation

---
 CMakeLists.txt                          |   2 +-
 DevicePersistent/Vector/Vector.H        |  41 ++---
 MatrixWrapper/Combination/Combination.H |  12 +-
 MatrixWrapper/Distributed/Distributed.H | 193 +++++++++++++++---------
 MatrixWrapper/LDUMatrix/HostMatrix.C    |  50 +++---
 MatrixWrapper/LDUMatrix/HostMatrix.H    |  12 +-
 StoppingCriterion/StoppingCriterion.C   |   5 +-
 StoppingCriterion/StoppingCriterion.H   |   3 +-
 lduLduBase/lduLduBase.H                 |  20 +--
 9 files changed, 192 insertions(+), 146 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 17ce88027..4c3cdadef 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,7 +73,7 @@ if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO))
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "sparse-communicator-cmake-modernization-combination-interface-ogl"
+    "sparse-communicator-cmake-modernization-combination-interface-ogl-rebase"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 706f08447..9796f711e 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -36,9 +36,8 @@ struct VectorInitFunctor {
 
 
     VectorInitFunctor(const ExecutorHandler &exec, const word name,
-                      const T *other,
-                      const label size,
-                      const label verbose, const bool on_device = false)
+                      const T *other, const label size, const label verbose,
+                      const bool on_device = false)
         : name_(name),
           exec_(exec),
           on_device_(on_device),
@@ -76,8 +75,8 @@ struct VectorInitFunctor {
                                              const_cast<T *>(other_));
 
         return gko::share(dist_vec::create(
-                exec, *exec_.get_communicator().get(),
-                vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)));
+            exec, *exec_.get_communicator().get(),
+            vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)));
     }
 };
 
@@ -120,10 +119,10 @@ public:
      * device or host
      * @param ranks_per_gpu
      */
-    PersistentVector(const T *memory, const label size, const word name, const objectRegistry &db,
-                     const ExecutorHandler &exec,
-                     const label verbose,
-                     const bool update, const bool init_on_device)
+    PersistentVector(const T *memory, const label size, const word name,
+                     const objectRegistry &db, const ExecutorHandler &exec,
+                     const label verbose, const bool update,
+                     const bool init_on_device)
         : PersistentBase<gko::experimental::distributed::Vector<T>,
                          VectorInitFunctor<T>>(
               name, db,
@@ -142,26 +141,12 @@ public:
      **/
     void copy_back(std::shared_ptr<const part_type> partition)
     {
-        const label local_size = size_;
-        auto global_size = partition->get_size();
+        auto ref_exec = exec_.get_ref_exec();
+        auto host_buffer_view = gko::array<T>::view(
+            exec_.get_device_exec(), size_, get_vector()->get_local_values());
 
-        auto comm = exec_.get_gko_mpi_host_comm();
-        auto host_device_vector =
-            dist_vec::create(exec_.get_ref_exec(), *comm.get());
-        host_device_vector->operator=(*get_vector().get());
-
-        // TODO why does it need its size
-        // repartitioner size should be sufficient
-        auto target_vector = dist_vec::create(exec_.get_ref_exec(), *comm.get(),
-                                              gko::dim<2>{global_size, 1},
-                                              gko::dim<2>{local_size, 1}, 1);
-        scalar *host_buffer = host_device_vector->get_local_values();
-
-        auto host_buffer_view =
-            gko::array<T>::view(exec_.get_ref_exec(), local_size, host_buffer);
-
-        auto to_view = gko::array<T>::view(exec_.get_ref_exec(), local_size,
-                                           const_cast<T *>(memory_));
+        auto to_view =
+            gko::array<T>::view(ref_exec, size_, const_cast<T *>(memory_));
         to_view = host_buffer_view;
     }
 
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index 2d30d2078..ff3a2847b 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -39,7 +39,6 @@ public:
         for (auto &interface : interface_ranges_) {
             auto length = interface.length();
             auto offset = interface.begin;
-            std::cout << " read interface " << length << " " << offset << "\n";
             auto op = gko::share(InnerMatrixType::create(exec));
 
             device_matrix_data in(
@@ -80,15 +79,8 @@ public:
      */
     CombinationMatrix &operator=(const CombinationMatrix &other)
     {
-        if (&other != this) {
-            gko::EnableLinOp<CombinationMatrix>::operator=(other);
-            interface_ranges_ = std::vector<gko::span>{};
-            for (auto &i_range : other.interface_ranges_) {
-                interface_ranges_.emplace_back(i_range.begin, i_range.end);
-            }
-            comb_ = other.comb_;
-        }
-        return *this;
+    FatalErrorInFunction << "Copying CombinationMatrix is not allowed"
+                         << abort(FatalError);
     }
 
     /**
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index bc5089bd3..1a8719f42 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -1,5 +1,7 @@
+#include <ginkgo/core/distributed/lin_op.hpp>
 #include <ginkgo/ginkgo.hpp>
 
+
 #include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 
 
@@ -7,13 +9,13 @@ template <typename DistMtxType>
 std::unique_ptr<DistMtxType> generate_dist_mtx_with_inner_type(
     word matrix_format, std::shared_ptr<gko::Executor> exec,
     std::shared_ptr<gko::experimental::distributed::sparse_communicator> comm,
-    const SparsityPattern &local_sparsity,
-    const SparsityPattern &non_local_sparsity)
+    std::shared_ptr<const SparsityPattern> local_sparsity,
+    std::shared_ptr<const SparsityPattern> non_local_sparsity)
 {
-    auto local_dim = local_sparsity.dim;
-    auto non_local_dim = non_local_sparsity.dim;
-    auto local_interfaces = local_sparsity.interface_spans;
-    auto non_local_interfaces = non_local_sparsity.interface_spans;
+    auto local_dim = local_sparsity->dim;
+    auto non_local_dim = non_local_sparsity->dim;
+    auto local_interfaces = local_sparsity->interface_spans;
+    auto non_local_interfaces = non_local_sparsity->interface_spans;
 
     if (matrix_format == "Ell") {
         using mtx_type = gko::matrix::Ell<scalar, label>;
@@ -61,10 +63,11 @@ std::unique_ptr<DistMtxType> generate_dist_mtx_with_inner_type(
 // matrix are of RepartDistMatrix type.
 template <typename ValueType, typename LocalIndexType, typename GlobalIndexType>
 class RepartDistMatrix
-    : public gko::EnableLinOp<
+    : public gko::experimental::EnableDistributedLinOp<
           RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>,
       public gko::EnableCreateMethod<
-          RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>> {
+          RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>,
+      public gko::experimental::distributed::DistributedBase {
     friend class gko::EnableCreateMethod<
         RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>;
     friend class gko::EnablePolymorphicObject<
@@ -76,24 +79,21 @@ class RepartDistMatrix
                                                GlobalIndexType>;
 
     using part_type = gko::experimental::distributed::Partition<label, label>;
-
-    using localized_partition =
+    using local_part_type =
         gko::experimental::distributed::localized_partition<label>;
 
-public:
-    using gko::EnableLinOp<RepartDistMatrix<ValueType, LocalIndexType,
-                                            GlobalIndexType>>::convert_to;
-    using gko::EnableLinOp<
-        RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>::move_to;
-
     using vec = gko::matrix::Dense<ValueType>;
-    using coef_type = gko::array<ValueType>;
     using device_matrix_data = gko::device_matrix_data<scalar, label>;
-    using matrix_data = gko::matrix_data<scalar, label>;
     using communicator = gko::experimental::mpi::communicator;
     using sparse_communicator =
         gko::experimental::distributed::sparse_communicator;
 
+public:
+    using gko::experimental::EnableDistributedLinOp<RepartDistMatrix<
+        ValueType, LocalIndexType, GlobalIndexType>>::convert_to;
+    using gko::experimental::EnableDistributedLinOp<
+        RepartDistMatrix<ValueType, LocalIndexType, GlobalIndexType>>::move_to;
+
     std::shared_ptr<const gko::LinOp> get_local_matrix()
     {
         this->dist_mtx_->get_local_matrix();
@@ -110,8 +110,9 @@ public:
      */
     RepartDistMatrix &operator=(const RepartDistMatrix &other)
     {
-        FatalErrorInFunction
-            << "Copying the RepartDistMatrix is disallowed for performance reasons" << abort(FatalError);
+        FatalErrorInFunction << "Copying the RepartDistMatrix is disallowed "
+                                "for performance reasons"
+                             << abort(FatalError);
     }
 
     /**
@@ -122,7 +123,8 @@ public:
     RepartDistMatrix &operator=(RepartDistMatrix &&other)
     {
         if (&other != this) {
-            gko::EnableLinOp<RepartDistMatrix>::operator=(std::move(other));
+            gko::experimental::EnableDistributedLinOp<
+                RepartDistMatrix>::operator=(std::move(other));
             this->dist_mtx_ = std::move(other.dist_mtx_);
             this->set_size(this->dist_mtx_->get_size());
         }
@@ -135,18 +137,16 @@ public:
     static std::shared_ptr<RepartDistMatrix> create(
         word matrix_format, std::shared_ptr<const HostMatrixWrapper> host_A)
     {
-        using part_type =
-            gko::experimental::distributed::localized_partition<label>;
-
         auto exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_device_exec();
+        auto comm = *exec_handler.get_communicator().get();
 
         // repartition things here first by creating device_matrix_data on
         // device with correct size on each rank
         //
         // NOTE Start by copying first and repartition later
-        auto local_sparsity = host_A->compute_local_sparsity(exec);
-        auto non_local_sparsity = host_A->compute_non_local_sparsity(exec);
+        auto local_sparsity_ = host_A->compute_local_sparsity(exec);
+        auto non_local_sparsity_ = host_A->compute_non_local_sparsity(exec);
 
         // create original communicator pattern
         auto src_comm_pattern = host_A.get()->create_communication_pattern();
@@ -167,46 +167,98 @@ public:
         //
         //
         auto comm_pattern = src_comm_pattern;
-        const label local_nnz = local_sparsity.size_;
-        const label non_local_nnz = non_local_sparsity.size_;
+        const label local_nnz = local_sparsity_->size_;
+        const label non_local_nnz = non_local_sparsity_->size_;
 
-        auto localized_partition = part_type::build_from_blocked_recv(
-            exec, local_sparsity.dim[0], comm_pattern.send_idxs, comm_pattern.target_ids,
-            comm_pattern.target_sizes);
+        auto localized_partition = local_part_type::build_from_blocked_recv(
+            exec, local_sparsity_->dim[0], comm_pattern.send_idxs,
+            comm_pattern.target_ids, comm_pattern.target_sizes);
 
-        auto comm = *exec_handler.get_communicator().get();
 
         auto sparse_comm =
-            gko::experimental::distributed::sparse_communicator::create(
-                comm, localized_partition);
+            sparse_communicator::create(comm, localized_partition);
 
         auto dist_A = generate_dist_mtx_with_inner_type<dist_mtx>(
-            matrix_format, exec, sparse_comm, local_sparsity,
-            non_local_sparsity);
+            matrix_format, exec, sparse_comm, local_sparsity_,
+            non_local_sparsity_);
 
         auto local_coeffs = gko::array<scalar>(exec, local_nnz);
         auto non_local_coeffs = gko::array<scalar>(exec, non_local_nnz);
 
-        host_A.get()->compute_local_coeffs(local_sparsity, local_coeffs);
-        host_A.get()->compute_non_local_coeffs(non_local_sparsity, non_local_coeffs);
+        host_A.get()->compute_local_coeffs(local_sparsity_, local_coeffs);
+        host_A.get()->compute_non_local_coeffs(non_local_sparsity_,
+                                               non_local_coeffs);
 
         dist_A->read_distributed(
-            device_matrix_data(exec,
-                local_sparsity.dim,
-                local_sparsity.row_idxs,
-                local_sparsity.col_idxs,
-                local_coeffs),
-            device_matrix_data(exec,
-                non_local_sparsity.dim,
-                non_local_sparsity.row_idxs,
-                non_local_sparsity.col_idxs,
-                non_local_coeffs),
-                sparse_comm);
-
-        return std::make_shared<RepartDistMatrix>(exec, dist_A->get_size(), std::move(dist_A));
+            device_matrix_data(exec, local_sparsity_->dim,
+                               local_sparsity_->row_idxs,
+                               local_sparsity_->col_idxs, local_coeffs),
+            device_matrix_data(exec, non_local_sparsity_->dim,
+                               non_local_sparsity_->row_idxs,
+                               non_local_sparsity_->col_idxs, non_local_coeffs),
+            sparse_comm);
+
+        return std::make_shared<RepartDistMatrix>(
+            exec, comm, host_A->get_size(), dist_A->get_size(),
+            std::move(dist_A));
     };
 
-    void update(std::shared_ptr<const HostMatrixWrapper> host_A){
+    // TODO matrix_format can be data member
+    void update(word matrix_format,
+                std::shared_ptr<const HostMatrixWrapper> host_A)
+    {
+        auto exec_handler = host_A.get()->get_exec_handler();
+        auto device_exec = exec_handler.get_device_exec();
+
+        scalar *local_value_ptr{};
+        scalar *non_local_value_ptr{};
+
+        if (matrix_format == "Csr") {
+            local_value_ptr =
+                (gko::matrix::Csr<scalar, label> *)dist_mtx_->get_local_matrix()
+                    ->get_values();
+
+            non_local_value_ptr = (gko::matrix::Csr<scalar, label> *)dist_mtx_
+                                      ->get_non_local_matrix()
+                                      ->get_values();
+        }
+        if (matrix_format == "Ell") {
+            local_value_ptr =
+                (gko::matrix::Ell<scalar, label> *)dist_mtx_->get_local_matrix()
+                    ->get_values();
+
+            non_local_value_ptr = (gko::matrix::Ell<scalar, label> *)dist_mtx_
+                                      ->get_non_local_matrix()
+                                      ->get_values();
+        }
+        if (matrix_format == "Coo") {
+            local_value_ptr =
+                (gko::matrix::Coo<scalar, label> *)dist_mtx_->get_local_matrix()
+                    ->get_values();
+
+            non_local_value_ptr = (gko::matrix::Coo<scalar, label> *)dist_mtx_
+                                      ->get_non_local_matrix()
+                                      ->get_values();
+        }
+
+        const label local_nnz = local_sparsity_->size_;
+        const label non_local_nnz = non_local_sparsity_->size_;
+        auto local_coeffs = gko::array<scalar>(device_exec, local_nnz);
+        auto non_local_coeffs = gko::array<scalar>(device_exec, non_local_nnz);
+
+        host_A.get()->compute_local_coeffs(local_sparsity_, local_coeffs);
+        host_A.get()->compute_non_local_coeffs(non_local_sparsity_,
+                                               non_local_coeffs);
+
+        auto value_view =
+            val_array::view(device_exec, local_nnz, local_value_ptr);
+
+        auto non_local_view =
+            val_array::view(device_exec, non_local_nnz, non_local_value_ptr);
+
+        value_view = local_coeffs;
+        non_local_view = non_local_coeffs;
+
         // On update dist_mtx already exists
         // and only local and non local coeffs need to be updated
         //
@@ -214,20 +266,25 @@ public:
     };
 
     RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
-                     gko::dim<2> size,
+                     communicator comm, gko::dim<2> local_size,
+                     gko::dim<2> global_size,
                      std::unique_ptr<dist_mtx> dist_mtx)
-        : gko::EnableLinOp<RepartDistMatrix>(exec),
+        : gko::experimental::EnableDistributedLinOp<RepartDistMatrix>(exec),
+          gko::experimental::distributed::DistributedBase(comm),
           dist_mtx_(std::move(dist_mtx))
     {
-        this->set_size(size);
+        this->set_size(global_size);
+        partition_ = gko::share(
+            gko::experimental::distributed::build_partition_from_local_size<
+                label, label>(exec, comm, local_size[0]));
     }
 
-    void write(const word field_name_, const objectRegistry &db_
-            ) const {
+    void write(const word field_name_, const objectRegistry &db_) const
+    {
         auto dist_matrix = gko::as<
             gko::experimental::distributed::Matrix<scalar, label, label>>(
             this->dist_mtx_.get());
-        word matrix_format_ { "Coo" };
+        word matrix_format_{"Coo"};
 
         if (matrix_format_ == "Coo") {
             using Coo = gko::matrix::Coo<scalar, label>;
@@ -283,17 +340,13 @@ public:
         }
     }
 
-    void solve(const gko::LinOp *b, gko::LinOp *x) const
-    {
-        std::cout << "call solve\n";
-        this->dist_mtx_->apply(b, x);
-    }
-
-protected:
-    RepartDistMatrix(std::shared_ptr<const gko::Executor> exec)
-        : gko::EnableLinOp<RepartDistMatrix>(exec)
+    RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
+                     communicator comm)
+        : gko::experimental::EnableDistributedLinOp<RepartDistMatrix>(exec),
+          gko::experimental::distributed::DistributedBase{comm}
     {}
 
+protected:
     // Here we implement the application of the linear operator, x = A * b.
     // apply_impl will be called by the apply method, after the arguments have
     // been moved to the correct executor and the operators checked for
@@ -304,7 +357,6 @@ protected:
     // are always true in this example).
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
-        std::cout << "call apply\n";
         this->dist_mtx_->apply(b, x);
     }
 
@@ -316,7 +368,6 @@ protected:
     void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
                     const gko::LinOp *beta, gko::LinOp *x) const override
     {
-        std::cout << "call apply 2\n";
         this->dist_mtx_->apply(alpha, b, beta, x);
     }
 
@@ -325,4 +376,8 @@ private:
     std::unique_ptr<dist_mtx> dist_mtx_;
 
     std::shared_ptr<part_type> partition_;
+
+    std::shared_ptr<SparsityPattern> local_sparsity_;
+
+    std::shared_ptr<SparsityPattern> non_local_sparsity_;
 };
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index 8a7337bb7..cf2c56104 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -338,17 +338,18 @@ HostMatrixWrapper::collect_cells_on_non_local_interface(
     return non_local_idxs;
 }
 
-SparsityPattern HostMatrixWrapper::compute_non_local_sparsity(
+std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
     std::shared_ptr<const gko::Executor> exec) const
 {
-    SparsityPattern sparsity{exec->get_master(), non_local_matrix_nnz_};
-    sparsity.dim = gko::dim<2>{nrows_, non_local_matrix_nnz_};
+    auto sparsity{std::make_shared<SparsityPattern>(exec->get_master(),
+                                                    non_local_matrix_nnz_)};
+    sparsity->dim = gko::dim<2>{nrows_, non_local_matrix_nnz_};
 
     auto non_local_row_indices =
         collect_cells_on_non_local_interface(interfaces_);
-    auto rows = sparsity.row_idxs.get_data();
-    auto cols = sparsity.col_idxs.get_data();
-    auto permute = sparsity.ldu_mapping.get_data();
+    auto rows = sparsity->row_idxs.get_data();
+    auto cols = sparsity->col_idxs.get_data();
+    auto permute = sparsity->ldu_mapping.get_data();
 
     label element_ctr = 0;
     // TODO currently we set permute eventhough this is not required
@@ -373,9 +374,9 @@ SparsityPattern HostMatrixWrapper::compute_non_local_sparsity(
         // reache
         if (interface_idx > prev_interface_ctr ||
             interface_ctr == non_local_interfaces.size() - 1) {
-            end = start + interface_ctr;
+            end = start + interface_ctr + 1;
             interface_ctr = 0;
-            sparsity.interface_spans.emplace_back(start, end);
+            sparsity->interface_spans.emplace_back(start, end);
             start = end;
             prev_interface_ctr = interface_idx;
         }
@@ -385,13 +386,13 @@ SparsityPattern HostMatrixWrapper::compute_non_local_sparsity(
     return sparsity;
 }
 
-SparsityPattern HostMatrixWrapper::compute_local_sparsity(
+std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_local_sparsity(
     std::shared_ptr<const gko::Executor> exec) const
 {
     LOG_1(verbose_, "start init host sparsity pattern")
 
-    SparsityPattern sparsity{exec->get_master(),
-                             local_matrix_w_interfaces_nnz_};
+    auto sparsity{std::make_shared<SparsityPattern>(
+        exec->get_master(), local_matrix_w_interfaces_nnz_)};
 
     auto lower_local = idx_array::view(
         exec, upper_nnz_, const_cast<label *>(addr_.lowerAddr().begin()));
@@ -405,9 +406,9 @@ SparsityPattern HostMatrixWrapper::compute_local_sparsity(
     // col of upper, row of lower
     const auto upper = upper_local.get_const_data();
 
-    auto rows = sparsity.row_idxs.get_data();
-    auto cols = sparsity.col_idxs.get_data();
-    const auto permute = sparsity.ldu_mapping.get_data();
+    auto rows = sparsity->row_idxs.get_data();
+    auto cols = sparsity->col_idxs.get_data();
+    const auto permute = sparsity->ldu_mapping.get_data();
 
     // Scan through given rows and insert row and column indices into array
     //
@@ -421,8 +422,8 @@ SparsityPattern HostMatrixWrapper::compute_local_sparsity(
     init_local_sparsity(nrows_, upper_nnz_, symmetric_, upper, lower, rows,
                         cols, permute);
 
-    sparsity.dim = gko::dim<2>{nrows_, nrows_};
-    sparsity.interface_spans.emplace_back(0, local_matrix_nnz_);
+    sparsity->dim = gko::dim<2>{nrows_, nrows_};
+    sparsity->interface_spans.emplace_back(0, local_matrix_nnz_);
 
     // if no local interfaces are present we are done here
     // otherwise we need to add local interfaces to local_sparsity in order
@@ -463,7 +464,7 @@ SparsityPattern HostMatrixWrapper::compute_local_sparsity(
                 local_interface_ctr + 1 == local_interfaces.size()) {
                 end = start + local_interface_ctr;
                 local_interface_ctr = 0;
-                sparsity.interface_spans.emplace_back(start, end);
+                sparsity->interface_spans.emplace_back(start, end);
                 start = end;
                 prev_interface_idx = interface_idx;
             }
@@ -475,12 +476,21 @@ SparsityPattern HostMatrixWrapper::compute_local_sparsity(
     return sparsity;
 }
 
+void HostMatrixWrapper::update(
+    const scalar *diag, const scalar *upper, const scalar *lower,
+    const FieldField<Field, scalar> &interfaceBouCoeffs,
+    const FieldField<Field, scalar> &interfaceIntCoeffs,
+    const lduInterfaceFieldPtrsList &interfaces)
+{
+    // TODO implement a regenerating update
+}
 
 void HostMatrixWrapper::compute_local_coeffs(
-    const SparsityPattern &sparsity, gko::array<scalar> &local_coeffs) const
+    std::shared_ptr<const SparsityPattern> sparsity,
+    gko::array<scalar> &local_coeffs) const
 {
     auto ref_exec = exec_.get_ref_exec();
-    auto permute = sparsity.ldu_mapping.get_data();
+    auto permute = sparsity->ldu_mapping.get_data();
 
     // // TODO this does not work for Ell
     if (reorder_on_copy_) {
@@ -585,7 +595,7 @@ void HostMatrixWrapper::compute_local_coeffs(
 
 
 void HostMatrixWrapper::compute_non_local_coeffs(
-    const SparsityPattern &, gko::array<scalar> &coeffs) const
+    std::shared_ptr<const SparsityPattern>, gko::array<scalar> &coeffs) const
 {
     auto ref_exec = exec_.get_ref_exec();
     auto interface_coeffs =
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index c4956855f..ca0ddc410 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -368,7 +368,7 @@ public:
      ** major ordering and the permutation index (local_sparsity_.ldu_mapping),
      ** which are required to create to a ginkgo matrix
      **/
-    SparsityPattern compute_local_sparsity(
+    std::shared_ptr<SparsityPattern> compute_local_sparsity(
         std::shared_ptr<const gko::Executor> exec) const;
 
     /** Copies the LDU matrix coefficients to local_coeffs without changing or
@@ -376,13 +376,13 @@ public:
      ** This uses the local_sparsity_.ldu_mapping to permute the data already
      ** on the host or device to be in row major order.
      **/
-    void compute_local_coeffs(const SparsityPattern &sparsity,
+    void compute_local_coeffs(std::shared_ptr<const SparsityPattern> sparsity,
                               gko::array<scalar> &target_coeffs) const;
 
     /** Copies the interface matrix coefficients to non_local_coeffs without
     ** changing or reinstantiating the sparsity pattern.
     **/
-    void compute_non_local_coeffs(const SparsityPattern &sparsity,
+    void compute_non_local_coeffs(std::shared_ptr<const SparsityPattern> sparsity,
                                   gko::array<scalar> &target_coeffs) const;
 
     /** Based on OpenFOAMs interfaces this function computes two
@@ -391,7 +391,7 @@ public:
      ** (non_local_sparsity_.ldu_mapping), which are required to create a ginkgo
      ** matrix
      **/
-    SparsityPattern compute_non_local_sparsity(
+    std::shared_ptr<SparsityPattern> compute_non_local_sparsity(
         std::shared_ptr<const gko::Executor> exec) const;
 
     /** Iterates all interfaces and counts the number of unique neighbour
@@ -408,9 +408,9 @@ public:
 
     label get_local_nrows() const { return nrows_; }
 
-    const ExecutorHandler &get_exec_handler() const { return exec_; }
+    gko::dim<2> get_size() const { return gko::dim<2>(nrows_, nrows_); }
 
-    gko::Array<scalar> &get_local_coeffs() const {};
+    const ExecutorHandler &get_exec_handler() const { return exec_; }
 };
 
 
diff --git a/StoppingCriterion/StoppingCriterion.C b/StoppingCriterion/StoppingCriterion.C
index 74406b5a6..8d3b3f082 100644
--- a/StoppingCriterion/StoppingCriterion.C
+++ b/StoppingCriterion/StoppingCriterion.C
@@ -31,7 +31,7 @@ void StoppingCriterion::OpenFOAMDistStoppingCriterion::compute_Axref_dist(
 
 scalar
 StoppingCriterion::OpenFOAMDistStoppingCriterion::compute_normfactor_dist(
-    std::shared_ptr<const gko::Executor> device_exec, const dist_vec *r,
+    std::shared_ptr<const gko::Executor> device_exec, const dist_vec* r,
     std::shared_ptr<const gko::LinOp> gkomatrix,
     std::shared_ptr<const dist_vec> x, std::shared_ptr<const dist_vec> b) const
 {
@@ -43,6 +43,7 @@ StoppingCriterion::OpenFOAMDistStoppingCriterion::compute_normfactor_dist(
 
     auto Axref = gko::share(
         dist_vec::create(device_exec, comm, global_size, local_size));
+    Axref->fill(0.0);
 
     compute_Axref_dist(global_size[0], local_size[0], device_exec, gkomatrix, x,
                        Axref);
@@ -89,7 +90,7 @@ bool StoppingCriterion::OpenFOAMDistStoppingCriterion::check_impl(
     auto start_eval = std::chrono::steady_clock::now();
     const auto exec = this->get_executor();
 
-    auto *dense_r = gko::as<dist_vec>(updater.residual_);
+    auto dense_r = gko::as<dist_vec>(updater.residual_);
     auto norm1 = vec::create(exec, gko::dim<2>{1});
     dense_r->compute_norm1(norm1.get());
     auto norm1_host = vec::create(exec->get_master(), gko::dim<2>{1});
diff --git a/StoppingCriterion/StoppingCriterion.H b/StoppingCriterion/StoppingCriterion.H
index 394673d40..aa7af9738 100644
--- a/StoppingCriterion/StoppingCriterion.H
+++ b/StoppingCriterion/StoppingCriterion.H
@@ -91,7 +91,8 @@ class StoppingCriterion {
          * or rewritten as || r - ( b - x* ) || + || (b - x*) ||
          *  */
         scalar compute_normfactor_dist(
-            std::shared_ptr<const gko::Executor> device_exec, const dist_vec *r,
+            std::shared_ptr<const gko::Executor> device_exec,
+            const dist_vec* r,
             std::shared_ptr<const gko::LinOp> gkomatrix,
             std::shared_ptr<const dist_vec> x,
             std::shared_ptr<const dist_vec> b) const;
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 311d17d18..c20490c4f 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -207,11 +207,12 @@ public:
     {}
 
 
-    solverPerformance solve_multi_gpu_impl(scalarField &psi,
+    solverPerformance solve_multi_gpu_impl(
+            ExecutorHandler &exec_handler,
+            scalarField &psi,
                                            const scalarField &source,
                                            solverPerformance &solverPerf) const
     {
-        auto exec_handler = host_A_.get()->get_exec_handler();
         auto ref_exec = exec_handler.get_ref_exec();
 
         // create a distributed matrix from the host matrix
@@ -264,13 +265,13 @@ public:
         TIME_WITH_FIELDNAME(
             verbose_, init_precond, field_name_,
             auto precond = this->init_preconditioner(
-                dist_A_v, exec_handler_.get_device_exec());)
+                dist_A_v, exec_handler.get_device_exec());)
 
         bool write(solver_controls_.lookupOrDefault<Switch>("export", false));
         if (write && db_.time().writeTime()) {
             LOG_0(verbose_, "Export system")
             dist_A.get()->write(field_name_, db_);
-            //dist_b.write(field_name_, db_);
+            dist_b.write();
         }
 
 
@@ -289,7 +290,7 @@ public:
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, field_name_,
                             dist_x.copy_back(partition);)
         auto bandwidth_copy_back = sizeof(scalar) * psi.size() /
-                                   delta_t_copy_x_back / 1000.0;
+                                   (delta_t_copy_x_back + SMALL) / 1000.0;
 
         solverPerf.initialResidual() = this->get_init_res_norm();
         solverPerf.finalResidual() = this->get_res_norm();
@@ -299,8 +300,8 @@ public:
         auto time_per_iter =
             delta_t_solve / max(this->get_number_of_iterations(), 1);
         scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
-        exec_handler_.get_gko_mpi_host_comm()->broadcast(
-            exec_handler_.get_ref_exec(), &prev_rel_res_cost, 1, 0);
+        exec_handler.get_gko_mpi_host_comm()->broadcast(
+            exec_handler.get_ref_exec(), &prev_rel_res_cost, 1, 0);
         this->set_prev_rel_res_cost(prev_rel_res_cost);
         auto time_per_iter_and_dof =
             time_per_iter * 1000.0 / partition->get_size();
@@ -322,17 +323,18 @@ public:
                                   const scalarField &source,
                                   const direction cmpt = 0) const
     {
+        auto exec_handler = host_A_.get()->get_exec_handler();
         // --- Setup class containing solver performance data
         solverPerformance solverPerf(
             lduMatrix::preconditioner::getName(solver_controls_) +
-                exec_handler_.get_exec_name() + typeName,
+                exec_handler.get_exec_name() + typeName,
             field_name_);
 
         // Solve system
         if (Pstream::parRun()) {
             TIME_WITH_FIELDNAME(
                 verbose_, solve_multi_gpu, field_name_,
-                auto res = solve_multi_gpu_impl(psi, source, solverPerf);)
+                auto res = solve_multi_gpu_impl(exec_handler, psi, source, solverPerf);)
             return res;
         } else {
             FatalErrorInFunction << "Only parallel runs are supported for OGL"

From 290b33309b510dcecf10d982b7b1de7216cf4f89 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 24 Apr 2024 09:00:16 +0200
Subject: [PATCH 43/90] WIP repartitioning

---
 CMakeFiles/3.20.2/CMakeCCompiler.cmake        |  78 ++
 CMakeFiles/3.20.2/CMakeCXXCompiler.cmake      |  91 +++
 .../3.20.2/CMakeDetermineCompilerABI_C.bin    | Bin 0 -> 17992 bytes
 .../3.20.2/CMakeDetermineCompilerABI_CXX.bin  | Bin 0 -> 18016 bytes
 CMakeFiles/3.20.2/CMakeSystem.cmake           |  15 +
 .../3.20.2/CompilerIdC/CMakeCCompilerId.c     | 752 ++++++++++++++++++
 CMakeFiles/3.20.2/CompilerIdC/a.out           | Bin 0 -> 18048 bytes
 .../CompilerIdCXX/CMakeCXXCompilerId.cpp      | 743 +++++++++++++++++
 CMakeFiles/3.20.2/CompilerIdCXX/a.out         | Bin 0 -> 18056 bytes
 CMakeFiles/CMakeError.log                     | 228 ++++++
 CMakeFiles/CMakeOutput.log                    | 506 ++++++++++++
 CMakeFiles/cmake.check_cache                  |   1 +
 CMakeLists.txt                                |   1 +
 CMakePresets.json                             |   2 +-
 .../ExecutorHandler/ExecutorHandler.H         |   6 +-
 .../CommunicationPattern.C                    |  91 +++
 .../CommunicationPattern.H                    |  19 +-
 MatrixWrapper/Distributed/Distributed.H       |  35 +-
 MatrixWrapper/LDUMatrix/HostMatrix.C          |   6 +-
 MatrixWrapper/LDUMatrix/HostMatrix.H          |   6 +-
 common/common.C                               |  10 +
 common/common.H                               |   2 +
 lduLduBase/lduLduBase.H                       |  17 +-
 23 files changed, 2589 insertions(+), 20 deletions(-)
 create mode 100644 CMakeFiles/3.20.2/CMakeCCompiler.cmake
 create mode 100644 CMakeFiles/3.20.2/CMakeCXXCompiler.cmake
 create mode 100755 CMakeFiles/3.20.2/CMakeDetermineCompilerABI_C.bin
 create mode 100755 CMakeFiles/3.20.2/CMakeDetermineCompilerABI_CXX.bin
 create mode 100644 CMakeFiles/3.20.2/CMakeSystem.cmake
 create mode 100644 CMakeFiles/3.20.2/CompilerIdC/CMakeCCompilerId.c
 create mode 100755 CMakeFiles/3.20.2/CompilerIdC/a.out
 create mode 100644 CMakeFiles/3.20.2/CompilerIdCXX/CMakeCXXCompilerId.cpp
 create mode 100755 CMakeFiles/3.20.2/CompilerIdCXX/a.out
 create mode 100644 CMakeFiles/CMakeError.log
 create mode 100644 CMakeFiles/CMakeOutput.log
 create mode 100644 CMakeFiles/cmake.check_cache
 create mode 100644 MatrixWrapper/CommunicationPattern/CommunicationPattern.C

diff --git a/CMakeFiles/3.20.2/CMakeCCompiler.cmake b/CMakeFiles/3.20.2/CMakeCCompiler.cmake
new file mode 100644
index 000000000..ef97b6dfd
--- /dev/null
+++ b/CMakeFiles/3.20.2/CMakeCCompiler.cmake
@@ -0,0 +1,78 @@
+set(CMAKE_C_COMPILER "/usr/lib64/ccache/cc")
+set(CMAKE_C_COMPILER_ARG1 "")
+set(CMAKE_C_COMPILER_ID "GNU")
+set(CMAKE_C_COMPILER_VERSION "8.5.0")
+set(CMAKE_C_COMPILER_VERSION_INTERNAL "")
+set(CMAKE_C_COMPILER_WRAPPER "")
+set(CMAKE_C_STANDARD_COMPUTED_DEFAULT "11")
+set(CMAKE_C_COMPILE_FEATURES "c_std_90;c_function_prototypes;c_std_99;c_restrict;c_variadic_macros;c_std_11;c_static_assert")
+set(CMAKE_C90_COMPILE_FEATURES "c_std_90;c_function_prototypes")
+set(CMAKE_C99_COMPILE_FEATURES "c_std_99;c_restrict;c_variadic_macros")
+set(CMAKE_C11_COMPILE_FEATURES "c_std_11;c_static_assert")
+
+set(CMAKE_C_PLATFORM_ID "Linux")
+set(CMAKE_C_SIMULATE_ID "")
+set(CMAKE_C_COMPILER_FRONTEND_VARIANT "")
+set(CMAKE_C_SIMULATE_VERSION "")
+
+
+
+
+set(CMAKE_AR "/usr/bin/ar")
+set(CMAKE_C_COMPILER_AR "/usr/bin/gcc-ar")
+set(CMAKE_RANLIB "/usr/bin/ranlib")
+set(CMAKE_C_COMPILER_RANLIB "/usr/bin/gcc-ranlib")
+set(CMAKE_LINKER "/usr/bin/ld")
+set(CMAKE_MT "")
+set(CMAKE_COMPILER_IS_GNUCC 1)
+set(CMAKE_C_COMPILER_LOADED 1)
+set(CMAKE_C_COMPILER_WORKS TRUE)
+set(CMAKE_C_ABI_COMPILED TRUE)
+set(CMAKE_COMPILER_IS_MINGW )
+set(CMAKE_COMPILER_IS_CYGWIN )
+if(CMAKE_COMPILER_IS_CYGWIN)
+  set(CYGWIN 1)
+  set(UNIX 1)
+endif()
+
+set(CMAKE_C_COMPILER_ENV_VAR "CC")
+
+if(CMAKE_COMPILER_IS_MINGW)
+  set(MINGW 1)
+endif()
+set(CMAKE_C_COMPILER_ID_RUN 1)
+set(CMAKE_C_SOURCE_FILE_EXTENSIONS c;m)
+set(CMAKE_C_IGNORE_EXTENSIONS h;H;o;O;obj;OBJ;def;DEF;rc;RC)
+set(CMAKE_C_LINKER_PREFERENCE 10)
+
+# Save compiler ABI information.
+set(CMAKE_C_SIZEOF_DATA_PTR "8")
+set(CMAKE_C_COMPILER_ABI "ELF")
+set(CMAKE_C_BYTE_ORDER "LITTLE_ENDIAN")
+set(CMAKE_C_LIBRARY_ARCHITECTURE "")
+
+if(CMAKE_C_SIZEOF_DATA_PTR)
+  set(CMAKE_SIZEOF_VOID_P "${CMAKE_C_SIZEOF_DATA_PTR}")
+endif()
+
+if(CMAKE_C_COMPILER_ABI)
+  set(CMAKE_INTERNAL_PLATFORM_ABI "${CMAKE_C_COMPILER_ABI}")
+endif()
+
+if(CMAKE_C_LIBRARY_ARCHITECTURE)
+  set(CMAKE_LIBRARY_ARCHITECTURE "")
+endif()
+
+set(CMAKE_C_CL_SHOWINCLUDES_PREFIX "")
+if(CMAKE_C_CL_SHOWINCLUDES_PREFIX)
+  set(CMAKE_CL_SHOWINCLUDES_PREFIX "${CMAKE_C_CL_SHOWINCLUDES_PREFIX}")
+endif()
+
+
+
+
+
+set(CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES "/opt/rocm-5.2.0/opencl/include;/opt/rocm-5.2.0/hip/include;/opt/rocm-5.2.0/include;/usr/lib/gcc/x86_64-redhat-linux/8/include;/usr/local/include;/usr/include")
+set(CMAKE_C_IMPLICIT_LINK_LIBRARIES "gcc;gcc_s;c;gcc;gcc_s")
+set(CMAKE_C_IMPLICIT_LINK_DIRECTORIES "/opt/rocm-5.2.0/lib64;/usr/lib/gcc/x86_64-redhat-linux/8;/usr/lib64;/lib64;/opt/rocm-5.2.0/opencl/lib;/opt/rocm-5.2.0/hip/lib;/opt/rocm-5.2.0/lib;/usr/lib")
+set(CMAKE_C_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES "")
diff --git a/CMakeFiles/3.20.2/CMakeCXXCompiler.cmake b/CMakeFiles/3.20.2/CMakeCXXCompiler.cmake
new file mode 100644
index 000000000..df8c84f1a
--- /dev/null
+++ b/CMakeFiles/3.20.2/CMakeCXXCompiler.cmake
@@ -0,0 +1,91 @@
+set(CMAKE_CXX_COMPILER "/usr/lib64/ccache/c++")
+set(CMAKE_CXX_COMPILER_ARG1 "")
+set(CMAKE_CXX_COMPILER_ID "GNU")
+set(CMAKE_CXX_COMPILER_VERSION "8.5.0")
+set(CMAKE_CXX_COMPILER_VERSION_INTERNAL "")
+set(CMAKE_CXX_COMPILER_WRAPPER "")
+set(CMAKE_CXX_STANDARD_COMPUTED_DEFAULT "14")
+set(CMAKE_CXX_COMPILE_FEATURES "cxx_std_98;cxx_template_template_parameters;cxx_std_11;cxx_alias_templates;cxx_alignas;cxx_alignof;cxx_attributes;cxx_auto_type;cxx_constexpr;cxx_decltype;cxx_decltype_incomplete_return_types;cxx_default_function_template_args;cxx_defaulted_functions;cxx_defaulted_move_initializers;cxx_delegating_constructors;cxx_deleted_functions;cxx_enum_forward_declarations;cxx_explicit_conversions;cxx_extended_friend_declarations;cxx_extern_templates;cxx_final;cxx_func_identifier;cxx_generalized_initializers;cxx_inheriting_constructors;cxx_inline_namespaces;cxx_lambdas;cxx_local_type_template_args;cxx_long_long_type;cxx_noexcept;cxx_nonstatic_member_init;cxx_nullptr;cxx_override;cxx_range_for;cxx_raw_string_literals;cxx_reference_qualified_functions;cxx_right_angle_brackets;cxx_rvalue_references;cxx_sizeof_member;cxx_static_assert;cxx_strong_enums;cxx_thread_local;cxx_trailing_return_types;cxx_unicode_literals;cxx_uniform_initialization;cxx_unrestricted_unions;cxx_user_literals;cxx_variadic_macros;cxx_variadic_templates;cxx_std_14;cxx_aggregate_default_initializers;cxx_attribute_deprecated;cxx_binary_literals;cxx_contextual_conversions;cxx_decltype_auto;cxx_digit_separators;cxx_generic_lambdas;cxx_lambda_init_captures;cxx_relaxed_constexpr;cxx_return_type_deduction;cxx_variable_templates;cxx_std_17;cxx_std_20")
+set(CMAKE_CXX98_COMPILE_FEATURES "cxx_std_98;cxx_template_template_parameters")
+set(CMAKE_CXX11_COMPILE_FEATURES "cxx_std_11;cxx_alias_templates;cxx_alignas;cxx_alignof;cxx_attributes;cxx_auto_type;cxx_constexpr;cxx_decltype;cxx_decltype_incomplete_return_types;cxx_default_function_template_args;cxx_defaulted_functions;cxx_defaulted_move_initializers;cxx_delegating_constructors;cxx_deleted_functions;cxx_enum_forward_declarations;cxx_explicit_conversions;cxx_extended_friend_declarations;cxx_extern_templates;cxx_final;cxx_func_identifier;cxx_generalized_initializers;cxx_inheriting_constructors;cxx_inline_namespaces;cxx_lambdas;cxx_local_type_template_args;cxx_long_long_type;cxx_noexcept;cxx_nonstatic_member_init;cxx_nullptr;cxx_override;cxx_range_for;cxx_raw_string_literals;cxx_reference_qualified_functions;cxx_right_angle_brackets;cxx_rvalue_references;cxx_sizeof_member;cxx_static_assert;cxx_strong_enums;cxx_thread_local;cxx_trailing_return_types;cxx_unicode_literals;cxx_uniform_initialization;cxx_unrestricted_unions;cxx_user_literals;cxx_variadic_macros;cxx_variadic_templates")
+set(CMAKE_CXX14_COMPILE_FEATURES "cxx_std_14;cxx_aggregate_default_initializers;cxx_attribute_deprecated;cxx_binary_literals;cxx_contextual_conversions;cxx_decltype_auto;cxx_digit_separators;cxx_generic_lambdas;cxx_lambda_init_captures;cxx_relaxed_constexpr;cxx_return_type_deduction;cxx_variable_templates")
+set(CMAKE_CXX17_COMPILE_FEATURES "cxx_std_17")
+set(CMAKE_CXX20_COMPILE_FEATURES "cxx_std_20")
+set(CMAKE_CXX23_COMPILE_FEATURES "")
+
+set(CMAKE_CXX_PLATFORM_ID "Linux")
+set(CMAKE_CXX_SIMULATE_ID "")
+set(CMAKE_CXX_COMPILER_FRONTEND_VARIANT "")
+set(CMAKE_CXX_SIMULATE_VERSION "")
+
+
+
+
+set(CMAKE_AR "/usr/bin/ar")
+set(CMAKE_CXX_COMPILER_AR "/usr/bin/gcc-ar")
+set(CMAKE_RANLIB "/usr/bin/ranlib")
+set(CMAKE_CXX_COMPILER_RANLIB "/usr/bin/gcc-ranlib")
+set(CMAKE_LINKER "/usr/bin/ld")
+set(CMAKE_MT "")
+set(CMAKE_COMPILER_IS_GNUCXX 1)
+set(CMAKE_CXX_COMPILER_LOADED 1)
+set(CMAKE_CXX_COMPILER_WORKS TRUE)
+set(CMAKE_CXX_ABI_COMPILED TRUE)
+set(CMAKE_COMPILER_IS_MINGW )
+set(CMAKE_COMPILER_IS_CYGWIN )
+if(CMAKE_COMPILER_IS_CYGWIN)
+  set(CYGWIN 1)
+  set(UNIX 1)
+endif()
+
+set(CMAKE_CXX_COMPILER_ENV_VAR "CXX")
+
+if(CMAKE_COMPILER_IS_MINGW)
+  set(MINGW 1)
+endif()
+set(CMAKE_CXX_COMPILER_ID_RUN 1)
+set(CMAKE_CXX_SOURCE_FILE_EXTENSIONS C;M;c++;cc;cpp;cxx;m;mm;mpp;CPP)
+set(CMAKE_CXX_IGNORE_EXTENSIONS inl;h;hpp;HPP;H;o;O;obj;OBJ;def;DEF;rc;RC)
+
+foreach (lang C OBJC OBJCXX)
+  if (CMAKE_${lang}_COMPILER_ID_RUN)
+    foreach(extension IN LISTS CMAKE_${lang}_SOURCE_FILE_EXTENSIONS)
+      list(REMOVE_ITEM CMAKE_CXX_SOURCE_FILE_EXTENSIONS ${extension})
+    endforeach()
+  endif()
+endforeach()
+
+set(CMAKE_CXX_LINKER_PREFERENCE 30)
+set(CMAKE_CXX_LINKER_PREFERENCE_PROPAGATES 1)
+
+# Save compiler ABI information.
+set(CMAKE_CXX_SIZEOF_DATA_PTR "8")
+set(CMAKE_CXX_COMPILER_ABI "ELF")
+set(CMAKE_CXX_BYTE_ORDER "LITTLE_ENDIAN")
+set(CMAKE_CXX_LIBRARY_ARCHITECTURE "")
+
+if(CMAKE_CXX_SIZEOF_DATA_PTR)
+  set(CMAKE_SIZEOF_VOID_P "${CMAKE_CXX_SIZEOF_DATA_PTR}")
+endif()
+
+if(CMAKE_CXX_COMPILER_ABI)
+  set(CMAKE_INTERNAL_PLATFORM_ABI "${CMAKE_CXX_COMPILER_ABI}")
+endif()
+
+if(CMAKE_CXX_LIBRARY_ARCHITECTURE)
+  set(CMAKE_LIBRARY_ARCHITECTURE "")
+endif()
+
+set(CMAKE_CXX_CL_SHOWINCLUDES_PREFIX "")
+if(CMAKE_CXX_CL_SHOWINCLUDES_PREFIX)
+  set(CMAKE_CL_SHOWINCLUDES_PREFIX "${CMAKE_CXX_CL_SHOWINCLUDES_PREFIX}")
+endif()
+
+
+
+
+
+set(CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES "/opt/rocm-5.2.0/opencl/include;/opt/rocm-5.2.0/hip/include;/opt/rocm-5.2.0/include;/usr/include/c++/8;/usr/include/c++/8/x86_64-redhat-linux;/usr/include/c++/8/backward;/usr/lib/gcc/x86_64-redhat-linux/8/include;/usr/local/include;/usr/include")
+set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "stdc++;m;gcc_s;gcc;c;gcc_s;gcc")
+set(CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES "/opt/rocm-5.2.0/lib64;/usr/lib/gcc/x86_64-redhat-linux/8;/usr/lib64;/lib64;/opt/rocm-5.2.0/opencl/lib;/opt/rocm-5.2.0/hip/lib;/opt/rocm-5.2.0/lib;/usr/lib")
+set(CMAKE_CXX_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES "")
diff --git a/CMakeFiles/3.20.2/CMakeDetermineCompilerABI_C.bin b/CMakeFiles/3.20.2/CMakeDetermineCompilerABI_C.bin
new file mode 100755
index 0000000000000000000000000000000000000000..e9bea52b28e3b9eb818abb45721ef86d431df7f5
GIT binary patch
literal 17992
zcmeHPeQX@X6`#BFSH8|iAT*GGFDgh$kk{vs1d}#!=g+&!u|w=YXmB>`yS2T_`R;nR
zhnz@NEs`P(1l&JbMXE&eR|{1?_=kY{M@gDOR8_4)Rf#{Kl3NiJl~gsN4IhQL@6En<
zx4X9+MO39l+$4K%=J!5k=FRNv%<PW8)i>B53<Ly+ptwh1YQ7;ZA?sI&<h?RUSeIBP
z;$pp6Cz=3hfGaLJNUa#t56Cpr7Lp4AMZK<|1#|`DGQxDd1w$TI;1VKjB+MdlN?Ejp
zGGHo1RPs>|%Nh&IB+!F0!t@elut=2kuz08!C%ri7F+EB0Ou2qoH@rrvext!Sq%a*v
zB6{U-m7*+$-51OZ#v{!#2{bMvOt)IFV9I*;fgYCgT0?{c_fUR*emPI|W6JL7$Qt9@
zw{~PR?O7vNo@t-y+TOl>t5V7<TV%b_KFSUZ?UgxUdy^|9{{~1ShWwu&SpU;UvKxMU
z=?6diaCzHw_DA0t`a0_J1<`8`1(1w-NUs2FwFmwrU{MeJ`#!jUdZ)3P+l8uPTrHVe
z(Nw23BPRw1Q{6r47G;aFO?2c7W=Aofo^Ia;=@=ySTsn(x+u>$S83j)qWQCd<-KA#q
zqCRPqOuaa|t0$Y!>7&|sR)+#iPUmw}0M!!p6ok42!25y9Kj>dn=km?QGHjV!D80~n
zP{yz?q5nBtwBg*2$Xm4G*q$6du;JWCk@sHftsf<iUTR4`(fExOQIY)iX*0NVK6&)l
zEoY@F@;m>QntSa)a_%q5qi-$l9!;G)y}KzclIPAWG$J|oFp#A+n?d~_^C*9ILTPO~
zCe`PY$9JaC?(xQMBzJ#cu7RvBFNZi}wQOmjwQUw9&k_aso22}<)krp7NuIe9O3p1N
z&s@AW8F)4M+7<H#Fz`Dv(6Y2J(c0Eq)yMp2cfJN}QNDg}a_+*#BTGw5uQt96NMP*j
z{Mm_W;ac0`vi|Qpj)HUFyp)_PFP!W97|ZiS-=*Y<)>E@5p{A|BJRRu!*g{@BCQc*t
zq61oRL8oQ~j1#Q)@z_}?<S{fW^GiICntMHQe`@Y>VsB#Zlf>xY@!PkyM8)3Z@phym
zgPSg3n_N_{EG;F^d=fI-I^Q7w49<NtICr^s?(d1E8{SMFJsU{g{ipI@vAy<>B_2wQ
zB@QH1dmUwaP`<LAD=jOFQ}cuU@@gS9)IYqVWE|1+6KbYl7Wc=x#_aM#sp0<d9p+(G
zFXr{k{$i?I9T@6M^$sP3l#}}7hfQ707vVPAKbRUF9qd#4hI&(pA&`)KZQR%o&p}iO
ztP6bw$5wpC%zm=8v>WL2Kr=umz&$fS&jURHG!8Aj02Bj`XBYxU_K3htB(QE}bIUy3
zKzN_wGi)Q+=s<zq<_Kaji~`2bGL-3s>uta^pdt~DJR0m-)f9XPELl7(Lk#lPk>4MV
zJRR%{uYD@i7mglp=nHRrvN0KsJ=&BEcO7jW2*-2bu0%MN2yg5TN4vvoyTg(0aEmO@
zPe6Si*x|g<Ts_o1Q1?LH19cD7J#aM-6kzU!Ax@O%h)i20SBOnSR}w8W#AR$1rS~;i
z>6<9c>#%i{=5^G~l<uN+7q17I|IaH+c@#Mjj3cS64PzlI&1<5Uf>wGt$Y6NEB4P#6
zwM5qtWztNM2$i4nJxDlAo@HQtc&y6uEK>e3MUZLs&myJYCO)bE&j_z$y$UeU)3YPG
z5msoMqFu^1B^KQh+tL}^*10vhagUyfCN<N-+dH?(=0;!Q+y)m81Bu%LcNRKh+wT(M
zW%3=<HqOkFYb=&?IU_e2olK`)842N*)pEJ~xRLXe<-SDuHx{zxNf6Cx)B27mDPq>#
z=9b*%f#C<l=0=!?a$g}_PE5H!Ti6}2*h?Q$(^;)FCB*fPdTdTlsD)zQ)YGPs&k5)#
z+}Fuo|L~sCRR4ow3Ds_N=-we$j{EkhiII^$kQyEu0lup|&A7g*%-N*xHpuSivX!4U
zOm(6NwN?vxBL}O-T3b!H7QPNH87x~?-HN)FH70Y2`?V+U4Ka1MQN2Cu?df$d*Y8gp
z7w@RwpY`z;`oV_!cxyd<Xuj~pd48U*&xfG@Z~olPhj2cJ^ILpf<}N?C=_}N>I47&h
zdd~;kI`#Qhef|vVruup);eEf<*F*Hcq;+$BeOq7O${t}okG}Bx6}TDV*niSGE8ns3
z-jkOM^O$<c@N=E1mkjT5K1y={FB#tVJV^VWUNYxNAK!Ch@S7Ll>u?Nv$=2s(9|rfa
z{tNqRvj4mG!mepw4HZAjeKntXq%1$TX&<%ewag>kI#;vLcDc`7i+#6_Uw`ENK&JKn
z`04g}KkNQjZ_jI5cYJQfmwJ2F+w=c<9r;Cn{(H^&_<79q)5`GN^NFj>L3!?}(;Nfm
zkXH*Kc!Aj!jMs>S;M=`XPdp?{PrRY>p5AUO1hX%GKkvkw1i#03;?2VA{dovpy7GH>
zr~EPzA%8gW<)Yw;uc)jWoboFL&)=Q+DlzNf%a90HzTbArw~AzQ+z!cj2v#rr9fX}3
zf;YAN{eu%<4d107v_mo;f;ZFr9j~1k!mkt~Uh-?iT2K5&vECE^lHF`29ujLsG;*?%
z#t1A;MT2-(zVo&-c48v9qVjD3^LF{*8sR**WcjiWe%uHDp${&<K8EtR?Z4q8|Gp2t
z5-aRs=XSz*&llA(>?EAmo)#*^gFbf3KKOTi@KZkc8$S5IeDF4${dm;tt3LP`;qLk#
zBb?XKT;Jya_v$AvkUXz(SpGMJ^LWYlp9y!j<A*-@8hjXe)N6|mKH`JteekD!@MnGS
zR|t34_btMCEync?G+5<#<59xh{@Dq*m!Fe>2Vtk-ylXuC8sUQW8aUrE!s9Ny1b6_x
zZsBnQyAy^-Bwjm?Jx6x<yE01}zR!~Kwd3I*0rx8ZMJc~Zek_kRZY4t@Sl9>POPqzK
zxJ06aZy;RII84WTgr9#lOZisO&|=vkGm?0frwin9H_AP3r=*?QaeEKRyT^ZB;<e9_
z8EL0>Jbw!C)zFT3VgiFy2l@;|q6oOcdlIi5|9=7ab>O#?WQV_t`z_!Rv8K|#Z0-c$
ztAn-A$3Gy>LN@v?*>OL2t^gjj$#Ha56O<>oN<2p8Vf<#qp`5dXHxOeJ;OGawk95LE
zp+iQXe@0!;wYyOs#sMC$pcpc|4-vG(<4FhM_kldx;eJlA7^8f7?@lAeIN;X>*VN>j
z13b#gHhL%833|*=I?AOYo)+qWFKjyC)Rnq@YkN`8OlhWd604)DLs6>#K@yH>rHf{#
zk{1en1g9D~!&K7lw5sPaZcLfVn{Eu{yl|zQ%NmFD>|sv@Kc*rNP?ZawN<KVApE10P
z<z#@mX{6Pnp3SH2^>yOTdOLZ}dOCTvesJ<V{NWUk25SA{6sq-)lP`?`R5a~AbjrB=
z<zz|&PQR&{tmQNf4$hEt!O*RuR2#}q4EvFuooJWcEI(3J0I;U-RWU4xm#ni5obrA=
z58t~KYyDmosP%c3=k4z*-{tEn!^_Xs5yEt-Y`H}?W7P$8!mz@2_{YYa07uufP1Uqy
zqb>{sC9@1C($b=Oq);`d;e%E<R9J!og_*oMnaz)DSv6zkizQVn&xmw>x{%dPJ)_)p
zXKahNkcxSzTCu3XNvxb%JS?maP&4J}>BFF6OUar#M0<8=hxDEv$i~R(#YA@s%4g&z
z@@n<mmO5@sqW=snSHsF0rkT|pTsY7LnfC8V?2<>lR5%>z%r+y`-Uo*gyHY()j@1Av
zoFUt*_9e+evUiVA2L^|`6NBn-fB#6|s5+YHhErsq3*&2A#RftLT=lf3tdZ>`n^5(P
zW@>Ch<}8nq#^d&Lq|hcHWgYSaS?7sQHB-u~Q(7*Ar$JN0AeAw4YPqCC{i;R7+fex2
zs~|UCDv^14h*eeLkSQLJL$BfKQ|}|CO6l;lsf`QhcSVa%aZG-54C8qYK89AHIonNb
zl2VhovNB#a;MlMMw>qY#@Kb6fb2ta~ENT`l&ck}KgkMriF-oYwgV9_=1)<1_DW*OH
z&jRUxrI?pKRrD$9a#I=5;FP6d^*u|2<6uskHsFakiH|$@Q3_dzSjNCXLV>$r8hW->
z8d(KJGtHtgUN+&*`Hushdw{74%)za|`z?Gvp3glHTgq}A>6hVZhKu!iUxv@0M}cP#
zJWpb*6C!qf-oG));B72O)UDqO_+>cT!1<pD#--#50=WD<ufqK;dl-TQ?pw1y?+3j^
zzzhk)v`E&S_16-9KTx?uqBQRt#Snz>f<-F7=xcj800}&+!TIz4(%YoZ=SH~vEXVW^
z=;PTBPV+w0BI!pdGk5t37Dph0K4tw#LtL^V<miJkRh`ef>wgURSSPfPp+Nct(&uwE
z7T3y)<47Ne%dXG!-Feb~#pN*hp510(-lfm`Wc#Q<_)gLu$ZQ4r9>g#Uu0PM`XG#C5
zilh~Zy3)@;+OE%kFM<D#!d92OJAXVAYuD%fxg_2c5KhbA>JR&$ky8-CH6%56Wgn0K
zu7X>h(@cK_y7*3<^?BXcMGI4w0}&ZG&Gh$xqN`Y+_XYXyR-B~vBX{EQ6dAZ3UxyTm
zus-h}F3^kmsFbe$us+i_fo0dP>@!jZe4nxf)?<3XrO)fawWRNEFRBik6YoF_ZE^W|
zUAF!j^%p7sSjD0hVf}U&&iOO_5ct$pq$KZy4pROv-QqOhRUhvAcF7jSN<B*l&G0<j
v2wdzp_A9cv?v{r2`#hY#!tfY3yp@z~MNvucERdIe*FBb0D_4}1TuJe7eOq;>

literal 0
HcmV?d00001

diff --git a/CMakeFiles/3.20.2/CMakeDetermineCompilerABI_CXX.bin b/CMakeFiles/3.20.2/CMakeDetermineCompilerABI_CXX.bin
new file mode 100755
index 0000000000000000000000000000000000000000..16bfa2aac92db91b95b3264cf9c0dfcde069315d
GIT binary patch
literal 18016
zcmeHPeT*B$6(8I86_WT&;OIe8Vp@>$q4W8UB$&{EeV=x9xeJLyLPOZwv-j>+@!2PP
zonE3=jhcWs=s~24D*v>-N)-aBsw!0k{%Lg~g{V@6K%(N0(CUhal$KPaYDpo5T;H30
z&)!|{rV&+XDV}8gW`6HuX5P&1&dko-xB7<${63%H;1~A_Of59TC1m~zF?yd&5*8E7
zL|m*9t3?wa4RFOJ2dNoj`T?0{+Cp*xpr{w~n?TGTml39GOc?So1D6nKBVi_qQ_7_4
zC<CTKge4#Ku&g2|lR!t!2+=iEjxbTy!zAj3NH0Y4OkbmdG3EMU-SA42y|h0LDNM(a
zh(7sSr6`MG$2lX5@o2M50*%WE(@iEUn6lpepeO4`{C`D4B(Rh6^YV*ee-NeWiq4!q
zwq;XiF58jQ^W~Y2nb?+&Et?{xLZn;P8||a);P7sl6Sg;-!ZP0o7h=di`_7ZE9(?J$
zH;@0gG&IoCcKM~>hUQV1FNj`qSPPP27wNA9w#o&^a)w>-dpvLf^)4CN%*Ksa*DXR(
zCZ-B`rDUi@Ls5j1N^e)PT2Y(OONLfVZ|}_&@>*IQ%V{Rhl>;`XOmh&M$Yhifrn)e%
z3}=IJ%4SN{^I~u))zhnVN4g`MMQ35!=qwg8QyrTj9fhQp&*ZR)wmMmp`m`$!vQT3m
zw2=?~Vaz9{fO0=6-=Z(W7MukP!(!{Gq}b<Bp2K+y&h3r7ixwQ)ox=wfocl8J{@Qx$
z<>b)|Ey*VuQ_I65`N)FdUwSQh^j9sXSyfV$>-uwQ_VtI8vu`JlzH@O$I(25@NK;%S
z&zuTLa^{)FxL9htd|g=l?KsMx9*?xHJtozUC+D`^jP~amS0TCk1EUSHd<A4N%VkT8
zt!s~<<Y}TH|3_rbZGIcc`YXv(R|3h|i^)^x?@Rh#Nxpu?xDgC|pA57tEsnRY?W^iz
z{zta`2-u>0!|vqlx$}pYmX=;=d<qcX{?o@#k5>!Vx;8HT@b2R%IQz{D$=UManf{Nk
zJWuprNS<gtd1L{q*!s%_U;jrY^1?B(h|mi*NWfFX3Y|O>MuhY}+J71fX*vYUm)Mh<
zeKYYuYW7lMcVhPAM0#lMjyIaaVs~<`1L>}z_2;l{&hNRhw3IycallyHb(Z`yG<$hy
z_EO*Mdx@nR-%1`m?MvSMyYio~-S+KIJe1g<csQY0>nPg;>Vs6ax1)uzM_w(Yh6hHr
zmh{6~VO+^h8^wLm*nX@0aB5^=Y^!ld(TW8vyRVq)Q3i+mQ+>k;A?2k0*daqx3PreY
z_YI}e>7jn5f4DD|7zPQ+SI6{y@Sa2k-|E0U`0T}Z(D9F#mUaML0Gb7Q5Zo~XbP?zY
zpgW*t7lC5H=N$&$;hn-a6Y{NI(cE$zZXCS7@EsQe8=WZ7*BnAD3RfC1%m)L?^ucur
zFcpDBF!ZRucV(0RA+Tifuncj?TSNXpF!YqaKiK|cpg$O%Yv>QI`%Ysr7=5%U8H^om
z9t_6w!B`>~O$68V1j9YS_MTv<C)gs(^Ak{?1UsBJnyZJp2kIWEd!X)t|1S^B!`KMJ
zG*KSQIjK2{ua5N;sZ~wmSP?_3NyO7cR}ekX5SOu)lzyzqOy5jto{z1jG|!`M;WW*=
zcs|Jd_pdA!P~?O^j-)a-j0Vg!j!!YX=#L{2*U4WY-Xgl3Xgkq1qD*KSCSp{6&Ucgq
zU{EXr>%(hPmZzQMVF)18?9YpoUO{|P|F03A$GR0@u(x+>cpc2p)`w$}&5>xhJKEh9
z-Q2Y)yl$tK4JTE@#5=k+uSb3KC5~NiLNJ)P-FN47S9Hr=LcB+HWr}k&8BDpxVmY7J
z^Alm1`8YBX!YQlf^Mx@z?<!mAOJsLCSDpaTygH?A4U;0~%WZC!bHm`s9<iYjhM7%f
zsi^!L<iL~rvnjg~=6RU|N+ze4CWW}cE}t1!ri%qb%NTkgFQB7vU#ENqMs}uC0}qNN
zRJ+NhyHU>O_U=^@yLR=1)X4BI;5*9GjPs_-m`v93HrX8=whB|ap^O)y*2;82&%-pd
z)>aeFg|CN82Gf>Vx1y%z^ocy;UhTO+?a9>HM)mfrx2M~^T)#i@xwx@@f7YM3&<}2_
zKX1*q55I3eH}B7j_3;q&{|%qH@eq#ZaD0oO%bexsHhrJk7RO{&S@-dPQ>Q-Os*j&x
z-c+9tCEV|q`h199nDzN>eSTYCuc#S6<6?+y{YmYr>|^1zCpQ`9F?EyS_c~KI8D8W3
zDUAW#WO&{4IIVxW$p~7D#qZV_yv7Cic^ktcL_aI*Fu0EOA6QqD>vfK`uxnaZL&Z;X
zUCm=0Da+4oTBbI=mT{z0=W6!ZF87&hvG3IJ>W{o0$h6)cKiN9(r`;dx?Rib}j?c{V
zrQV+P_WWO-M}FSl|88SGejhXaq%u7A{P|VJpgi_`n#LG7hP+w`zz58jKVBmefW3KX
zS3DpLSG=L}o!)9J0HZJdK5xgH1b@f3<ITeD`*{F9y7G5-yZo}s_j@~jo!~LM9baCV
zH`wtNg2(T6e5IIo@nrxuRr9`WyL_vdY>rzY84uv>B{=GWUoTo*@l~)d-A(?6%Klud
zu>kHu47tg-iFQ}~Cb7m9|AN(QB_0s%A{?5pq%i_Z6{11>MSk<PGPYsDzr3<HfO*?J
zaFuW#Te5uF1E2H2pYg!uu3{*U)Bafx`M-PME3m>YcJ3gY*L+bO!#2Ws?rEYzJm_Jk
z?14Y+fuHoi&wAkh@W9vN=*OjAU-7{A6Yi|<F~WHs&Gmf_aJPQ)JjwGMhvk1wI6p5L
z{{!L9cKpx-Z^M_7OTD^1@Le8w!2^HF1Ao>7f0=M+ecvIR=VDx6UxQhGCmtr;>7Q+Y
zyZJc@xF1&<Xw8A`f0gj41K&?L?;B(J65u}A-NMfe>`oZIE%Dmt*mGou_o126LOd_!
zYoCX20Pa@)3sQciSZIn%6TFYh4-<PI?8I3l{8q}~ON0x04u=WvCp=6z9~biS+fFIp
zDsFFylZ7gAE9WT4;~^yHd!Z!l)IQI@A>~6g&;L2Xo$rfzX{Yu%e-iMJXsfgfm$CsC
z)x&^0pZ~uA`Rl=N^W-;huMEEdd=>N)UYk-0e;wuhE5)-UPq4Tj?630IzBhl5cHq9_
z=K~~VXhC_f&-ZU9;de+oweRaoGT%0M?~Ib9N&Fy>^>x-O4IMHB{nIJmD&?#FmW{2Z
z^6>qr5o06ZVHUE`L6Sd0e&BlTk+?;Wq6a<fJmP_i&T^@UCwMwxhfOD(x>B}m>L_a2
zN!2h<Vs*wkBa!NVkc4AenWE7ZDToMdf>ZRoZbUN9w4&v+PAoE6Fq|05x#5v=KBphh
za)(?MyqFPK$1NJ=X;&o=UPPPG-HT;sfV!b)l%kd^WUTeI<Mw*ndG>nRd9{A9^IiO5
z7mx;O{bCoY^^ct|jR0i#p<Tw|FFR8ju=`EP=1ixla4?3Xr*+LNO0}W9#IPS}x$zF!
z%?i7!3INvhy()$Uag(*TfnDB<=i+;ZVy)k+0<}J`^4$Gh<vV;`Ww`lSf%`L~WJ+bz
zEwUM_E>Ik7n}00Kad31^+f;QW8+G9^P%_GJ!Ym`IXA~7<3O2ODDa8^TX3Q3piCke!
z%_&)<P%J5Gc}8RkQ`0%k(6W)c?u>T33n`d~q85uPoV3at#Y4i}fSN5&O&tOiOG?(%
zCfd7QJ)rgO-P;Qp={c>K=t)H~)6+uv+ECXPJwIMhEC;+0?KaWsu`OjxpFsEOYF_g1
zmRc5mPB)C4W)X==2{_mVg&WwJ*e;KJ;Q>xK9=2QQPZF5y+bNX32Zs~eQ@y~$4qYjF
z)()6$rwk5_^dyFqk%56-{b?ng=z-H@Lea9Sp~4B!5xASOdR{4)w5*WlIKk(of-<S*
zvv@?*{l0^Y;3O#?N|Pl-Tjo(vV2qVYmAI`lUHvSoy?1~esVnQCV7cOP>j6`@Q?9Nj
zSw*CDXv$E>fEq=UPI63cMTW<G9`;B_pp82Wb%Ih8`Eq2etix$y9i9M~n#3K}k?f&7
z*fXh7G&u*gVhJ}`OEF3)z=MapiV7lvtt29bHUq!dFM(H({*P#r)DtJOpus6q!)#Ym
zgX3UMozme&Ie~9M_=!yCAYvK=&x;7$98=KY&C<v!L{!5l>SJXCZm54h;CTobvcTBf
z47?7*=jQo51hJ(o$B}*+u4cGcpVw>nTzVLI*1+RD#=0P4)#r5`gACrzf<&GAeSlwr
zGY_2q34dHlo*;nB&*LOq_p*jzNZ@)l>+?F%iv-M&FbtPu-C4h#@CSg(sS~AnJt>MH
z1U?61rc4_gNBS^ac!q=X=XIwQR8c-p!sTZ<rUyVD&xCNA*QeS^KMZ^f&hne65Qiax
zqO2clh)Y(C^haf?`u);b|6{<%{)6^0@Osw?(&zIwCfCf1<47Ne%c{@g@&(d=8JU(K
zzrS13$02Uj=k>G4s6Z<%vKS?^74&-$Lw&CQd{bQV=1KoanX3MTt5F&I0sLC^dEUi;
zUtyC&-kCq1g|+JQy4@t+6c85VZ}o@$&&WxL;2e^=TXlVpCYDZlPBVQ8bn#m`>+?Eb
z%Qfo13cBbj*5~y?{(Ba$VdWr@JMq|x4BU=yLJCD#pU?d-(g*yol&=1;K2yBEtooJp
zN6LWjQ?|f*OwT#=c|O-p`p)*E>aaQSF2v9lm!H=q)?B0hMan-~v1mqEzr%rZ{!Bjv
zK6Mo-$?Ky-l>dugv>R}%5BGhmWJ|<MJxd4B@SI#4F7_My71>;OQ^WlI8Jx?)@I!8R
aGb!7OqLSblAvgVzdrhfUt|%!vlH#9Z%8z&e

literal 0
HcmV?d00001

diff --git a/CMakeFiles/3.20.2/CMakeSystem.cmake b/CMakeFiles/3.20.2/CMakeSystem.cmake
new file mode 100644
index 000000000..866431272
--- /dev/null
+++ b/CMakeFiles/3.20.2/CMakeSystem.cmake
@@ -0,0 +1,15 @@
+set(CMAKE_HOST_SYSTEM "Linux-4.18.0-408.el8.x86_64")
+set(CMAKE_HOST_SYSTEM_NAME "Linux")
+set(CMAKE_HOST_SYSTEM_VERSION "4.18.0-408.el8.x86_64")
+set(CMAKE_HOST_SYSTEM_PROCESSOR "x86_64")
+
+
+
+set(CMAKE_SYSTEM "Linux-4.18.0-408.el8.x86_64")
+set(CMAKE_SYSTEM_NAME "Linux")
+set(CMAKE_SYSTEM_VERSION "4.18.0-408.el8.x86_64")
+set(CMAKE_SYSTEM_PROCESSOR "x86_64")
+
+set(CMAKE_CROSSCOMPILING "FALSE")
+
+set(CMAKE_SYSTEM_LOADED 1)
diff --git a/CMakeFiles/3.20.2/CompilerIdC/CMakeCCompilerId.c b/CMakeFiles/3.20.2/CompilerIdC/CMakeCCompilerId.c
new file mode 100644
index 000000000..7e3bd5309
--- /dev/null
+++ b/CMakeFiles/3.20.2/CompilerIdC/CMakeCCompilerId.c
@@ -0,0 +1,752 @@
+#ifdef __cplusplus
+# error "A C++ compiler has been selected for C."
+#endif
+
+#if defined(__18CXX)
+# define ID_VOID_MAIN
+#endif
+#if defined(__CLASSIC_C__)
+/* cv-qualifiers did not exist in K&R C */
+# define const
+# define volatile
+#endif
+
+
+/* Version number components: V=Version, R=Revision, P=Patch
+   Version date components:   YYYY=Year, MM=Month,   DD=Day  */
+
+#if defined(__INTEL_COMPILER) || defined(__ICC)
+# define COMPILER_ID "Intel"
+# if defined(_MSC_VER)
+#  define SIMULATE_ID "MSVC"
+# endif
+# if defined(__GNUC__)
+#  define SIMULATE_ID "GNU"
+# endif
+  /* __INTEL_COMPILER = VRP prior to 2021, and then VVVV for 2021 and later,
+     except that a few beta releases use the old format with V=2021.  */
+# if __INTEL_COMPILER < 2021 || __INTEL_COMPILER == 202110 || __INTEL_COMPILER == 202111
+#  define COMPILER_VERSION_MAJOR DEC(__INTEL_COMPILER/100)
+#  define COMPILER_VERSION_MINOR DEC(__INTEL_COMPILER/10 % 10)
+#  if defined(__INTEL_COMPILER_UPDATE)
+#   define COMPILER_VERSION_PATCH DEC(__INTEL_COMPILER_UPDATE)
+#  else
+#   define COMPILER_VERSION_PATCH DEC(__INTEL_COMPILER   % 10)
+#  endif
+# else
+#  define COMPILER_VERSION_MAJOR DEC(__INTEL_COMPILER)
+#  define COMPILER_VERSION_MINOR DEC(__INTEL_COMPILER_UPDATE)
+   /* The third version component from --version is an update index,
+      but no macro is provided for it.  */
+#  define COMPILER_VERSION_PATCH DEC(0)
+# endif
+# if defined(__INTEL_COMPILER_BUILD_DATE)
+   /* __INTEL_COMPILER_BUILD_DATE = YYYYMMDD */
+#  define COMPILER_VERSION_TWEAK DEC(__INTEL_COMPILER_BUILD_DATE)
+# endif
+# if defined(_MSC_VER)
+   /* _MSC_VER = VVRR */
+#  define SIMULATE_VERSION_MAJOR DEC(_MSC_VER / 100)
+#  define SIMULATE_VERSION_MINOR DEC(_MSC_VER % 100)
+# endif
+# if defined(__GNUC__)
+#  define SIMULATE_VERSION_MAJOR DEC(__GNUC__)
+# elif defined(__GNUG__)
+#  define SIMULATE_VERSION_MAJOR DEC(__GNUG__)
+# endif
+# if defined(__GNUC_MINOR__)
+#  define SIMULATE_VERSION_MINOR DEC(__GNUC_MINOR__)
+# endif
+# if defined(__GNUC_PATCHLEVEL__)
+#  define SIMULATE_VERSION_PATCH DEC(__GNUC_PATCHLEVEL__)
+# endif
+
+#elif (defined(__clang__) && defined(__INTEL_CLANG_COMPILER)) || defined(__INTEL_LLVM_COMPILER)
+# define COMPILER_ID "IntelLLVM"
+#if defined(_MSC_VER)
+# define SIMULATE_ID "MSVC"
+#endif
+#if defined(__GNUC__)
+# define SIMULATE_ID "GNU"
+#endif
+/* __INTEL_LLVM_COMPILER = VVVVRP prior to 2021.2.0, VVVVRRPP for 2021.2.0 and
+ * later.  Look for 6 digit vs. 8 digit version number to decide encoding.
+ * VVVV is no smaller than the current year when a versio is released.
+ */
+#if __INTEL_LLVM_COMPILER < 1000000L
+# define COMPILER_VERSION_MAJOR DEC(__INTEL_LLVM_COMPILER/100)
+# define COMPILER_VERSION_MINOR DEC(__INTEL_LLVM_COMPILER/10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__INTEL_LLVM_COMPILER    % 10)
+#else
+# define COMPILER_VERSION_MAJOR DEC(__INTEL_LLVM_COMPILER/10000)
+# define COMPILER_VERSION_MINOR DEC(__INTEL_LLVM_COMPILER/100 % 100)
+# define COMPILER_VERSION_PATCH DEC(__INTEL_LLVM_COMPILER     % 100)
+#endif
+#if defined(_MSC_VER)
+  /* _MSC_VER = VVRR */
+# define SIMULATE_VERSION_MAJOR DEC(_MSC_VER / 100)
+# define SIMULATE_VERSION_MINOR DEC(_MSC_VER % 100)
+#endif
+#if defined(__GNUC__)
+# define SIMULATE_VERSION_MAJOR DEC(__GNUC__)
+#elif defined(__GNUG__)
+# define SIMULATE_VERSION_MAJOR DEC(__GNUG__)
+#endif
+#if defined(__GNUC_MINOR__)
+# define SIMULATE_VERSION_MINOR DEC(__GNUC_MINOR__)
+#endif
+#if defined(__GNUC_PATCHLEVEL__)
+# define SIMULATE_VERSION_PATCH DEC(__GNUC_PATCHLEVEL__)
+#endif
+
+#elif defined(__PATHCC__)
+# define COMPILER_ID "PathScale"
+# define COMPILER_VERSION_MAJOR DEC(__PATHCC__)
+# define COMPILER_VERSION_MINOR DEC(__PATHCC_MINOR__)
+# if defined(__PATHCC_PATCHLEVEL__)
+#  define COMPILER_VERSION_PATCH DEC(__PATHCC_PATCHLEVEL__)
+# endif
+
+#elif defined(__BORLANDC__) && defined(__CODEGEARC_VERSION__)
+# define COMPILER_ID "Embarcadero"
+# define COMPILER_VERSION_MAJOR HEX(__CODEGEARC_VERSION__>>24 & 0x00FF)
+# define COMPILER_VERSION_MINOR HEX(__CODEGEARC_VERSION__>>16 & 0x00FF)
+# define COMPILER_VERSION_PATCH DEC(__CODEGEARC_VERSION__     & 0xFFFF)
+
+#elif defined(__BORLANDC__)
+# define COMPILER_ID "Borland"
+  /* __BORLANDC__ = 0xVRR */
+# define COMPILER_VERSION_MAJOR HEX(__BORLANDC__>>8)
+# define COMPILER_VERSION_MINOR HEX(__BORLANDC__ & 0xFF)
+
+#elif defined(__WATCOMC__) && __WATCOMC__ < 1200
+# define COMPILER_ID "Watcom"
+   /* __WATCOMC__ = VVRR */
+# define COMPILER_VERSION_MAJOR DEC(__WATCOMC__ / 100)
+# define COMPILER_VERSION_MINOR DEC((__WATCOMC__ / 10) % 10)
+# if (__WATCOMC__ % 10) > 0
+#  define COMPILER_VERSION_PATCH DEC(__WATCOMC__ % 10)
+# endif
+
+#elif defined(__WATCOMC__)
+# define COMPILER_ID "OpenWatcom"
+   /* __WATCOMC__ = VVRP + 1100 */
+# define COMPILER_VERSION_MAJOR DEC((__WATCOMC__ - 1100) / 100)
+# define COMPILER_VERSION_MINOR DEC((__WATCOMC__ / 10) % 10)
+# if (__WATCOMC__ % 10) > 0
+#  define COMPILER_VERSION_PATCH DEC(__WATCOMC__ % 10)
+# endif
+
+#elif defined(__SUNPRO_C)
+# define COMPILER_ID "SunPro"
+# if __SUNPRO_C >= 0x5100
+   /* __SUNPRO_C = 0xVRRP */
+#  define COMPILER_VERSION_MAJOR HEX(__SUNPRO_C>>12)
+#  define COMPILER_VERSION_MINOR HEX(__SUNPRO_C>>4 & 0xFF)
+#  define COMPILER_VERSION_PATCH HEX(__SUNPRO_C    & 0xF)
+# else
+   /* __SUNPRO_CC = 0xVRP */
+#  define COMPILER_VERSION_MAJOR HEX(__SUNPRO_C>>8)
+#  define COMPILER_VERSION_MINOR HEX(__SUNPRO_C>>4 & 0xF)
+#  define COMPILER_VERSION_PATCH HEX(__SUNPRO_C    & 0xF)
+# endif
+
+#elif defined(__HP_cc)
+# define COMPILER_ID "HP"
+  /* __HP_cc = VVRRPP */
+# define COMPILER_VERSION_MAJOR DEC(__HP_cc/10000)
+# define COMPILER_VERSION_MINOR DEC(__HP_cc/100 % 100)
+# define COMPILER_VERSION_PATCH DEC(__HP_cc     % 100)
+
+#elif defined(__DECC)
+# define COMPILER_ID "Compaq"
+  /* __DECC_VER = VVRRTPPPP */
+# define COMPILER_VERSION_MAJOR DEC(__DECC_VER/10000000)
+# define COMPILER_VERSION_MINOR DEC(__DECC_VER/100000  % 100)
+# define COMPILER_VERSION_PATCH DEC(__DECC_VER         % 10000)
+
+#elif defined(__IBMC__) && defined(__COMPILER_VER__)
+# define COMPILER_ID "zOS"
+  /* __IBMC__ = VRP */
+# define COMPILER_VERSION_MAJOR DEC(__IBMC__/100)
+# define COMPILER_VERSION_MINOR DEC(__IBMC__/10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__IBMC__    % 10)
+
+#elif defined(__ibmxl__) && defined(__clang__)
+# define COMPILER_ID "XLClang"
+# define COMPILER_VERSION_MAJOR DEC(__ibmxl_version__)
+# define COMPILER_VERSION_MINOR DEC(__ibmxl_release__)
+# define COMPILER_VERSION_PATCH DEC(__ibmxl_modification__)
+# define COMPILER_VERSION_TWEAK DEC(__ibmxl_ptf_fix_level__)
+
+
+#elif defined(__IBMC__) && !defined(__COMPILER_VER__) && __IBMC__ >= 800
+# define COMPILER_ID "XL"
+  /* __IBMC__ = VRP */
+# define COMPILER_VERSION_MAJOR DEC(__IBMC__/100)
+# define COMPILER_VERSION_MINOR DEC(__IBMC__/10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__IBMC__    % 10)
+
+#elif defined(__IBMC__) && !defined(__COMPILER_VER__) && __IBMC__ < 800
+# define COMPILER_ID "VisualAge"
+  /* __IBMC__ = VRP */
+# define COMPILER_VERSION_MAJOR DEC(__IBMC__/100)
+# define COMPILER_VERSION_MINOR DEC(__IBMC__/10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__IBMC__    % 10)
+
+#elif defined(__NVCOMPILER)
+# define COMPILER_ID "NVHPC"
+# define COMPILER_VERSION_MAJOR DEC(__NVCOMPILER_MAJOR__)
+# define COMPILER_VERSION_MINOR DEC(__NVCOMPILER_MINOR__)
+# if defined(__NVCOMPILER_PATCHLEVEL__)
+#  define COMPILER_VERSION_PATCH DEC(__NVCOMPILER_PATCHLEVEL__)
+# endif
+
+#elif defined(__PGI)
+# define COMPILER_ID "PGI"
+# define COMPILER_VERSION_MAJOR DEC(__PGIC__)
+# define COMPILER_VERSION_MINOR DEC(__PGIC_MINOR__)
+# if defined(__PGIC_PATCHLEVEL__)
+#  define COMPILER_VERSION_PATCH DEC(__PGIC_PATCHLEVEL__)
+# endif
+
+#elif defined(_CRAYC)
+# define COMPILER_ID "Cray"
+# define COMPILER_VERSION_MAJOR DEC(_RELEASE_MAJOR)
+# define COMPILER_VERSION_MINOR DEC(_RELEASE_MINOR)
+
+#elif defined(__TI_COMPILER_VERSION__)
+# define COMPILER_ID "TI"
+  /* __TI_COMPILER_VERSION__ = VVVRRRPPP */
+# define COMPILER_VERSION_MAJOR DEC(__TI_COMPILER_VERSION__/1000000)
+# define COMPILER_VERSION_MINOR DEC(__TI_COMPILER_VERSION__/1000   % 1000)
+# define COMPILER_VERSION_PATCH DEC(__TI_COMPILER_VERSION__        % 1000)
+
+#elif defined(__FUJITSU) || defined(__FCC_VERSION) || defined(__fcc_version)
+# define COMPILER_ID "Fujitsu"
+
+#elif defined(__ghs__)
+# define COMPILER_ID "GHS"
+/* __GHS_VERSION_NUMBER = VVVVRP */
+# ifdef __GHS_VERSION_NUMBER
+# define COMPILER_VERSION_MAJOR DEC(__GHS_VERSION_NUMBER / 100)
+# define COMPILER_VERSION_MINOR DEC(__GHS_VERSION_NUMBER / 10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__GHS_VERSION_NUMBER      % 10)
+# endif
+
+#elif defined(__TINYC__)
+# define COMPILER_ID "TinyCC"
+
+#elif defined(__BCC__)
+# define COMPILER_ID "Bruce"
+
+#elif defined(__SCO_VERSION__)
+# define COMPILER_ID "SCO"
+
+#elif defined(__ARMCC_VERSION) && !defined(__clang__)
+# define COMPILER_ID "ARMCC"
+#if __ARMCC_VERSION >= 1000000
+  /* __ARMCC_VERSION = VRRPPPP */
+  # define COMPILER_VERSION_MAJOR DEC(__ARMCC_VERSION/1000000)
+  # define COMPILER_VERSION_MINOR DEC(__ARMCC_VERSION/10000 % 100)
+  # define COMPILER_VERSION_PATCH DEC(__ARMCC_VERSION     % 10000)
+#else
+  /* __ARMCC_VERSION = VRPPPP */
+  # define COMPILER_VERSION_MAJOR DEC(__ARMCC_VERSION/100000)
+  # define COMPILER_VERSION_MINOR DEC(__ARMCC_VERSION/10000 % 10)
+  # define COMPILER_VERSION_PATCH DEC(__ARMCC_VERSION    % 10000)
+#endif
+
+
+#elif defined(__clang__) && defined(__apple_build_version__)
+# define COMPILER_ID "AppleClang"
+# if defined(_MSC_VER)
+#  define SIMULATE_ID "MSVC"
+# endif
+# define COMPILER_VERSION_MAJOR DEC(__clang_major__)
+# define COMPILER_VERSION_MINOR DEC(__clang_minor__)
+# define COMPILER_VERSION_PATCH DEC(__clang_patchlevel__)
+# if defined(_MSC_VER)
+   /* _MSC_VER = VVRR */
+#  define SIMULATE_VERSION_MAJOR DEC(_MSC_VER / 100)
+#  define SIMULATE_VERSION_MINOR DEC(_MSC_VER % 100)
+# endif
+# define COMPILER_VERSION_TWEAK DEC(__apple_build_version__)
+
+#elif defined(__clang__) && defined(__ARMCOMPILER_VERSION)
+# define COMPILER_ID "ARMClang"
+  # define COMPILER_VERSION_MAJOR DEC(__ARMCOMPILER_VERSION/1000000)
+  # define COMPILER_VERSION_MINOR DEC(__ARMCOMPILER_VERSION/10000 % 100)
+  # define COMPILER_VERSION_PATCH DEC(__ARMCOMPILER_VERSION     % 10000)
+# define COMPILER_VERSION_INTERNAL DEC(__ARMCOMPILER_VERSION)
+
+#elif defined(__clang__)
+# define COMPILER_ID "Clang"
+# if defined(_MSC_VER)
+#  define SIMULATE_ID "MSVC"
+# endif
+# define COMPILER_VERSION_MAJOR DEC(__clang_major__)
+# define COMPILER_VERSION_MINOR DEC(__clang_minor__)
+# define COMPILER_VERSION_PATCH DEC(__clang_patchlevel__)
+# if defined(_MSC_VER)
+   /* _MSC_VER = VVRR */
+#  define SIMULATE_VERSION_MAJOR DEC(_MSC_VER / 100)
+#  define SIMULATE_VERSION_MINOR DEC(_MSC_VER % 100)
+# endif
+
+#elif defined(__GNUC__)
+# define COMPILER_ID "GNU"
+# define COMPILER_VERSION_MAJOR DEC(__GNUC__)
+# if defined(__GNUC_MINOR__)
+#  define COMPILER_VERSION_MINOR DEC(__GNUC_MINOR__)
+# endif
+# if defined(__GNUC_PATCHLEVEL__)
+#  define COMPILER_VERSION_PATCH DEC(__GNUC_PATCHLEVEL__)
+# endif
+
+#elif defined(_MSC_VER)
+# define COMPILER_ID "MSVC"
+  /* _MSC_VER = VVRR */
+# define COMPILER_VERSION_MAJOR DEC(_MSC_VER / 100)
+# define COMPILER_VERSION_MINOR DEC(_MSC_VER % 100)
+# if defined(_MSC_FULL_VER)
+#  if _MSC_VER >= 1400
+    /* _MSC_FULL_VER = VVRRPPPPP */
+#   define COMPILER_VERSION_PATCH DEC(_MSC_FULL_VER % 100000)
+#  else
+    /* _MSC_FULL_VER = VVRRPPPP */
+#   define COMPILER_VERSION_PATCH DEC(_MSC_FULL_VER % 10000)
+#  endif
+# endif
+# if defined(_MSC_BUILD)
+#  define COMPILER_VERSION_TWEAK DEC(_MSC_BUILD)
+# endif
+
+#elif defined(__VISUALDSPVERSION__) || defined(__ADSPBLACKFIN__) || defined(__ADSPTS__) || defined(__ADSP21000__)
+# define COMPILER_ID "ADSP"
+#if defined(__VISUALDSPVERSION__)
+  /* __VISUALDSPVERSION__ = 0xVVRRPP00 */
+# define COMPILER_VERSION_MAJOR HEX(__VISUALDSPVERSION__>>24)
+# define COMPILER_VERSION_MINOR HEX(__VISUALDSPVERSION__>>16 & 0xFF)
+# define COMPILER_VERSION_PATCH HEX(__VISUALDSPVERSION__>>8  & 0xFF)
+#endif
+
+#elif defined(__IAR_SYSTEMS_ICC__) || defined(__IAR_SYSTEMS_ICC)
+# define COMPILER_ID "IAR"
+# if defined(__VER__) && defined(__ICCARM__)
+#  define COMPILER_VERSION_MAJOR DEC((__VER__) / 1000000)
+#  define COMPILER_VERSION_MINOR DEC(((__VER__) / 1000) % 1000)
+#  define COMPILER_VERSION_PATCH DEC((__VER__) % 1000)
+#  define COMPILER_VERSION_INTERNAL DEC(__IAR_SYSTEMS_ICC__)
+# elif defined(__VER__) && (defined(__ICCAVR__) || defined(__ICCRX__) || defined(__ICCRH850__) || defined(__ICCRL78__) || defined(__ICC430__) || defined(__ICCRISCV__) || defined(__ICCV850__) || defined(__ICC8051__) || defined(__ICCSTM8__))
+#  define COMPILER_VERSION_MAJOR DEC((__VER__) / 100)
+#  define COMPILER_VERSION_MINOR DEC((__VER__) - (((__VER__) / 100)*100))
+#  define COMPILER_VERSION_PATCH DEC(__SUBVERSION__)
+#  define COMPILER_VERSION_INTERNAL DEC(__IAR_SYSTEMS_ICC__)
+# endif
+
+#elif defined(__SDCC_VERSION_MAJOR) || defined(SDCC)
+# define COMPILER_ID "SDCC"
+# if defined(__SDCC_VERSION_MAJOR)
+#  define COMPILER_VERSION_MAJOR DEC(__SDCC_VERSION_MAJOR)
+#  define COMPILER_VERSION_MINOR DEC(__SDCC_VERSION_MINOR)
+#  define COMPILER_VERSION_PATCH DEC(__SDCC_VERSION_PATCH)
+# else
+  /* SDCC = VRP */
+#  define COMPILER_VERSION_MAJOR DEC(SDCC/100)
+#  define COMPILER_VERSION_MINOR DEC(SDCC/10 % 10)
+#  define COMPILER_VERSION_PATCH DEC(SDCC    % 10)
+# endif
+
+
+/* These compilers are either not known or too old to define an
+  identification macro.  Try to identify the platform and guess that
+  it is the native compiler.  */
+#elif defined(__hpux) || defined(__hpua)
+# define COMPILER_ID "HP"
+
+#else /* unknown compiler */
+# define COMPILER_ID ""
+#endif
+
+/* Construct the string literal in pieces to prevent the source from
+   getting matched.  Store it in a pointer rather than an array
+   because some compilers will just produce instructions to fill the
+   array rather than assigning a pointer to a static array.  */
+char const* info_compiler = "INFO" ":" "compiler[" COMPILER_ID "]";
+#ifdef SIMULATE_ID
+char const* info_simulate = "INFO" ":" "simulate[" SIMULATE_ID "]";
+#endif
+
+#ifdef __QNXNTO__
+char const* qnxnto = "INFO" ":" "qnxnto[]";
+#endif
+
+#if defined(__CRAYXT_COMPUTE_LINUX_TARGET)
+char const *info_cray = "INFO" ":" "compiler_wrapper[CrayPrgEnv]";
+#endif
+
+#define STRINGIFY_HELPER(X) #X
+#define STRINGIFY(X) STRINGIFY_HELPER(X)
+
+/* Identify known platforms by name.  */
+#if defined(__linux) || defined(__linux__) || defined(linux)
+# define PLATFORM_ID "Linux"
+
+#elif defined(__CYGWIN__)
+# define PLATFORM_ID "Cygwin"
+
+#elif defined(__MINGW32__)
+# define PLATFORM_ID "MinGW"
+
+#elif defined(__APPLE__)
+# define PLATFORM_ID "Darwin"
+
+#elif defined(_WIN32) || defined(__WIN32__) || defined(WIN32)
+# define PLATFORM_ID "Windows"
+
+#elif defined(__FreeBSD__) || defined(__FreeBSD)
+# define PLATFORM_ID "FreeBSD"
+
+#elif defined(__NetBSD__) || defined(__NetBSD)
+# define PLATFORM_ID "NetBSD"
+
+#elif defined(__OpenBSD__) || defined(__OPENBSD)
+# define PLATFORM_ID "OpenBSD"
+
+#elif defined(__sun) || defined(sun)
+# define PLATFORM_ID "SunOS"
+
+#elif defined(_AIX) || defined(__AIX) || defined(__AIX__) || defined(__aix) || defined(__aix__)
+# define PLATFORM_ID "AIX"
+
+#elif defined(__hpux) || defined(__hpux__)
+# define PLATFORM_ID "HP-UX"
+
+#elif defined(__HAIKU__)
+# define PLATFORM_ID "Haiku"
+
+#elif defined(__BeOS) || defined(__BEOS__) || defined(_BEOS)
+# define PLATFORM_ID "BeOS"
+
+#elif defined(__QNX__) || defined(__QNXNTO__)
+# define PLATFORM_ID "QNX"
+
+#elif defined(__tru64) || defined(_tru64) || defined(__TRU64__)
+# define PLATFORM_ID "Tru64"
+
+#elif defined(__riscos) || defined(__riscos__)
+# define PLATFORM_ID "RISCos"
+
+#elif defined(__sinix) || defined(__sinix__) || defined(__SINIX__)
+# define PLATFORM_ID "SINIX"
+
+#elif defined(__UNIX_SV__)
+# define PLATFORM_ID "UNIX_SV"
+
+#elif defined(__bsdos__)
+# define PLATFORM_ID "BSDOS"
+
+#elif defined(_MPRAS) || defined(MPRAS)
+# define PLATFORM_ID "MP-RAS"
+
+#elif defined(__osf) || defined(__osf__)
+# define PLATFORM_ID "OSF1"
+
+#elif defined(_SCO_SV) || defined(SCO_SV) || defined(sco_sv)
+# define PLATFORM_ID "SCO_SV"
+
+#elif defined(__ultrix) || defined(__ultrix__) || defined(_ULTRIX)
+# define PLATFORM_ID "ULTRIX"
+
+#elif defined(__XENIX__) || defined(_XENIX) || defined(XENIX)
+# define PLATFORM_ID "Xenix"
+
+#elif defined(__WATCOMC__)
+# if defined(__LINUX__)
+#  define PLATFORM_ID "Linux"
+
+# elif defined(__DOS__)
+#  define PLATFORM_ID "DOS"
+
+# elif defined(__OS2__)
+#  define PLATFORM_ID "OS2"
+
+# elif defined(__WINDOWS__)
+#  define PLATFORM_ID "Windows3x"
+
+# elif defined(__VXWORKS__)
+#  define PLATFORM_ID "VxWorks"
+
+# else /* unknown platform */
+#  define PLATFORM_ID
+# endif
+
+#elif defined(__INTEGRITY)
+# if defined(INT_178B)
+#  define PLATFORM_ID "Integrity178"
+
+# else /* regular Integrity */
+#  define PLATFORM_ID "Integrity"
+# endif
+
+#else /* unknown platform */
+# define PLATFORM_ID
+
+#endif
+
+/* For windows compilers MSVC and Intel we can determine
+   the architecture of the compiler being used.  This is because
+   the compilers do not have flags that can change the architecture,
+   but rather depend on which compiler is being used
+*/
+#if defined(_WIN32) && defined(_MSC_VER)
+# if defined(_M_IA64)
+#  define ARCHITECTURE_ID "IA64"
+
+# elif defined(_M_ARM64EC)
+#  define ARCHITECTURE_ID "ARM64EC"
+
+# elif defined(_M_X64) || defined(_M_AMD64)
+#  define ARCHITECTURE_ID "x64"
+
+# elif defined(_M_IX86)
+#  define ARCHITECTURE_ID "X86"
+
+# elif defined(_M_ARM64)
+#  define ARCHITECTURE_ID "ARM64"
+
+# elif defined(_M_ARM)
+#  if _M_ARM == 4
+#   define ARCHITECTURE_ID "ARMV4I"
+#  elif _M_ARM == 5
+#   define ARCHITECTURE_ID "ARMV5I"
+#  else
+#   define ARCHITECTURE_ID "ARMV" STRINGIFY(_M_ARM)
+#  endif
+
+# elif defined(_M_MIPS)
+#  define ARCHITECTURE_ID "MIPS"
+
+# elif defined(_M_SH)
+#  define ARCHITECTURE_ID "SHx"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#elif defined(__WATCOMC__)
+# if defined(_M_I86)
+#  define ARCHITECTURE_ID "I86"
+
+# elif defined(_M_IX86)
+#  define ARCHITECTURE_ID "X86"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#elif defined(__IAR_SYSTEMS_ICC__) || defined(__IAR_SYSTEMS_ICC)
+# if defined(__ICCARM__)
+#  define ARCHITECTURE_ID "ARM"
+
+# elif defined(__ICCRX__)
+#  define ARCHITECTURE_ID "RX"
+
+# elif defined(__ICCRH850__)
+#  define ARCHITECTURE_ID "RH850"
+
+# elif defined(__ICCRL78__)
+#  define ARCHITECTURE_ID "RL78"
+
+# elif defined(__ICCRISCV__)
+#  define ARCHITECTURE_ID "RISCV"
+
+# elif defined(__ICCAVR__)
+#  define ARCHITECTURE_ID "AVR"
+
+# elif defined(__ICC430__)
+#  define ARCHITECTURE_ID "MSP430"
+
+# elif defined(__ICCV850__)
+#  define ARCHITECTURE_ID "V850"
+
+# elif defined(__ICC8051__)
+#  define ARCHITECTURE_ID "8051"
+
+# elif defined(__ICCSTM8__)
+#  define ARCHITECTURE_ID "STM8"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#elif defined(__ghs__)
+# if defined(__PPC64__)
+#  define ARCHITECTURE_ID "PPC64"
+
+# elif defined(__ppc__)
+#  define ARCHITECTURE_ID "PPC"
+
+# elif defined(__ARM__)
+#  define ARCHITECTURE_ID "ARM"
+
+# elif defined(__x86_64__)
+#  define ARCHITECTURE_ID "x64"
+
+# elif defined(__i386__)
+#  define ARCHITECTURE_ID "X86"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#elif defined(__TI_COMPILER_VERSION__)
+# if defined(__TI_ARM__)
+#  define ARCHITECTURE_ID "ARM"
+
+# elif defined(__MSP430__)
+#  define ARCHITECTURE_ID "MSP430"
+
+# elif defined(__TMS320C28XX__)
+#  define ARCHITECTURE_ID "TMS320C28x"
+
+# elif defined(__TMS320C6X__) || defined(_TMS320C6X)
+#  define ARCHITECTURE_ID "TMS320C6x"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#else
+#  define ARCHITECTURE_ID
+#endif
+
+/* Convert integer to decimal digit literals.  */
+#define DEC(n)                   \
+  ('0' + (((n) / 10000000)%10)), \
+  ('0' + (((n) / 1000000)%10)),  \
+  ('0' + (((n) / 100000)%10)),   \
+  ('0' + (((n) / 10000)%10)),    \
+  ('0' + (((n) / 1000)%10)),     \
+  ('0' + (((n) / 100)%10)),      \
+  ('0' + (((n) / 10)%10)),       \
+  ('0' +  ((n) % 10))
+
+/* Convert integer to hex digit literals.  */
+#define HEX(n)             \
+  ('0' + ((n)>>28 & 0xF)), \
+  ('0' + ((n)>>24 & 0xF)), \
+  ('0' + ((n)>>20 & 0xF)), \
+  ('0' + ((n)>>16 & 0xF)), \
+  ('0' + ((n)>>12 & 0xF)), \
+  ('0' + ((n)>>8  & 0xF)), \
+  ('0' + ((n)>>4  & 0xF)), \
+  ('0' + ((n)     & 0xF))
+
+/* Construct a string literal encoding the version number components. */
+#ifdef COMPILER_VERSION_MAJOR
+char const info_version[] = {
+  'I', 'N', 'F', 'O', ':',
+  'c','o','m','p','i','l','e','r','_','v','e','r','s','i','o','n','[',
+  COMPILER_VERSION_MAJOR,
+# ifdef COMPILER_VERSION_MINOR
+  '.', COMPILER_VERSION_MINOR,
+#  ifdef COMPILER_VERSION_PATCH
+   '.', COMPILER_VERSION_PATCH,
+#   ifdef COMPILER_VERSION_TWEAK
+    '.', COMPILER_VERSION_TWEAK,
+#   endif
+#  endif
+# endif
+  ']','\0'};
+#endif
+
+/* Construct a string literal encoding the internal version number. */
+#ifdef COMPILER_VERSION_INTERNAL
+char const info_version_internal[] = {
+  'I', 'N', 'F', 'O', ':',
+  'c','o','m','p','i','l','e','r','_','v','e','r','s','i','o','n','_',
+  'i','n','t','e','r','n','a','l','[',
+  COMPILER_VERSION_INTERNAL,']','\0'};
+#endif
+
+/* Construct a string literal encoding the version number components. */
+#ifdef SIMULATE_VERSION_MAJOR
+char const info_simulate_version[] = {
+  'I', 'N', 'F', 'O', ':',
+  's','i','m','u','l','a','t','e','_','v','e','r','s','i','o','n','[',
+  SIMULATE_VERSION_MAJOR,
+# ifdef SIMULATE_VERSION_MINOR
+  '.', SIMULATE_VERSION_MINOR,
+#  ifdef SIMULATE_VERSION_PATCH
+   '.', SIMULATE_VERSION_PATCH,
+#   ifdef SIMULATE_VERSION_TWEAK
+    '.', SIMULATE_VERSION_TWEAK,
+#   endif
+#  endif
+# endif
+  ']','\0'};
+#endif
+
+/* Construct the string literal in pieces to prevent the source from
+   getting matched.  Store it in a pointer rather than an array
+   because some compilers will just produce instructions to fill the
+   array rather than assigning a pointer to a static array.  */
+char const* info_platform = "INFO" ":" "platform[" PLATFORM_ID "]";
+char const* info_arch = "INFO" ":" "arch[" ARCHITECTURE_ID "]";
+
+
+
+#if !defined(__STDC__)
+# if (defined(_MSC_VER) && !defined(__clang__)) \
+  || (defined(__ibmxl__) || defined(__IBMC__))
+#  define C_DIALECT "90"
+# else
+#  define C_DIALECT
+# endif
+#elif __STDC_VERSION__ >= 201000L
+# define C_DIALECT "11"
+#elif __STDC_VERSION__ >= 199901L
+# define C_DIALECT "99"
+#else
+# define C_DIALECT "90"
+#endif
+const char* info_language_dialect_default =
+  "INFO" ":" "dialect_default[" C_DIALECT "]";
+
+/*--------------------------------------------------------------------------*/
+
+#ifdef ID_VOID_MAIN
+void main() {}
+#else
+# if defined(__CLASSIC_C__)
+int main(argc, argv) int argc; char *argv[];
+# else
+int main(int argc, char* argv[])
+# endif
+{
+  int require = 0;
+  require += info_compiler[argc];
+  require += info_platform[argc];
+  require += info_arch[argc];
+#ifdef COMPILER_VERSION_MAJOR
+  require += info_version[argc];
+#endif
+#ifdef COMPILER_VERSION_INTERNAL
+  require += info_version_internal[argc];
+#endif
+#ifdef SIMULATE_ID
+  require += info_simulate[argc];
+#endif
+#ifdef SIMULATE_VERSION_MAJOR
+  require += info_simulate_version[argc];
+#endif
+#if defined(__CRAYXT_COMPUTE_LINUX_TARGET)
+  require += info_cray[argc];
+#endif
+  require += info_language_dialect_default[argc];
+  (void)argv;
+  return require;
+}
+#endif
diff --git a/CMakeFiles/3.20.2/CompilerIdC/a.out b/CMakeFiles/3.20.2/CompilerIdC/a.out
new file mode 100755
index 0000000000000000000000000000000000000000..700412c0ec41d3fae265706afc29140fe2b2544b
GIT binary patch
literal 18048
zcmeHPeQX@X6`#Aa9fu_6ONgl<v^fg2PLR)MlLV6zxbs)mtrOG4;X|p}tnb$LDd)T6
zZjU-tZPO+onUIpIABu!pB2}uOs@i`_ME_}%rV#ayRzYgTA5bYRghmC5D@qD9<oe#s
zdw08gyHSK%T9lh)@6G()$IQH$-JO}8@mGgNhl7ED;1CqI3rrnqN=Qik3bFSNnIx=F
zEEWlItym>m0BM3NAvs7@jOjaMnrSP^g@B@7Ur+&k!Gw%3y;i}HhYDOmq|JmWlAx5L
z3Ce(}5K+lTJuGWHER#TY$q3U3m7{|w>tPc05~P<PJ*G#fU`)AwSU0@J$=-M{0Vzx;
zkcdI~Tcaq8;m$Kw8snpjWD;mXMwqTsuwcr1p9eiG=cR@S32vkO{QPo?>c^ClqC1<J
z+_0`Yo9@bHa^<<MxxNiu8`i~2`B<;4H`+(p&0BZKoUpywwOHnra3O~L9Y6ilo^{Lr
z_N^zbzw6DW$490%l&|kZUA`cC$)Ny}Q4i@QfUWSr9|bJxfj{np3#fMrtGPkwI>z;q
zWfU!a*2v_<=FyS<0lhcY8(S~B^98HBm`}}it%r0Rl4dTIMYnBqvt}{{PaI^0J~F;V
zPn$(^I#aUD;`o+<Y(8g>8<SZR3NSsJ&rtz%CF&^%bqRp?1J!>*aQm09$t=c}`8fID
zblWZ&!@h+6=Wy1Bb2}n$!G>dda(Ksvb00<CTWz0sPuur;t9GFI{v}bNJ#@?pR!(XA
ze%*RPsv^JVjgk51@6+c0tnGVc;m+}qlgA1z389@lE+jemKyyM=+Sh{mKMtY%iK$rI
z>itsvkaloW2JIefo<wr%J61bnb?yq4)#A$Ow$*!4@&r+kzg5bw??tlaf_D5uNSj~K
zj-S0l3p}Sif5BP_27XHhS}Uig+Ex$N^fCXvo8AVtC||iln?G~*!Ahm_T=UC-1SU=#
zIx$r%T-)k|tp96|py2$MU)Sc#r%w)@$MR?g+P?`Z2ZmnP4r_r^ZAV4hFOCI<&IA56
zAN+nVTpDOQy7x4=s%l}(Tk<@V56hptdu0BF<Xt24=aM^;^Y16eM-Se(vNbApXa~EH
zZXaE92HWiH9TzGU?fCm4Yjw|y<iFAR_eST>4bH!rtgL)V+jk<M-TKG!U$H%RP9(pO
zoJii6)a^cy?FIEkD*HjT<<SfE0sHO6!pPR)v5l$xY$201i#y?IGU1eV@bw4GVkwi)
z?To9?7o*5}j>IQ~)F@;PYbsxy-8qU+Hv)}fYGx<J(-|XcrYyKMr;Kve+S$`HAp)yH
zpNT{E_<kCEzfu_mY60B^bS=2f0{Szc-v)XE^pcl=;&Tu~;K6MoFc%4|TDqw95Zrip
zf8x7u;e4gijRJ#<B8Z{=rC^iafhYsrQTzn3d7vU0jyxP3Sk@BU50(@!4R8nKYa)L*
z9C<7_6z=#&Xeb;#*fbRGe6(2$#~*Id!hQP|Z4M`L;l5-zo(y;Phok-Bj{b0@Kin$I
z^J7p?fE~^o%{4;90}T%}JkapK<>7$>jJ+@<i1HYbX`AE<v4-eUqJ^e}j4h+|9E?ve
zTrGcbUbaidR#BSgQP)tqkNEh$!@&HvFI4iFJ{(LSsm=}KA(iGi(KA7nz5--0JgtaW
zLbQWuJ5eSy4HFS6Kj*uP1QruzeRw^}@+?sP@S2xt_Rj*PUnM^0^S`1zkM+y{7`4-i
zMF}bLX`(k1-8?X`G1>{^t~JrV7!0GLz46|j`1+o8(avpVI;t6#!n=Cb%PxWb#&HZz
zE;c7`2;5xgiEp??h+mR_nXZ<(RppwC<y<b4n~qMWQm%}IaLXFGTz)c>^OWWOM)@}v
zvgK(I%^9=i#waOb*4!_Y+}h1!cZ;>nFcjtfM7SKBa-Ub&O)w!$?bcISqckJLm5zF>
zO-<>AV%~z0BMd{a6Z8E*_J+r{jgJi9D=MgVl|%O?IlH{)9zD5z`w&QtZQTxhS9uoU
z9IQG%lRmyqc3GFL{A|Y3r;1Q(y^zo3U}jlws|DxF%i)qi*-~{YnnpG=okQHOJ$e0z
zsk@CD?b&EguY0+1f8ulTrpEo*c-}%kxUTWMRc{{}FI+zF&tr}85cL1GAGz@mj^}WE
zi=Wio<>xkimf9A_WHnju@qk;WG2Uv7pJCqAm=7hr@0Z4Wh+dd#e%+YgHs-ggL#X%B
zWxrm5iy@BnC!?pj*M--fykwZi)Juln>rB05cpda%8UuLA@Y?8JT6^=7IYs*TEgXa2
zxBx$yV<;2-sI0@_I@W(-T}}3X*IL*mt*fEphq<oiGmezy=QcH|O)q5}>DIZJeYVSe
z=2Gmtb^Q7xuLm-1^v4gj&ii5a$3}Zz(!Ap%^L%NvXQMs;pXZU6`}^N(%*XFzrXN&>
z$DRi+G6v<b=R-8cz%is>2*Jn8zF?w`9fG}k<DPg(Se|%O^*g@ZSO`X5{QcgEw+Q~u
z@5C1gukY(2_~6Rl<(=}2tKavX_!WZ3>rQ-0b<W_#mkJ)cJMm@Wh=(6TB3#|y?UZj5
zd+GbS6OUB)NjUN4g7-%_@fEOdeU}}Q@eq8K=6$|)W(ap9M!e+PMTaMTl}LEvAG4dS
z#zUe*Xptk;G)7=rDw@RK<@atoV-qHVORD<<n773THwfo(CCit6@Pj`14}EaCI~mI3
zw*R7!{6Bs0rC4DPJ2w)}>%FLsVH4pz?^LJ|_xji=`{3X4!H@djFZ$sB^1)Z*$j75z
zxB1``guCmzpKzW-bA6uz+^e5FP4dEZfBlAVepa%bKN0S3$9H}3c6<?e)T`GA-|mCw
zeelP8@F#unX9;)L_Z7l<9>(<zG^z5t@hIVL|7-%>%g-9%L0DxtMehu@|5?HXtwS(A
zK{)RdW4r`-0Cu+Ua|62*hObM!{yFv(+2MU>N*eZWN%{Kc;qL+WD!*U7{vz$PiQ8He
zY`RLl?7R)~xPs(<-nPKvKmbmdxSzLea?zl^oS&3<{qujlv{U~a{v6;d!2fs>0)tcs
zx)UN%1YF?>iPt~3zX*6lv{(JlYC8a55v+fIeFbqAveEC69rt_TM}S9davc3R;LC%{
z#FJEh#-BqRc0nB>T)G#Se~@^)=%n`xj&(4+3^@8hki4m2F%BIm0{u#Jy^r2PdANW1
zIRi-<en58kInYh`k`^payNJ5-y&7<|&;6Ff#f^Ytd3YU8?!gD<TC&sUvZEm$toQ#o
z;88Yeqi@La*N;PT(*81Wn*7j9I$xrE<F0%~cezx=Q%2pe>!uq{Yv~)-brsF@jA5x0
zT-|-$u~_XtNW$^1RMF~*<wcA(#_5?{#)_rfY2D1F-B@fUZ@Dp)^TK20TsE`Y%<l12
z@MFg0!K-q?Q^|)HGv_ki#d0!0-O8l&qM6O7?Dcix&U!m}&U!j|^?q>jJ^bMmkOu1g
z;uNa)kCQKr093T>K6J{s{N-dy15Ur`>8x^^VVQU?4V_rXn5rnXhVm1`eq?5+x@0%Y
zZ?7o;Sl9Pz7#74!*4YM5c|V?q?_G-Zey<7C`@F{U_IHi%@^y{j<!5#DFjXomx5#F!
zxqwb4R@n~!*qBq`=(@J4nU-wSg~ve2D#J;(l&Bp=)U8?A<O+uoOK=b|o!6(c`AH+I
zr>%Ukq#NZqfv2*vre&sMx7-}>^%l}G58WshjXh9JtGGv~O{wYf?Cc&;v87~99ijtU
zjNRtI03H4sNkj2;eRx}P%aA^_bx?;Bkxreo&<F3`n%pun;N&Rp={oFU)`v8*pbc&l
z`sUHG{^Y1WHaxt2Xj~sp_KyysRd_h2bo4Hk{^Cb8&9q?|tRgd#M@16}`&m)&A4sW#
zoFMBw)v2dTd40ynrSZh)$QVeaGdaCnGSecHo675aRE;y?Ru?{ZnbP7TbD%X@DpA(*
z5UVtrHFDEsW7^bR=iERXj-ax8@ocK+fm3ON4%?~x@a&urvC^Jd%a{ae6&0P~nB0U6
z5Az)Cj*cb!N4hLyno`rba%{4kfwRULxX&>)gL|uE={-5Hr>Ip_9L`jVQ9=hEJjxAJ
z5HWnXh?r&0!LMw;STQdfDQ3=4f162z2B(yU>Uc_n<6zF1&A{t$8sByB6DwpPqKtu8
zMGU+=3*A~3O;#ahSXMDJS+?M|`A-BqF9Aaz7@MoW>o9z-p3h4VTgq}A>Gg2o83)$q
z^%_369tEB~@c4|e9*EfWd0odMgEz1sQMdje;OF2>1m}M^n2?f(3E=YcxQ)kwQV4#w
z!i8(utk3I2&k!(2f-o$Sb!Yt!!tVkqr%sgS^`tn05cr&kN+}y0M|vMzc*cYC=XIx7
zNuSS~aQRt|>2A=+Gb5bl^{EBYj{+ZqyZj0j;z5X@Pgy_Gl#r|lIeM2&)#eNC`tJum
z_8+v5p+Nct(&zIyimUSCIMPSpvg`AB`Vi@#L8dLpZ|}DBA&A@cdHrmT3WVP&?Safz
zq3=Qr^|}5$k2pg5cxKfeqV@#DzYmvPpa0GR|NVt^E_rwUXy2~S>vntbrhsrv{?>lj
z|BM`k2>xjaYVPX#9{)WCw>+nr{t9&QJ2~s~ytI%0UIoj6hzy)&`a3|;RjkkRSN?k!
zN3e1b$eq=5SKN*-01ri2pVt*n)5rTL@a%!}W%?2@?fQb|Eil=Vf$vkc!Fo*3xb%5G
z+CloAtVopg7=8^Rh;jLOewQG9w>-In)x`qD5Q|q46=D4@7tZ=j-vvH(6)DN<qXo+U
z8UDizWXIkA&<^)~5|N}crc%e~fEu2oyAv+15Bn9_Tz93Re$T?0EDR5E!>gpMD@7&2
T^Fdzvv$rd$Hm)csxsu}F3c8L4

literal 0
HcmV?d00001

diff --git a/CMakeFiles/3.20.2/CompilerIdCXX/CMakeCXXCompilerId.cpp b/CMakeFiles/3.20.2/CompilerIdCXX/CMakeCXXCompilerId.cpp
new file mode 100644
index 000000000..92e7655ae
--- /dev/null
+++ b/CMakeFiles/3.20.2/CompilerIdCXX/CMakeCXXCompilerId.cpp
@@ -0,0 +1,743 @@
+/* This source file must have a .cpp extension so that all C++ compilers
+   recognize the extension without flags.  Borland does not know .cxx for
+   example.  */
+#ifndef __cplusplus
+# error "A C compiler has been selected for C++."
+#endif
+
+
+/* Version number components: V=Version, R=Revision, P=Patch
+   Version date components:   YYYY=Year, MM=Month,   DD=Day  */
+
+#if defined(__COMO__)
+# define COMPILER_ID "Comeau"
+  /* __COMO_VERSION__ = VRR */
+# define COMPILER_VERSION_MAJOR DEC(__COMO_VERSION__ / 100)
+# define COMPILER_VERSION_MINOR DEC(__COMO_VERSION__ % 100)
+
+#elif defined(__INTEL_COMPILER) || defined(__ICC)
+# define COMPILER_ID "Intel"
+# if defined(_MSC_VER)
+#  define SIMULATE_ID "MSVC"
+# endif
+# if defined(__GNUC__)
+#  define SIMULATE_ID "GNU"
+# endif
+  /* __INTEL_COMPILER = VRP prior to 2021, and then VVVV for 2021 and later,
+     except that a few beta releases use the old format with V=2021.  */
+# if __INTEL_COMPILER < 2021 || __INTEL_COMPILER == 202110 || __INTEL_COMPILER == 202111
+#  define COMPILER_VERSION_MAJOR DEC(__INTEL_COMPILER/100)
+#  define COMPILER_VERSION_MINOR DEC(__INTEL_COMPILER/10 % 10)
+#  if defined(__INTEL_COMPILER_UPDATE)
+#   define COMPILER_VERSION_PATCH DEC(__INTEL_COMPILER_UPDATE)
+#  else
+#   define COMPILER_VERSION_PATCH DEC(__INTEL_COMPILER   % 10)
+#  endif
+# else
+#  define COMPILER_VERSION_MAJOR DEC(__INTEL_COMPILER)
+#  define COMPILER_VERSION_MINOR DEC(__INTEL_COMPILER_UPDATE)
+   /* The third version component from --version is an update index,
+      but no macro is provided for it.  */
+#  define COMPILER_VERSION_PATCH DEC(0)
+# endif
+# if defined(__INTEL_COMPILER_BUILD_DATE)
+   /* __INTEL_COMPILER_BUILD_DATE = YYYYMMDD */
+#  define COMPILER_VERSION_TWEAK DEC(__INTEL_COMPILER_BUILD_DATE)
+# endif
+# if defined(_MSC_VER)
+   /* _MSC_VER = VVRR */
+#  define SIMULATE_VERSION_MAJOR DEC(_MSC_VER / 100)
+#  define SIMULATE_VERSION_MINOR DEC(_MSC_VER % 100)
+# endif
+# if defined(__GNUC__)
+#  define SIMULATE_VERSION_MAJOR DEC(__GNUC__)
+# elif defined(__GNUG__)
+#  define SIMULATE_VERSION_MAJOR DEC(__GNUG__)
+# endif
+# if defined(__GNUC_MINOR__)
+#  define SIMULATE_VERSION_MINOR DEC(__GNUC_MINOR__)
+# endif
+# if defined(__GNUC_PATCHLEVEL__)
+#  define SIMULATE_VERSION_PATCH DEC(__GNUC_PATCHLEVEL__)
+# endif
+
+#elif (defined(__clang__) && defined(__INTEL_CLANG_COMPILER)) || defined(__INTEL_LLVM_COMPILER)
+# define COMPILER_ID "IntelLLVM"
+#if defined(_MSC_VER)
+# define SIMULATE_ID "MSVC"
+#endif
+#if defined(__GNUC__)
+# define SIMULATE_ID "GNU"
+#endif
+/* __INTEL_LLVM_COMPILER = VVVVRP prior to 2021.2.0, VVVVRRPP for 2021.2.0 and
+ * later.  Look for 6 digit vs. 8 digit version number to decide encoding.
+ * VVVV is no smaller than the current year when a versio is released.
+ */
+#if __INTEL_LLVM_COMPILER < 1000000L
+# define COMPILER_VERSION_MAJOR DEC(__INTEL_LLVM_COMPILER/100)
+# define COMPILER_VERSION_MINOR DEC(__INTEL_LLVM_COMPILER/10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__INTEL_LLVM_COMPILER    % 10)
+#else
+# define COMPILER_VERSION_MAJOR DEC(__INTEL_LLVM_COMPILER/10000)
+# define COMPILER_VERSION_MINOR DEC(__INTEL_LLVM_COMPILER/100 % 100)
+# define COMPILER_VERSION_PATCH DEC(__INTEL_LLVM_COMPILER     % 100)
+#endif
+#if defined(_MSC_VER)
+  /* _MSC_VER = VVRR */
+# define SIMULATE_VERSION_MAJOR DEC(_MSC_VER / 100)
+# define SIMULATE_VERSION_MINOR DEC(_MSC_VER % 100)
+#endif
+#if defined(__GNUC__)
+# define SIMULATE_VERSION_MAJOR DEC(__GNUC__)
+#elif defined(__GNUG__)
+# define SIMULATE_VERSION_MAJOR DEC(__GNUG__)
+#endif
+#if defined(__GNUC_MINOR__)
+# define SIMULATE_VERSION_MINOR DEC(__GNUC_MINOR__)
+#endif
+#if defined(__GNUC_PATCHLEVEL__)
+# define SIMULATE_VERSION_PATCH DEC(__GNUC_PATCHLEVEL__)
+#endif
+
+#elif defined(__PATHCC__)
+# define COMPILER_ID "PathScale"
+# define COMPILER_VERSION_MAJOR DEC(__PATHCC__)
+# define COMPILER_VERSION_MINOR DEC(__PATHCC_MINOR__)
+# if defined(__PATHCC_PATCHLEVEL__)
+#  define COMPILER_VERSION_PATCH DEC(__PATHCC_PATCHLEVEL__)
+# endif
+
+#elif defined(__BORLANDC__) && defined(__CODEGEARC_VERSION__)
+# define COMPILER_ID "Embarcadero"
+# define COMPILER_VERSION_MAJOR HEX(__CODEGEARC_VERSION__>>24 & 0x00FF)
+# define COMPILER_VERSION_MINOR HEX(__CODEGEARC_VERSION__>>16 & 0x00FF)
+# define COMPILER_VERSION_PATCH DEC(__CODEGEARC_VERSION__     & 0xFFFF)
+
+#elif defined(__BORLANDC__)
+# define COMPILER_ID "Borland"
+  /* __BORLANDC__ = 0xVRR */
+# define COMPILER_VERSION_MAJOR HEX(__BORLANDC__>>8)
+# define COMPILER_VERSION_MINOR HEX(__BORLANDC__ & 0xFF)
+
+#elif defined(__WATCOMC__) && __WATCOMC__ < 1200
+# define COMPILER_ID "Watcom"
+   /* __WATCOMC__ = VVRR */
+# define COMPILER_VERSION_MAJOR DEC(__WATCOMC__ / 100)
+# define COMPILER_VERSION_MINOR DEC((__WATCOMC__ / 10) % 10)
+# if (__WATCOMC__ % 10) > 0
+#  define COMPILER_VERSION_PATCH DEC(__WATCOMC__ % 10)
+# endif
+
+#elif defined(__WATCOMC__)
+# define COMPILER_ID "OpenWatcom"
+   /* __WATCOMC__ = VVRP + 1100 */
+# define COMPILER_VERSION_MAJOR DEC((__WATCOMC__ - 1100) / 100)
+# define COMPILER_VERSION_MINOR DEC((__WATCOMC__ / 10) % 10)
+# if (__WATCOMC__ % 10) > 0
+#  define COMPILER_VERSION_PATCH DEC(__WATCOMC__ % 10)
+# endif
+
+#elif defined(__SUNPRO_CC)
+# define COMPILER_ID "SunPro"
+# if __SUNPRO_CC >= 0x5100
+   /* __SUNPRO_CC = 0xVRRP */
+#  define COMPILER_VERSION_MAJOR HEX(__SUNPRO_CC>>12)
+#  define COMPILER_VERSION_MINOR HEX(__SUNPRO_CC>>4 & 0xFF)
+#  define COMPILER_VERSION_PATCH HEX(__SUNPRO_CC    & 0xF)
+# else
+   /* __SUNPRO_CC = 0xVRP */
+#  define COMPILER_VERSION_MAJOR HEX(__SUNPRO_CC>>8)
+#  define COMPILER_VERSION_MINOR HEX(__SUNPRO_CC>>4 & 0xF)
+#  define COMPILER_VERSION_PATCH HEX(__SUNPRO_CC    & 0xF)
+# endif
+
+#elif defined(__HP_aCC)
+# define COMPILER_ID "HP"
+  /* __HP_aCC = VVRRPP */
+# define COMPILER_VERSION_MAJOR DEC(__HP_aCC/10000)
+# define COMPILER_VERSION_MINOR DEC(__HP_aCC/100 % 100)
+# define COMPILER_VERSION_PATCH DEC(__HP_aCC     % 100)
+
+#elif defined(__DECCXX)
+# define COMPILER_ID "Compaq"
+  /* __DECCXX_VER = VVRRTPPPP */
+# define COMPILER_VERSION_MAJOR DEC(__DECCXX_VER/10000000)
+# define COMPILER_VERSION_MINOR DEC(__DECCXX_VER/100000  % 100)
+# define COMPILER_VERSION_PATCH DEC(__DECCXX_VER         % 10000)
+
+#elif defined(__IBMCPP__) && defined(__COMPILER_VER__)
+# define COMPILER_ID "zOS"
+  /* __IBMCPP__ = VRP */
+# define COMPILER_VERSION_MAJOR DEC(__IBMCPP__/100)
+# define COMPILER_VERSION_MINOR DEC(__IBMCPP__/10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__IBMCPP__    % 10)
+
+#elif defined(__ibmxl__) && defined(__clang__)
+# define COMPILER_ID "XLClang"
+# define COMPILER_VERSION_MAJOR DEC(__ibmxl_version__)
+# define COMPILER_VERSION_MINOR DEC(__ibmxl_release__)
+# define COMPILER_VERSION_PATCH DEC(__ibmxl_modification__)
+# define COMPILER_VERSION_TWEAK DEC(__ibmxl_ptf_fix_level__)
+
+
+#elif defined(__IBMCPP__) && !defined(__COMPILER_VER__) && __IBMCPP__ >= 800
+# define COMPILER_ID "XL"
+  /* __IBMCPP__ = VRP */
+# define COMPILER_VERSION_MAJOR DEC(__IBMCPP__/100)
+# define COMPILER_VERSION_MINOR DEC(__IBMCPP__/10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__IBMCPP__    % 10)
+
+#elif defined(__IBMCPP__) && !defined(__COMPILER_VER__) && __IBMCPP__ < 800
+# define COMPILER_ID "VisualAge"
+  /* __IBMCPP__ = VRP */
+# define COMPILER_VERSION_MAJOR DEC(__IBMCPP__/100)
+# define COMPILER_VERSION_MINOR DEC(__IBMCPP__/10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__IBMCPP__    % 10)
+
+#elif defined(__NVCOMPILER)
+# define COMPILER_ID "NVHPC"
+# define COMPILER_VERSION_MAJOR DEC(__NVCOMPILER_MAJOR__)
+# define COMPILER_VERSION_MINOR DEC(__NVCOMPILER_MINOR__)
+# if defined(__NVCOMPILER_PATCHLEVEL__)
+#  define COMPILER_VERSION_PATCH DEC(__NVCOMPILER_PATCHLEVEL__)
+# endif
+
+#elif defined(__PGI)
+# define COMPILER_ID "PGI"
+# define COMPILER_VERSION_MAJOR DEC(__PGIC__)
+# define COMPILER_VERSION_MINOR DEC(__PGIC_MINOR__)
+# if defined(__PGIC_PATCHLEVEL__)
+#  define COMPILER_VERSION_PATCH DEC(__PGIC_PATCHLEVEL__)
+# endif
+
+#elif defined(_CRAYC)
+# define COMPILER_ID "Cray"
+# define COMPILER_VERSION_MAJOR DEC(_RELEASE_MAJOR)
+# define COMPILER_VERSION_MINOR DEC(_RELEASE_MINOR)
+
+#elif defined(__TI_COMPILER_VERSION__)
+# define COMPILER_ID "TI"
+  /* __TI_COMPILER_VERSION__ = VVVRRRPPP */
+# define COMPILER_VERSION_MAJOR DEC(__TI_COMPILER_VERSION__/1000000)
+# define COMPILER_VERSION_MINOR DEC(__TI_COMPILER_VERSION__/1000   % 1000)
+# define COMPILER_VERSION_PATCH DEC(__TI_COMPILER_VERSION__        % 1000)
+
+#elif defined(__FUJITSU) || defined(__FCC_VERSION) || defined(__fcc_version)
+# define COMPILER_ID "Fujitsu"
+
+#elif defined(__ghs__)
+# define COMPILER_ID "GHS"
+/* __GHS_VERSION_NUMBER = VVVVRP */
+# ifdef __GHS_VERSION_NUMBER
+# define COMPILER_VERSION_MAJOR DEC(__GHS_VERSION_NUMBER / 100)
+# define COMPILER_VERSION_MINOR DEC(__GHS_VERSION_NUMBER / 10 % 10)
+# define COMPILER_VERSION_PATCH DEC(__GHS_VERSION_NUMBER      % 10)
+# endif
+
+#elif defined(__SCO_VERSION__)
+# define COMPILER_ID "SCO"
+
+#elif defined(__ARMCC_VERSION) && !defined(__clang__)
+# define COMPILER_ID "ARMCC"
+#if __ARMCC_VERSION >= 1000000
+  /* __ARMCC_VERSION = VRRPPPP */
+  # define COMPILER_VERSION_MAJOR DEC(__ARMCC_VERSION/1000000)
+  # define COMPILER_VERSION_MINOR DEC(__ARMCC_VERSION/10000 % 100)
+  # define COMPILER_VERSION_PATCH DEC(__ARMCC_VERSION     % 10000)
+#else
+  /* __ARMCC_VERSION = VRPPPP */
+  # define COMPILER_VERSION_MAJOR DEC(__ARMCC_VERSION/100000)
+  # define COMPILER_VERSION_MINOR DEC(__ARMCC_VERSION/10000 % 10)
+  # define COMPILER_VERSION_PATCH DEC(__ARMCC_VERSION    % 10000)
+#endif
+
+
+#elif defined(__clang__) && defined(__apple_build_version__)
+# define COMPILER_ID "AppleClang"
+# if defined(_MSC_VER)
+#  define SIMULATE_ID "MSVC"
+# endif
+# define COMPILER_VERSION_MAJOR DEC(__clang_major__)
+# define COMPILER_VERSION_MINOR DEC(__clang_minor__)
+# define COMPILER_VERSION_PATCH DEC(__clang_patchlevel__)
+# if defined(_MSC_VER)
+   /* _MSC_VER = VVRR */
+#  define SIMULATE_VERSION_MAJOR DEC(_MSC_VER / 100)
+#  define SIMULATE_VERSION_MINOR DEC(_MSC_VER % 100)
+# endif
+# define COMPILER_VERSION_TWEAK DEC(__apple_build_version__)
+
+#elif defined(__clang__) && defined(__ARMCOMPILER_VERSION)
+# define COMPILER_ID "ARMClang"
+  # define COMPILER_VERSION_MAJOR DEC(__ARMCOMPILER_VERSION/1000000)
+  # define COMPILER_VERSION_MINOR DEC(__ARMCOMPILER_VERSION/10000 % 100)
+  # define COMPILER_VERSION_PATCH DEC(__ARMCOMPILER_VERSION     % 10000)
+# define COMPILER_VERSION_INTERNAL DEC(__ARMCOMPILER_VERSION)
+
+#elif defined(__clang__)
+# define COMPILER_ID "Clang"
+# if defined(_MSC_VER)
+#  define SIMULATE_ID "MSVC"
+# endif
+# define COMPILER_VERSION_MAJOR DEC(__clang_major__)
+# define COMPILER_VERSION_MINOR DEC(__clang_minor__)
+# define COMPILER_VERSION_PATCH DEC(__clang_patchlevel__)
+# if defined(_MSC_VER)
+   /* _MSC_VER = VVRR */
+#  define SIMULATE_VERSION_MAJOR DEC(_MSC_VER / 100)
+#  define SIMULATE_VERSION_MINOR DEC(_MSC_VER % 100)
+# endif
+
+#elif defined(__GNUC__) || defined(__GNUG__)
+# define COMPILER_ID "GNU"
+# if defined(__GNUC__)
+#  define COMPILER_VERSION_MAJOR DEC(__GNUC__)
+# else
+#  define COMPILER_VERSION_MAJOR DEC(__GNUG__)
+# endif
+# if defined(__GNUC_MINOR__)
+#  define COMPILER_VERSION_MINOR DEC(__GNUC_MINOR__)
+# endif
+# if defined(__GNUC_PATCHLEVEL__)
+#  define COMPILER_VERSION_PATCH DEC(__GNUC_PATCHLEVEL__)
+# endif
+
+#elif defined(_MSC_VER)
+# define COMPILER_ID "MSVC"
+  /* _MSC_VER = VVRR */
+# define COMPILER_VERSION_MAJOR DEC(_MSC_VER / 100)
+# define COMPILER_VERSION_MINOR DEC(_MSC_VER % 100)
+# if defined(_MSC_FULL_VER)
+#  if _MSC_VER >= 1400
+    /* _MSC_FULL_VER = VVRRPPPPP */
+#   define COMPILER_VERSION_PATCH DEC(_MSC_FULL_VER % 100000)
+#  else
+    /* _MSC_FULL_VER = VVRRPPPP */
+#   define COMPILER_VERSION_PATCH DEC(_MSC_FULL_VER % 10000)
+#  endif
+# endif
+# if defined(_MSC_BUILD)
+#  define COMPILER_VERSION_TWEAK DEC(_MSC_BUILD)
+# endif
+
+#elif defined(__VISUALDSPVERSION__) || defined(__ADSPBLACKFIN__) || defined(__ADSPTS__) || defined(__ADSP21000__)
+# define COMPILER_ID "ADSP"
+#if defined(__VISUALDSPVERSION__)
+  /* __VISUALDSPVERSION__ = 0xVVRRPP00 */
+# define COMPILER_VERSION_MAJOR HEX(__VISUALDSPVERSION__>>24)
+# define COMPILER_VERSION_MINOR HEX(__VISUALDSPVERSION__>>16 & 0xFF)
+# define COMPILER_VERSION_PATCH HEX(__VISUALDSPVERSION__>>8  & 0xFF)
+#endif
+
+#elif defined(__IAR_SYSTEMS_ICC__) || defined(__IAR_SYSTEMS_ICC)
+# define COMPILER_ID "IAR"
+# if defined(__VER__) && defined(__ICCARM__)
+#  define COMPILER_VERSION_MAJOR DEC((__VER__) / 1000000)
+#  define COMPILER_VERSION_MINOR DEC(((__VER__) / 1000) % 1000)
+#  define COMPILER_VERSION_PATCH DEC((__VER__) % 1000)
+#  define COMPILER_VERSION_INTERNAL DEC(__IAR_SYSTEMS_ICC__)
+# elif defined(__VER__) && (defined(__ICCAVR__) || defined(__ICCRX__) || defined(__ICCRH850__) || defined(__ICCRL78__) || defined(__ICC430__) || defined(__ICCRISCV__) || defined(__ICCV850__) || defined(__ICC8051__) || defined(__ICCSTM8__))
+#  define COMPILER_VERSION_MAJOR DEC((__VER__) / 100)
+#  define COMPILER_VERSION_MINOR DEC((__VER__) - (((__VER__) / 100)*100))
+#  define COMPILER_VERSION_PATCH DEC(__SUBVERSION__)
+#  define COMPILER_VERSION_INTERNAL DEC(__IAR_SYSTEMS_ICC__)
+# endif
+
+
+/* These compilers are either not known or too old to define an
+  identification macro.  Try to identify the platform and guess that
+  it is the native compiler.  */
+#elif defined(__hpux) || defined(__hpua)
+# define COMPILER_ID "HP"
+
+#else /* unknown compiler */
+# define COMPILER_ID ""
+#endif
+
+/* Construct the string literal in pieces to prevent the source from
+   getting matched.  Store it in a pointer rather than an array
+   because some compilers will just produce instructions to fill the
+   array rather than assigning a pointer to a static array.  */
+char const* info_compiler = "INFO" ":" "compiler[" COMPILER_ID "]";
+#ifdef SIMULATE_ID
+char const* info_simulate = "INFO" ":" "simulate[" SIMULATE_ID "]";
+#endif
+
+#ifdef __QNXNTO__
+char const* qnxnto = "INFO" ":" "qnxnto[]";
+#endif
+
+#if defined(__CRAYXT_COMPUTE_LINUX_TARGET)
+char const *info_cray = "INFO" ":" "compiler_wrapper[CrayPrgEnv]";
+#endif
+
+#define STRINGIFY_HELPER(X) #X
+#define STRINGIFY(X) STRINGIFY_HELPER(X)
+
+/* Identify known platforms by name.  */
+#if defined(__linux) || defined(__linux__) || defined(linux)
+# define PLATFORM_ID "Linux"
+
+#elif defined(__CYGWIN__)
+# define PLATFORM_ID "Cygwin"
+
+#elif defined(__MINGW32__)
+# define PLATFORM_ID "MinGW"
+
+#elif defined(__APPLE__)
+# define PLATFORM_ID "Darwin"
+
+#elif defined(_WIN32) || defined(__WIN32__) || defined(WIN32)
+# define PLATFORM_ID "Windows"
+
+#elif defined(__FreeBSD__) || defined(__FreeBSD)
+# define PLATFORM_ID "FreeBSD"
+
+#elif defined(__NetBSD__) || defined(__NetBSD)
+# define PLATFORM_ID "NetBSD"
+
+#elif defined(__OpenBSD__) || defined(__OPENBSD)
+# define PLATFORM_ID "OpenBSD"
+
+#elif defined(__sun) || defined(sun)
+# define PLATFORM_ID "SunOS"
+
+#elif defined(_AIX) || defined(__AIX) || defined(__AIX__) || defined(__aix) || defined(__aix__)
+# define PLATFORM_ID "AIX"
+
+#elif defined(__hpux) || defined(__hpux__)
+# define PLATFORM_ID "HP-UX"
+
+#elif defined(__HAIKU__)
+# define PLATFORM_ID "Haiku"
+
+#elif defined(__BeOS) || defined(__BEOS__) || defined(_BEOS)
+# define PLATFORM_ID "BeOS"
+
+#elif defined(__QNX__) || defined(__QNXNTO__)
+# define PLATFORM_ID "QNX"
+
+#elif defined(__tru64) || defined(_tru64) || defined(__TRU64__)
+# define PLATFORM_ID "Tru64"
+
+#elif defined(__riscos) || defined(__riscos__)
+# define PLATFORM_ID "RISCos"
+
+#elif defined(__sinix) || defined(__sinix__) || defined(__SINIX__)
+# define PLATFORM_ID "SINIX"
+
+#elif defined(__UNIX_SV__)
+# define PLATFORM_ID "UNIX_SV"
+
+#elif defined(__bsdos__)
+# define PLATFORM_ID "BSDOS"
+
+#elif defined(_MPRAS) || defined(MPRAS)
+# define PLATFORM_ID "MP-RAS"
+
+#elif defined(__osf) || defined(__osf__)
+# define PLATFORM_ID "OSF1"
+
+#elif defined(_SCO_SV) || defined(SCO_SV) || defined(sco_sv)
+# define PLATFORM_ID "SCO_SV"
+
+#elif defined(__ultrix) || defined(__ultrix__) || defined(_ULTRIX)
+# define PLATFORM_ID "ULTRIX"
+
+#elif defined(__XENIX__) || defined(_XENIX) || defined(XENIX)
+# define PLATFORM_ID "Xenix"
+
+#elif defined(__WATCOMC__)
+# if defined(__LINUX__)
+#  define PLATFORM_ID "Linux"
+
+# elif defined(__DOS__)
+#  define PLATFORM_ID "DOS"
+
+# elif defined(__OS2__)
+#  define PLATFORM_ID "OS2"
+
+# elif defined(__WINDOWS__)
+#  define PLATFORM_ID "Windows3x"
+
+# elif defined(__VXWORKS__)
+#  define PLATFORM_ID "VxWorks"
+
+# else /* unknown platform */
+#  define PLATFORM_ID
+# endif
+
+#elif defined(__INTEGRITY)
+# if defined(INT_178B)
+#  define PLATFORM_ID "Integrity178"
+
+# else /* regular Integrity */
+#  define PLATFORM_ID "Integrity"
+# endif
+
+#else /* unknown platform */
+# define PLATFORM_ID
+
+#endif
+
+/* For windows compilers MSVC and Intel we can determine
+   the architecture of the compiler being used.  This is because
+   the compilers do not have flags that can change the architecture,
+   but rather depend on which compiler is being used
+*/
+#if defined(_WIN32) && defined(_MSC_VER)
+# if defined(_M_IA64)
+#  define ARCHITECTURE_ID "IA64"
+
+# elif defined(_M_ARM64EC)
+#  define ARCHITECTURE_ID "ARM64EC"
+
+# elif defined(_M_X64) || defined(_M_AMD64)
+#  define ARCHITECTURE_ID "x64"
+
+# elif defined(_M_IX86)
+#  define ARCHITECTURE_ID "X86"
+
+# elif defined(_M_ARM64)
+#  define ARCHITECTURE_ID "ARM64"
+
+# elif defined(_M_ARM)
+#  if _M_ARM == 4
+#   define ARCHITECTURE_ID "ARMV4I"
+#  elif _M_ARM == 5
+#   define ARCHITECTURE_ID "ARMV5I"
+#  else
+#   define ARCHITECTURE_ID "ARMV" STRINGIFY(_M_ARM)
+#  endif
+
+# elif defined(_M_MIPS)
+#  define ARCHITECTURE_ID "MIPS"
+
+# elif defined(_M_SH)
+#  define ARCHITECTURE_ID "SHx"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#elif defined(__WATCOMC__)
+# if defined(_M_I86)
+#  define ARCHITECTURE_ID "I86"
+
+# elif defined(_M_IX86)
+#  define ARCHITECTURE_ID "X86"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#elif defined(__IAR_SYSTEMS_ICC__) || defined(__IAR_SYSTEMS_ICC)
+# if defined(__ICCARM__)
+#  define ARCHITECTURE_ID "ARM"
+
+# elif defined(__ICCRX__)
+#  define ARCHITECTURE_ID "RX"
+
+# elif defined(__ICCRH850__)
+#  define ARCHITECTURE_ID "RH850"
+
+# elif defined(__ICCRL78__)
+#  define ARCHITECTURE_ID "RL78"
+
+# elif defined(__ICCRISCV__)
+#  define ARCHITECTURE_ID "RISCV"
+
+# elif defined(__ICCAVR__)
+#  define ARCHITECTURE_ID "AVR"
+
+# elif defined(__ICC430__)
+#  define ARCHITECTURE_ID "MSP430"
+
+# elif defined(__ICCV850__)
+#  define ARCHITECTURE_ID "V850"
+
+# elif defined(__ICC8051__)
+#  define ARCHITECTURE_ID "8051"
+
+# elif defined(__ICCSTM8__)
+#  define ARCHITECTURE_ID "STM8"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#elif defined(__ghs__)
+# if defined(__PPC64__)
+#  define ARCHITECTURE_ID "PPC64"
+
+# elif defined(__ppc__)
+#  define ARCHITECTURE_ID "PPC"
+
+# elif defined(__ARM__)
+#  define ARCHITECTURE_ID "ARM"
+
+# elif defined(__x86_64__)
+#  define ARCHITECTURE_ID "x64"
+
+# elif defined(__i386__)
+#  define ARCHITECTURE_ID "X86"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#elif defined(__TI_COMPILER_VERSION__)
+# if defined(__TI_ARM__)
+#  define ARCHITECTURE_ID "ARM"
+
+# elif defined(__MSP430__)
+#  define ARCHITECTURE_ID "MSP430"
+
+# elif defined(__TMS320C28XX__)
+#  define ARCHITECTURE_ID "TMS320C28x"
+
+# elif defined(__TMS320C6X__) || defined(_TMS320C6X)
+#  define ARCHITECTURE_ID "TMS320C6x"
+
+# else /* unknown architecture */
+#  define ARCHITECTURE_ID ""
+# endif
+
+#else
+#  define ARCHITECTURE_ID
+#endif
+
+/* Convert integer to decimal digit literals.  */
+#define DEC(n)                   \
+  ('0' + (((n) / 10000000)%10)), \
+  ('0' + (((n) / 1000000)%10)),  \
+  ('0' + (((n) / 100000)%10)),   \
+  ('0' + (((n) / 10000)%10)),    \
+  ('0' + (((n) / 1000)%10)),     \
+  ('0' + (((n) / 100)%10)),      \
+  ('0' + (((n) / 10)%10)),       \
+  ('0' +  ((n) % 10))
+
+/* Convert integer to hex digit literals.  */
+#define HEX(n)             \
+  ('0' + ((n)>>28 & 0xF)), \
+  ('0' + ((n)>>24 & 0xF)), \
+  ('0' + ((n)>>20 & 0xF)), \
+  ('0' + ((n)>>16 & 0xF)), \
+  ('0' + ((n)>>12 & 0xF)), \
+  ('0' + ((n)>>8  & 0xF)), \
+  ('0' + ((n)>>4  & 0xF)), \
+  ('0' + ((n)     & 0xF))
+
+/* Construct a string literal encoding the version number components. */
+#ifdef COMPILER_VERSION_MAJOR
+char const info_version[] = {
+  'I', 'N', 'F', 'O', ':',
+  'c','o','m','p','i','l','e','r','_','v','e','r','s','i','o','n','[',
+  COMPILER_VERSION_MAJOR,
+# ifdef COMPILER_VERSION_MINOR
+  '.', COMPILER_VERSION_MINOR,
+#  ifdef COMPILER_VERSION_PATCH
+   '.', COMPILER_VERSION_PATCH,
+#   ifdef COMPILER_VERSION_TWEAK
+    '.', COMPILER_VERSION_TWEAK,
+#   endif
+#  endif
+# endif
+  ']','\0'};
+#endif
+
+/* Construct a string literal encoding the internal version number. */
+#ifdef COMPILER_VERSION_INTERNAL
+char const info_version_internal[] = {
+  'I', 'N', 'F', 'O', ':',
+  'c','o','m','p','i','l','e','r','_','v','e','r','s','i','o','n','_',
+  'i','n','t','e','r','n','a','l','[',
+  COMPILER_VERSION_INTERNAL,']','\0'};
+#endif
+
+/* Construct a string literal encoding the version number components. */
+#ifdef SIMULATE_VERSION_MAJOR
+char const info_simulate_version[] = {
+  'I', 'N', 'F', 'O', ':',
+  's','i','m','u','l','a','t','e','_','v','e','r','s','i','o','n','[',
+  SIMULATE_VERSION_MAJOR,
+# ifdef SIMULATE_VERSION_MINOR
+  '.', SIMULATE_VERSION_MINOR,
+#  ifdef SIMULATE_VERSION_PATCH
+   '.', SIMULATE_VERSION_PATCH,
+#   ifdef SIMULATE_VERSION_TWEAK
+    '.', SIMULATE_VERSION_TWEAK,
+#   endif
+#  endif
+# endif
+  ']','\0'};
+#endif
+
+/* Construct the string literal in pieces to prevent the source from
+   getting matched.  Store it in a pointer rather than an array
+   because some compilers will just produce instructions to fill the
+   array rather than assigning a pointer to a static array.  */
+char const* info_platform = "INFO" ":" "platform[" PLATFORM_ID "]";
+char const* info_arch = "INFO" ":" "arch[" ARCHITECTURE_ID "]";
+
+
+
+#if defined(__INTEL_COMPILER) && defined(_MSVC_LANG) && _MSVC_LANG < 201403L
+#  if defined(__INTEL_CXX11_MODE__)
+#    if defined(__cpp_aggregate_nsdmi)
+#      define CXX_STD 201402L
+#    else
+#      define CXX_STD 201103L
+#    endif
+#  else
+#    define CXX_STD 199711L
+#  endif
+#elif defined(_MSC_VER) && defined(_MSVC_LANG)
+#  define CXX_STD _MSVC_LANG
+#else
+#  define CXX_STD __cplusplus
+#endif
+
+const char* info_language_dialect_default = "INFO" ":" "dialect_default["
+#if CXX_STD > 202002L
+  "23"
+#elif CXX_STD > 201703L
+  "20"
+#elif CXX_STD >= 201703L
+  "17"
+#elif CXX_STD >= 201402L
+  "14"
+#elif CXX_STD >= 201103L
+  "11"
+#else
+  "98"
+#endif
+"]";
+
+/*--------------------------------------------------------------------------*/
+
+int main(int argc, char* argv[])
+{
+  int require = 0;
+  require += info_compiler[argc];
+  require += info_platform[argc];
+#ifdef COMPILER_VERSION_MAJOR
+  require += info_version[argc];
+#endif
+#ifdef COMPILER_VERSION_INTERNAL
+  require += info_version_internal[argc];
+#endif
+#ifdef SIMULATE_ID
+  require += info_simulate[argc];
+#endif
+#ifdef SIMULATE_VERSION_MAJOR
+  require += info_simulate_version[argc];
+#endif
+#if defined(__CRAYXT_COMPUTE_LINUX_TARGET)
+  require += info_cray[argc];
+#endif
+  require += info_language_dialect_default[argc];
+  (void)argv;
+  return require;
+}
diff --git a/CMakeFiles/3.20.2/CompilerIdCXX/a.out b/CMakeFiles/3.20.2/CompilerIdCXX/a.out
new file mode 100755
index 0000000000000000000000000000000000000000..ec937b83b59a04bf2e6fc52beff11b9fe0854b4d
GIT binary patch
literal 18056
zcmeHPdu$xV8K1lJD~WS9A*L=#>q7-K5BdBcF-K_N&d+u0BoI3g8q8*Wx3*8YSMT;{
zf>aeXA*!K)RB9@#Rjd3{TBM49C=q{ArvcKcQWd0D`~j8HibzqRB8URvkzBuT=KJn$
z_jc0=wG_lnviHsWzQ@dbGdnvoyW{VUOpN+{KEc5+ZWowZYKTfm{R;8@J7kiuh*&0~
zVy##sngD5lD=IliRgCFHGR?Gw<N`oZFXC4~#2=LrrfU@pd8oi8MA}H0B2h{yx|cFw
zDnv-~Q4h;1f-(touZj?jQaLoDtcOX|YbU*Sl4p8`3dWS{hjqhiitJ7KqmaUM6p0v?
zzZHtI816i0B`}_9mPw#d8DY9f!GbC4-35BGe#HMzq@4tIQhr{35$q45R9(@NO3n;y
z>PaQKQ^`zmfA{{#K=;6=a3LG+m-R;bC_A=&m&^&<n@wSvSHp!E@>^SfIMDRrp~Gua
zH~;bA#%&i@J@wNP>hcBAs}5Zt8FG>SHejn<a4cuY1;5P$7f|nlm56WLh;<zhx;~fA
zX7qw(<Sktn`uNl~Jz?g}xn#jI^HbY~QrV0-Wz3{Z#dGC=&9pKHvAK9$FJP(<^NMpe
z7^iH~MlvJDCdLPc^!{*vc(drq=B%E4HlFU@4C!7-nwfYCn`n!ZHJ{A6;vfq(_CXu@
z;2*|(A`O)LN%8vRGHk&Gz%ZO`+bb#dIh5ycz6$5|M&5-g9NV44#VVZpGV<PT`|=0c
z!8cm8M;pgigoO6+lI1U*(GEV}a*9<YMY+CzjxW4)zqasq?ci${?wlGwy>zrGDzwul
zLz0|+qA@B;9Um+YiGLkM`BSsuwyr}`{it?i>#b=2NMk>eTQ6E2kmZXYi&-u!oo(xS
z2qjMu1^H!CezPCR`peqM%K>fSf_C!!9h&b&?WN1sYB2CKGSE^wJKNSZT+zq;AKLl?
zuto9OUE0FA^ADCvr578Y2E;dg>gcK2O5xhNqS6nqKZ1e_-+4n@D4sn%@)4Ftd$hv<
zl}AV3&=xh{nYI(6?YSl2$VY(R<%Ubawi6Et7*EQQx427wG<^yRtsI2qjomZ8@N(?#
z@r8F|yJ8EMVp9`GZhF2sBz9>>x{*#!tUrfsb$;w}sid8}6tKGbULk)?EPODr@b2)!
zo3YaBSG9wueA=ylE&c=BanE$@zSwl^{+M1}XW2ebKcuq#Yg!y@UkTV_uN20&kM7tK
z&!%(9l$qZHqs_Ek+Q!!(F!P0EHnXQ!g-Dnpn>o@uEu=;+WmvP>e0t9Wj(!9h`S|=E
ziYJmr%8Xku7R(yOl(naC)3ord3Eb8T+2iwT>QbpR0rU{iy+F5s>nxzZ0s14LH$ty?
z6)29A7<>=z6u$lKzBR4QEl1&o!~0S`8$rJZ1%{j35kvcH0pn*R%0TxM&jI!=pduD*
zKkOe`+2l`wCB;hs+zI)b$R7>1Kkgq1c0Lvu35Jd|j0D&Hpiv9<9&XZtk%P@+!DuEJ
zi3NLO!F7Yd&|t80FxWmAY?0;pHK<3y4(E;L>Y?s|x(DhWsC(ek>4D=gH^Pu3%5ym<
zRkQfYTu+x;l{C&3F|;WnjuCAoy4Vnvv6Y-|Qt2;Jn%84%D9!7r>p4y9F8Ry@jQQ_e
zE@e?<(H})pUK{oXR2t`}7@qY<k%;B;R|tHzVpySw=p@=flnE`vM1;!E`R*kFm=w#v
z`tbOZ<>@4Oc#O+5`||>&TZvEV|2M+xSg!({rFQCXj!Ka+qLZ8+8rl+C2lJ}+p-32}
zNumDU{=VMLeVan-cAANhW>^aE?%TW`4WPeqo`Q>pvDl5in{$1=1GfnACi#OYuH$4-
za*g?7CX>v}g<y5$$VdpMtdYrNXObCL*>Zm)ySY?x4n#9X+T0Q%Ma-A`g(~NUu^sn_
z4UI6>Y*MA7@^6rnPww+db|b6><NNe@$|%eWajjiGKC9>QSqo-{FwMkH%=ZK3GrD8v
z)cEMVqJ(PK*>pF`72odNdTerX1f+Isp9H?6Jk7YaD$l=U9dD3b)?q7~PFnhG9%`-U
zvdIiARcmcE;d=QRxMWbaRNeBXkxI^G5cg`&C2CKm&NiyIXT3e$?&bRZiR0qN`u$lS
zZ=oODP#<sA(}$n8pPT3Bh5CF5`u~Q{+<XY<b2z`nS8C4kbDO?HZHsfVimdy5z^PN8
zZ`J3|ux_fahhpycOMN{=56t@dw!Xft?^o2!pK&w9w*O@GmA`f2y(c#r<}r1X;paM2
zHyPdseTwD)t}?XecATi2jG+Bme7B9kYhHk_z%d*n`dQhB!F{a%!M>W@uXF5$UDduC
zDt?mtY98}QS$=NQhpA1kW*+I(xsrXh%YEi*>^pV5`XlcLGOhQ=kGIeJN%zNkdtTML
z<1;h9)Z4S(o}cD*<mdhQ?>6V-=P}cdE5mcor>`&v<+<lCXpVt%NUsopmzNQLw1ypk
zZ}X;H@qn;g@rLqy{AyzXn0fL0dpq7F_&vWJZx(Lv*8}jvmEX(T<(HM;@7wX^g6HdY
zd_{T9V8>g<qKm%*@RoPU6%Urb@3z}%6UXWOx*dm=OK@*BB;x^iOWWd#uY&K^-Q=$=
zf4^I8EP!7dw!6u9h)!4hIuUinzff(q91p-(7Fzr9avCGBO(7b@Tk?H(HDfC#{42`e
z1u$=$2W}9~^GcR4df-Pq@FzTQ`PDI$$7%l+5Bc{z@K&s_i=CSY=lxz($FP-fUUw=~
zh<iQk6g}``9{33l{1p%ULl3+QXFe|V`kDtmO}MkZhY062G}re@z}@=EGbA5!++Tkn
zoX1MG^Ebks?fAY2-hmGymwNSk;FBJB)&qas1OJr={sQ67`o2auufw>$z6Mo(Cmtf)
z>7T8DyZKoI+>g5qwBNw?zfO3s1D__Gzbj+;0^mOQvW3SD>`oZIC-K^G>`Ai2->E5S
zA)b-)wd3KR0e36^>r#HD5Hzm#lOeDAUXt={;=YzBXIZ9h_E*Twf!gu>%Yggf<cV_}
z{;HI3uNk)^ggeLiZ%8|}<Ff&HyXYwUlM6Wy_%(n>Xne-G0*1E%4*~8R&ksxa%5w^;
zFT*jwS3$e;K9l^095_EgdH+iB`=+Q8q~BjkI~`)3Bj0BT?{(l&*sx#auYGR*7W0Ms
zj>iW`%J4GT2|4V%hw@NPzOQ>oezmk;`y4-qcKo$|dkb*%r_(=E&{5l=-)fXEnHQI(
ze5F4#z8v#~^6-6X661QnLo8H92S^@Iu3!MOGK@-`5>>?Xuw!}PqNiBM<H?^M`0Azy
zPHX7{o4WI6V&1US39g<<PdHrp50Y@aE1tLd!dVff599P?CTWG^&a`f35>6~UpS7GA
z%DLg;VkVW`XQmFgDtIx&u(zAHiaA#$4_?^ZpL8#lodN1rGOp*%R5o5+UpsEEx1DFN
zr=3^p2Rq-zA9ew0pw=&Tp<4gg`O*kLb|2bh9R9L1r2)I&^h8QI&9F>7tcFg^B~4Y7
zN<(>xVLvicv)!_rWhW~N0M_)q3Wf!7leM>jUEYi5;(Lc;t=}sGwLY)#-2GkQJA7SX
zxcOO!`!lY`3q|D?*^Ctz=nl5cKUK_GaCA-ERCFa9bzvANSVcIQ78jMXjk=YFk6ht&
zV*w69CbIfmDm!DO^n{ho7j&bzU*M^$lxdlX@GUp@_PYz|n1^oU^Tq+Frj<V+)JLg_
zVmf^QRH{<4rZ&-`ZN@%xX!q_RI^H#&2*-0dp?`CtuRocY&GK<Burs<dwrxZo**=VC
zG<En~Ss&2|)P{EoefZw(v2Ei+z{9>*&Lv?7l$-Uji5-Kn34O=t=;X+hJ{23B7y+wh
z!mtb>^Gz9<xuP*=>W))ja0GM*+!l#sMlTl3gpdb0!Fh$OK5t|aa7I+B^Wid;!c%3!
z<Bm}he7p?`3kOciwZ!tM6QjVGDHO_aTi?0zkyU$V06S7w`a#FiN2Ar}P~8s0x*m8H
z;lhEmWy}Dz@`}!LOn!|Fqk0BDlMcrQ$Ga_Kj#6`(VtA&QghR(kxZyE1j~`iw69+P2
zPf;tcICPZHme7F*W4eI~B8)99!j`!oez6+@FDu(HY|c~1n@@lSr<8{38%l%YV9rP<
z;kh`64?Xw^=TZ<+#=!F;3{8}V{;f(Qs}MFUE1#SxT5$jT7XqG}z_~OoV=(Z344<#(
za}$cIWI2xXD{wW##rnK&!{^sSz^e{CKV+;ABGvl5zhjZX8(EO3Q-2uncj0UV=fCKW
zO36h6xcodX;`yu;f}ibh;hr|@^M28@1neh4m>x+#v3@7vcLSA6C`$9bQZIrK_`HZp
z)%x##VBlE~&Y$<6TB)LZ?u5(Fa!mJuKAs)nH1AV&l75IXc9x%D@gPJ{l=UMGQOSys
z{$81?%qyMsKLmX2KWHBV?|UthKA+1`T$LBckv;-fwLZ_omq`CF$gB$T9eh>#D8#Gv
zc^_?&3Ir=RTOhM#?8gv8eXjpfQ&jSnNFUFpR)<h^0^&c1t6HD`-U9y}hD{E6XZ~ov
zTA%mtj^j-MVM+d0e%SwvoPY@ai3#d%m3=<?yAw`%PBZ--=;Hfw*601emaEkN6X>F=
zSfAIc{C6+TVC5i?JIm*<xE)^x9*VF&|9yb7^b$V=yz0Q^W{US$wSIXYk}}}?lx?sc
z({m1eUcYsc{yJ79%6bgH4iUt-{JcLAC4Hwnxr5ck1&ATmTSiob^}8K7>oa{H_|#RT
zB=3{vDF0`_WH;dKe`tsMezoMfuu3h_K{Y&2cNbh-ANDJ<x$a6s{ho)jSr{JUhF3{h
YSBgr4=Y-t!AGlpfwQ)sB$&nQQ28g4e)Bpeg

literal 0
HcmV?d00001

diff --git a/CMakeFiles/CMakeError.log b/CMakeFiles/CMakeError.log
new file mode 100644
index 000000000..b2c53d98e
--- /dev/null
+++ b/CMakeFiles/CMakeError.log
@@ -0,0 +1,228 @@
+Performing C SOURCE FILE Test OGL_C_TSAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_3f5c5/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_3f5c5.dir/build.make CMakeFiles/cmTC_3f5c5.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building C object CMakeFiles/cmTC_3f5c5.dir/src.c.o
+/usr/lib64/ccache/cc -DOGL_C_TSAN_SUPPORTED  -fsanitize=thread -static-libtsan -fno-omit-frame-pointer -fPIC -o CMakeFiles/cmTC_3f5c5.dir/src.c.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.c
+Linking C executable cmTC_3f5c5
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_3f5c5.dir/link.txt --verbose=1
+/usr/lib64/ccache/cc CMakeFiles/cmTC_3f5c5.dir/src.c.o -o cmTC_3f5c5  -fsanitize=thread -static-libtsan -fno-omit-frame-pointer -fPIC 
+/usr/bin/ld: cannot find -ltsan
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_3f5c5.dir/build.make:99: cmTC_3f5c5] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_3f5c5/fast] Error 2
+
+
+Source file was:
+int main(void) { return 0; }
+Performing C SOURCE FILE Test OGL_C_ASAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_b8d05/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_b8d05.dir/build.make CMakeFiles/cmTC_b8d05.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building C object CMakeFiles/cmTC_b8d05.dir/src.c.o
+/usr/lib64/ccache/cc -DOGL_C_ASAN_SUPPORTED  -fsanitize=address -fno-omit-frame-pointer -o CMakeFiles/cmTC_b8d05.dir/src.c.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.c
+Linking C executable cmTC_b8d05
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_b8d05.dir/link.txt --verbose=1
+/usr/lib64/ccache/cc CMakeFiles/cmTC_b8d05.dir/src.c.o -o cmTC_b8d05  -fsanitize=address -fno-omit-frame-pointer 
+/usr/bin/ld: cannot find /usr/lib64/libasan.so.5.0.0
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_b8d05.dir/build.make:99: cmTC_b8d05] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_b8d05/fast] Error 2
+
+
+Source file was:
+int main(void) { return 0; }
+Performing C SOURCE FILE Test OGL_C_LSAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_3dddb/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_3dddb.dir/build.make CMakeFiles/cmTC_3dddb.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building C object CMakeFiles/cmTC_3dddb.dir/src.c.o
+/usr/lib64/ccache/cc -DOGL_C_LSAN_SUPPORTED  -fsanitize=leak -o CMakeFiles/cmTC_3dddb.dir/src.c.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.c
+Linking C executable cmTC_3dddb
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_3dddb.dir/link.txt --verbose=1
+/usr/lib64/ccache/cc CMakeFiles/cmTC_3dddb.dir/src.c.o -o cmTC_3dddb  -fsanitize=leak 
+/usr/bin/ld: cannot find /usr/lib64/liblsan.so.0.0.0
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_3dddb.dir/build.make:99: cmTC_3dddb] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_3dddb/fast] Error 2
+
+
+Source file was:
+int main(void) { return 0; }
+Performing C SOURCE FILE Test OGL_C_UBSAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_98217/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_98217.dir/build.make CMakeFiles/cmTC_98217.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building C object CMakeFiles/cmTC_98217.dir/src.c.o
+/usr/lib64/ccache/cc -DOGL_C_UBSAN_SUPPORTED  -fsanitize=undefined -static-libubsan -o CMakeFiles/cmTC_98217.dir/src.c.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.c
+Linking C executable cmTC_98217
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_98217.dir/link.txt --verbose=1
+/usr/lib64/ccache/cc CMakeFiles/cmTC_98217.dir/src.c.o -o cmTC_98217  -fsanitize=undefined -static-libubsan 
+/usr/bin/ld: cannot find -lubsan
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_98217.dir/build.make:99: cmTC_98217] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_98217/fast] Error 2
+
+
+Source file was:
+int main(void) { return 0; }
+Performing C++ SOURCE FILE Test OGL_CXX_TSAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_eb8da/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_eb8da.dir/build.make CMakeFiles/cmTC_eb8da.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_eb8da.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_CXX_TSAN_SUPPORTED  -fsanitize=thread -static-libtsan -fno-omit-frame-pointer -fPIC -o CMakeFiles/cmTC_eb8da.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_eb8da
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_eb8da.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_eb8da.dir/src.cxx.o -o cmTC_eb8da  -fsanitize=thread -static-libtsan -fno-omit-frame-pointer -fPIC 
+/usr/bin/ld: cannot find -ltsan
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_eb8da.dir/build.make:99: cmTC_eb8da] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_eb8da/fast] Error 2
+
+
+Source file was:
+int main() { return 0; }
+Performing C++ SOURCE FILE Test OGL_CXX_ASAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_25176/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_25176.dir/build.make CMakeFiles/cmTC_25176.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_25176.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_CXX_ASAN_SUPPORTED  -fsanitize=address -fno-omit-frame-pointer -o CMakeFiles/cmTC_25176.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_25176
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_25176.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_25176.dir/src.cxx.o -o cmTC_25176  -fsanitize=address -fno-omit-frame-pointer 
+/usr/bin/ld: cannot find /usr/lib64/libasan.so.5.0.0
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_25176.dir/build.make:99: cmTC_25176] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_25176/fast] Error 2
+
+
+Source file was:
+int main() { return 0; }
+Performing C++ SOURCE FILE Test OGL_CXX_LSAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_b4bbc/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_b4bbc.dir/build.make CMakeFiles/cmTC_b4bbc.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_b4bbc.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_CXX_LSAN_SUPPORTED  -fsanitize=leak -o CMakeFiles/cmTC_b4bbc.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_b4bbc
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_b4bbc.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_b4bbc.dir/src.cxx.o -o cmTC_b4bbc  -fsanitize=leak 
+/usr/bin/ld: cannot find /usr/lib64/liblsan.so.0.0.0
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_b4bbc.dir/build.make:99: cmTC_b4bbc] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_b4bbc/fast] Error 2
+
+
+Source file was:
+int main() { return 0; }
+Performing C++ SOURCE FILE Test OGL_CXX_UBSAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_6c762/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_6c762.dir/build.make CMakeFiles/cmTC_6c762.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_6c762.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_CXX_UBSAN_SUPPORTED  -fsanitize=undefined -static-libubsan -o CMakeFiles/cmTC_6c762.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_6c762
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_6c762.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_6c762.dir/src.cxx.o -o cmTC_6c762  -fsanitize=undefined -static-libubsan 
+/usr/bin/ld: cannot find -lubsan
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_6c762.dir/build.make:99: cmTC_6c762] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_6c762/fast] Error 2
+
+
+Source file was:
+int main() { return 0; }
+Performing C++ SOURCE FILE Test OGL_HIP_TSAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_132bb/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_132bb.dir/build.make CMakeFiles/cmTC_132bb.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_132bb.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_HIP_TSAN_SUPPORTED  -fsanitize=thread -static-libtsan -fno-omit-frame-pointer -fPIC -o CMakeFiles/cmTC_132bb.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_132bb
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_132bb.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_132bb.dir/src.cxx.o -o cmTC_132bb  -fsanitize=thread -static-libtsan -fno-omit-frame-pointer -fPIC 
+/usr/bin/ld: cannot find -ltsan
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_132bb.dir/build.make:99: cmTC_132bb] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_132bb/fast] Error 2
+
+
+Source file was:
+int main() { return 0; }
+Performing C++ SOURCE FILE Test OGL_HIP_ASAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_e61e9/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_e61e9.dir/build.make CMakeFiles/cmTC_e61e9.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_e61e9.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_HIP_ASAN_SUPPORTED  -fsanitize=address -fno-omit-frame-pointer -o CMakeFiles/cmTC_e61e9.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_e61e9
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_e61e9.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_e61e9.dir/src.cxx.o -o cmTC_e61e9  -fsanitize=address -fno-omit-frame-pointer 
+/usr/bin/ld: cannot find /usr/lib64/libasan.so.5.0.0
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_e61e9.dir/build.make:99: cmTC_e61e9] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_e61e9/fast] Error 2
+
+
+Source file was:
+int main() { return 0; }
+Performing C++ SOURCE FILE Test OGL_HIP_LSAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_26bce/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_26bce.dir/build.make CMakeFiles/cmTC_26bce.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_26bce.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_HIP_LSAN_SUPPORTED  -fsanitize=leak -o CMakeFiles/cmTC_26bce.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_26bce
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_26bce.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_26bce.dir/src.cxx.o -o cmTC_26bce  -fsanitize=leak 
+/usr/bin/ld: cannot find /usr/lib64/liblsan.so.0.0.0
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_26bce.dir/build.make:99: cmTC_26bce] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_26bce/fast] Error 2
+
+
+Source file was:
+int main() { return 0; }
+Performing C++ SOURCE FILE Test OGL_HIP_UBSAN_SUPPORTED failed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_dd64e/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_dd64e.dir/build.make CMakeFiles/cmTC_dd64e.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_dd64e.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_HIP_UBSAN_SUPPORTED  -fsanitize=undefined -static-libubsan -o CMakeFiles/cmTC_dd64e.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_dd64e
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_dd64e.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_dd64e.dir/src.cxx.o -o cmTC_dd64e  -fsanitize=undefined -static-libubsan 
+/usr/bin/ld: cannot find -lubsan
+collect2: error: ld returned 1 exit status
+gmake[1]: *** [CMakeFiles/cmTC_dd64e.dir/build.make:99: cmTC_dd64e] Error 1
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+gmake: *** [Makefile:127: cmTC_dd64e/fast] Error 2
+
+
+Source file was:
+int main() { return 0; }
diff --git a/CMakeFiles/CMakeOutput.log b/CMakeFiles/CMakeOutput.log
new file mode 100644
index 000000000..191d8cbbe
--- /dev/null
+++ b/CMakeFiles/CMakeOutput.log
@@ -0,0 +1,506 @@
+The system is: Linux - 4.18.0-408.el8.x86_64 - x86_64
+Compiling the C compiler identification source file "CMakeCCompilerId.c" succeeded.
+Compiler: /usr/lib64/ccache/cc 
+Build flags: 
+Id flags:  
+
+The output was:
+0
+
+
+Compilation of the C compiler identification source "CMakeCCompilerId.c" produced "a.out"
+
+The C compiler identification is GNU, found in "/home/greole/data/code/OGL/CMakeFiles/3.20.2/CompilerIdC/a.out"
+
+Compiling the CXX compiler identification source file "CMakeCXXCompilerId.cpp" succeeded.
+Compiler: /usr/lib64/ccache/c++ 
+Build flags: 
+Id flags:  
+
+The output was:
+0
+
+
+Compilation of the CXX compiler identification source "CMakeCXXCompilerId.cpp" produced "a.out"
+
+The CXX compiler identification is GNU, found in "/home/greole/data/code/OGL/CMakeFiles/3.20.2/CompilerIdCXX/a.out"
+
+Detecting C compiler ABI info compiled with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_94b6b/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_94b6b.dir/build.make CMakeFiles/cmTC_94b6b.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building C object CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o
+/usr/lib64/ccache/cc   -v -o CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o -c /usr/share/cmake/Modules/CMakeCCompilerABI.c
+Using built-in specs.
+COLLECT_GCC=/usr/bin/cc
+OFFLOAD_TARGET_NAMES=nvptx-none
+OFFLOAD_TARGET_DEFAULT=1
+Target: x86_64-redhat-linux
+Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,fortran,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --disable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux
+Thread model: posix
+gcc version 8.5.0 20210514 (Red Hat 8.5.0-15) (GCC) 
+COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o' '-mtune=generic' '-march=x86-64'
+ /usr/libexec/gcc/x86_64-redhat-linux/8/cc1 -quiet -v /usr/share/cmake/Modules/CMakeCCompilerABI.c -quiet -dumpbase CMakeCCompilerABI.c -mtune=generic -march=x86-64 -auxbase-strip CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o -version -o /tmp/ccUkmE2k.s
+GNU C17 (GCC) version 8.5.0 20210514 (Red Hat 8.5.0-15) (x86_64-redhat-linux)
+	compiled by GNU C version 8.5.0 20210514 (Red Hat 8.5.0-15), GMP version 6.1.2, MPFR version 3.1.6-p2, MPC version 1.1.0, isl version isl-0.16.1-GMP
+
+GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072
+ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/8/include-fixed"
+ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/8/../../../../x86_64-redhat-linux/include"
+#include "..." search starts here:
+#include <...> search starts here:
+ /opt/rocm-5.2.0/opencl/include
+ /opt/rocm-5.2.0/hip/include
+ /opt/rocm-5.2.0/include
+ /usr/lib/gcc/x86_64-redhat-linux/8/include
+ /usr/local/include
+ /usr/include
+End of search list.
+GNU C17 (GCC) version 8.5.0 20210514 (Red Hat 8.5.0-15) (x86_64-redhat-linux)
+	compiled by GNU C version 8.5.0 20210514 (Red Hat 8.5.0-15), GMP version 6.1.2, MPFR version 3.1.6-p2, MPC version 1.1.0, isl version isl-0.16.1-GMP
+
+GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072
+Compiler executable checksum: 546828b79bce5595f67138c9f33121d6
+COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o' '-mtune=generic' '-march=x86-64'
+ as -v --64 -o CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o /tmp/ccUkmE2k.s
+GNU assembler version 2.30 (x86_64-redhat-linux) using BFD version version 2.30-117.el8
+COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/
+LIBRARY_PATH=/opt/rocm-5.2.0/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/opt/rocm-5.2.0/opencl/lib/:/opt/rocm-5.2.0/hip/lib/:/opt/rocm-5.2.0/lib/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../:/lib/:/usr/lib/
+COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o' '-mtune=generic' '-march=x86-64'
+Linking C executable cmTC_94b6b
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_94b6b.dir/link.txt --verbose=1
+/usr/lib64/ccache/cc  -v CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o -o cmTC_94b6b 
+Using built-in specs.
+COLLECT_GCC=/usr/bin/cc
+COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper
+OFFLOAD_TARGET_NAMES=nvptx-none
+OFFLOAD_TARGET_DEFAULT=1
+Target: x86_64-redhat-linux
+Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,fortran,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --disable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux
+Thread model: posix
+gcc version 8.5.0 20210514 (Red Hat 8.5.0-15) (GCC) 
+COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/
+LIBRARY_PATH=/opt/rocm-5.2.0/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/opt/rocm-5.2.0/opencl/lib/:/opt/rocm-5.2.0/hip/lib/:/opt/rocm-5.2.0/lib/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../:/lib/:/usr/lib/
+COLLECT_GCC_OPTIONS='-v' '-o' 'cmTC_94b6b' '-mtune=generic' '-march=x86-64'
+ /usr/libexec/gcc/x86_64-redhat-linux/8/collect2 -plugin /usr/libexec/gcc/x86_64-redhat-linux/8/liblto_plugin.so -plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper -plugin-opt=-fresolution=/tmp/cc5zvrd2.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s --build-id --no-add-needed --eh-frame-hdr --hash-style=gnu -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o cmTC_94b6b /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o /usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o -L/opt/rocm-5.2.0/lib/../lib64 -L/usr/lib/gcc/x86_64-redhat-linux/8 -L/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/opt/rocm-5.2.0/opencl/lib -L/opt/rocm-5.2.0/hip/lib -L/opt/rocm-5.2.0/lib -L/usr/lib/gcc/x86_64-redhat-linux/8/../../.. CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/lib/gcc/x86_64-redhat-linux/8/crtend.o /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o
+COLLECT_GCC_OPTIONS='-v' '-o' 'cmTC_94b6b' '-mtune=generic' '-march=x86-64'
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+
+
+
+Parsed C implicit include dir info from above output: rv=done
+  found start of include info
+  found start of implicit include info
+    add: [/opt/rocm-5.2.0/opencl/include]
+    add: [/opt/rocm-5.2.0/hip/include]
+    add: [/opt/rocm-5.2.0/include]
+    add: [/usr/lib/gcc/x86_64-redhat-linux/8/include]
+    add: [/usr/local/include]
+    add: [/usr/include]
+  end of search list found
+  collapse include dir [/opt/rocm-5.2.0/opencl/include] ==> [/opt/rocm-5.2.0/opencl/include]
+  collapse include dir [/opt/rocm-5.2.0/hip/include] ==> [/opt/rocm-5.2.0/hip/include]
+  collapse include dir [/opt/rocm-5.2.0/include] ==> [/opt/rocm-5.2.0/include]
+  collapse include dir [/usr/lib/gcc/x86_64-redhat-linux/8/include] ==> [/usr/lib/gcc/x86_64-redhat-linux/8/include]
+  collapse include dir [/usr/local/include] ==> [/usr/local/include]
+  collapse include dir [/usr/include] ==> [/usr/include]
+  implicit include dirs: [/opt/rocm-5.2.0/opencl/include;/opt/rocm-5.2.0/hip/include;/opt/rocm-5.2.0/include;/usr/lib/gcc/x86_64-redhat-linux/8/include;/usr/local/include;/usr/include]
+
+
+Parsed C implicit link information from above output:
+  link line regex: [^( *|.*[/\])(ld|CMAKE_LINK_STARTFILE-NOTFOUND|([^/\]+-)?ld|collect2)[^/\]*( |$)]
+  ignore line: [Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp]
+  ignore line: []
+  ignore line: [Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_94b6b/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_94b6b.dir/build.make CMakeFiles/cmTC_94b6b.dir/build]
+  ignore line: [gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp']
+  ignore line: [Building C object CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o]
+  ignore line: [/usr/lib64/ccache/cc   -v -o CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o -c /usr/share/cmake/Modules/CMakeCCompilerABI.c]
+  ignore line: [Using built-in specs.]
+  ignore line: [COLLECT_GCC=/usr/bin/cc]
+  ignore line: [OFFLOAD_TARGET_NAMES=nvptx-none]
+  ignore line: [OFFLOAD_TARGET_DEFAULT=1]
+  ignore line: [Target: x86_64-redhat-linux]
+  ignore line: [Configured with: ../configure --enable-bootstrap --enable-languages=c c++ fortran lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --disable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux]
+  ignore line: [Thread model: posix]
+  ignore line: [gcc version 8.5.0 20210514 (Red Hat 8.5.0-15) (GCC) ]
+  ignore line: [COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o' '-mtune=generic' '-march=x86-64']
+  ignore line: [ /usr/libexec/gcc/x86_64-redhat-linux/8/cc1 -quiet -v /usr/share/cmake/Modules/CMakeCCompilerABI.c -quiet -dumpbase CMakeCCompilerABI.c -mtune=generic -march=x86-64 -auxbase-strip CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o -version -o /tmp/ccUkmE2k.s]
+  ignore line: [GNU C17 (GCC) version 8.5.0 20210514 (Red Hat 8.5.0-15) (x86_64-redhat-linux)]
+  ignore line: [	compiled by GNU C version 8.5.0 20210514 (Red Hat 8.5.0-15)  GMP version 6.1.2  MPFR version 3.1.6-p2  MPC version 1.1.0  isl version isl-0.16.1-GMP]
+  ignore line: []
+  ignore line: [GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072]
+  ignore line: [ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/8/include-fixed"]
+  ignore line: [ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/8/../../../../x86_64-redhat-linux/include"]
+  ignore line: [#include "..." search starts here:]
+  ignore line: [#include <...> search starts here:]
+  ignore line: [ /opt/rocm-5.2.0/opencl/include]
+  ignore line: [ /opt/rocm-5.2.0/hip/include]
+  ignore line: [ /opt/rocm-5.2.0/include]
+  ignore line: [ /usr/lib/gcc/x86_64-redhat-linux/8/include]
+  ignore line: [ /usr/local/include]
+  ignore line: [ /usr/include]
+  ignore line: [End of search list.]
+  ignore line: [GNU C17 (GCC) version 8.5.0 20210514 (Red Hat 8.5.0-15) (x86_64-redhat-linux)]
+  ignore line: [	compiled by GNU C version 8.5.0 20210514 (Red Hat 8.5.0-15)  GMP version 6.1.2  MPFR version 3.1.6-p2  MPC version 1.1.0  isl version isl-0.16.1-GMP]
+  ignore line: []
+  ignore line: [GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072]
+  ignore line: [Compiler executable checksum: 546828b79bce5595f67138c9f33121d6]
+  ignore line: [COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o' '-mtune=generic' '-march=x86-64']
+  ignore line: [ as -v --64 -o CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o /tmp/ccUkmE2k.s]
+  ignore line: [GNU assembler version 2.30 (x86_64-redhat-linux) using BFD version version 2.30-117.el8]
+  ignore line: [COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/]
+  ignore line: [LIBRARY_PATH=/opt/rocm-5.2.0/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/opt/rocm-5.2.0/opencl/lib/:/opt/rocm-5.2.0/hip/lib/:/opt/rocm-5.2.0/lib/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../:/lib/:/usr/lib/]
+  ignore line: [COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o' '-mtune=generic' '-march=x86-64']
+  ignore line: [Linking C executable cmTC_94b6b]
+  ignore line: [/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_94b6b.dir/link.txt --verbose=1]
+  ignore line: [/usr/lib64/ccache/cc  -v CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o -o cmTC_94b6b ]
+  ignore line: [Using built-in specs.]
+  ignore line: [COLLECT_GCC=/usr/bin/cc]
+  ignore line: [COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper]
+  ignore line: [OFFLOAD_TARGET_NAMES=nvptx-none]
+  ignore line: [OFFLOAD_TARGET_DEFAULT=1]
+  ignore line: [Target: x86_64-redhat-linux]
+  ignore line: [Configured with: ../configure --enable-bootstrap --enable-languages=c c++ fortran lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --disable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux]
+  ignore line: [Thread model: posix]
+  ignore line: [gcc version 8.5.0 20210514 (Red Hat 8.5.0-15) (GCC) ]
+  ignore line: [COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/]
+  ignore line: [LIBRARY_PATH=/opt/rocm-5.2.0/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/opt/rocm-5.2.0/opencl/lib/:/opt/rocm-5.2.0/hip/lib/:/opt/rocm-5.2.0/lib/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../:/lib/:/usr/lib/]
+  ignore line: [COLLECT_GCC_OPTIONS='-v' '-o' 'cmTC_94b6b' '-mtune=generic' '-march=x86-64']
+  link line: [ /usr/libexec/gcc/x86_64-redhat-linux/8/collect2 -plugin /usr/libexec/gcc/x86_64-redhat-linux/8/liblto_plugin.so -plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper -plugin-opt=-fresolution=/tmp/cc5zvrd2.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s --build-id --no-add-needed --eh-frame-hdr --hash-style=gnu -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o cmTC_94b6b /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o /usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o -L/opt/rocm-5.2.0/lib/../lib64 -L/usr/lib/gcc/x86_64-redhat-linux/8 -L/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/opt/rocm-5.2.0/opencl/lib -L/opt/rocm-5.2.0/hip/lib -L/opt/rocm-5.2.0/lib -L/usr/lib/gcc/x86_64-redhat-linux/8/../../.. CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/lib/gcc/x86_64-redhat-linux/8/crtend.o /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o]
+    arg [/usr/libexec/gcc/x86_64-redhat-linux/8/collect2] ==> ignore
+    arg [-plugin] ==> ignore
+    arg [/usr/libexec/gcc/x86_64-redhat-linux/8/liblto_plugin.so] ==> ignore
+    arg [-plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper] ==> ignore
+    arg [-plugin-opt=-fresolution=/tmp/cc5zvrd2.res] ==> ignore
+    arg [-plugin-opt=-pass-through=-lgcc] ==> ignore
+    arg [-plugin-opt=-pass-through=-lgcc_s] ==> ignore
+    arg [-plugin-opt=-pass-through=-lc] ==> ignore
+    arg [-plugin-opt=-pass-through=-lgcc] ==> ignore
+    arg [-plugin-opt=-pass-through=-lgcc_s] ==> ignore
+    arg [--build-id] ==> ignore
+    arg [--no-add-needed] ==> ignore
+    arg [--eh-frame-hdr] ==> ignore
+    arg [--hash-style=gnu] ==> ignore
+    arg [-m] ==> ignore
+    arg [elf_x86_64] ==> ignore
+    arg [-dynamic-linker] ==> ignore
+    arg [/lib64/ld-linux-x86-64.so.2] ==> ignore
+    arg [-o] ==> ignore
+    arg [cmTC_94b6b] ==> ignore
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o]
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o]
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o]
+    arg [-L/opt/rocm-5.2.0/lib/../lib64] ==> dir [/opt/rocm-5.2.0/lib/../lib64]
+    arg [-L/usr/lib/gcc/x86_64-redhat-linux/8] ==> dir [/usr/lib/gcc/x86_64-redhat-linux/8]
+    arg [-L/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64] ==> dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64]
+    arg [-L/lib/../lib64] ==> dir [/lib/../lib64]
+    arg [-L/usr/lib/../lib64] ==> dir [/usr/lib/../lib64]
+    arg [-L/opt/rocm-5.2.0/opencl/lib] ==> dir [/opt/rocm-5.2.0/opencl/lib]
+    arg [-L/opt/rocm-5.2.0/hip/lib] ==> dir [/opt/rocm-5.2.0/hip/lib]
+    arg [-L/opt/rocm-5.2.0/lib] ==> dir [/opt/rocm-5.2.0/lib]
+    arg [-L/usr/lib/gcc/x86_64-redhat-linux/8/../../..] ==> dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../..]
+    arg [CMakeFiles/cmTC_94b6b.dir/CMakeCCompilerABI.c.o] ==> ignore
+    arg [-lgcc] ==> lib [gcc]
+    arg [--as-needed] ==> ignore
+    arg [-lgcc_s] ==> lib [gcc_s]
+    arg [--no-as-needed] ==> ignore
+    arg [-lc] ==> lib [c]
+    arg [-lgcc] ==> lib [gcc]
+    arg [--as-needed] ==> ignore
+    arg [-lgcc_s] ==> lib [gcc_s]
+    arg [--no-as-needed] ==> ignore
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/crtend.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/crtend.o]
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o]
+  collapse obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o] ==> [/usr/lib64/crt1.o]
+  collapse obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o] ==> [/usr/lib64/crti.o]
+  collapse obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o] ==> [/usr/lib64/crtn.o]
+  collapse library dir [/opt/rocm-5.2.0/lib/../lib64] ==> [/opt/rocm-5.2.0/lib64]
+  collapse library dir [/usr/lib/gcc/x86_64-redhat-linux/8] ==> [/usr/lib/gcc/x86_64-redhat-linux/8]
+  collapse library dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64] ==> [/usr/lib64]
+  collapse library dir [/lib/../lib64] ==> [/lib64]
+  collapse library dir [/usr/lib/../lib64] ==> [/usr/lib64]
+  collapse library dir [/opt/rocm-5.2.0/opencl/lib] ==> [/opt/rocm-5.2.0/opencl/lib]
+  collapse library dir [/opt/rocm-5.2.0/hip/lib] ==> [/opt/rocm-5.2.0/hip/lib]
+  collapse library dir [/opt/rocm-5.2.0/lib] ==> [/opt/rocm-5.2.0/lib]
+  collapse library dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../..] ==> [/usr/lib]
+  implicit libs: [gcc;gcc_s;c;gcc;gcc_s]
+  implicit objs: [/usr/lib64/crt1.o;/usr/lib64/crti.o;/usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o;/usr/lib/gcc/x86_64-redhat-linux/8/crtend.o;/usr/lib64/crtn.o]
+  implicit dirs: [/opt/rocm-5.2.0/lib64;/usr/lib/gcc/x86_64-redhat-linux/8;/usr/lib64;/lib64;/opt/rocm-5.2.0/opencl/lib;/opt/rocm-5.2.0/hip/lib;/opt/rocm-5.2.0/lib;/usr/lib]
+  implicit fwks: []
+
+
+Detecting CXX compiler ABI info compiled with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_90c22/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_90c22.dir/build.make CMakeFiles/cmTC_90c22.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o
+/usr/lib64/ccache/c++   -v -o CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o -c /usr/share/cmake/Modules/CMakeCXXCompilerABI.cpp
+Using built-in specs.
+COLLECT_GCC=/usr/bin/c++
+OFFLOAD_TARGET_NAMES=nvptx-none
+OFFLOAD_TARGET_DEFAULT=1
+Target: x86_64-redhat-linux
+Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,fortran,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --disable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux
+Thread model: posix
+gcc version 8.5.0 20210514 (Red Hat 8.5.0-15) (GCC) 
+COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o' '-shared-libgcc' '-mtune=generic' '-march=x86-64'
+ /usr/libexec/gcc/x86_64-redhat-linux/8/cc1plus -quiet -v -D_GNU_SOURCE /usr/share/cmake/Modules/CMakeCXXCompilerABI.cpp -quiet -dumpbase CMakeCXXCompilerABI.cpp -mtune=generic -march=x86-64 -auxbase-strip CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o -version -o /tmp/ccEoBZDv.s
+GNU C++14 (GCC) version 8.5.0 20210514 (Red Hat 8.5.0-15) (x86_64-redhat-linux)
+	compiled by GNU C version 8.5.0 20210514 (Red Hat 8.5.0-15), GMP version 6.1.2, MPFR version 3.1.6-p2, MPC version 1.1.0, isl version isl-0.16.1-GMP
+
+GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072
+ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/8/include-fixed"
+ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/8/../../../../x86_64-redhat-linux/include"
+#include "..." search starts here:
+#include <...> search starts here:
+ /opt/rocm-5.2.0/opencl/include
+ /opt/rocm-5.2.0/hip/include
+ /opt/rocm-5.2.0/include
+ /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8
+ /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/x86_64-redhat-linux
+ /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/backward
+ /usr/lib/gcc/x86_64-redhat-linux/8/include
+ /usr/local/include
+ /usr/include
+End of search list.
+GNU C++14 (GCC) version 8.5.0 20210514 (Red Hat 8.5.0-15) (x86_64-redhat-linux)
+	compiled by GNU C version 8.5.0 20210514 (Red Hat 8.5.0-15), GMP version 6.1.2, MPFR version 3.1.6-p2, MPC version 1.1.0, isl version isl-0.16.1-GMP
+
+GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072
+Compiler executable checksum: eb7b5bd1ebb9974a42b8082997aecb2a
+COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o' '-shared-libgcc' '-mtune=generic' '-march=x86-64'
+ as -v --64 -o CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o /tmp/ccEoBZDv.s
+GNU assembler version 2.30 (x86_64-redhat-linux) using BFD version version 2.30-117.el8
+COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/
+LIBRARY_PATH=/opt/rocm-5.2.0/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/opt/rocm-5.2.0/opencl/lib/:/opt/rocm-5.2.0/hip/lib/:/opt/rocm-5.2.0/lib/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../:/lib/:/usr/lib/
+COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o' '-shared-libgcc' '-mtune=generic' '-march=x86-64'
+Linking CXX executable cmTC_90c22
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_90c22.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++  -v CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o -o cmTC_90c22 
+Using built-in specs.
+COLLECT_GCC=/usr/bin/c++
+COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper
+OFFLOAD_TARGET_NAMES=nvptx-none
+OFFLOAD_TARGET_DEFAULT=1
+Target: x86_64-redhat-linux
+Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,fortran,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --disable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux
+Thread model: posix
+gcc version 8.5.0 20210514 (Red Hat 8.5.0-15) (GCC) 
+COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/
+LIBRARY_PATH=/opt/rocm-5.2.0/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/opt/rocm-5.2.0/opencl/lib/:/opt/rocm-5.2.0/hip/lib/:/opt/rocm-5.2.0/lib/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../:/lib/:/usr/lib/
+COLLECT_GCC_OPTIONS='-v' '-o' 'cmTC_90c22' '-shared-libgcc' '-mtune=generic' '-march=x86-64'
+ /usr/libexec/gcc/x86_64-redhat-linux/8/collect2 -plugin /usr/libexec/gcc/x86_64-redhat-linux/8/liblto_plugin.so -plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper -plugin-opt=-fresolution=/tmp/cczL0CNF.res -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc --build-id --no-add-needed --eh-frame-hdr --hash-style=gnu -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o cmTC_90c22 /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o /usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o -L/opt/rocm-5.2.0/lib/../lib64 -L/usr/lib/gcc/x86_64-redhat-linux/8 -L/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/opt/rocm-5.2.0/opencl/lib -L/opt/rocm-5.2.0/hip/lib -L/opt/rocm-5.2.0/lib -L/usr/lib/gcc/x86_64-redhat-linux/8/../../.. CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o -lstdc++ -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /usr/lib/gcc/x86_64-redhat-linux/8/crtend.o /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o
+COLLECT_GCC_OPTIONS='-v' '-o' 'cmTC_90c22' '-shared-libgcc' '-mtune=generic' '-march=x86-64'
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+
+
+
+Parsed CXX implicit include dir info from above output: rv=done
+  found start of include info
+  found start of implicit include info
+    add: [/opt/rocm-5.2.0/opencl/include]
+    add: [/opt/rocm-5.2.0/hip/include]
+    add: [/opt/rocm-5.2.0/include]
+    add: [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8]
+    add: [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/x86_64-redhat-linux]
+    add: [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/backward]
+    add: [/usr/lib/gcc/x86_64-redhat-linux/8/include]
+    add: [/usr/local/include]
+    add: [/usr/include]
+  end of search list found
+  collapse include dir [/opt/rocm-5.2.0/opencl/include] ==> [/opt/rocm-5.2.0/opencl/include]
+  collapse include dir [/opt/rocm-5.2.0/hip/include] ==> [/opt/rocm-5.2.0/hip/include]
+  collapse include dir [/opt/rocm-5.2.0/include] ==> [/opt/rocm-5.2.0/include]
+  collapse include dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8] ==> [/usr/include/c++/8]
+  collapse include dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/x86_64-redhat-linux] ==> [/usr/include/c++/8/x86_64-redhat-linux]
+  collapse include dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/backward] ==> [/usr/include/c++/8/backward]
+  collapse include dir [/usr/lib/gcc/x86_64-redhat-linux/8/include] ==> [/usr/lib/gcc/x86_64-redhat-linux/8/include]
+  collapse include dir [/usr/local/include] ==> [/usr/local/include]
+  collapse include dir [/usr/include] ==> [/usr/include]
+  implicit include dirs: [/opt/rocm-5.2.0/opencl/include;/opt/rocm-5.2.0/hip/include;/opt/rocm-5.2.0/include;/usr/include/c++/8;/usr/include/c++/8/x86_64-redhat-linux;/usr/include/c++/8/backward;/usr/lib/gcc/x86_64-redhat-linux/8/include;/usr/local/include;/usr/include]
+
+
+Parsed CXX implicit link information from above output:
+  link line regex: [^( *|.*[/\])(ld|CMAKE_LINK_STARTFILE-NOTFOUND|([^/\]+-)?ld|collect2)[^/\]*( |$)]
+  ignore line: [Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp]
+  ignore line: []
+  ignore line: [Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_90c22/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_90c22.dir/build.make CMakeFiles/cmTC_90c22.dir/build]
+  ignore line: [gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp']
+  ignore line: [Building CXX object CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o]
+  ignore line: [/usr/lib64/ccache/c++   -v -o CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o -c /usr/share/cmake/Modules/CMakeCXXCompilerABI.cpp]
+  ignore line: [Using built-in specs.]
+  ignore line: [COLLECT_GCC=/usr/bin/c++]
+  ignore line: [OFFLOAD_TARGET_NAMES=nvptx-none]
+  ignore line: [OFFLOAD_TARGET_DEFAULT=1]
+  ignore line: [Target: x86_64-redhat-linux]
+  ignore line: [Configured with: ../configure --enable-bootstrap --enable-languages=c c++ fortran lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --disable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux]
+  ignore line: [Thread model: posix]
+  ignore line: [gcc version 8.5.0 20210514 (Red Hat 8.5.0-15) (GCC) ]
+  ignore line: [COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o' '-shared-libgcc' '-mtune=generic' '-march=x86-64']
+  ignore line: [ /usr/libexec/gcc/x86_64-redhat-linux/8/cc1plus -quiet -v -D_GNU_SOURCE /usr/share/cmake/Modules/CMakeCXXCompilerABI.cpp -quiet -dumpbase CMakeCXXCompilerABI.cpp -mtune=generic -march=x86-64 -auxbase-strip CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o -version -o /tmp/ccEoBZDv.s]
+  ignore line: [GNU C++14 (GCC) version 8.5.0 20210514 (Red Hat 8.5.0-15) (x86_64-redhat-linux)]
+  ignore line: [	compiled by GNU C version 8.5.0 20210514 (Red Hat 8.5.0-15)  GMP version 6.1.2  MPFR version 3.1.6-p2  MPC version 1.1.0  isl version isl-0.16.1-GMP]
+  ignore line: []
+  ignore line: [GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072]
+  ignore line: [ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/8/include-fixed"]
+  ignore line: [ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/8/../../../../x86_64-redhat-linux/include"]
+  ignore line: [#include "..." search starts here:]
+  ignore line: [#include <...> search starts here:]
+  ignore line: [ /opt/rocm-5.2.0/opencl/include]
+  ignore line: [ /opt/rocm-5.2.0/hip/include]
+  ignore line: [ /opt/rocm-5.2.0/include]
+  ignore line: [ /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8]
+  ignore line: [ /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/x86_64-redhat-linux]
+  ignore line: [ /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/backward]
+  ignore line: [ /usr/lib/gcc/x86_64-redhat-linux/8/include]
+  ignore line: [ /usr/local/include]
+  ignore line: [ /usr/include]
+  ignore line: [End of search list.]
+  ignore line: [GNU C++14 (GCC) version 8.5.0 20210514 (Red Hat 8.5.0-15) (x86_64-redhat-linux)]
+  ignore line: [	compiled by GNU C version 8.5.0 20210514 (Red Hat 8.5.0-15)  GMP version 6.1.2  MPFR version 3.1.6-p2  MPC version 1.1.0  isl version isl-0.16.1-GMP]
+  ignore line: []
+  ignore line: [GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072]
+  ignore line: [Compiler executable checksum: eb7b5bd1ebb9974a42b8082997aecb2a]
+  ignore line: [COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o' '-shared-libgcc' '-mtune=generic' '-march=x86-64']
+  ignore line: [ as -v --64 -o CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o /tmp/ccEoBZDv.s]
+  ignore line: [GNU assembler version 2.30 (x86_64-redhat-linux) using BFD version version 2.30-117.el8]
+  ignore line: [COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/]
+  ignore line: [LIBRARY_PATH=/opt/rocm-5.2.0/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/opt/rocm-5.2.0/opencl/lib/:/opt/rocm-5.2.0/hip/lib/:/opt/rocm-5.2.0/lib/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../:/lib/:/usr/lib/]
+  ignore line: [COLLECT_GCC_OPTIONS='-v' '-c' '-o' 'CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o' '-shared-libgcc' '-mtune=generic' '-march=x86-64']
+  ignore line: [Linking CXX executable cmTC_90c22]
+  ignore line: [/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_90c22.dir/link.txt --verbose=1]
+  ignore line: [/usr/lib64/ccache/c++  -v CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o -o cmTC_90c22 ]
+  ignore line: [Using built-in specs.]
+  ignore line: [COLLECT_GCC=/usr/bin/c++]
+  ignore line: [COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper]
+  ignore line: [OFFLOAD_TARGET_NAMES=nvptx-none]
+  ignore line: [OFFLOAD_TARGET_DEFAULT=1]
+  ignore line: [Target: x86_64-redhat-linux]
+  ignore line: [Configured with: ../configure --enable-bootstrap --enable-languages=c c++ fortran lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --disable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux]
+  ignore line: [Thread model: posix]
+  ignore line: [gcc version 8.5.0 20210514 (Red Hat 8.5.0-15) (GCC) ]
+  ignore line: [COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/8/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/]
+  ignore line: [LIBRARY_PATH=/opt/rocm-5.2.0/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/8/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/opt/rocm-5.2.0/opencl/lib/:/opt/rocm-5.2.0/hip/lib/:/opt/rocm-5.2.0/lib/:/usr/lib/gcc/x86_64-redhat-linux/8/../../../:/lib/:/usr/lib/]
+  ignore line: [COLLECT_GCC_OPTIONS='-v' '-o' 'cmTC_90c22' '-shared-libgcc' '-mtune=generic' '-march=x86-64']
+  link line: [ /usr/libexec/gcc/x86_64-redhat-linux/8/collect2 -plugin /usr/libexec/gcc/x86_64-redhat-linux/8/liblto_plugin.so -plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper -plugin-opt=-fresolution=/tmp/cczL0CNF.res -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc --build-id --no-add-needed --eh-frame-hdr --hash-style=gnu -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o cmTC_90c22 /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o /usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o -L/opt/rocm-5.2.0/lib/../lib64 -L/usr/lib/gcc/x86_64-redhat-linux/8 -L/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/opt/rocm-5.2.0/opencl/lib -L/opt/rocm-5.2.0/hip/lib -L/opt/rocm-5.2.0/lib -L/usr/lib/gcc/x86_64-redhat-linux/8/../../.. CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o -lstdc++ -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /usr/lib/gcc/x86_64-redhat-linux/8/crtend.o /usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o]
+    arg [/usr/libexec/gcc/x86_64-redhat-linux/8/collect2] ==> ignore
+    arg [-plugin] ==> ignore
+    arg [/usr/libexec/gcc/x86_64-redhat-linux/8/liblto_plugin.so] ==> ignore
+    arg [-plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper] ==> ignore
+    arg [-plugin-opt=-fresolution=/tmp/cczL0CNF.res] ==> ignore
+    arg [-plugin-opt=-pass-through=-lgcc_s] ==> ignore
+    arg [-plugin-opt=-pass-through=-lgcc] ==> ignore
+    arg [-plugin-opt=-pass-through=-lc] ==> ignore
+    arg [-plugin-opt=-pass-through=-lgcc_s] ==> ignore
+    arg [-plugin-opt=-pass-through=-lgcc] ==> ignore
+    arg [--build-id] ==> ignore
+    arg [--no-add-needed] ==> ignore
+    arg [--eh-frame-hdr] ==> ignore
+    arg [--hash-style=gnu] ==> ignore
+    arg [-m] ==> ignore
+    arg [elf_x86_64] ==> ignore
+    arg [-dynamic-linker] ==> ignore
+    arg [/lib64/ld-linux-x86-64.so.2] ==> ignore
+    arg [-o] ==> ignore
+    arg [cmTC_90c22] ==> ignore
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o]
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o]
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o]
+    arg [-L/opt/rocm-5.2.0/lib/../lib64] ==> dir [/opt/rocm-5.2.0/lib/../lib64]
+    arg [-L/usr/lib/gcc/x86_64-redhat-linux/8] ==> dir [/usr/lib/gcc/x86_64-redhat-linux/8]
+    arg [-L/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64] ==> dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64]
+    arg [-L/lib/../lib64] ==> dir [/lib/../lib64]
+    arg [-L/usr/lib/../lib64] ==> dir [/usr/lib/../lib64]
+    arg [-L/opt/rocm-5.2.0/opencl/lib] ==> dir [/opt/rocm-5.2.0/opencl/lib]
+    arg [-L/opt/rocm-5.2.0/hip/lib] ==> dir [/opt/rocm-5.2.0/hip/lib]
+    arg [-L/opt/rocm-5.2.0/lib] ==> dir [/opt/rocm-5.2.0/lib]
+    arg [-L/usr/lib/gcc/x86_64-redhat-linux/8/../../..] ==> dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../..]
+    arg [CMakeFiles/cmTC_90c22.dir/CMakeCXXCompilerABI.cpp.o] ==> ignore
+    arg [-lstdc++] ==> lib [stdc++]
+    arg [-lm] ==> lib [m]
+    arg [-lgcc_s] ==> lib [gcc_s]
+    arg [-lgcc] ==> lib [gcc]
+    arg [-lc] ==> lib [c]
+    arg [-lgcc_s] ==> lib [gcc_s]
+    arg [-lgcc] ==> lib [gcc]
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/crtend.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/crtend.o]
+    arg [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o] ==> obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o]
+  collapse obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crt1.o] ==> [/usr/lib64/crt1.o]
+  collapse obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crti.o] ==> [/usr/lib64/crti.o]
+  collapse obj [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64/crtn.o] ==> [/usr/lib64/crtn.o]
+  collapse library dir [/opt/rocm-5.2.0/lib/../lib64] ==> [/opt/rocm-5.2.0/lib64]
+  collapse library dir [/usr/lib/gcc/x86_64-redhat-linux/8] ==> [/usr/lib/gcc/x86_64-redhat-linux/8]
+  collapse library dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../../../lib64] ==> [/usr/lib64]
+  collapse library dir [/lib/../lib64] ==> [/lib64]
+  collapse library dir [/usr/lib/../lib64] ==> [/usr/lib64]
+  collapse library dir [/opt/rocm-5.2.0/opencl/lib] ==> [/opt/rocm-5.2.0/opencl/lib]
+  collapse library dir [/opt/rocm-5.2.0/hip/lib] ==> [/opt/rocm-5.2.0/hip/lib]
+  collapse library dir [/opt/rocm-5.2.0/lib] ==> [/opt/rocm-5.2.0/lib]
+  collapse library dir [/usr/lib/gcc/x86_64-redhat-linux/8/../../..] ==> [/usr/lib]
+  implicit libs: [stdc++;m;gcc_s;gcc;c;gcc_s;gcc]
+  implicit objs: [/usr/lib64/crt1.o;/usr/lib64/crti.o;/usr/lib/gcc/x86_64-redhat-linux/8/crtbegin.o;/usr/lib/gcc/x86_64-redhat-linux/8/crtend.o;/usr/lib64/crtn.o]
+  implicit dirs: [/opt/rocm-5.2.0/lib64;/usr/lib/gcc/x86_64-redhat-linux/8;/usr/lib64;/lib64;/opt/rocm-5.2.0/opencl/lib;/opt/rocm-5.2.0/hip/lib;/opt/rocm-5.2.0/lib;/usr/lib]
+  implicit fwks: []
+
+
+Performing C SOURCE FILE Test OGL_C_COVERAGE_SUPPORTED succeeded with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_cb78e/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_cb78e.dir/build.make CMakeFiles/cmTC_cb78e.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building C object CMakeFiles/cmTC_cb78e.dir/src.c.o
+/usr/lib64/ccache/cc -DOGL_C_COVERAGE_SUPPORTED  --coverage -o CMakeFiles/cmTC_cb78e.dir/src.c.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.c
+Linking C executable cmTC_cb78e
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_cb78e.dir/link.txt --verbose=1
+/usr/lib64/ccache/cc CMakeFiles/cmTC_cb78e.dir/src.c.o -o cmTC_cb78e  --coverage 
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+
+
+Source file was:
+int main(void) { return 0; }
+Performing C++ SOURCE FILE Test OGL_CXX_COVERAGE_SUPPORTED succeeded with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_20207/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_20207.dir/build.make CMakeFiles/cmTC_20207.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_20207.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_CXX_COVERAGE_SUPPORTED  --coverage -o CMakeFiles/cmTC_20207.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_20207
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_20207.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_20207.dir/src.cxx.o -o cmTC_20207  --coverage 
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+
+
+Source file was:
+int main() { return 0; }
+Performing C++ SOURCE FILE Test OGL_HIP_COVERAGE_SUPPORTED succeeded with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_a8137/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_a8137.dir/build.make CMakeFiles/cmTC_a8137.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_a8137.dir/src.cxx.o
+/usr/lib64/ccache/c++ -DOGL_HIP_COVERAGE_SUPPORTED  --coverage -o CMakeFiles/cmTC_a8137.dir/src.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/src.cxx
+Linking CXX executable cmTC_a8137
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_a8137.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_a8137.dir/src.cxx.o -o cmTC_a8137  --coverage 
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+
+
+Source file was:
+int main() { return 0; }
+Determining if the include file cxxabi.h exists passed with the following output:
+Change Dir: /home/greole/data/code/OGL/CMakeFiles/CMakeTmp
+
+Run Build Command(s):/usr/bin/gmake -f Makefile cmTC_c0af7/fast && /usr/bin/gmake  -f CMakeFiles/cmTC_c0af7.dir/build.make CMakeFiles/cmTC_c0af7.dir/build
+gmake[1]: Entering directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+Building CXX object CMakeFiles/cmTC_c0af7.dir/CheckIncludeFile.cxx.o
+/usr/lib64/ccache/c++    -o CMakeFiles/cmTC_c0af7.dir/CheckIncludeFile.cxx.o -c /home/greole/data/code/OGL/CMakeFiles/CMakeTmp/CheckIncludeFile.cxx
+Linking CXX executable cmTC_c0af7
+/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_c0af7.dir/link.txt --verbose=1
+/usr/lib64/ccache/c++ CMakeFiles/cmTC_c0af7.dir/CheckIncludeFile.cxx.o -o cmTC_c0af7 
+gmake[1]: Leaving directory '/home/greole/data/code/OGL/CMakeFiles/CMakeTmp'
+
+
+
diff --git a/CMakeFiles/cmake.check_cache b/CMakeFiles/cmake.check_cache
new file mode 100644
index 000000000..3dccd7317
--- /dev/null
+++ b/CMakeFiles/cmake.check_cache
@@ -0,0 +1 @@
+# This file is generated by cmake for dependency checking of the CMakeCache.txt file
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4c3cdadef..c77803271 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -171,6 +171,7 @@ target_sources(
           BaseWrapper/CoupledLduBase/GKOCoupledLduBase.C
           MatrixWrapper/LDUMatrix/HostMatrix.C
           MatrixWrapper/LDUMatrix/HostMatrixFreeFunctions.C
+          MatrixWrapper/CommunicationPattern/CommunicationPattern.C
           Solver/CG/GKOCG.C
           Solver/BiCGStab/GKOBiCGStab.C
           # Solver / IR / GKOIR.C Solver / Multigrid / GKOMultigrid.C
diff --git a/CMakePresets.json b/CMakePresets.json
index f46b6bc2c..6f9dfc3e4 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -13,7 +13,7 @@
       "description": "Default build using Ninja generator",
       "generator": "Ninja",
       "cacheVariables": {
-        "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
         "CMAKE_INSTALL_PREFIX": "$env{FOAM_USER_LIBBIN}/.."
       }
     },
diff --git a/DevicePersistent/ExecutorHandler/ExecutorHandler.H b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
index e7a8fe7d0..87a6dec49 100644
--- a/DevicePersistent/ExecutorHandler/ExecutorHandler.H
+++ b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
@@ -12,6 +12,8 @@
 
 namespace Foam {
 
+
+
 struct ExecutorInitFunctor {
     mutable std::shared_ptr<gko::experimental::mpi::communicator> comm_;
 
@@ -30,15 +32,15 @@ struct ExecutorInitFunctor {
                     ? std::make_shared<gko::experimental::mpi::communicator>(
                           MPI_COMM_WORLD, force_host_buffer)
                     : NULL),
-          device_id_((Pstream::parRun()) ? comm_->rank() / gpus_per_rank : 0),
+          device_id_( comm_->rank() / gpus_per_rank ),
           executor_name_(executor_name),
           field_name_(field_name),
           verbose_(verbose)
     {}
 
-
     void update(std::shared_ptr<gko::Executor>) const {}
 
+
     const std::string not_compiled_tag = "not compiled";
     const gko::version_info version = gko::version_info::get();
     std::shared_ptr<gko::Executor> init() const
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
new file mode 100644
index 000000000..c9bcab210
--- /dev/null
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -0,0 +1,91 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::IOSortingIdxHandler
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    CommunicationPattern.H
+
+\*---------------------------------------------------------------------------*/
+
+#include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
+#include "common/common.H"
+
+std::ostream& operator<<(std::ostream& out, const CommunicationPattern& e) {
+    out << "CommunicationPattern: for rank: " << e.comm.rank(); 
+    out << " {"; 
+    out << "\ntarget_ids: " << e.target_ids;
+    out << "\ntarget_sizes: " << e.target_sizes;
+    out << "}\n";
+    return out;
+}
+
+CommunicationPattern repartition_comm_pattern(
+    label ranks_per_gpu, 
+    CommunicationPattern &src_comm_pattern)
+{
+    using comm_size_type = label;
+
+    if (ranks_per_gpu == 1) {
+        return CommunicationPattern(src_comm_pattern);
+    }
+
+
+    auto exec = src_comm_pattern.target_ids.get_executor();
+    auto target_rank = [ranks_per_gpu](label src_rank, label dst_rank) {
+        // all ranks 0 .. ranks_per_gpu go to 0
+        return src_rank % ranks_per_gpu;
+    };
+
+    // find ranks which are now local
+    std::vector<label> repart_target_ids{};
+    std::vector<label> repart_target_sizes{};
+    std::vector<std::pair<gko::array<label>, comm_size_type>> repart_send_idxs;
+    label procs_ctr{0};
+    label rank {src_comm_pattern.comm.rank()};
+    for (int i = 0; i < src_comm_pattern.target_ids.get_size(); i++) {
+        label target_id = src_comm_pattern.target_ids.get_data()[i];
+        // check if communication partner is local after repartitioning
+        // if it is not local we keep it
+        if (target_rank(rank, target_id) != rank ) {
+            repart_target_ids.push_back(target_id);
+            repart_target_sizes.push_back(
+                src_comm_pattern.target_sizes.get_data()[i]);
+            repart_send_idxs.push_back(src_comm_pattern.send_idxs[i]);
+            procs_ctr++;
+        } else {
+            // store reparted ranks
+            std::cout <<  "!!! reparted rank [" << rank << "] " <<  src_comm_pattern.target_ids.get_data()[i] << "\n";
+
+        }
+        // we also have to get the communication partner from ranks that are now local
+    }
+
+    gko::array<comm_size_type> target_ids{exec, repart_target_ids.begin(),
+                                          repart_target_ids.end()};
+    gko::array<comm_size_type> target_sizes{exec, repart_target_sizes.begin(),
+                                            repart_target_sizes.end()};
+
+    auto ret =  CommunicationPattern{src_comm_pattern.comm, target_ids, target_sizes, repart_send_idxs};
+    std::cout << ret;
+    return ret;
+}
+
+
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 2c369d7de..1fb5a002e 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -26,9 +26,15 @@ SourceFiles
 \*---------------------------------------------------------------------------*/
 
 #pragma once
+#include <ginkgo/ginkgo.hpp>
+#include "fvCFD.H"
+
 
 struct CommunicationPattern {
-    using comm_size_type = int;
+    using comm_size_type = label;
+
+    gko::experimental::mpi::communicator comm;
+
     // an array storing to which rank to communicate
     gko::array<comm_size_type> target_ids;
 
@@ -40,3 +46,14 @@ struct CommunicationPattern {
     // are owned by the interface and the corresponding target rank
     std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
 };
+
+std::ostream& operator<<(std::ostream& out, const CommunicationPattern& e); 
+
+// TODO NOTE currently we have just one communication pattern for multiple proc
+// interfaces
+//  hence this will communicate vectors which are too long, FIXME potentially
+//  have the wrong idxs and will communictate the same remote buffer several
+//  times through the combination matrix.
+CommunicationPattern repartition_comm_pattern(
+    label ranks_per_gpu,
+    CommunicationPattern &src_comm_pattern);
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 1a8719f42..e7dea7d14 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -1,8 +1,36 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::IOSortingIdxHandler
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    CommunicationPattern.H
+
+\*---------------------------------------------------------------------------*/
+
 #include <ginkgo/core/distributed/lin_op.hpp>
 #include <ginkgo/ginkgo.hpp>
 
 
 #include "MatrixWrapper/LDUMatrix/HostMatrix.H"
+#include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
 
 
 template <typename DistMtxType>
@@ -134,12 +162,15 @@ public:
 
     std::shared_ptr<part_type> get_partition() { return this->partition_; }
 
+
     static std::shared_ptr<RepartDistMatrix> create(
         word matrix_format, std::shared_ptr<const HostMatrixWrapper> host_A)
     {
         auto exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_device_exec();
         auto comm = *exec_handler.get_communicator().get();
+        // TODO FIXME
+        label ranks_per_gpu = 2;
 
         // repartition things here first by creating device_matrix_data on
         // device with correct size on each rank
@@ -149,8 +180,8 @@ public:
         auto non_local_sparsity_ = host_A->compute_non_local_sparsity(exec);
 
         // create original communicator pattern
-        auto src_comm_pattern = host_A.get()->create_communication_pattern();
-        // auto dst_comm_pattern = repartition_comm_pattern(src_comm_pattern);
+        auto src_comm_pattern = host_A.get()->create_communication_pattern(comm);
+        auto dst_comm_pattern = repartition_comm_pattern(ranks_per_gpu, src_comm_pattern);
 
         // // if accumulate_ranks
         // // 1. move to shared buffer
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index cf2c56104..80f43eefc 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -159,7 +159,9 @@ void neg_interface_iterator(const lduInterfaceFieldPtrsList &interfaces,
 }
 
 
-CommunicationPattern HostMatrixWrapper::create_communication_pattern() const
+CommunicationPattern HostMatrixWrapper::create_communication_pattern(
+    const gko::experimental::mpi::communicator& comm
+    ) const
 {
     using comm_size_type = CommunicationPattern::comm_size_type;
     // temp map, mapping from neighbour rank interface cells
@@ -207,7 +209,7 @@ CommunicationPattern HostMatrixWrapper::create_communication_pattern() const
         iter++;
     }
 
-    return CommunicationPattern{target_ids, target_sizes, send_idxs};
+    return CommunicationPattern{comm, target_ids, target_sizes, send_idxs};
 }
 
 
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index ca0ddc410..2da21f125 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -278,8 +278,6 @@ private:
 
     const FieldField<Field, scalar> &interfaceBouCoeffs_;
 
-    const CommunicationPattern communication_pattern_;
-
     /* Iterates all interfaces and collects the coefficients into a vector
     **
     ** @param local whether local or non local coefficients should be collected
@@ -401,7 +399,9 @@ public:
      ** @param interfaces The list of interfaces for the search
      ** @return the CommunicationPattern
      */
-    CommunicationPattern create_communication_pattern() const;
+    CommunicationPattern create_communication_pattern(
+        const gko::experimental::mpi::communicator& comm
+            ) const;
 
 
     bool get_verbose() const { return verbose_; }
diff --git a/common/common.C b/common/common.C
index 9f7ec3650..33b1b3605 100644
--- a/common/common.C
+++ b/common/common.C
@@ -157,6 +157,16 @@ label get_solve_prev_iters(word sys_matrix_name, const objectRegistry &db,
     return get_gko_solver_property(sys_matrix_name, iters_name, db, label(1));
 }
 
+std::ostream& operator<<(std::ostream& out, const gko::array<label>& e){
+    out << " array: [";
+    for (int i = 0; i < e.get_size(); i++) {
+        out << e.get_const_data()[i] << " ";
+
+    }
+    out << "]";
+    return out;
+} 
+
 std::ostream &operator<<(std::ostream &os,
                          const std::shared_ptr<gko::matrix::Dense<scalar>> in)
 {
diff --git a/common/common.H b/common/common.H
index f0cf66a57..b319329f7 100644
--- a/common/common.H
+++ b/common/common.H
@@ -92,6 +92,8 @@ namespace Foam {
 
 #define OGL_NOT_IMPLEMENTED GKO_NOT_IMPLEMENTED
 
+std::ostream& operator<<(std::ostream& out, const gko::array<label>& e); 
+
 std::ostream &operator<<(
     std::ostream &os,
     const std::shared_ptr<gko::matrix::Dense<scalar>> array_in);
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index c20490c4f..7d55c697e 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -323,6 +323,10 @@ public:
                                   const scalarField &source,
                                   const direction cmpt = 0) const
     {
+        if (!Pstream::parRun()) {
+            FatalErrorInFunction << "Only parallel runs are supported for OGL"
+                                 << exit(FatalError);
+        }
         auto exec_handler = host_A_.get()->get_exec_handler();
         // --- Setup class containing solver performance data
         solverPerformance solverPerf(
@@ -331,15 +335,10 @@ public:
             field_name_);
 
         // Solve system
-        if (Pstream::parRun()) {
-            TIME_WITH_FIELDNAME(
-                verbose_, solve_multi_gpu, field_name_,
-                auto res = solve_multi_gpu_impl(exec_handler, psi, source, solverPerf);)
-            return res;
-        } else {
-            FatalErrorInFunction << "Only parallel runs are supported for OGL"
-                                 << exit(FatalError);
-        }
+        TIME_WITH_FIELDNAME(
+            verbose_, solve_multi_gpu, field_name_,
+            auto res = solve_multi_gpu_impl(exec_handler, psi, source, solverPerf);)
+        return res;
 
         return solverPerf;
     };

From a0513a76137bf8f8722e7f6f256f73838fcca964 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 24 Apr 2024 12:11:14 +0200
Subject: [PATCH 44/90] move OpenFOAM detection to separate cmake file

---
 CMakeLists.txt           | 13 +++----------
 cmake/OpenFOAM.cmake     | 21 +++++++++++++++++++++
 unitTests/CMakeLists.txt | 13 ++++---------
 3 files changed, 28 insertions(+), 19 deletions(-)
 create mode 100644 cmake/OpenFOAM.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c77803271..4ee728109 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,6 +20,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 # Configuration options
 include(cmake/build_type_helpers.cmake)
+include(cmake/OpenFOAM.cmake)
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/")
 
 include(CheckIncludeFileCXX)
@@ -205,17 +206,9 @@ enable_sanitizers(
   ${OGL_ENABLE_SANITIZE_UB} ${OGL_ENABLE_SANITIZE_THREAD}
   ${OGL_ENABLE_SANITIZE_MEMORY})
 
-target_include_directories(
-  OGL SYSTEM
-  PUBLIC $ENV{FOAM_SRC}/finiteVolume/lnInclude
-         $ENV{FOAM_SRC}/meshTools/lnInclude $ENV{FOAM_SRC}/OpenFOAM/lnInclude
-         $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude ${CMAKE_CURRENT_SOURCE_DIR})
+target_include_directories(OGL SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
-target_link_libraries(
-  OGL
-  PRIVATE $ENV{FOAM_LIBBIN}/libOpenFOAM.so $ENV{FOAM_LIBBIN}/libfiniteVolume.so
-          $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so Ginkgo::ginkgo
-          stdc++fs)
+target_link_libraries(OGL PRIVATE OpenFOAM Ginkgo::ginkgo stdc++fs)
 
 if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)
diff --git a/cmake/OpenFOAM.cmake b/cmake/OpenFOAM.cmake
new file mode 100644
index 000000000..78ee6cba6
--- /dev/null
+++ b/cmake/OpenFOAM.cmake
@@ -0,0 +1,21 @@
+add_library(OpenFOAM SHARED IMPORTED)
+
+target_include_directories(
+  OpenFOAM
+  INTERFACE $ENV{FOAM_SRC}/finiteVolume/lnInclude
+            $ENV{FOAM_SRC}/meshTools/lnInclude
+            $ENV{FOAM_SRC}/OpenFOAM/lnInclude
+            $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude)
+if(APPLE)
+  set_target_properties(
+    OpenFOAM
+    PROPERTIES IMPORTED_LOCATION $ENV{FOAM_LIBBIN}/libOpenFOAM.dylib
+               $ENV{FOAM_LIBBIN}/libfiniteVolume.dylib
+               $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.dylib)
+else()
+  set_target_properties(
+    OpenFOAM
+    PROPERTIES IMPORTED_LOCATION $ENV{FOAM_LIBBIN}/libOpenFOAM.so
+               $ENV{FOAM_LIBBIN}/libfiniteVolume.so
+               $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so)
+endif()
diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 57d05a909..b07d436b1 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -18,22 +18,17 @@ add_executable(matrixConversion "test_HostMatrix.C")
 find_package(MPI REQUIRED)
 
 target_include_directories(
-    matrixConversion SYSTEM
+  matrixConversion SYSTEM
   PUBLIC $ENV{FOAM_SRC}/finiteVolume/lnInclude
          $ENV{FOAM_SRC}/meshTools/lnInclude $ENV{FOAM_SRC}/OpenFOAM/lnInclude
          $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude ${CMAKE_CURRENT_SOURCE_DIR})
 
-target_compile_definitions(matrixConversion PUBLIC WM_LABEL_SIZE=32 WM_ARCH_OPTION=64
-                              NoRepository WM_DP)
+target_compile_definitions(
+  matrixConversion PUBLIC WM_LABEL_SIZE=32 WM_ARCH_OPTION=64 NoRepository WM_DP)
 
 target_link_libraries(
   matrixConversion
-  PUBLIC GTest::gtest_main
-         $ENV{FOAM_LIBBIN}/libOpenFOAM.so
-         $ENV{FOAM_LIBBIN}/libfiniteVolume.so
-         $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so
-         OGL
-         Ginkgo::ginkgo
+  PUBLIC GTest::gtest_main OGL OpenFOAM Ginkgo::ginkgo
          # MPI::MPI_CXX
          ${CMAKE_DL_LIBS})
 

From 772947f024229d673fd68f97d62c750f01df583b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 24 Apr 2024 13:33:25 +0200
Subject: [PATCH 45/90] fix mac related compilation issues

---
 CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4ee728109..7e2a1295f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -92,7 +92,7 @@ add_subdirectory(third_party)
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0 -ggdb")
 set(CMAKE_CXX_FLAGS_RELEASE
     "${CMAKE_CXX_FLAGS_RELEASE} \
-  -Wall -Wpedantic -Wextra -march=native -fopenmp -fPIC -Wno-undefined-var-template"
+  -Wall -Wpedantic -Wextra -Wextramarch=native -fopenmp -fPIC -Wno-undefined-var-template"
 )
 
 if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
@@ -143,6 +143,7 @@ if(APPLE)
   target_compile_definitions(OGL PRIVATE APPLE arm64)
 else()
   target_compile_definitions(OGL PRIVATE linux64)
+  target_link_libraries(OGL PRIVATE stdc++fs)
 endif()
 
 # Add a custom command that produces version.cpp, plus a dummy output that's not
@@ -208,7 +209,7 @@ enable_sanitizers(
 
 target_include_directories(OGL SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
-target_link_libraries(OGL PRIVATE OpenFOAM Ginkgo::ginkgo stdc++fs)
+target_link_libraries(OGL PRIVATE OpenFOAM Ginkgo::ginkgo)
 
 if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)

From d5ec320ffe082f9f42f69ebb9e10abef4a4ef890 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 2 May 2024 11:32:17 +0200
Subject: [PATCH 46/90] Implement repartitioning of comm pattern

---
 CMakeLists.txt                                |   2 +-
 .../CommunicationPattern.C                    | 251 +++++++++++++++---
 .../CommunicationPattern.H                    |   9 +-
 MatrixWrapper/Distributed/Distributed.H       | 111 ++++----
 MatrixWrapper/LDUMatrix/HostMatrix.H          |   6 +-
 common/common.C                               |  12 +
 common/common.H                               |  15 ++
 lduLduBase/lduLduBase.H                       |   2 +-
 8 files changed, 320 insertions(+), 88 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7e2a1295f..026fc00c9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -92,7 +92,7 @@ add_subdirectory(third_party)
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0 -ggdb")
 set(CMAKE_CXX_FLAGS_RELEASE
     "${CMAKE_CXX_FLAGS_RELEASE} \
-  -Wall -Wpedantic -Wextra -Wextramarch=native -fopenmp -fPIC -Wno-undefined-var-template"
+  -Wall -Wpedantic -Wextra -Wextra -march=native -fopenmp -fPIC -Wno-undefined-var-template"
 )
 
 if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index c9bcab210..9cc5fa6d9 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -28,18 +28,42 @@ SourceFiles
 #include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
 #include "common/common.H"
 
-std::ostream& operator<<(std::ostream& out, const CommunicationPattern& e) {
-    out << "CommunicationPattern: for rank: " << e.comm.rank(); 
-    out << " {"; 
+std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e)
+{
+    out << "CommunicationPattern: for rank: " << e.comm.rank();
+    out << " {";
     out << "\ntarget_ids: " << e.target_ids;
     out << "\ntarget_sizes: " << e.target_sizes;
     out << "}\n";
     return out;
 }
 
+template <typename T, typename Compare>
+std::vector<label> sort_permutation(const std::vector<T> &vec, Compare compare)
+{
+    std::vector<label> p(vec.size());
+    std::iota(p.begin(), p.end(), 0);
+    std::stable_sort(p.begin(), p.end(), [&](std::size_t i, std::size_t j) {
+        return compare(vec[i], vec[j]);
+    });
+    return p;
+}
+
+template <typename T>
+std::vector<T> apply_permutation(const std::vector<T> vec,
+                                 const std::vector<label> &p)
+{
+    std::vector<T> sorted_vec(vec.size());
+    std::transform(p.begin(), p.end(), sorted_vec.begin(),
+                   [&](label i) { return vec[i]; });
+    return sorted_vec;
+}
+
 CommunicationPattern repartition_comm_pattern(
-    label ranks_per_gpu, 
-    CommunicationPattern &src_comm_pattern)
+    label ranks_per_gpu, CommunicationPattern &src_comm_pattern,
+    std::shared_ptr<
+        const gko::experimental::distributed::Partition<label, label>>
+        partition)
 {
     using comm_size_type = label;
 
@@ -47,45 +71,204 @@ CommunicationPattern repartition_comm_pattern(
         return CommunicationPattern(src_comm_pattern);
     }
 
-
     auto exec = src_comm_pattern.target_ids.get_executor();
-    auto target_rank = [ranks_per_gpu](label src_rank, label dst_rank) {
-        // all ranks 0 .. ranks_per_gpu go to 0
-        return src_rank % ranks_per_gpu;
+    auto target_rank = [ranks_per_gpu](label rank) {
+        return rank - (rank % ranks_per_gpu);
     };
 
+    auto comm = src_comm_pattern.comm;
+
     // find ranks which are now local
-    std::vector<label> repart_target_ids{};
-    std::vector<label> repart_target_sizes{};
-    std::vector<std::pair<gko::array<label>, comm_size_type>> repart_send_idxs;
-    label procs_ctr{0};
-    label rank {src_comm_pattern.comm.rank()};
+    // the target_ids, target_sizes etc need to be communicated
+    // with new owner. thus we use maps to store them where the index
+    // is the new owner.
+    std::vector<label> target_ids{};
+    std::vector<label> target_sizes{};
+    std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
+    label rank{src_comm_pattern.comm.rank()};
+    label owner_rank = target_rank(rank);
+    bool owner = rank == owner_rank;
+
+    // map for exchanging communication data where the index is the target rank
     for (int i = 0; i < src_comm_pattern.target_ids.get_size(); i++) {
         label target_id = src_comm_pattern.target_ids.get_data()[i];
-        // check if communication partner is local after repartitioning
-        // if it is not local we keep it
-        if (target_rank(rank, target_id) != rank ) {
-            repart_target_ids.push_back(target_id);
-            repart_target_sizes.push_back(
-                src_comm_pattern.target_sizes.get_data()[i]);
-            repart_send_idxs.push_back(src_comm_pattern.send_idxs[i]);
-            procs_ctr++;
-        } else {
-            // store reparted ranks
-            std::cout <<  "!!! reparted rank [" << rank << "] " <<  src_comm_pattern.target_ids.get_data()[i] << "\n";
+        // Step 1. Check if communication partner is non-local after
+        // repartitioning. If it is non-local we keep it. Otherwise local
+        // communication can be discarded or merged. Here non-local means: the
+        // communication target proc id is != repartition proc id
+        //
+        // [ 1 | 2 | 3 ] [ 4 | 5 | 6 ] where [] - bounds after repart
+        //                                    | - bounds before repart
+        // e.g. rank 3 has rank 2 and 4 as comm neighbours
+        //      rank 2 can be discarded but rank 4 is non-local and needs
+        //      to be communicated to rank 1 to be merged into the new
+        //      communicaton pattern. This needs an all to all communication.
+
+        label repart_target_id = target_rank(target_id);
+        label target_size = src_comm_pattern.target_sizes.get_const_data()[i];
+        auto send_idx = src_comm_pattern.send_idxs[i];
 
+        if (repart_target_id != owner_rank) {
+            // communication pattern is non local, hence we keep it
+            // after repartitioning we now have to communicate with a different
+            // rank
+            // store new owner rank to which repart_target_* needs to be send to
+            target_ids.push_back(repart_target_id);
+            target_sizes.push_back(target_size);
+            send_idxs.push_back(send_idx);
         }
-        // we also have to get the communication partner from ranks that are now local
     }
 
-    gko::array<comm_size_type> target_ids{exec, repart_target_ids.begin(),
-                                          repart_target_ids.end()};
-    gko::array<comm_size_type> target_sizes{exec, repart_target_sizes.begin(),
-                                            repart_target_sizes.end()};
+    label total_ranks{comm.size()};
+    std::vector<label> recv_elements(ranks_per_gpu, 0);
+    // where to put elements from recv
+    std::vector<label> recv_offsets(total_ranks, 0);
+    std::vector<label> recv_counts(total_ranks, 0);
+    std::vector<label> send_counts(total_ranks, 0);
 
-    auto ret =  CommunicationPattern{src_comm_pattern.comm, target_ids, target_sizes, repart_send_idxs};
-    std::cout << ret;
-    return ret;
-}
+    // communicate with all neighbours which are to be merged
+    // how many elements the owner rank is to receive
+    label tot_recv_elements{0};
+    label comm_elements_buffer{0};
+    if (owner) {
+        for (int i = 1; i < ranks_per_gpu; i++) {
+            comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
+            tot_recv_elements += comm_elements_buffer;
+            recv_offsets[rank + i] = tot_recv_elements;
+            recv_counts[rank + i] = comm_elements_buffer;
+        }
+        send_counts[owner_rank] = target_ids.size();
+        recv_counts[owner_rank] = target_ids.size();
+    } else {
+        comm_elements_buffer = target_ids.size();
+        comm.send(exec, &comm_elements_buffer, 1, owner_rank, owner_rank);
+        tot_recv_elements += comm_elements_buffer;
+        send_counts[owner_rank] = tot_recv_elements;
+    }
+
+    // all send buffer start at zero
+    std::vector<label> send_offsets(total_ranks, 0);
+    std::vector<label> target_recv_buffer(tot_recv_elements + 1);
+
+    comm.all_to_all_v(exec, target_ids.data(), send_counts.data(),
+                      send_offsets.data(), target_recv_buffer.data(),
+                      recv_counts.data(), recv_offsets.data());
+
+    target_ids = target_recv_buffer;
+
+    comm.all_to_all_v(exec, target_sizes.data(), send_counts.data(),
+                      send_offsets.data(), target_recv_buffer.data(),
+                      recv_counts.data(), recv_offsets.data());
+
+    target_sizes = target_recv_buffer;
+
+    // communicate the send indices to owner rank
+    // send indices are packed into std::vector of gko::arrays thus we send them
+    // one by one
+    // additionally we already have then target_sizes and send_counts and
+    // recv_counts which can be reused to for sending the index vectors
+
+    if (owner) {
+        // retrieve from all neighbours
+        for (int i = 1; i < ranks_per_gpu; i++) {
+            // retrieve all send_idxs
+            for (int j = 0; j < recv_counts[rank + i]; j++) {
+                auto target_size = target_sizes[j + 1];
+                auto target_id = target_ids[j + 1];
+                std::vector<label> recv_buffer(target_size);
 
+                comm.recv(exec, recv_buffer.data(), target_size, rank + i,
+                          rank);
 
+                // the new offset is
+                auto offset = partition->get_range_bounds()[rank + i] -
+                              partition->get_range_bounds()[rank];
+
+                std::transform(recv_buffer.begin(), recv_buffer.end(),
+                               recv_buffer.begin(),
+                               [&](label idx) { return idx + offset; });
+
+                send_idxs.push_back({gko::array<label>{
+                                         exec,
+                                         recv_buffer.begin(),
+                                         recv_buffer.end(),
+                                     },
+                                     target_id});
+            }
+        }
+    } else {
+        // send all comm pairs to new owner
+        auto send_count = send_counts[owner_rank];
+        for (int i = 0; i < send_count; i++) {
+            auto send_buffer = send_idxs[i].first;
+            comm.send(exec, send_buffer.get_const_data(),
+                      send_buffer.get_size(), owner_rank, owner_rank);
+        }
+    }
+
+
+    // clear communcation neighbors on non owning rank
+    if (!owner) {
+        target_ids.clear();
+        target_sizes.clear();
+        send_idxs.clear();
+    }
+
+    // Step 3. merge communication
+    // we now might have communication partners multiple times, thus we can
+    // merge them sort repart_target_ids and use the sorting for target_sizes
+    // and repart_send_idxs
+    std::vector<label> merged_target_ids{};
+    std::vector<label> merged_target_sizes{};
+    std::vector<std::vector<label>> merged_send_idxs{};
+    if (owner) {
+        auto p = sort_permutation(target_ids,
+                                  [](label a, label b) { return a < b; });
+
+        target_sizes = apply_permutation(target_sizes, p);
+        target_ids = apply_permutation(target_ids, p);
+        send_idxs = apply_permutation(send_idxs, p);
+
+        // Step 4.
+        // Merge communication pattern with corresponding neighbours
+        if (target_ids.size() > 1) {
+            merged_target_ids.push_back(target_ids[0]);
+            merged_target_sizes.push_back(target_sizes[0]);
+            merged_send_idxs.emplace_back(std::vector<label>(
+                send_idxs[0].first.get_data(),
+                send_idxs[0].first.get_data() + send_idxs[0].first.get_size()));
+        }
+
+        for (int i = 1; i < target_ids.size(); i++) {
+            if (target_ids[i] == merged_target_ids.back()) {
+                merged_target_sizes.back() += target_sizes[i];
+                merged_send_idxs.back().insert(
+                    merged_send_idxs.back().end(),
+                    send_idxs[i].first.get_data(),
+                    send_idxs[i].first.get_data() +
+                        send_idxs[i].first.get_size());
+            } else {
+                merged_target_ids.push_back(target_ids[i]);
+            }
+        }
+    }
+
+    // recompute send_idxs
+    send_idxs.clear();
+
+    for (int i = 0; i < merged_target_ids.size(); i++) {
+        label target_id = merged_target_ids[i];
+        send_idxs.emplace_back(std::pair<gko::array<label>, comm_size_type>{
+            gko::array<label>{exec, merged_send_idxs[i].begin(),
+                              merged_send_idxs[i].end()},
+            target_id});
+    }
+
+    return CommunicationPattern{
+        src_comm_pattern.comm,
+        gko::array<comm_size_type>{exec, merged_target_ids.begin(),
+                                   merged_target_ids.end()},
+        gko::array<comm_size_type>{exec, merged_target_sizes.begin(),
+                                   merged_target_sizes.end()},
+        send_idxs};
+}
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 1fb5a002e..01988e780 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -29,6 +29,8 @@ SourceFiles
 #include <ginkgo/ginkgo.hpp>
 #include "fvCFD.H"
 
+// TODO move this file to top level since it is not really a matrix-wrapper
+
 
 struct CommunicationPattern {
     using comm_size_type = label;
@@ -47,7 +49,7 @@ struct CommunicationPattern {
     std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
 };
 
-std::ostream& operator<<(std::ostream& out, const CommunicationPattern& e); 
+std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e);
 
 // TODO NOTE currently we have just one communication pattern for multiple proc
 // interfaces
@@ -55,5 +57,6 @@ std::ostream& operator<<(std::ostream& out, const CommunicationPattern& e);
 //  have the wrong idxs and will communictate the same remote buffer several
 //  times through the combination matrix.
 CommunicationPattern repartition_comm_pattern(
-    label ranks_per_gpu,
-    CommunicationPattern &src_comm_pattern);
+    label ranks_per_gpu, CommunicationPattern &src_comm_pattern,
+std::shared_ptr<const gko::experimental::distributed::Partition<label, label>> partition
+);
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index e7dea7d14..27267f6f2 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -29,8 +29,8 @@ SourceFiles
 #include <ginkgo/ginkgo.hpp>
 
 
-#include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 #include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
+#include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 
 
 template <typename DistMtxType>
@@ -162,7 +162,6 @@ public:
 
     std::shared_ptr<part_type> get_partition() { return this->partition_; }
 
-
     static std::shared_ptr<RepartDistMatrix> create(
         word matrix_format, std::shared_ptr<const HostMatrixWrapper> host_A)
     {
@@ -179,24 +178,34 @@ public:
         auto local_sparsity_ = host_A->compute_local_sparsity(exec);
         auto non_local_sparsity_ = host_A->compute_non_local_sparsity(exec);
 
+        auto orig_partition = gko::share(
+            gko::experimental::distributed::build_partition_from_local_size<
+                label, label>(exec, comm, local_sparsity_->dim[0]));
+
         // create original communicator pattern
-        auto src_comm_pattern = host_A.get()->create_communication_pattern(comm);
-        auto dst_comm_pattern = repartition_comm_pattern(ranks_per_gpu, src_comm_pattern);
-
-        // // if accumulate_ranks
-        // // 1. move to shared buffer
-        // // 2. switch interfaces to local_data
-        // // 3. modify communication pattern
-        // auto [local_sparsity_pattern_r, non_local_sparsity_pattern_r] =
-        //     repartition_indices(ranks_per_gpu, source_partition,
-        //                         local_sparsity_pattern,
-        //                         non_local_sparsity_pattern);
-        //
-        // auto [local_coeff_r, non_local_coeff_r] =
-        //     repartition_coeffs(ranks_per_gpu, source_partition,
-        //                        local_device_data, non_local_device_data);
-        //
-        //
+        auto src_comm_pattern =
+            host_A.get()->create_communication_pattern(comm);
+        // create partiton here and pass to constructor
+        auto dst_comm_pattern =
+            repartition_comm_pattern(ranks_per_gpu, src_comm_pattern, orig_partition);
+
+        auto repartition_sparsity = [](label, std::shared_ptr<SparsityPattern>,
+                                       std::shared_ptr<SparsityPattern>,
+                                       const CommunicationPattern &) {
+            // this is not implemented yet, but we can't fail here for debugging reasons
+            //FatalErrorInFunction << "Not implemented" << abort(FatalError);
+            return std::make_pair<std::shared_ptr<SparsityPattern>,
+                                  std::shared_ptr<SparsityPattern>>({}, {});
+        };
+
+        auto [repart_loc_sparsity, repart_non_loc_sparsity] =
+            repartition_sparsity(ranks_per_gpu, local_sparsity_,
+                                 non_local_sparsity_, dst_comm_pattern);
+
+        local_sparsity_ = repart_loc_sparsity;
+        non_local_sparsity_ = repart_non_loc_sparsity;
+
+
         auto comm_pattern = src_comm_pattern;
         const label local_nnz = local_sparsity_->size_;
         const label non_local_nnz = non_local_sparsity_->size_;
@@ -244,33 +253,43 @@ public:
         scalar *local_value_ptr{};
         scalar *non_local_value_ptr{};
 
-        if (matrix_format == "Csr") {
-            local_value_ptr =
-                (gko::matrix::Csr<scalar, label> *)dist_mtx_->get_local_matrix()
-                    ->get_values();
-
-            non_local_value_ptr = (gko::matrix::Csr<scalar, label> *)dist_mtx_
-                                      ->get_non_local_matrix()
-                                      ->get_values();
-        }
-        if (matrix_format == "Ell") {
-            local_value_ptr =
-                (gko::matrix::Ell<scalar, label> *)dist_mtx_->get_local_matrix()
-                    ->get_values();
-
-            non_local_value_ptr = (gko::matrix::Ell<scalar, label> *)dist_mtx_
-                                      ->get_non_local_matrix()
-                                      ->get_values();
-        }
-        if (matrix_format == "Coo") {
-            local_value_ptr =
-                (gko::matrix::Coo<scalar, label> *)dist_mtx_->get_local_matrix()
-                    ->get_values();
-
-            non_local_value_ptr = (gko::matrix::Coo<scalar, label> *)dist_mtx_
-                                      ->get_non_local_matrix()
-                                      ->get_values();
-        }
+        FatalErrorInFunction << "Not implemented" << abort(FatalError);
+        // TODO local matrix is now of type Combination<Csr>
+        // Combination should get an update(gko::array<> coeffs function)
+        //
+        // if (matrix_format == "Csr") {
+        //     local_value_ptr =
+        //         (gko::matrix::Csr<scalar, label>
+        //         *)(dist_mtx_->get_local_matrix())
+        //             ->get_values();
+
+        //     non_local_value_ptr = (gko::matrix::Csr<scalar, label>
+        //     *)dist_mtx_
+        //                               ->get_non_local_matrix()
+        //                               ->get_values();
+        // }
+        // if (matrix_format == "Ell") {
+        //     local_value_ptr =
+        //         (gko::matrix::Ell<scalar, label>
+        //         *)dist_mtx_->get_local_matrix()
+        //             ->get_values();
+
+        //     non_local_value_ptr = (gko::matrix::Ell<scalar, label>
+        //     *)dist_mtx_
+        //                               ->get_non_local_matrix()
+        //                               ->get_values();
+        // }
+        // if (matrix_format == "Coo") {
+        //     local_value_ptr =
+        //         (gko::matrix::Coo<scalar, label>
+        //         *)dist_mtx_->get_local_matrix()
+        //             ->get_values();
+
+        //     non_local_value_ptr = (gko::matrix::Coo<scalar, label>
+        //     *)dist_mtx_
+        //                               ->get_non_local_matrix()
+        //                               ->get_values();
+        // }
 
         const label local_nnz = local_sparsity_->size_;
         const label non_local_nnz = non_local_sparsity_->size_;
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 2da21f125..7b71ff1c4 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -44,9 +44,9 @@ namespace Foam {
 struct SparsityPattern {
     SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size)
         : size_(size),
-          col_idxs{exec, size},
-          row_idxs{exec, size},
-          ldu_mapping{exec, size}
+          col_idxs{exec, static_cast<gko::size_type>(size)},
+          row_idxs{exec, static_cast<gko::size_type>(size)},
+          ldu_mapping{exec, static_cast<gko::size_type>(size)}
     {}
 
     // TODO rename the public members and remove trailing _
diff --git a/common/common.C b/common/common.C
index 33b1b3605..ff6dacf4f 100644
--- a/common/common.C
+++ b/common/common.C
@@ -157,6 +157,16 @@ label get_solve_prev_iters(word sys_matrix_name, const objectRegistry &db,
     return get_gko_solver_property(sys_matrix_name, iters_name, db, label(1));
 }
 
+std::ostream& operator<<(std::ostream& out, const std::vector<label>& e){
+    out << " vector: [";
+    for (int i = 0; i < e.size(); i++) {
+        out << e[i] << " ";
+
+    }
+    out << "]";
+    return out;
+}
+
 std::ostream& operator<<(std::ostream& out, const gko::array<label>& e){
     out << " array: [";
     for (int i = 0; i < e.get_size(); i++) {
@@ -191,4 +201,6 @@ std::ostream &operator<<(std::ostream &os,
     }
     return os;
 }
+
+
 }  // namespace Foam
diff --git a/common/common.H b/common/common.H
index b319329f7..68a23808c 100644
--- a/common/common.H
+++ b/common/common.H
@@ -92,12 +92,27 @@ namespace Foam {
 
 #define OGL_NOT_IMPLEMENTED GKO_NOT_IMPLEMENTED
 
+std::ostream& operator<<(std::ostream& out, const std::vector<label>& e);
+
 std::ostream& operator<<(std::ostream& out, const gko::array<label>& e); 
 
 std::ostream &operator<<(
     std::ostream &os,
     const std::shared_ptr<gko::matrix::Dense<scalar>> array_in);
 
+
+// template <typename T, typename Compare>
+// std::vector<std::size_t> sort_permutation(
+//     const std::vector<T>& vec,
+//     Compare compare);
+
+// template <typename T>
+// std::vector<T> apply_permutation(
+//     const std::vector<T>& vec,
+//     const std::vector<std::size_t>& p);
+
+
+// TODO move to somewhere else
 void export_system(const word fieldName, const gko::matrix::Csr<scalar> *A,
                    const gko::matrix::Dense<scalar> *x,
                    const gko::matrix::Dense<scalar> *b, const word time);
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 7d55c697e..54f509ca7 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -139,7 +139,7 @@ private:
 
     const label verbose_;
 
-    const ExecutorHandler& exec_handler_;
+    const ExecutorHandler exec_handler_;
 
     // A thin wrapper around OpenFOAMs LDU matrix class
     PersistentHostMatrixAdapter host_A_;

From 3b3551d415786a8faf252e166124f2ac6d998634 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 9 May 2024 14:53:49 +0200
Subject: [PATCH 47/90] wip repartition sparsity

---
 MatrixWrapper/Combination/Combination.H       |   2 +
 .../CommunicationPattern.C                    |  70 ++--
 .../CommunicationPattern.H                    |  13 +-
 MatrixWrapper/Distributed/Distributed.H       | 341 ++++++++++++++++--
 MatrixWrapper/LDUMatrix/HostMatrix.C          |  61 ++--
 MatrixWrapper/LDUMatrix/HostMatrix.H          |  52 ++-
 common/common.C                               |   2 +
 common/common.H                               |   6 +
 8 files changed, 455 insertions(+), 92 deletions(-)

diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index ff3a2847b..03a718068 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -33,10 +33,12 @@ public:
 
     void read(const device_matrix_data &data) override
     {
+        std::cout << __FILE__ << " read combination \n";
         auto exec = this->get_executor();
         auto data_exec = data.get_executor();
 
         for (auto &interface : interface_ranges_) {
+            std::cout << __FILE__ << " read combination  interface \n";
             auto length = interface.length();
             auto offset = interface.begin;
             auto op = gko::share(InnerMatrixType::create(exec));
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index 9cc5fa6d9..c7a2b0063 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -28,6 +28,44 @@ SourceFiles
 #include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
 #include "common/common.H"
 
+std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+           std::vector<label>, std::vector<label>>
+compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
+                         const gko::experimental::mpi::communicator &comm,
+                         std::shared_ptr<const gko::Executor> exec)
+{
+    label total_ranks{comm.size()};
+    label rank{comm.rank()};
+    std::vector<label> send_counts(total_ranks, 0);
+    std::vector<label> recv_counts(total_ranks, 0);
+    std::vector<label> send_offsets(total_ranks, 0);
+    std::vector<label> recv_offsets(total_ranks, 0);
+
+    label tot_recv_elements{0};
+    label comm_elements_buffer{0};
+    if (rank == owner_rank) {
+        // send and recv to it self
+        send_counts[owner_rank] = size;
+        recv_counts[owner_rank] = size;
+        tot_recv_elements = size;
+
+        for (int i = 1; i < ranks_per_gpu; i++) {
+            comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
+            recv_offsets[rank + i] = tot_recv_elements;
+            recv_counts[rank + i] = comm_elements_buffer;
+            tot_recv_elements += comm_elements_buffer;
+        }
+    } else {
+        comm.send(exec, &size, 1, owner_rank, owner_rank);
+        send_counts[owner_rank] = size;
+    }
+
+    std::vector<label> target_recv_buffer(tot_recv_elements, 0);
+
+    return std::make_tuple(send_counts, recv_counts, send_offsets, recv_offsets,
+                           target_recv_buffer);
+}
+
 std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e)
 {
     out << "CommunicationPattern: for rank: " << e.comm.rank();
@@ -119,36 +157,14 @@ CommunicationPattern repartition_comm_pattern(
         }
     }
 
-    label total_ranks{comm.size()};
-    std::vector<label> recv_elements(ranks_per_gpu, 0);
     // where to put elements from recv
-    std::vector<label> recv_offsets(total_ranks, 0);
-    std::vector<label> recv_counts(total_ranks, 0);
-    std::vector<label> send_counts(total_ranks, 0);
-
     // communicate with all neighbours which are to be merged
     // how many elements the owner rank is to receive
-    label tot_recv_elements{0};
-    label comm_elements_buffer{0};
-    if (owner) {
-        for (int i = 1; i < ranks_per_gpu; i++) {
-            comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
-            tot_recv_elements += comm_elements_buffer;
-            recv_offsets[rank + i] = tot_recv_elements;
-            recv_counts[rank + i] = comm_elements_buffer;
-        }
-        send_counts[owner_rank] = target_ids.size();
-        recv_counts[owner_rank] = target_ids.size();
-    } else {
-        comm_elements_buffer = target_ids.size();
-        comm.send(exec, &comm_elements_buffer, 1, owner_rank, owner_rank);
-        tot_recv_elements += comm_elements_buffer;
-        send_counts[owner_rank] = tot_recv_elements;
-    }
+    auto [send_counts, recv_counts, send_offsets, recv_offsets,
+          target_recv_buffer] =
+        compute_send_recv_counts(ranks_per_gpu, owner_rank, target_ids.size(),
+                                 comm, exec);
 
-    // all send buffer start at zero
-    std::vector<label> send_offsets(total_ranks, 0);
-    std::vector<label> target_recv_buffer(tot_recv_elements + 1);
 
     comm.all_to_all_v(exec, target_ids.data(), send_counts.data(),
                       send_offsets.data(), target_recv_buffer.data(),
@@ -168,6 +184,8 @@ CommunicationPattern repartition_comm_pattern(
     // additionally we already have then target_sizes and send_counts and
     // recv_counts which can be reused to for sending the index vectors
 
+    // TODO this could be refactored to separtate function
+    // this is also used distributed
     if (owner) {
         // retrieve from all neighbours
         for (int i = 1; i < ranks_per_gpu; i++) {
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 01988e780..6ae08a363 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -32,6 +32,14 @@ SourceFiles
 // TODO move this file to top level since it is not really a matrix-wrapper
 
 
+// This function computes the send and recv counts vectors for repartitioning
+//
+std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+           std::vector<label>, std::vector<label>>
+compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
+                         const gko::experimental::mpi::communicator &comm,
+                         std::shared_ptr<const gko::Executor> exec);
+
 struct CommunicationPattern {
     using comm_size_type = label;
 
@@ -58,5 +66,6 @@ std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e);
 //  times through the combination matrix.
 CommunicationPattern repartition_comm_pattern(
     label ranks_per_gpu, CommunicationPattern &src_comm_pattern,
-std::shared_ptr<const gko::experimental::distributed::Partition<label, label>> partition
-);
+    std::shared_ptr<
+        const gko::experimental::distributed::Partition<label, label>>
+        partition);
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 27267f6f2..c4408500f 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -36,7 +36,8 @@ SourceFiles
 template <typename DistMtxType>
 std::unique_ptr<DistMtxType> generate_dist_mtx_with_inner_type(
     word matrix_format, std::shared_ptr<gko::Executor> exec,
-    std::shared_ptr<gko::experimental::distributed::sparse_communicator> comm,
+    std::shared_ptr<const gko::experimental::distributed::sparse_communicator>
+        comm,
     std::shared_ptr<const SparsityPattern> local_sparsity,
     std::shared_ptr<const SparsityPattern> non_local_sparsity)
 {
@@ -168,7 +169,7 @@ public:
         auto exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_device_exec();
         auto comm = *exec_handler.get_communicator().get();
-        // TODO FIXME
+        // TODO FIXME read from dictionary
         label ranks_per_gpu = 2;
 
         // repartition things here first by creating device_matrix_data on
@@ -186,34 +187,311 @@ public:
         auto src_comm_pattern =
             host_A.get()->create_communication_pattern(comm);
         // create partiton here and pass to constructor
-        auto dst_comm_pattern =
-            repartition_comm_pattern(ranks_per_gpu, src_comm_pattern, orig_partition);
-
-        auto repartition_sparsity = [](label, std::shared_ptr<SparsityPattern>,
-                                       std::shared_ptr<SparsityPattern>,
-                                       const CommunicationPattern &) {
-            // this is not implemented yet, but we can't fail here for debugging reasons
-            //FatalErrorInFunction << "Not implemented" << abort(FatalError);
-            return std::make_pair<std::shared_ptr<SparsityPattern>,
-                                  std::shared_ptr<SparsityPattern>>({}, {});
-        };
+        auto dst_comm_pattern = repartition_comm_pattern(
+            ranks_per_gpu, src_comm_pattern, orig_partition);
+
+        label rank{src_comm_pattern.comm.rank()};
+
+        auto repartition_sparsity =
+            [ranks_per_gpu, src_comm_pattern, exec, orig_partition](
+                std::shared_ptr<SparsityPattern> src_local_pattern,
+                std::shared_ptr<SparsityPattern> src_non_local_pattern) {
+                // 1. obtain send recv sizes vector
+                // here we can reuse code from repartition_comm_pattern
+                //
+                // 2. initialize send and recv buffer
+                // if we keep interfaces separated row and col indices can be
+                //
+                // 3. ldu mapping needs to be update?
+                // 4. dim needs to be updated
+                // this is not implemented yet, but we can't fail here for
+                // debugging reasons
+                // FatalErrorInFunction << "Not implemented" <<
+                // abort(FatalError);
+                auto target_rank = [ranks_per_gpu](label rank) {
+                    return rank - (rank % ranks_per_gpu);
+                };
+
+
+                auto comm = src_comm_pattern.comm;
+
+                // find ranks which are now local
+                // the target_ids, target_sizes etc need to be communicated
+                // with new owner. thus we use maps to store them where the
+                // index is the new owner.
+                label rank{src_comm_pattern.comm.rank()};
+                label owner_rank = target_rank(rank);
+                bool owner = rank == owner_rank;
+
+                auto local_comm_pattern = compute_send_recv_counts(
+                    ranks_per_gpu, owner_rank, src_local_pattern->size_, comm,
+                    exec);
+
+                auto gather_to_owner = [&](label offset, label *data,
+                                           label size, auto &comm_pattern) {
+                    if (offset > 0) {
+                        std::transform(data, data + size, data,
+                                       [&](label idx) { return idx + offset; });
+                    }
+
+                    auto [send_counts, recv_counts, send_offsets, recv_offsets,
+                          recv_buffer] = comm_pattern;
+
+                    comm.all_to_all_v(exec, data, send_counts.data(),
+                                      send_offsets.data(), recv_buffer.data(),
+                                      recv_counts.data(), recv_offsets.data());
+
+                    return std::vector<label>{recv_buffer};
+                };
+
+
+                label offset = orig_partition->get_range_bounds()[rank] -
+                               orig_partition->get_range_bounds()[owner_rank];
+
+                std::vector<label> merged_local_rows = gather_to_owner(
+                    offset, src_local_pattern->row_idxs.get_data(),
+                    src_local_pattern->row_idxs.get_size(), local_comm_pattern);
+
+                std::vector<label> merged_local_cols = gather_to_owner(
+                    offset, src_local_pattern->col_idxs.get_data(),
+                    src_local_pattern->row_idxs.get_size(), local_comm_pattern);
+
+                std::vector<label> merged_ldu_mapping = gather_to_owner(
+                    0, src_local_pattern->ldu_mapping.get_data(),
+                    src_local_pattern->row_idxs.get_size(), local_comm_pattern);
+
+
+                label rows = (owner) ? merged_local_rows.back() + 1 : 0;
+                gko::dim<2> merged_local_dim{rows, rows};
+
+                std::vector<label> merged_begin{0};
+                std::vector<label> merged_end{ merged_local_rows.size()};
+                std::vector<label> merged_ranks{rank};
+
+                // TODO remove
+                // std::cout << __FILE__ << " on rank " << rank
+                //           << " merged_local_rows " << merged_local_rows
+                //            << " merged_local_cols " << merged_local_cols
+                //         << "\n";
+
+                auto non_local_comm_pattern = compute_send_recv_counts(
+                    ranks_per_gpu, owner_rank, src_non_local_pattern->size_,
+                    comm, exec);
+
+                std::vector<label> merged_non_local_rows = gather_to_owner(
+                    offset, src_non_local_pattern->row_idxs.get_data(),
+                    src_non_local_pattern->row_idxs.get_size(),
+                    non_local_comm_pattern);
+
+                std::vector<label> merged_non_local_cols = gather_to_owner(
+                    offset, src_non_local_pattern->col_idxs.get_data(),
+                    src_non_local_pattern->row_idxs.get_size(),
+                    non_local_comm_pattern);
+
+                std::vector<label> merged_non_local_ldu_mapping =
+                    gather_to_owner(
+                        0, src_non_local_pattern->ldu_mapping.get_data(),
+                        src_non_local_pattern->row_idxs.get_size(),
+                        non_local_comm_pattern);
+
+                // TODO remove
+                // std::cout << __FILE__ << " on rank " << rank
+                //           << " merged_non_local_rows" <<
+                //           merged_non_local_rows
+                //           << " merged_non_local_cols " <<
+                //           merged_non_local_cols
+                //           << "\n";
+
+                auto [span_send_counts, span_recv_counts, span_send_offsets,
+                      span_recv_offsets, span_recv_buffer] =
+                    compute_send_recv_counts(
+                        ranks_per_gpu, owner_rank,
+                        src_non_local_pattern->interface_spans.size(), comm,
+                        exec);
+
+                std::vector<label> spans_begin;
+                std::vector<label> spans_end;
+
+                for (auto elem : src_non_local_pattern->interface_spans) {
+                    spans_begin.push_back(elem.begin);
+                    spans_end.push_back(elem.end);
+                }
+
+                comm.all_to_all_v(
+                    exec, spans_begin.data(), span_send_counts.data(),
+                    span_send_offsets.data(), span_recv_buffer.data(),
+                    span_recv_counts.data(), span_recv_offsets.data());
+
+                std::vector<label> merged_non_local_begin{span_recv_buffer};
+
+                comm.all_to_all_v(
+                    exec, spans_end.data(), span_send_counts.data(),
+                    span_send_offsets.data(), span_recv_buffer.data(),
+                    span_recv_counts.data(), span_recv_offsets.data());
+                std::vector<label> merged_non_local_end{span_recv_buffer};
+
+                comm.all_to_all_v(
+                    exec, src_non_local_pattern->rank.data(),
+                    span_send_counts.data(), span_send_offsets.data(),
+                    span_recv_buffer.data(), span_recv_counts.data(),
+                    span_recv_offsets.data());
+                std::vector<label> merged_non_local_ranks{span_recv_buffer};
+
+                std::cout << __FILE__ << " on rank " << rank << " merged_begin"
+                          << merged_begin << " merged_end " << merged_end
+                          << " merged_ranks " << merged_non_local_ranks << "\n";
+
+                // TODO shorten this
+                std::vector<label> gathered_non_local_rows;
+                std::vector<label> gathered_non_local_cols;
+                std::vector<label> gathered_non_local_mapping;
+                std::vector<label> gathered_ranks;
+                std::vector<label> gathered_begin;
+                std::vector<label> gathered_end;
+                label merged_ranks_size = merged_non_local_ranks.size();
+                for (label i=0;i<merged_ranks_size;i++) {
+                    auto merged_target_rank = merged_non_local_ranks[i];
+                    label to_rank = target_rank(merged_target_rank);
+                    bool local = rank == to_rank;
+                    std::cout << __FILE__ << " on rank " << rank
+                              << " merged_target_rank " << merged_target_rank
+                              << " is local " << local << " target_rank "
+                              << to_rank << "\n";
+                    auto begin = merged_begin[i];
+                    auto end = merged_end[i];
+                    if (local) {
+                        label size_before = merged_local_rows.size();
+                        merged_local_rows.insert(
+                            merged_local_rows.end(),
+                            merged_non_local_rows.begin() + begin,
+                            merged_non_local_rows.begin() + end);
+                        merged_local_cols.insert(
+                            merged_local_cols.end(),
+                            merged_non_local_cols.begin() + begin,
+                            merged_non_local_cols.begin() + end);
+                        merged_ldu_mapping.insert(
+                            merged_ldu_mapping.end(),
+                            merged_non_local_ldu_mapping.data() + begin,
+                            merged_non_local_ldu_mapping.data() + end);
+
+                        label size_after = merged_local_rows.size();
+                        merged_begin.push_back(size_before);
+                        merged_end.push_back(size_after);
+                        merged_ranks.push_back(rank);
+                    } else {
+                        label size_before = gathered_non_local_rows.size();
+                        gathered_non_local_rows.insert(
+                            gathered_non_local_rows.end(),
+                            merged_non_local_rows.begin() + begin,
+                            merged_non_local_rows.begin() + end);
+                        gathered_non_local_cols.insert(
+                            gathered_non_local_cols.end(),
+                            merged_non_local_cols.begin() + begin,
+                            merged_non_local_cols.begin() + end);
+                        gathered_non_local_mapping.insert(
+                            gathered_non_local_mapping.end(),
+                            merged_non_local_cols.begin() + begin,
+                            merged_non_local_cols.begin() + end);
+                        label size_after = gathered_non_local_rows.size();
+                        std::cout << __FILE__ << " on rank " << rank
+                                << " non_local size_before " << size_before
+                                << " non_local size_after " << size_after
+                                << "\n";
+                        gathered_begin.push_back(size_before);
+                        gathered_end.push_back(size_after);
+                        gathered_ranks.push_back(to_rank);
+                    }
+                }
+
+                // add interfaces
+                // to add interfaces we go through the non_local_sparsity
+                // pattern and check if interface is still non_local
+                //
+                // iterate interfaces and send to owner.
+                // on owner decide to move to local or non_local
+                // for sending to owner we can also use the all_to_all_v
+                // approach we than have
+                //
+                // row [ 4 , 8,  12 | 16 , 17, 18 ] <- local row
+                // col [ 1 , 2, 3 | 1, 2, 3 ] <- just the interface ctr
+                // from repartition_comm_pattern we could get
+                //
+                // or we split local / non-local first
+                // for this we could store ranges when computing
+                // repartition_comm_pattern
+                //
+
+
+                if (owner) {
+                    auto new_local_spars_pattern =
+                        std::make_shared<SparsityPattern>(
+                            exec, merged_local_rows.size(), merged_local_dim,
+                            merged_local_rows, merged_local_cols,
+                            merged_ldu_mapping, merged_begin, merged_end,
+                            merged_ranks);
+
+                    std::cout << __FILE__ << " on rank " << rank
+                              << "local dim ["
+                              << new_local_spars_pattern->dim[0] << " x "
+                              << new_local_spars_pattern->dim[1] << "] \n";
+
+                    auto new_non_local_spars_pattern =
+                        std::make_shared<SparsityPattern>(
+                            exec, gathered_non_local_rows.size(),
+                            gko::dim<2>{merged_local_dim[0],
+                                        gathered_non_local_rows.size()},
+                            gathered_non_local_rows, gathered_non_local_cols,
+                            gathered_non_local_mapping, gathered_begin,
+                            gathered_end, gathered_ranks);
+
+                    std::cout << __FILE__ << " on rank " << rank
+                              << " done_sparsity \n";
+                    return std::make_pair<std::shared_ptr<SparsityPattern>,
+                                          std::shared_ptr<SparsityPattern>>(
+                        std::move(new_local_spars_pattern),
+                        std::move(new_non_local_spars_pattern));
+                } else {
+                    auto new_local_spars_pattern =
+                        std::make_shared<SparsityPattern>(exec, 0);
+
+                    auto new_non_local_spars_pattern =
+                        std::make_shared<SparsityPattern>(exec, 0);
+
+                    new_local_spars_pattern->dim = merged_local_dim;
+                    new_non_local_spars_pattern->dim = merged_local_dim;
+
+                    return std::make_pair<std::shared_ptr<SparsityPattern>,
+                                          std::shared_ptr<SparsityPattern>>(
+                        std::move(new_local_spars_pattern),
+                        std::move(new_non_local_spars_pattern));
+                }
+            };
 
         auto [repart_loc_sparsity, repart_non_loc_sparsity] =
-            repartition_sparsity(ranks_per_gpu, local_sparsity_,
-                                 non_local_sparsity_, dst_comm_pattern);
+            repartition_sparsity(local_sparsity_, non_local_sparsity_);
 
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
-
-        auto comm_pattern = src_comm_pattern;
-        const label local_nnz = local_sparsity_->size_;
-        const label non_local_nnz = non_local_sparsity_->size_;
+        // const label local_nnz = local_sparsity_->size_;
+        // const label non_local_nnz = non_local_sparsity_->size_;
 
         auto localized_partition = local_part_type::build_from_blocked_recv(
-            exec, local_sparsity_->dim[0], comm_pattern.send_idxs,
-            comm_pattern.target_ids, comm_pattern.target_sizes);
-
+            exec, local_sparsity_->dim[0], dst_comm_pattern.send_idxs,
+            dst_comm_pattern.target_ids, dst_comm_pattern.target_sizes);
+
+
+        std::cout << __FILE__ << " rank " << rank << " local sparsity dim"
+                  << local_sparsity_->dim[0] << "non_local sparsity dim"
+                  << non_local_sparsity_->dim[0] << "x"
+                  << non_local_sparsity_->dim[1] << " target_ids "
+                  << dst_comm_pattern.target_ids << " target_sizes "
+                  << dst_comm_pattern.target_sizes << " target_send_idxs.size "
+                  << dst_comm_pattern.send_idxs.size()
+                  << " non_local_sparsity.size " << non_local_sparsity_->size_
+                  << " get_recv_indices "
+                  << localized_partition->get_recv_indices().get_num_elems()
+                  << " \n";
 
         auto sparse_comm =
             sparse_communicator::create(comm, localized_partition);
@@ -221,14 +499,19 @@ public:
         auto dist_A = generate_dist_mtx_with_inner_type<dist_mtx>(
             matrix_format, exec, sparse_comm, local_sparsity_,
             non_local_sparsity_);
+        std::cout << __FILE__ << " rank " << rank
+                  << " done generate dist_mtx\n";
 
-        auto local_coeffs = gko::array<scalar>(exec, local_nnz);
-        auto non_local_coeffs = gko::array<scalar>(exec, non_local_nnz);
+        auto local_coeffs = gko::array<scalar>(exec, local_sparsity_->size_);
+        auto non_local_coeffs =
+            gko::array<scalar>(exec, non_local_sparsity_->size_);
 
-        host_A.get()->compute_local_coeffs(local_sparsity_, local_coeffs);
-        host_A.get()->compute_non_local_coeffs(non_local_sparsity_,
-                                               non_local_coeffs);
+        // host_A.get()->compute_local_coeffs(local_sparsity_, local_coeffs);
+        // host_A.get()->compute_non_local_coeffs(non_local_sparsity_,
+        //                                        non_local_coeffs);
 
+        std::cout << __FILE__ << " on rank " << rank
+                  << " done compute local coeffs \n";
         dist_A->read_distributed(
             device_matrix_data(exec, local_sparsity_->dim,
                                local_sparsity_->row_idxs,
@@ -237,6 +520,9 @@ public:
                                non_local_sparsity_->row_idxs,
                                non_local_sparsity_->col_idxs, non_local_coeffs),
             sparse_comm);
+        std::cout << __FILE__ << " on rank " << rank
+                  << " done read_distributed  dist_A size" << dist_A->get_size()
+                  << " \n";
 
         return std::make_shared<RepartDistMatrix>(
             exec, comm, host_A->get_size(), dist_A->get_size(),
@@ -334,6 +620,7 @@ public:
         auto dist_matrix = gko::as<
             gko::experimental::distributed::Matrix<scalar, label, label>>(
             this->dist_mtx_.get());
+        // TODO FIXME dont leave hardcoded value here
         word matrix_format_{"Coo"};
 
         if (matrix_format_ == "Coo") {
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index 80f43eefc..f9a9521a0 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -314,24 +314,32 @@ HostMatrixWrapper::collect_local_interface_indices(
     return local_interface_idxs;
 }
 
-std::vector<std::tuple<label, label, label>>
+std::vector<std::tuple<label, label, label, label>>
 HostMatrixWrapper::collect_cells_on_non_local_interface(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // vector of neighbour cell idx connected to interface
-    std::vector<std::tuple<label, label, label>> non_local_idxs{};
+    std::vector<std::tuple<label, label, label, label>> non_local_idxs{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
-    label interface_ctr = 0;
     interface_iterator<
         processorFvPatch>(interfaces, [&](label, label interface_id,
                                           label interface_size,
                                           const processorLduInterface &,
                                           const lduInterfaceField *iface) {
         const auto &face_cells{iface->interface().faceCells()};
+
+        const processorLduInterface &pldui =
+                refCast<const processorLduInterface>(iface->interface());
+        const label neighbProcNo = pldui.neighbProcNo();
+        pldui.send(Pstream::commsTypes::blocking, face_cells);
+
+        auto otherSide_tmp = pldui.receive<label>(
+                Pstream::commsTypes::blocking, interface_size);
+
         for (label cellI = 0; cellI < interface_size; cellI++) {
             auto local_row = face_cells[cellI];
-            non_local_idxs.push_back({interface_id, interface_ctr, local_row});
-            interface_ctr += 1;
+            auto col = otherSide_tmp()[cellI];
+            non_local_idxs.push_back({interface_id, col, local_row, neighbProcNo});
         }
     });
 
@@ -347,42 +355,40 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
                                                     non_local_matrix_nnz_)};
     sparsity->dim = gko::dim<2>{nrows_, non_local_matrix_nnz_};
 
-    auto non_local_row_indices =
+    auto non_local_indices =
         collect_cells_on_non_local_interface(interfaces_);
     auto rows = sparsity->row_idxs.get_data();
     auto cols = sparsity->col_idxs.get_data();
     auto permute = sparsity->ldu_mapping.get_data();
+    label prev_interface_ctr{0};
+    label end{0};
+    label start{0};
 
     label element_ctr = 0;
+    label interface_ctr{0};
+    label prev_rank{0};
+
     // TODO currently we set permute eventhough this is not required
     // anymore, remove permute from non_local_interfaces
-    for (auto [interface_id, interface_idx, row] : non_local_row_indices) {
+    for (auto [interface_idx, col, row, rank] : non_local_indices) {
         rows[element_ctr] = row;
-        cols[element_ctr] = interface_idx;
+        cols[element_ctr] = col;
         permute[element_ctr] = element_ctr;
-        element_ctr += 1;
-    }
 
-    // TODO this can be merged with algorithm in update_local_matrix_data
-    auto non_local_interfaces =
-        collect_cells_on_non_local_interface(interfaces_);
-    label interface_ctr{0};
-    label prev_interface_ctr{0};
-    label end{0};
-    label start{0};
-    for (auto [interface_idx, interface_cell_ctr, interface_row] :
-         non_local_interfaces) {
-        // check if a new interface has started or final interface has been
-        // reache
-        if (interface_idx > prev_interface_ctr ||
-            interface_ctr == non_local_interfaces.size() - 1) {
-            end = start + interface_ctr + 1;
-            interface_ctr = 0;
+        // a new interface started or the last element on last interface has been reached
+        bool last_element = element_ctr == non_local_indices.size() - 1;
+        if (interface_idx > prev_interface_ctr || last_element) {
+            // this check will be reached one element earlier if we reached
+            // the end of the non_local_indices thus we need to increment
+            // the elment_ctr once more
+            end = (last_element) ? element_ctr + 1 : element_ctr;
             sparsity->interface_spans.emplace_back(start, end);
+            sparsity->rank.emplace_back(prev_rank);
             start = end;
             prev_interface_ctr = interface_idx;
         }
-        interface_ctr++;
+        prev_rank = rank;
+        element_ctr++;
     }
 
     return sparsity;
@@ -494,8 +500,9 @@ void HostMatrixWrapper::compute_local_coeffs(
     auto ref_exec = exec_.get_ref_exec();
     auto permute = sparsity->ldu_mapping.get_data();
 
-    // // TODO this does not work for Ell
     if (reorder_on_copy_) {
+        // TODO assert array is on master
+        // TODO assert
         auto dense = local_coeffs.get_data();
         if (local_interface_nnz_) {
             auto couple_coeffs = collect_interface_coeffs(
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 7b71ff1c4..b9979ca48 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -44,19 +44,51 @@ namespace Foam {
 struct SparsityPattern {
     SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size)
         : size_(size),
-          col_idxs{exec, static_cast<gko::size_type>(size)},
           row_idxs{exec, static_cast<gko::size_type>(size)},
+          col_idxs{exec, static_cast<gko::size_type>(size)},
           ldu_mapping{exec, static_cast<gko::size_type>(size)}
     {}
 
+    SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size,
+                    gko::dim<2> dim, std::vector<label> &rows,
+                    std::vector<label> &cols, std::vector<label> &map,
+                    std::vector<label> &spans_begin,
+                    std::vector<label> &spans_end, std::vector<label> &ranks)
+        : size_(size),
+          dim(dim),
+          row_idxs{exec, static_cast<gko::size_type>(size)},
+          col_idxs{exec, static_cast<gko::size_type>(size)},
+          ldu_mapping{exec, static_cast<gko::size_type>(size)}
+    {
+        ASSERT_EQ(size, rows.size());
+        std::copy(rows.begin(), rows.end(), row_idxs.get_data());
+
+        ASSERT_EQ(size, cols.size());
+        std::copy(cols.begin(), cols.end(), col_idxs.get_data());
+
+        ASSERT_EQ(size, map.size());
+        std::copy(map.begin(), map.end(), ldu_mapping.get_data());
+
+        ASSERT_EQ(spans_begin.size(), spans_end.size());
+        std::transform(spans_begin.begin(), spans_begin.end(),
+                       spans_end.begin(), std::back_inserter(interface_spans),
+                       [](const auto begin, const auto end) {
+                           return gko::span{static_cast<gko::size_type>(begin),
+                                            static_cast<gko::size_type>(end)};
+                       });
+
+        ASSERT_EQ(spans_begin.size(), ranks.size());
+        rank = ranks;
+    }
+
     // TODO rename the public members and remove trailing _
     // number of nnz entries
     const label size_;
 
-    mutable gko::Array<label> col_idxs;
-
     mutable gko::Array<label> row_idxs;
 
+    mutable gko::Array<label> col_idxs;
+
     // ldu_mapping[sorted_(csr)_position] =
     // unsorted_(consecutive_ldu_)_position
     mutable gko::Array<label> ldu_mapping;
@@ -68,7 +100,8 @@ struct SparsityPattern {
     // this is used to keep all col and row idx consecutive in memory
     mutable std::vector<gko::span> interface_spans;
 
-    // keep track of original rank of each interface or matrix block
+    // keep track of original (communication) rank of each interface or matrix
+    // block
     mutable std::vector<label> rank;
 };
 
@@ -317,7 +350,7 @@ private:
     ** ret = [(1,2,1),(2,20, 2), (3, 20, 2) ...]
     **         i0   i1,   i...
     */
-    std::vector<std::tuple<label, label, label>>
+    std::vector<std::tuple<label, label, label, label>>
     collect_cells_on_non_local_interface(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
@@ -325,7 +358,6 @@ private:
     // updater
 
 
-
 public:
     // segregated matrix wrapper constructor
     HostMatrixWrapper(const ExecutorHandler &exec, const objectRegistry &db,
@@ -380,8 +412,9 @@ public:
     /** Copies the interface matrix coefficients to non_local_coeffs without
     ** changing or reinstantiating the sparsity pattern.
     **/
-    void compute_non_local_coeffs(std::shared_ptr<const SparsityPattern> sparsity,
-                                  gko::array<scalar> &target_coeffs) const;
+    void compute_non_local_coeffs(
+        std::shared_ptr<const SparsityPattern> sparsity,
+        gko::array<scalar> &target_coeffs) const;
 
     /** Based on OpenFOAMs interfaces this function computes two
      ** consecutive index arrays (non_local_sparsisty_.row_idxs and col_idxs) in
@@ -400,8 +433,7 @@ public:
      ** @return the CommunicationPattern
      */
     CommunicationPattern create_communication_pattern(
-        const gko::experimental::mpi::communicator& comm
-            ) const;
+        const gko::experimental::mpi::communicator &comm) const;
 
 
     bool get_verbose() const { return verbose_; }
diff --git a/common/common.C b/common/common.C
index ff6dacf4f..ac1320cea 100644
--- a/common/common.C
+++ b/common/common.C
@@ -46,6 +46,8 @@ void export_mtx(const word fieldName,
     std::string folder{db.time().timePath()};
     std::filesystem::create_directories(folder);
 
+    std::cout << __FILE__ << " exporting to " << folder << " " << As.size() << "\n";
+
     for (int i = 0; i < As.size(); i++) {
         std::string fn{folder + "/" + fieldName + "_A_" + std::to_string(i) +
                        ".mtx"};
diff --git a/common/common.H b/common/common.H
index 68a23808c..e46c78ada 100644
--- a/common/common.H
+++ b/common/common.H
@@ -90,6 +90,12 @@ namespace Foam {
 
 #define UNUSED(x) (void)(x)
 
+#define ASSERT_EQ(_val1, _val2)                                            \
+    if (_val1 != _val2) {                                                      \
+        throw ::gko::ValueMismatch(__FILE__, __LINE__, __func__, _val1, _val2, \
+                                   "expected equal values");                   \
+    }
+
 #define OGL_NOT_IMPLEMENTED GKO_NOT_IMPLEMENTED
 
 std::ostream& operator<<(std::ostream& out, const std::vector<label>& e);

From 0b38189a354d052542daf4bc4636c1f3e875882c Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 15 May 2024 11:25:55 +0200
Subject: [PATCH 48/90] fixup! size of non_local_matrix cols

---
 MatrixWrapper/Distributed/Distributed.H | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index c4408500f..e1cadc058 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -265,7 +265,7 @@ public:
                 gko::dim<2> merged_local_dim{rows, rows};
 
                 std::vector<label> merged_begin{0};
-                std::vector<label> merged_end{ merged_local_rows.size()};
+                std::vector<label> merged_end{merged_local_rows.size()};
                 std::vector<label> merged_ranks{rank};
 
                 // TODO remove
@@ -349,6 +349,7 @@ public:
                 std::vector<label> gathered_begin;
                 std::vector<label> gathered_end;
                 label merged_ranks_size = merged_non_local_ranks.size();
+                // redistribute interfaces
                 for (label i=0;i<merged_ranks_size;i++) {
                     auto merged_target_rank = merged_non_local_ranks[i];
                     label to_rank = target_rank(merged_target_rank);
@@ -357,8 +358,8 @@ public:
                               << " merged_target_rank " << merged_target_rank
                               << " is local " << local << " target_rank "
                               << to_rank << "\n";
-                    auto begin = merged_begin[i];
-                    auto end = merged_end[i];
+                    auto begin = merged_non_local_begin[i];
+                    auto end = merged_non_local_end[i];
                     if (local) {
                         label size_before = merged_local_rows.size();
                         merged_local_rows.insert(
@@ -379,6 +380,8 @@ public:
                         merged_end.push_back(size_after);
                         merged_ranks.push_back(rank);
                     } else {
+                        // FIXME begin and end are from the merged local
+                        // here we need non local
                         label size_before = gathered_non_local_rows.size();
                         gathered_non_local_rows.insert(
                             gathered_non_local_rows.end(),
@@ -394,6 +397,7 @@ public:
                             merged_non_local_cols.begin() + end);
                         label size_after = gathered_non_local_rows.size();
                         std::cout << __FILE__ << " on rank " << rank
+                                << " begin " << begin << " end " << end
                                 << " non_local size_before " << size_before
                                 << " non_local size_after " << size_after
                                 << "\n";
@@ -431,6 +435,8 @@ public:
                             merged_ranks);
 
                     std::cout << __FILE__ << " on rank " << rank
+                              << " merged_begin " <<  merged_begin
+                              << " merged_end " <<  merged_end
                               << "local dim ["
                               << new_local_spars_pattern->dim[0] << " x "
                               << new_local_spars_pattern->dim[1] << "] \n";

From 87afdc9656eb832deb5aaaf582aff9334f356523 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 May 2024 15:06:53 +0200
Subject: [PATCH 49/90] wip repartition sparsity

---
 MatrixWrapper/Combination/Combination.H |   9 +-
 MatrixWrapper/Distributed/Distributed.H | 170 +++++++++++++++++-------
 MatrixWrapper/LDUMatrix/HostMatrix.C    |  31 -----
 3 files changed, 131 insertions(+), 79 deletions(-)

diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index 03a718068..c5b29f9f6 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -38,10 +38,17 @@ public:
         auto data_exec = data.get_executor();
 
         for (auto &interface : interface_ranges_) {
-            std::cout << __FILE__ << " read combination  interface \n";
             auto length = interface.length();
             auto offset = interface.begin;
             auto op = gko::share(InnerMatrixType::create(exec));
+            auto rows = gko::array<IndexType>(
+                    exec,
+                    make_const_array_view(data_exec, length,
+                                          data.get_const_row_idxs() + offset)
+                        .copy_to_array());
+            std::cout << __FILE__ << " read combination  interface"
+                      << " offset: " <<  offset << " rows " << rows << " \n";
+
 
             device_matrix_data in(
                 exec, comb_->get_size(),
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index e1cadc058..ee788e26e 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -33,6 +33,24 @@ SourceFiles
 #include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 
 
+template <typename Type>
+void gather_array_to_owner(
+    label *data, label size,
+    std::shared_ptr<const gko::Executor> exec,
+    std::shared_ptr<const gko::experimental::distributed::sparse_communicator>
+        comm,
+    const CommunicationPattern &comm_pattern)
+{
+    // this will need a specific comm_pattern
+
+    auto [send_counts, recv_counts, send_offsets, recv_offsets, recv_buffer] =
+        comm_pattern;
+
+    comm.all_to_all_v(exec, data, send_counts.data(), send_offsets.data(),
+                      recv_buffer.data(), recv_counts.data(),
+                      recv_offsets.data());
+};
+
 template <typename DistMtxType>
 std::unique_ptr<DistMtxType> generate_dist_mtx_with_inner_type(
     word matrix_format, std::shared_ptr<gko::Executor> exec,
@@ -268,12 +286,6 @@ public:
                 std::vector<label> merged_end{merged_local_rows.size()};
                 std::vector<label> merged_ranks{rank};
 
-                // TODO remove
-                // std::cout << __FILE__ << " on rank " << rank
-                //           << " merged_local_rows " << merged_local_rows
-                //            << " merged_local_cols " << merged_local_cols
-                //         << "\n";
-
                 auto non_local_comm_pattern = compute_send_recv_counts(
                     ranks_per_gpu, owner_rank, src_non_local_pattern->size_,
                     comm, exec);
@@ -283,8 +295,27 @@ public:
                     src_non_local_pattern->row_idxs.get_size(),
                     non_local_comm_pattern);
 
+                // the non local cols are in local idx of other side
+                // thus we need the new offset of the other side
+                for (label i = 0; i < src_non_local_pattern->rank.size(); i++) {
+                    auto comm_rank = src_non_local_pattern->rank[i];
+                    auto [begin, end] =
+                        src_non_local_pattern->interface_spans[i];
+                    label local_offset =
+                        orig_partition->get_range_bounds()[comm_rank] -
+                        orig_partition->get_range_bounds()[owner_rank];
+                    auto *data =
+                        src_non_local_pattern->col_idxs.get_data() + begin;
+                    auto size = end - begin;
+                    std::transform(data, data + size, data, [&](label idx) {
+                        return idx + local_offset;
+                    });
+                }
+
+                // we now dealed with the new offset thus we dont need an offset
+                // now
                 std::vector<label> merged_non_local_cols = gather_to_owner(
-                    offset, src_non_local_pattern->col_idxs.get_data(),
+                    0, src_non_local_pattern->col_idxs.get_data(),
                     src_non_local_pattern->row_idxs.get_size(),
                     non_local_comm_pattern);
 
@@ -294,14 +325,6 @@ public:
                         src_non_local_pattern->row_idxs.get_size(),
                         non_local_comm_pattern);
 
-                // TODO remove
-                // std::cout << __FILE__ << " on rank " << rank
-                //           << " merged_non_local_rows" <<
-                //           merged_non_local_rows
-                //           << " merged_non_local_cols " <<
-                //           merged_non_local_cols
-                //           << "\n";
-
                 auto [span_send_counts, span_recv_counts, span_send_offsets,
                       span_recv_offsets, span_recv_buffer] =
                     compute_send_recv_counts(
@@ -337,9 +360,45 @@ public:
                     span_recv_offsets.data());
                 std::vector<label> merged_non_local_ranks{span_recv_buffer};
 
-                std::cout << __FILE__ << " on rank " << rank << " merged_begin"
-                          << merged_begin << " merged_end " << merged_end
-                          << " merged_ranks " << merged_non_local_ranks << "\n";
+
+                // fix the received merged_non_local_begin and ends
+                // we received begins and ends of the interfaces from
+                // the neighbour ranks. however the begin and ends are
+                // computed base on the non merged interfaces. Now we
+                // need to offset these values
+                // before merge
+                // rank 0 begin [ 0 10 ], end [10 20]
+                // rank 1 begin [ 0 10 ], end [10 20]
+                // merged begin [ 0 10 0 10 ], end [10 20 10 20]
+                // after fix
+                // merged begin [ 0 10 20 30 ], end [10 20 30 40]
+                for (label i = 1; i < ranks_per_gpu; i++) {
+                    auto recv_offset = span_recv_offsets[rank + i];
+                    auto recv_count = span_recv_counts[rank + i];
+                    auto range_offset = merged_non_local_end[recv_offset - 1];
+                    for (label j = 0; j < recv_count; j++) {
+                        merged_non_local_begin[recv_offset + j] += range_offset;
+                        merged_non_local_end[recv_offset + j] += range_offset;
+                    }
+                }
+
+
+                // TODO remove
+                //     if (rank == 0) {
+                // std::cout << __FILE__ << " on rank " << rank << "
+                // merged_begin"
+                //           << merged_begin
+                //           << " span_recv_counts " << span_recv_counts
+                //           << " span_recv_offsets " << span_recv_offsets
+                //           << " merged_non_local_begins " <<
+                //           merged_non_local_begin
+                //           << " merged_non_local_end " << merged_non_local_end
+                //           << " merged_end " << merged_end
+                //           << " merged_non_local_rows " <<
+                //           merged_non_local_rows
+                //           << " merged_ranks " << merged_non_local_ranks <<
+                //           "\n";
+                // }
 
                 // TODO shorten this
                 std::vector<label> gathered_non_local_rows;
@@ -350,17 +409,18 @@ public:
                 std::vector<label> gathered_end;
                 label merged_ranks_size = merged_non_local_ranks.size();
                 // redistribute interfaces
-                for (label i=0;i<merged_ranks_size;i++) {
+                for (label i = 0; i < merged_ranks_size; i++) {
                     auto merged_target_rank = merged_non_local_ranks[i];
                     label to_rank = target_rank(merged_target_rank);
                     bool local = rank == to_rank;
-                    std::cout << __FILE__ << " on rank " << rank
-                              << " merged_target_rank " << merged_target_rank
-                              << " is local " << local << " target_rank "
-                              << to_rank << "\n";
+                    // these are the begin ends before merging they need to be
+                    // offseted;
                     auto begin = merged_non_local_begin[i];
                     auto end = merged_non_local_end[i];
                     if (local) {
+                        // TODO depending on the interface simple non
+                        // transforming and interpolating interfaces like now
+                        // local processor interfaces could be merged
                         label size_before = merged_local_rows.size();
                         merged_local_rows.insert(
                             merged_local_rows.end(),
@@ -374,19 +434,30 @@ public:
                             merged_ldu_mapping.end(),
                             merged_non_local_ldu_mapping.data() + begin,
                             merged_non_local_ldu_mapping.data() + end);
+                        // TODO store from rank ie from which rank it came
+                        merged_ranks.push_back(to_rank);
 
                         label size_after = merged_local_rows.size();
                         merged_begin.push_back(size_before);
                         merged_end.push_back(size_after);
-                        merged_ranks.push_back(rank);
+                        // TODO remove
+                        // if (rank == 0) {
+                        //     std::cout << __FILE__ << " on rank " << rank
+                        //             << " begin " << begin << " end " << end
+                        //             << " insert merged_local_rows " <<
+                        //             merged_local_rows
+                        //             << "\n";
+                        //     }
                     } else {
-                        // FIXME begin and end are from the merged local
-                        // here we need non local
                         label size_before = gathered_non_local_rows.size();
                         gathered_non_local_rows.insert(
                             gathered_non_local_rows.end(),
                             merged_non_local_rows.begin() + begin,
                             merged_non_local_rows.begin() + end);
+                        // FIXME
+                        // up until now non_local_cols are stil using
+                        // local_idx and we need to convert them to
+                        // consequtive compressed idxs
                         gathered_non_local_cols.insert(
                             gathered_non_local_cols.end(),
                             merged_non_local_cols.begin() + begin,
@@ -396,11 +467,15 @@ public:
                             merged_non_local_cols.begin() + begin,
                             merged_non_local_cols.begin() + end);
                         label size_after = gathered_non_local_rows.size();
-                        std::cout << __FILE__ << " on rank " << rank
-                                << " begin " << begin << " end " << end
-                                << " non_local size_before " << size_before
-                                << " non_local size_after " << size_after
-                                << "\n";
+                        // if (rank == 0) {
+                        // std::cout << __FILE__ << " on rank " << rank
+                        //         << " gathered_non_local_rows " <<
+                        //         gathered_non_local_rows
+                        //         << " begin " << begin << " end " << end
+                        //         << " non_local size_before " << size_before
+                        //         << " non_local size_after " << size_after
+                        //         << "\n";
+                        // }
                         gathered_begin.push_back(size_before);
                         gathered_end.push_back(size_after);
                         gathered_ranks.push_back(to_rank);
@@ -434,12 +509,15 @@ public:
                             merged_ldu_mapping, merged_begin, merged_end,
                             merged_ranks);
 
-                    std::cout << __FILE__ << " on rank " << rank
-                              << " merged_begin " <<  merged_begin
-                              << " merged_end " <<  merged_end
-                              << "local dim ["
-                              << new_local_spars_pattern->dim[0] << " x "
-                              << new_local_spars_pattern->dim[1] << "] \n";
+                    // if (rank == 0) {
+                    // std::cout << __FILE__ << " on rank " << rank
+                    //           << merged_local_rows
+                    //           << " merged_begin " <<  merged_begin
+                    //           << " merged_end " <<  merged_end
+                    //           << "local dim ["
+                    //           << new_local_spars_pattern->dim[0] << " x "
+                    //           << new_local_spars_pattern->dim[1] << "] \n";
+                    //         }
 
                     auto new_non_local_spars_pattern =
                         std::make_shared<SparsityPattern>(
@@ -450,8 +528,6 @@ public:
                             gathered_non_local_mapping, gathered_begin,
                             gathered_end, gathered_ranks);
 
-                    std::cout << __FILE__ << " on rank " << rank
-                              << " done_sparsity \n";
                     return std::make_pair<std::shared_ptr<SparsityPattern>,
                                           std::shared_ptr<SparsityPattern>>(
                         std::move(new_local_spars_pattern),
@@ -512,12 +588,9 @@ public:
         auto non_local_coeffs =
             gko::array<scalar>(exec, non_local_sparsity_->size_);
 
-        // host_A.get()->compute_local_coeffs(local_sparsity_, local_coeffs);
-        // host_A.get()->compute_non_local_coeffs(non_local_sparsity_,
-        //                                        non_local_coeffs);
+        local_coeffs.fill(0.0);
+        non_local_coeffs.fill(0.0);
 
-        std::cout << __FILE__ << " on rank " << rank
-                  << " done compute local coeffs \n";
         dist_A->read_distributed(
             device_matrix_data(exec, local_sparsity_->dim,
                                local_sparsity_->row_idxs,
@@ -526,9 +599,12 @@ public:
                                non_local_sparsity_->row_idxs,
                                non_local_sparsity_->col_idxs, non_local_coeffs),
             sparse_comm);
-        std::cout << __FILE__ << " on rank " << rank
-                  << " done read_distributed  dist_A size" << dist_A->get_size()
-                  << " \n";
+
+
+        // We now have an distribute matrix which needs to be filled/updated
+        // first we copy L+D+U over
+        // here we can reuse the gather to owner implementation
+
 
         return std::make_shared<RepartDistMatrix>(
             exec, comm, host_A->get_size(), dist_A->get_size(),
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index f9a9521a0..a121b5a29 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -500,35 +500,6 @@ void HostMatrixWrapper::compute_local_coeffs(
     auto ref_exec = exec_.get_ref_exec();
     auto permute = sparsity->ldu_mapping.get_data();
 
-    if (reorder_on_copy_) {
-        // TODO assert array is on master
-        // TODO assert
-        auto dense = local_coeffs.get_data();
-        if (local_interface_nnz_) {
-            auto couple_coeffs = collect_interface_coeffs(
-                interfaces_, interfaceBouCoeffs_, true);
-            if (symmetric_) {
-                symmetric_update_w_interface(
-                    local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_, permute,
-                    scaling_, diag_, upper_, couple_coeffs.data(), dense);
-
-            } else {
-                non_symmetric_update_w_interface(
-                    local_matrix_w_interfaces_nnz_, nrows_, upper_nnz_, permute,
-                    scaling_, diag_, upper_, lower_, couple_coeffs.data(),
-                    dense);
-            }
-        } else {
-            if (symmetric_) {
-                symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-                                 scaling_, diag_, upper_, dense);
-            } else {
-                non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-                                     scaling_, diag_, upper_, lower_, dense);
-            }
-        }
-    }
-    // } else {
     //     // TODO
     //     // - this should be moved to separate function to avoid making this
     //     // too long
@@ -598,8 +569,6 @@ void HostMatrixWrapper::compute_local_coeffs(
     //                           row_collection.get());
     //
     //     dense_vec->copy_from(row_collection);
-    // }
-    //
 }
 
 

From fff32cf3d91e553604fbe161aa02d37d83f80c33 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sat, 18 May 2024 09:56:19 +0200
Subject: [PATCH 50/90] clean up

---
 MatrixWrapper/Distributed/Distributed.H | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index ee788e26e..23b45cfaa 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -33,24 +33,6 @@ SourceFiles
 #include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 
 
-template <typename Type>
-void gather_array_to_owner(
-    label *data, label size,
-    std::shared_ptr<const gko::Executor> exec,
-    std::shared_ptr<const gko::experimental::distributed::sparse_communicator>
-        comm,
-    const CommunicationPattern &comm_pattern)
-{
-    // this will need a specific comm_pattern
-
-    auto [send_counts, recv_counts, send_offsets, recv_offsets, recv_buffer] =
-        comm_pattern;
-
-    comm.all_to_all_v(exec, data, send_counts.data(), send_offsets.data(),
-                      recv_buffer.data(), recv_counts.data(),
-                      recv_offsets.data());
-};
-
 template <typename DistMtxType>
 std::unique_ptr<DistMtxType> generate_dist_mtx_with_inner_type(
     word matrix_format, std::shared_ptr<gko::Executor> exec,

From 32d8cfcb0654368e85184b690fd7755e33108dc8 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 23 May 2024 08:01:23 +0200
Subject: [PATCH 51/90] copy and reorder ldu, fix compressed non-local cols

---
 .../DistributedMatrixAdapter.H                |   1 +
 .../CommunicationPattern.C                    |  97 +++-
 .../CommunicationPattern.C.bck                | 335 +++++++++++++
 .../CommunicationPattern.H                    |  30 +-
 .../CommunicationPattern.H.bck                |  76 +++
 MatrixWrapper/Distributed/Distributed.H       | 169 +++++--
 MatrixWrapper/LDUMatrix/HostMatrix.C          |   5 +-
 MatrixWrapper/LDUMatrix/HostMatrix.H          |  24 +
 MatrixWrapper/LDUMatrix/HostMatrix.H.bck      | 457 ++++++++++++++++++
 .../LDUMatrix/HostMatrixFreeFunctions.C       |   2 +-
 lduLduBase/lduLduBase.H                       |  10 +-
 11 files changed, 1153 insertions(+), 53 deletions(-)
 create mode 100644 MatrixWrapper/CommunicationPattern/CommunicationPattern.C.bck
 create mode 100644 MatrixWrapper/CommunicationPattern/CommunicationPattern.H.bck
 create mode 100644 MatrixWrapper/LDUMatrix/HostMatrix.H.bck

diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 249a9d7f9..5e58ba2fd 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -72,6 +72,7 @@ struct DistributedMatrixInitFunctor {
     {
         return RepartDistMatrix<scalar, label, label>::create(
               solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
+              solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
               host_A_.get());
     }
 };
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index c7a2b0063..22be2a031 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -30,10 +30,23 @@ SourceFiles
 
 std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
            std::vector<label>, std::vector<label>>
-compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
+compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
                          const gko::experimental::mpi::communicator &comm,
-                         std::shared_ptr<const gko::Executor> exec)
+                         std::shared_ptr<const gko::Executor> exec, label size)
 {
+    return compute_send_recv_counts(ranks_per_gpu, owner_rank, comm, exec, size,
+                                    size, 0, 0);
+}
+
+std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+           std::vector<label>, std::vector<label>>
+compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
+                         const gko::experimental::mpi::communicator &comm,
+                         std::shared_ptr<const gko::Executor> exec, label size,
+                         label total_size, label padding_before,
+                         label padding_after)
+{
+    ASSERT_EQ(total_size, size + padding_before + padding_after);
     label total_ranks{comm.size()};
     label rank{comm.rank()};
     std::vector<label> send_counts(total_ranks, 0);
@@ -45,19 +58,39 @@ compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
     label comm_elements_buffer{0};
     if (rank == owner_rank) {
         // send and recv to it self
+        recv_offsets[owner_rank] = padding_before;
         send_counts[owner_rank] = size;
         recv_counts[owner_rank] = size;
-        tot_recv_elements = size;
+        // the start of the next rank data
+        tot_recv_elements = padding_before + size + padding_after;
 
         for (int i = 1; i < ranks_per_gpu; i++) {
+            // receive the recv counts
             comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
-            recv_offsets[rank + i] = tot_recv_elements;
             recv_counts[rank + i] = comm_elements_buffer;
+            label recv_size = comm_elements_buffer;
+
+            // receive the padding before
+            comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
+            tot_recv_elements += comm_elements_buffer;
+            recv_offsets[rank + i] = tot_recv_elements;
+
+            // receive the padding after and keep for next rank
+            // as new offset
+            tot_recv_elements += recv_size;
+            comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
             tot_recv_elements += comm_elements_buffer;
         }
     } else {
+        // send how many elements to communicate
         comm.send(exec, &size, 1, owner_rank, owner_rank);
         send_counts[owner_rank] = size;
+
+        // send how much padding before is needed
+        comm.send(exec, &padding_before, 1, owner_rank, owner_rank);
+
+        // send how much padding after is needed
+        comm.send(exec, &padding_after, 1, owner_rank, owner_rank);
     }
 
     std::vector<label> target_recv_buffer(tot_recv_elements, 0);
@@ -66,6 +99,58 @@ compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
                            target_recv_buffer);
 }
 
+void gather_to_owner(
+    std::shared_ptr<const gko::Executor> exec,
+    gko::experimental::mpi::communicator &comm,
+    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+               std::vector<label>, std::vector<label>> &comm_pattern,
+    const scalar *send_buffer, scalar *recv_buffer)
+{
+    auto [send_counts, recv_counts, send_offsets, recv_offsets, recv_buffer_] =
+        comm_pattern;
+    // TODO add some sanity checks for comm pattern
+    // 1. length needs to be same as mpi ranks
+    // 2. recv_buffer length should match ie at least length of recv_counts
+
+    // NOTE old holes are captured by comm_pattern
+    // the default comm pattern needs to be adjusted
+    // since we might also need to leave "holes" between the
+    // gathered blocks thus the offset / distance between data
+    // should be before repartitioning
+    //
+    // send_buffer should be on the host
+    // recv_buffer should be on the device
+    auto rank = comm.rank();
+
+    comm.all_to_all_v(exec, send_buffer, send_counts.data(),
+                      send_offsets.data(), recv_buffer, recv_counts.data(),
+                      recv_offsets.data());
+};
+
+std::vector<label> gather_to_owner(
+    std::shared_ptr<const gko::Executor> exec,
+    gko::experimental::mpi::communicator &comm,
+    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+               std::vector<label>, std::vector<label>> &comm_pattern,
+    label size,
+    scalar *send_buffer, label offset = 0)
+{
+    // TODO don't modify send_buffer
+    if (offset > 0) {
+        std::transform(send_buffer, send_buffer + size, send_buffer,
+                       [&](label idx) { return idx + offset; });
+    }
+
+    auto & [send_counts, recv_counts, send_offsets, recv_offsets, recv_buffer] =
+        comm_pattern;
+
+    comm.all_to_all_v(exec, data, send_counts.data(), send_offsets.data(),
+                      recv_buffer.data(), recv_counts.data(),
+                      recv_offsets.data());
+
+    return std::vector<label>{recv_buffer};
+};
+
 std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e)
 {
     out << "CommunicationPattern: for rank: " << e.comm.rank();
@@ -162,8 +247,8 @@ CommunicationPattern repartition_comm_pattern(
     // how many elements the owner rank is to receive
     auto [send_counts, recv_counts, send_offsets, recv_offsets,
           target_recv_buffer] =
-        compute_send_recv_counts(ranks_per_gpu, owner_rank, target_ids.size(),
-                                 comm, exec);
+        compute_send_recv_counts(ranks_per_gpu, owner_rank, comm, exec,
+                                 target_ids.size());
 
 
     comm.all_to_all_v(exec, target_ids.data(), send_counts.data(),
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C.bck b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C.bck
new file mode 100644
index 000000000..0b14cb92e
--- /dev/null
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C.bck
@@ -0,0 +1,335 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::IOSortingIdxHandler
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    CommunicationPattern.H
+
+\*---------------------------------------------------------------------------*/
+
+#include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
+#include "common/common.H"
+
+std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+           std::vector<label>, std::vector<label>>
+compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
+                         const gko::experimental::mpi::communicator &comm,
+                         std::shared_ptr<const gko::Executor> exec,
+                         label padding_before,
+                         label padding_after
+)
+{
+    label total_ranks{comm.size()};
+    label rank{comm.rank()};
+    std::vector<label> send_counts(total_ranks, 0);
+    std::vector<label> recv_counts(total_ranks, 0);
+    std::vector<label> send_offsets(total_ranks, 0);
+    std::vector<label> recv_offsets(total_ranks, 0);
+
+    label tot_recv_elements{0};
+    label comm_elements_buffer{0};
+    if (rank == owner_rank) {
+        // send and recv to it self
+        recv_offsets[owner_rank] = padding_before;
+        send_counts[owner_rank] = size;
+        recv_counts[owner_rank] = size;
+        // the start of the next rank data
+        tot_recv_elements = padding_before + size + padding_after;
+
+        for (int i = 1; i < ranks_per_gpu; i++) {
+            // receive the recv counts
+            comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
+            recv_counts[rank + i] = comm_elements_buffer;
+
+            // receive the padding before
+            comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
+            tot_recv_elements += comm_elements_buffer;
+            recv_offsets[rank + i] = tot_recv_elements;
+
+            // receive the padding after and keep for next rank
+            // as new offset
+            comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
+            tot_recv_elements += comm_elements_buffer;
+        }
+    } else {
+        // send how many elements to communicate
+        comm.send(exec, &size, 1, owner_rank, owner_rank);
+        send_counts[owner_rank] = size;
+
+        // send how much padding before is needed
+        comm.send(exec, &padding_before, 1, owner_rank, owner_rank);
+
+        // send how much padding after is needed
+        comm.send(exec, &padding_after, 1, owner_rank, owner_rank);
+    }
+
+    std::vector<label> target_recv_buffer(tot_recv_elements, 0);
+
+    return std::make_tuple(send_counts, recv_counts, send_offsets, recv_offsets,
+                           target_recv_buffer);
+}
+
+std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e)
+{
+    out << "CommunicationPattern: for rank: " << e.comm.rank();
+    out << " {";
+    out << "\ntarget_ids: " << e.target_ids;
+    out << "\ntarget_sizes: " << e.target_sizes;
+    out << "}\n";
+    return out;
+}
+
+template <typename T, typename Compare>
+std::vector<label> sort_permutation(const std::vector<T> &vec, Compare compare)
+{
+    std::vector<label> p(vec.size());
+    std::iota(p.begin(), p.end(), 0);
+    std::stable_sort(p.begin(), p.end(), [&](std::size_t i, std::size_t j) {
+        return compare(vec[i], vec[j]);
+    });
+    return p;
+}
+
+template <typename T>
+std::vector<T> apply_permutation(const std::vector<T> vec,
+                                 const std::vector<label> &p)
+{
+    std::vector<T> sorted_vec(vec.size());
+    std::transform(p.begin(), p.end(), sorted_vec.begin(),
+                   [&](label i) { return vec[i]; });
+    return sorted_vec;
+}
+
+CommunicationPattern repartition_comm_pattern(
+    label ranks_per_gpu, CommunicationPattern &src_comm_pattern,
+    std::shared_ptr<
+        const gko::experimental::distributed::Partition<label, label>>
+        partition)
+{
+    using comm_size_type = label;
+
+    if (ranks_per_gpu == 1) {
+        return CommunicationPattern(src_comm_pattern);
+    }
+
+    auto exec = src_comm_pattern.target_ids.get_executor();
+    auto target_rank = [ranks_per_gpu](label rank) {
+        return rank - (rank % ranks_per_gpu);
+    };
+
+    auto comm = src_comm_pattern.comm;
+
+    // find ranks which are now local
+    // the target_ids, target_sizes etc need to be communicated
+    // with new owner. thus we use maps to store them where the index
+    // is the new owner.
+    std::vector<label> target_ids{};
+    std::vector<label> target_sizes{};
+    std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
+    label rank{src_comm_pattern.comm.rank()};
+    label owner_rank = target_rank(rank);
+    bool owner = rank == owner_rank;
+
+    std::cout << __FILE__ << " on rank " << rank << "before loop \n";
+
+    // map for exchanging communication data where the index is the target rank
+    for (int i = 0; i < src_comm_pattern.target_ids.get_size(); i++) {
+        label target_id = src_comm_pattern.target_ids.get_data()[i];
+        // Step 1. Check if communication partner is non-local after
+        // repartitioning. If it is non-local we keep it. Otherwise local
+        // communication can be discarded or merged. Here non-local means: the
+        // communication target proc id is != repartition proc id
+        //
+        // [ 1 | 2 | 3 ] [ 4 | 5 | 6 ] where [] - bounds after repart
+        //                                    | - bounds before repart
+        // e.g. rank 3 has rank 2 and 4 as comm neighbours
+        //      rank 2 can be discarded but rank 4 is non-local and needs
+        //      to be communicated to rank 1 to be merged into the new
+        //      communicaton pattern. This needs an all to all communication.
+
+        label repart_target_id = target_rank(target_id);
+        label target_size = src_comm_pattern.target_sizes.get_const_data()[i];
+        auto send_idx = src_comm_pattern.send_idxs[i];
+
+        if (repart_target_id != owner_rank) {
+            // communication pattern is non local, hence we keep it
+            // after repartitioning we now have to communicate with a different
+            // rank
+            // store new owner rank to which repart_target_* needs to be send to
+            target_ids.push_back(repart_target_id);
+            target_sizes.push_back(target_size);
+            send_idxs.push_back(send_idx);
+        }
+    }
+
+    std::cout << __FILE__ << " on rank " << rank << "after loop \n";
+
+    // where to put elements from recv
+    // communicate with all neighbours which are to be merged
+    // how many elements the owner rank is to receive
+    auto [send_counts, recv_counts, send_offsets, recv_offsets,
+          target_recv_buffer] =
+        compute_send_recv_counts(ranks_per_gpu, owner_rank, target_ids.size(),
+                                 comm, exec);
+
+
+    comm.all_to_all_v(exec, target_ids.data(), send_counts.data(),
+                      send_offsets.data(), target_recv_buffer.data(),
+                      recv_counts.data(), recv_offsets.data());
+
+
+    target_ids = target_recv_buffer;
+
+    comm.all_to_all_v(exec, target_sizes.data(), send_counts.data(),
+                      send_offsets.data(), target_recv_buffer.data(),
+                      recv_counts.data(), recv_offsets.data());
+
+    target_sizes = target_recv_buffer;
+
+    // communicate the send indices to owner rank
+    // send indices are packed into std::vector of gko::arrays thus we send them
+    // one by one
+    // additionally we already have then target_sizes and send_counts and
+    // recv_counts which can be reused to for sending the index vectors
+
+    // TODO this could be refactored to separate function
+    // this is also used distributed
+    if (owner) {
+        // retrieve from all neighbours
+        std::cout << __FILE__ << " on rank "
+                  << rank << " owner "
+                  << ranks_per_gpu
+                  << " recv_counts " << recv_counts
+                  << " \n";
+        for (int i = 1; i < ranks_per_gpu; i++) {
+            // retrieve all send_idxs
+
+            for (int j = 0; j < recv_counts[rank + i]; j++) {
+                auto target_size = target_sizes[j + 1];
+                auto target_id = target_ids[j + 1];
+                std::vector<label> recv_buffer(target_size);
+                std::cout << __FILE__ << " on rank " << rank
+                          << " recv from  i " << rank + i
+                          <<  " interface j " << j
+                    << " target_size " << target_size
+                    << " target_id " << target_id
+                    << "\n";
+
+                comm.recv(exec, recv_buffer.data(), target_size, rank + i,
+                          rank);
+                std::cout << __FILE__ << " on rank " << rank
+                    << " after recv \n";
+
+
+                // the new offset is
+                auto offset = partition->get_range_bounds()[rank + i] -
+                              partition->get_range_bounds()[rank];
+                std::transform(recv_buffer.begin(), recv_buffer.end(),
+                               recv_buffer.begin(),
+                               [&](label idx) { return idx + offset; });
+
+                send_idxs.push_back({gko::array<label>{
+                                         exec,
+                                         recv_buffer.begin(),
+                                         recv_buffer.end(),
+                                     },
+                                     target_id});
+            }
+        }
+    } else {
+        // send all comm pairs to new owner
+        auto send_count = send_counts[owner_rank];
+        for (int i = 0; i < send_count; i++) {
+            auto send_buffer = send_idxs[i].first;
+            comm.send(exec, send_buffer.get_const_data(),
+                      send_buffer.get_size(), owner_rank, owner_rank);
+        }
+    }
+    std::cout << __FILE__ << " on rank " << rank << "after loop 2 \n";
+
+
+    // clear communcation neighbors on non owning rank
+    if (!owner) {
+        target_ids.clear();
+        target_sizes.clear();
+        send_idxs.clear();
+    }
+
+    // Step 3. merge communication
+    // we now might have communication partners multiple times, thus we can
+    // merge them sort repart_target_ids and use the sorting for target_sizes
+    // and repart_send_idxs
+    std::vector<label> merged_target_ids{};
+    std::vector<label> merged_target_sizes{};
+    std::vector<std::vector<label>> merged_send_idxs{};
+    if (owner) {
+        auto p = sort_permutation(target_ids,
+                                  [](label a, label b) { return a < b; });
+
+        target_sizes = apply_permutation(target_sizes, p);
+        target_ids = apply_permutation(target_ids, p);
+        send_idxs = apply_permutation(send_idxs, p);
+
+        // Step 4.
+        // Merge communication pattern with corresponding neighbours
+        if (target_ids.size() > 1) {
+            merged_target_ids.push_back(target_ids[0]);
+            merged_target_sizes.push_back(target_sizes[0]);
+            merged_send_idxs.emplace_back(std::vector<label>(
+                send_idxs[0].first.get_data(),
+                send_idxs[0].first.get_data() + send_idxs[0].first.get_size()));
+        }
+
+        for (int i = 1; i < target_ids.size(); i++) {
+            if (target_ids[i] == merged_target_ids.back()) {
+                merged_target_sizes.back() += target_sizes[i];
+                merged_send_idxs.back().insert(
+                    merged_send_idxs.back().end(),
+                    send_idxs[i].first.get_data(),
+                    send_idxs[i].first.get_data() +
+                        send_idxs[i].first.get_size());
+            } else {
+                merged_target_ids.push_back(target_ids[i]);
+            }
+        }
+    }
+    std::cout << __FILE__ << " on rank " << rank << "after loop 3 \n";
+
+    // recompute send_idxs
+    send_idxs.clear();
+
+    for (int i = 0; i < merged_target_ids.size(); i++) {
+        label target_id = merged_target_ids[i];
+        send_idxs.emplace_back(std::pair<gko::array<label>, comm_size_type>{
+            gko::array<label>{exec, merged_send_idxs[i].begin(),
+                              merged_send_idxs[i].end()},
+            target_id});
+    }
+    std::cout << __FILE__ << " on rank " << rank << "after loop 4 \n";
+
+    return CommunicationPattern{
+        src_comm_pattern.comm,
+        gko::array<comm_size_type>{exec, merged_target_ids.begin(),
+                                   merged_target_ids.end()},
+        gko::array<comm_size_type>{exec, merged_target_sizes.begin(),
+                                   merged_target_sizes.end()},
+        send_idxs};
+}
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 6ae08a363..836d38541 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -33,12 +33,36 @@ SourceFiles
 
 
 // This function computes the send and recv counts vectors for repartitioning
-//
 std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
            std::vector<label>, std::vector<label>>
-compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
+compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
                          const gko::experimental::mpi::communicator &comm,
-                         std::shared_ptr<const gko::Executor> exec);
+                         std::shared_ptr<const gko::Executor> exec, label size,
+                         label total_size, label padding_before,
+                         label padding_after);
+
+// This function computes the send and recv counts vectors for repartitioning
+std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+           std::vector<label>, std::vector<label>>
+compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
+                         const gko::experimental::mpi::communicator &comm,
+                         std::shared_ptr<const gko::Executor> exec, label size);
+
+void gather_to_owner(
+    std::shared_ptr<const gko::Executor> exec,
+    gko::experimental::mpi::communicator &comm,
+    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+               std::vector<label>, std::vector<label>> &comm_pattern,
+    const scalar *send_buffer, scalar *recv_buffer);
+
+std::vector<label> gather_to_owner(
+    std::shared_ptr<const gko::Executor> exec,
+    gko::experimental::mpi::communicator &comm,
+    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+               std::vector<label>, std::vector<label>> &comm_pattern,
+    const scalar *send_buffer, scalar *recv_buffer,
+    label offset=0
+);
 
 struct CommunicationPattern {
     using comm_size_type = label;
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H.bck b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H.bck
new file mode 100644
index 000000000..e46aa555a
--- /dev/null
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H.bck
@@ -0,0 +1,76 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::IOSortingIdxHandler
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    CommunicationPattern.H
+
+\*---------------------------------------------------------------------------*/
+
+#pragma once
+#include <ginkgo/ginkgo.hpp>
+#include "fvCFD.H"
+
+// TODO move this file to top level since it is not really a matrix-wrapper
+
+
+/** This function computes the send and recv counts vectors for repartitioning
+ *
+ * @parameter padding allow padding in recv buffer
+ * */
+std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+           std::vector<label>, std::vector<label>>
+compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
+                         const gko::experimental::mpi::communicator &comm,
+                         std::shared_ptr<const gko::Executor> exec,
+                         label padding_before=0,
+                         label padding_after=0
+);
+
+struct CommunicationPattern {
+    using comm_size_type = label;
+
+    gko::experimental::mpi::communicator comm;
+
+    // an array storing to which rank to communicate
+    gko::array<comm_size_type> target_ids;
+
+    // an array storing how many elements to communicate
+    // to the corresponding target_id
+    gko::array<comm_size_type> target_sizes;
+
+    // send_idx stores the index_set ie which cells
+    // are owned by the interface and the corresponding target rank
+    std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
+};
+
+std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e);
+
+// TODO NOTE currently we have just one communication pattern for multiple proc
+// interfaces
+//  hence this will communicate vectors which are too long, FIXME potentially
+//  have the wrong idxs and will communictate the same remote buffer several
+//  times through the combination matrix.
+CommunicationPattern repartition_comm_pattern(
+    label ranks_per_gpu, CommunicationPattern &src_comm_pattern,
+    std::shared_ptr<
+        const gko::experimental::distributed::Partition<label, label>>
+        partition);
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 23b45cfaa..f8e74a4a8 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -24,6 +24,7 @@ SourceFiles
     CommunicationPattern.H
 
 \*---------------------------------------------------------------------------*/
+#include <map>
 
 #include <ginkgo/core/distributed/lin_op.hpp>
 #include <ginkgo/ginkgo.hpp>
@@ -164,13 +165,12 @@ public:
     std::shared_ptr<part_type> get_partition() { return this->partition_; }
 
     static std::shared_ptr<RepartDistMatrix> create(
-        word matrix_format, std::shared_ptr<const HostMatrixWrapper> host_A)
+        word matrix_format, label ranks_per_gpu,
+        std::shared_ptr<const HostMatrixWrapper> host_A)
     {
         auto exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_device_exec();
         auto comm = *exec_handler.get_communicator().get();
-        // TODO FIXME read from dictionary
-        label ranks_per_gpu = 2;
 
         // repartition things here first by creating device_matrix_data on
         // device with correct size on each rank
@@ -192,8 +192,16 @@ public:
 
         label rank{src_comm_pattern.comm.rank()};
 
+        auto target_rank = [ranks_per_gpu](label rank) {
+            return rank - (rank % ranks_per_gpu);
+        };
+
+        label owner_rank = target_rank(rank);
+        bool owner = rank == owner_rank;
+
         auto repartition_sparsity =
-            [ranks_per_gpu, src_comm_pattern, exec, orig_partition](
+            [rank, owner, owner_rank, comm, ranks_per_gpu, src_comm_pattern,
+             exec, orig_partition, target_rank](
                 std::shared_ptr<SparsityPattern> src_local_pattern,
                 std::shared_ptr<SparsityPattern> src_non_local_pattern) {
                 // 1. obtain send recv sizes vector
@@ -208,24 +216,11 @@ public:
                 // debugging reasons
                 // FatalErrorInFunction << "Not implemented" <<
                 // abort(FatalError);
-                auto target_rank = [ranks_per_gpu](label rank) {
-                    return rank - (rank % ranks_per_gpu);
-                };
-
 
-                auto comm = src_comm_pattern.comm;
 
-                // find ranks which are now local
-                // the target_ids, target_sizes etc need to be communicated
-                // with new owner. thus we use maps to store them where the
-                // index is the new owner.
-                label rank{src_comm_pattern.comm.rank()};
-                label owner_rank = target_rank(rank);
-                bool owner = rank == owner_rank;
-
-                auto local_comm_pattern = compute_send_recv_counts(
-                    ranks_per_gpu, owner_rank, src_local_pattern->size_, comm,
-                    exec);
+                auto local_comm_pattern =
+                    compute_send_recv_counts(ranks_per_gpu, owner_rank, comm,
+                                             exec, src_local_pattern->size_);
 
                 auto gather_to_owner = [&](label offset, label *data,
                                            label size, auto &comm_pattern) {
@@ -260,6 +255,23 @@ public:
                     0, src_local_pattern->ldu_mapping.get_data(),
                     src_local_pattern->row_idxs.get_size(), local_comm_pattern);
 
+                // after gathering on owner we need to make ldu_mapping_idx
+                // consecutive
+                {
+                    auto [send_counts, recv_counts, send_offsets, recv_offsets,
+                          recv_buffer] = local_comm_pattern;
+
+                    label ldu_offset = 0;
+                    auto *data = src_local_pattern->ldu_mapping.get_data();
+                    for (label i = 0; i < ranks_per_gpu; i++) {
+                        auto size = recv_counts[i];
+                        std::transform(
+                            data + ldu_offset, data + size, data + ldu_offset,
+                            [&](label idx) { return idx + ldu_offset; });
+                        ldu_offset += size;
+                    }
+                }
+
 
                 label rows = (owner) ? merged_local_rows.back() + 1 : 0;
                 gko::dim<2> merged_local_dim{rows, rows};
@@ -269,8 +281,8 @@ public:
                 std::vector<label> merged_ranks{rank};
 
                 auto non_local_comm_pattern = compute_send_recv_counts(
-                    ranks_per_gpu, owner_rank, src_non_local_pattern->size_,
-                    comm, exec);
+                    ranks_per_gpu, owner_rank, comm, exec,
+                    src_non_local_pattern->size_);
 
                 std::vector<label> merged_non_local_rows = gather_to_owner(
                     offset, src_non_local_pattern->row_idxs.get_data(),
@@ -310,9 +322,8 @@ public:
                 auto [span_send_counts, span_recv_counts, span_send_offsets,
                       span_recv_offsets, span_recv_buffer] =
                     compute_send_recv_counts(
-                        ranks_per_gpu, owner_rank,
-                        src_non_local_pattern->interface_spans.size(), comm,
-                        exec);
+                        ranks_per_gpu, owner_rank, comm, exec,
+                        src_non_local_pattern->interface_spans.size());
 
                 std::vector<label> spans_begin;
                 std::vector<label> spans_end;
@@ -444,6 +455,7 @@ public:
                             gathered_non_local_cols.end(),
                             merged_non_local_cols.begin() + begin,
                             merged_non_local_cols.begin() + end);
+                        // TODO we don't do anything with non local mapping
                         gathered_non_local_mapping.insert(
                             gathered_non_local_mapping.end(),
                             merged_non_local_cols.begin() + begin,
@@ -464,6 +476,27 @@ public:
                     }
                 }
 
+                // for compressed cols we compute a mapping first
+                std::map<label, label> compressed_col_idxs{};
+                label unique_idx = 0;
+                for (auto idx : gathered_non_local_cols) {
+                    auto search = compressed_col_idxs.find(idx);
+                    if (search == compressed_col_idxs.end()) {
+                        std::cout << " insert " << idx << " " << unique_idx
+                                  << "\n";
+                        compressed_col_idxs[idx] = unique_idx;
+                        unique_idx += 1;
+                    };
+                }
+
+                for (int i = 0; i < gathered_non_local_cols.size(); i++) {
+                    label uncompressed = gathered_non_local_cols[i];
+                    label compressed = compressed_col_idxs[uncompressed];
+                    std::cout << " i " << i << " uncompressed " << uncompressed
+                              << " compressed " << compressed << "\n";
+                    gathered_non_local_cols[i] = compressed;
+                }
+
                 // add interfaces
                 // to add interfaces we go through the non_local_sparsity
                 // pattern and check if interface is still non_local
@@ -583,10 +616,71 @@ public:
             sparse_comm);
 
 
-        // We now have an distribute matrix which needs to be filled/updated
-        // first we copy L+D+U over
-        // here we can reuse the gather to owner implementation
+        label nrows = host_A->get_local_nrows();
+        label local_matrix_nnz = host_A->get_local_matrix_nnz();
+        // size + padding has to be local_matrix_nnz
+        auto diag_comm_pattern = compute_send_recv_counts(
+            ranks_per_gpu, owner_rank, comm, exec, nrows, local_matrix_nnz,
+            local_matrix_nnz - nrows, 0);
+
+        label upper_nnz = host_A->get_upper_nnz();
+        auto upper_comm_pattern = compute_send_recv_counts(
+            ranks_per_gpu, owner_rank, comm, exec, upper_nnz, local_matrix_nnz,
+            0, local_matrix_nnz - upper_nnz);
+        auto lower_comm_pattern = compute_send_recv_counts(
+            ranks_per_gpu, owner_rank, comm, exec, upper_nnz, local_matrix_nnz,
+            upper_nnz, nrows);
+
+        scalar *local_ptr;
+
+        if (owner) {
+            using Coo = gko::matrix::Coo<scalar, label>;
+            std::shared_ptr<const Coo> local =
+                gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
+                                 dist_A->get_local_matrix())
+                                 ->get_combination()
+                                 ->get_operators()[0]);
+
+            local_ptr = const_cast<scalar *>(local->get_const_values());
+        }
+        gather_to_owner(exec, comm, diag_comm_pattern, host_A->get_diag(),
+                        local_ptr);
+        gather_to_owner(exec, comm, upper_comm_pattern, host_A->get_upper(),
+                        local_ptr);
+        if (host_A->symmetric()) {
+            // TODO FIXME
+            // if symmetric we can reuse already copied data
+            gather_to_owner(exec, comm, lower_comm_pattern, host_A->get_lower(),
+                            local_ptr);
+        } else {
+            gather_to_owner(exec, comm, lower_comm_pattern, host_A->get_lower(),
+                            local_ptr);
+        }
+
+
+        // next update local and non_local interfaces
+
+
+        // TODO make sure this doesn't copy
+        // create a non owning dense matrix of local_values
+        using dim_type = gko::dim<2>::dimension_type;
+        auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
+            exec,
+            gko::dim<2>{static_cast<dim_type>(repart_loc_sparsity->size_), 1},
+            gko::array<scalar>::view(exec, repart_loc_sparsity->size_,
+                                     local_ptr),
+            1));
+        if (rank == 0) {
+            std::cout << __FILE__ << " on rank " << rank
+                      << " created row collection of size  "
+                      << repart_loc_sparsity->size_ << " ldu_mapping "
+                      << local_sparsity_->ldu_mapping << "\n";
+        }
 
+        // TODO this needs to copy ldu_mapping to the device
+        auto dense_vec = row_collection->clone();
+        dense_vec->row_gather(&local_sparsity_->ldu_mapping,
+                              row_collection.get());
 
         return std::make_shared<RepartDistMatrix>(
             exec, comm, host_A->get_size(), dist_A->get_size(),
@@ -749,23 +843,24 @@ public:
 
 protected:
     // Here we implement the application of the linear operator, x = A * b.
-    // apply_impl will be called by the apply method, after the arguments have
-    // been moved to the correct executor and the operators checked for
+    // apply_impl will be called by the apply method, after the arguments
+    // have been moved to the correct executor and the operators checked for
     // conforming sizes.
     //
-    // For simplicity, we assume that there is always only one right hand side
-    // and the stride of consecutive elements in the vectors is 1 (both of these
-    // are always true in this example).
+    // For simplicity, we assume that there is always only one right hand
+    // side and the stride of consecutive elements in the vectors is 1 (both
+    // of these are always true in this example).
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
         this->dist_mtx_->apply(b, x);
     }
 
 
-    // There is also a version of the apply function which does the operation
-    // x = alpha * A * b + beta * x. This function is commonly used and can
-    // often be better optimized than implementing it using x = A * b. However,
-    // for simplicity, we will implement it exactly like that in this example.
+    // There is also a version of the apply function which does the
+    // operation x = alpha * A * b + beta * x. This function is commonly
+    // used and can often be better optimized than implementing it using x =
+    // A * b. However, for simplicity, we will implement it exactly like
+    // that in this example.
     void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
                     const gko::LinOp *beta, gko::LinOp *x) const override
     {
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index a121b5a29..b5dded6e3 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -426,7 +426,7 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_local_sparsity(
     // TODO in order to simplify when local interfaces exists set
     // local_sparsity to size of nrows_w_interfaces, if interfaces exist
     // local_sparsity is only valid till nrows_
-    label after_neighbours = (symmetric_) ? upper_nnz_ : 2 * upper_nnz_;
+    label after_neighbours = 2 * upper_nnz_;
     init_local_sparsity(nrows_, upper_nnz_, symmetric_, upper, lower, rows,
                         cols, permute);
 
@@ -439,6 +439,9 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_local_sparsity(
     // idx = [d_1, u_1, l_2, d_2, u_2, ... d_n, i_11, i_12, .., i_nn] where
     // i_j,k j=interface index and k cell index on the interface
     if (local_interface_nnz_) {
+        // we now treat interfaces differently
+        FatalErrorInFunction << "Not implemented" << abort(FatalError);
+
         // NOTE currently, this copies the interface indizes first to a vector
         // of tuples before inserting it into the persistent arrays. We could
         // remove the unnessary copy via the vector of tuples and
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index b9979ca48..6a2fa02b9 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -440,6 +440,30 @@ public:
 
     label get_local_nrows() const { return nrows_; }
 
+    label get_upper_nnz() const { return  upper_nnz_;}
+
+    label get_local_matrix_nnz() const { return local_matrix_nnz_; }
+
+    /** Create a view into host diag ptr  */
+    const scalar* get_diag() const {
+        return diag_;
+    }
+
+    /** Create a view into host lower ptr  */
+    const scalar* get_lower() const {
+        return lower_;
+    }
+
+    /** Create a view into host lower ptr  */
+    const scalar* get_upper() const {
+        return upper_;
+    }
+
+    /** Create a view into host lower ptr  */
+    bool symmetric() const {
+        return symmetric_;
+    }
+
     gko::dim<2> get_size() const { return gko::dim<2>(nrows_, nrows_); }
 
     const ExecutorHandler &get_exec_handler() const { return exec_; }
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H.bck b/MatrixWrapper/LDUMatrix/HostMatrix.H.bck
new file mode 100644
index 000000000..8163daf00
--- /dev/null
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H.bck
@@ -0,0 +1,457 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    HostMatrix.H
+
+\*---------------------------------------------------------------------------*/
+#ifndef OGL_HostMatrix_INCLUDED_H
+#define OGL_HostMatrix_INCLUDED_H
+#include <ginkgo/ginkgo.hpp>
+
+#include "fvCFD.H"
+#include "processorLduInterface.H"
+
+#include <vector>
+#include "DevicePersistent/Array/Array.H"
+#include "DevicePersistent/DeviceIdGuard/DeviceIdGuard.H"
+#include "DevicePersistent/IOGlobalIndex/gkoGlobalIndex.H"
+#include "MatrixWrapper/Combination/Combination.H"
+#include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
+#include "common/common.H"
+
+
+namespace Foam {
+
+
+struct SparsityPattern {
+    SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size)
+        : size_(size),
+          row_idxs{exec, static_cast<gko::size_type>(size)},
+          col_idxs{exec, static_cast<gko::size_type>(size)},
+          ldu_mapping{exec, static_cast<gko::size_type>(size)}
+    {}
+
+    SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size,
+                    gko::dim<2> dim, std::vector<label> &rows,
+                    std::vector<label> &cols, std::vector<label> &map,
+                    std::vector<label> &spans_begin,
+                    std::vector<label> &spans_end, std::vector<label> &ranks)
+        : size_(size),
+          dim(dim),
+          row_idxs{exec, static_cast<gko::size_type>(size)},
+          col_idxs{exec, static_cast<gko::size_type>(size)},
+          ldu_mapping{exec, static_cast<gko::size_type>(size)}
+    {
+        ASSERT_EQ(size, rows.size());
+        std::copy(rows.begin(), rows.end(), row_idxs.get_data());
+
+        ASSERT_EQ(size, cols.size());
+        std::copy(cols.begin(), cols.end(), col_idxs.get_data());
+
+        ASSERT_EQ(size, map.size());
+        std::copy(map.begin(), map.end(), ldu_mapping.get_data());
+
+        ASSERT_EQ(spans_begin.size(), spans_end.size());
+        std::transform(spans_begin.begin(), spans_begin.end(),
+                       spans_end.begin(), std::back_inserter(interface_spans),
+                       [](const auto begin, const auto end) {
+                           return gko::span{static_cast<gko::size_type>(begin),
+                                            static_cast<gko::size_type>(end)};
+                       });
+
+        ASSERT_EQ(spans_begin.size(), ranks.size());
+        rank = ranks;
+    }
+
+    // TODO rename the public members and remove trailing _
+    // number of nnz entries
+    const label size_;
+
+    mutable gko::Array<label> row_idxs;
+
+    mutable gko::Array<label> col_idxs;
+
+    // ldu_mapping[sorted_(csr)_position] =
+    // unsorted_(consecutive_ldu_)_position
+    mutable gko::Array<label> ldu_mapping;
+
+    // dimensions of the resulting matrix
+    mutable gko::dim<2> dim;
+
+    // A vector of spans indicating begin and end of each interface
+    // this is used to keep all col and row idx consecutive in memory
+    mutable std::vector<gko::span> interface_spans;
+
+    // keep track of original (communication) rank of each interface or matrix
+    // block
+    mutable std::vector<label> rank;
+};
+
+// Free functions
+const lduInterfaceField *interface_getter(
+    const lduInterfaceFieldPtrsList &interfaces, const label i);
+
+/** Write contiguous row and col indices from OpenFOAM lower and upper indices
+ ** For details on the lower triangular based indexing see
+ ** https://openfoamwiki.net/index.php/OpenFOAM_guide/Matrices_in_OpenFOA
+ ** Note that the order of the indices are depicted wrong on the wiki
+ ** In general the upper triangular matrix is traversed in row major
+ ** and the lower triangular matrix in column major order.
+ **
+ ** @param nrows number of rows
+ ** @param upper_nnz number of non zeros in the upper triangular matrix
+ ** @param is_symmetric whether matrix is symmetric, in the symmetric case
+ ** the lower elements indices in the permute array are computed differently
+ ** @param upper pointer to OFs rows array
+ ** @param lower pointer to OFs cols array
+ ** @param rows pointer to rows array
+ ** @param cols pointer to columns array
+ ** @param permute pointer to permuter array
+ */
+void init_local_sparsity(const label nrows, const label upper_nnz,
+                         const bool is_symmetric, const label *upper,
+                         const label *lower, label *rows, label *cols,
+                         label *permute);
+
+/** Perform update of the coefficients array in row major order from a symmetric
+ *ldu matrix
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ *upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param out pointer to data of output matrix
+ **/
+void symmetric_update(const label total_nnz, const label upper_nnz,
+                      const label *permute, const scalar scale,
+                      const scalar *diag, const scalar *upper, scalar *out);
+
+
+/** Perform update of the coefficients array in row major order from a
+ *non-symmetric ldu matrix
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ *upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for lower entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param lower pointer to lower triangular entries of the original ldu matrix
+ ** @param out pointer to data of output matrix
+ **/
+void non_symmetric_update(const label total_nnz, const label upper_nnz,
+                          const label *permute, const scalar scale,
+                          const scalar *diag, const scalar *upper,
+                          const scalar *lower, scalar *dense);
+
+
+/** Perform update of the coefficients array in row major order from a
+ ** symmetric ldu matrix including local interface data.
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ ** upper
+ **
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for interface entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param interface pointer to interface entries
+ ** @param out pointer to data of output matrix
+ **/
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
+                                  const label upper_nnz, const label *permute,
+                                  const scalar scale, const scalar *diag,
+                                  const scalar *upper, const scalar *interface,
+                                  scalar *dense);
+
+
+/** Perform update of the coefficients array in row major order from a
+ ** non-symmetric ldu matrix including local interface data.
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ ** upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for lower entries
+ **        2*upper_nnz + diag_nnz <= permute for interface entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param lower pointer to lower triangular entries of the original ldu matrix
+ ** @param interface pointer to interface entries
+ ** @param out pointer to data of output matrix
+ **/
+void non_symmetric_update_w_interface(
+    const label total_nnz, const label diag_nnz, const label upper_nnz,
+    const label *permute, const scalar scale, const scalar *diag,
+    const scalar *upper, const scalar *lower, const scalar *couple_coeffs,
+    scalar *dense);
+
+
+/* The HostMatrixWrapper class handles the conversion from OpenFOAMs lduMatrix
+ * format into Ginkgo array data structures
+ *
+ * It mainly generates:
+ * - sparsity pattern of the local_matrix and
+ *   non_local_matrix (with global columns)
+ * - CommunicationPattern
+ * - implements an interfaces so that
+ *   RepartDistMatrix can read_distributed(A_host)
+ * */
+class HostMatrixWrapper {
+private:
+    using vec = gko::matrix::Dense<scalar>;
+    using idx_array = gko::array<label>;
+
+    const ExecutorHandler exec_;
+
+    const DeviceIdGuardHandler device_id_guard_;
+
+    const label verbose_;
+
+    const word field_name_;
+
+    // Whether the matrix coefficients should be reordered
+    // during copy or on device
+    const bool reorder_on_copy_;
+
+    const lduAddressing &addr_;
+
+    const scalar *diag_;
+
+    const scalar *upper_;
+
+    const scalar *lower_;
+
+    // multiply the complete system by this factor, ie sAx=sb
+    // NOTE this needed to avoid negative diagonal matrix entries, but
+    // this could be also achieved by just fliping the sign
+    const scalar scaling_;
+
+    // number of local matrix rows
+    const label nrows_;
+
+    // number of local upper elements
+    // ie coefficients which column_idx < nrows_
+    const label upper_nnz_;
+
+    const bool symmetric_;
+
+    // number of local elements on interfaces
+    // ie number of interfaces which column_idx is < nrows_
+    // and have to be sorted into local matrix
+    const label local_interface_nnz_;
+
+    // total number of local upper and lower elements
+    // ie 2*upper_nnz_ since the sparsity pattern is symmetric
+    const label non_diag_nnz_;
+
+    // nnz of local matrix wo local interfaces
+    const label local_matrix_nnz_;
+
+    // nnzs of local matrix including local interfaces
+    const label local_matrix_w_interfaces_nnz_;
+
+    // TODO remove
+    // mutable SparsityPattern local_sparsity_pattern_;
+
+    // non-local indices
+    const label non_local_matrix_nnz_;
+
+    const lduInterfaceFieldPtrsList &interfaces_;
+
+    const FieldField<Field, scalar> &interfaceBouCoeffs_;
+
+    /* Iterates all interfaces and collects the coefficients into a vector
+    **
+    ** @param local whether local or non local coefficients should be collected
+    */
+    std::vector<scalar> collect_interface_coeffs(
+        const lduInterfaceFieldPtrsList &interfaces_,
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        const bool local) const;
+
+
+    /* Iterates all interfaces and counts the number of elements
+    **
+    ** @param interfaces The list of interfaces for the search
+    ** @param proc_interfaces Count only elements on (true)
+    *processorLduInterfaces or exclude processorLduInterfaces (false)
+    */
+    label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
+                              bool proc_interfaces) const;
+
+    /** Iterates all local interfaces and returns the relative order and
+    **corresponding row and column indices
+    **
+    ** @return vector of tuples containing the interface number, the local row,
+    *the local column
+    **/
+    std::vector<std::tuple<label, label, label>>
+    collect_local_interface_indices(
+        const lduInterfaceFieldPtrsList &interfaces_) const;
+
+
+    /** Iterates all interfaces and collect the corresponding cell id (row)
+    ** and a unique counter
+    **
+    ** @return vector of size nnz_non_local, with a running index, row
+    ** index, and a unique compressed column index
+    ** sections for each interface
+    ** ret = [(1,2,1),(2,20, 2), (3, 20, 2) ...]
+    **         i0   i1,   i...
+    */
+    std::vector<std::tuple<label, label, label, label>>
+    collect_cells_on_non_local_interface(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+
+    // updater
+
+
+public:
+    // segregated matrix wrapper constructor
+    HostMatrixWrapper(const ExecutorHandler &exec, const objectRegistry &db,
+                      label nrows, label upper_nnz, bool symmetric,
+                      const scalar *diag, const scalar *upper,
+                      const scalar *lower, const lduAddressing &addr,
+                      const FieldField<Field, scalar> &interfaceBouCoeffs,
+                      const FieldField<Field, scalar> &interfaceIntCoeffs,
+                      const lduInterfaceFieldPtrsList &interfaces,
+                      const dictionary &solverControls, const word &fieldName,
+                      label verbose);
+
+    // getter
+
+public:
+    static std::shared_ptr<HostMatrixWrapper> create(
+        const ExecutorHandler &exec, const objectRegistry &db, label nrows,
+        label upper_nnz, bool symmetric, const scalar *diag,
+        const scalar *upper, const scalar *lower, const lduAddressing &addr,
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        const FieldField<Field, scalar> &interfaceIntCoeffs,
+        const lduInterfaceFieldPtrsList &interfaces,
+        const dictionary &solverControls, const word &fieldName, label verbose)
+    {
+        return std::make_shared<HostMatrixWrapper>(
+            exec, db, nrows, upper_nnz, symmetric, diag, upper, lower, addr,
+            interfaceBouCoeffs, interfaceIntCoeffs, interfaces, solverControls,
+            fieldName, verbose);
+    };
+
+    void update(const scalar *diag, const scalar *upper, const scalar *lower,
+                const FieldField<Field, scalar> &interfaceBouCoeffs,
+                const FieldField<Field, scalar> &interfaceIntCoeffs,
+                const lduInterfaceFieldPtrsList &interfaces);
+
+    /** Based on OpenFOAMs ldu matrix format this function computes two
+     ** consecutive index arrays (local_sparsisty_.row_idxs and col_idxs) in row
+     ** major ordering and the permutation index (local_sparsity_.ldu_mapping),
+     ** which are required to create to a ginkgo matrix
+     **/
+    std::shared_ptr<SparsityPattern> compute_local_sparsity(
+        std::shared_ptr<const gko::Executor> exec) const;
+
+    /** Copies the LDU matrix coefficients to local_coeffs without changing or
+     ** reinstantiating the sparsity pattern.
+     ** This uses the local_sparsity_.ldu_mapping to permute the data already
+     ** on the host or device to be in row major order.
+     **/
+    void compute_local_coeffs(std::shared_ptr<const SparsityPattern> sparsity,
+                              gko::array<scalar> &target_coeffs) const;
+
+    /** Copies the interface matrix coefficients to non_local_coeffs without
+    ** changing or reinstantiating the sparsity pattern.
+    **/
+    void compute_non_local_coeffs(
+        std::shared_ptr<const SparsityPattern> sparsity,
+        gko::array<scalar> &target_coeffs) const;
+
+    /** Based on OpenFOAMs interfaces this function computes two
+     ** consecutive index arrays (non_local_sparsisty_.row_idxs and col_idxs) in
+     ** row *major ordering and the permutation index
+     ** (non_local_sparsity_.ldu_mapping), which are required to create a ginkgo
+     ** matrix
+     **/
+    std::shared_ptr<SparsityPattern> compute_non_local_sparsity(
+        std::shared_ptr<const gko::Executor> exec) const;
+
+    /** Iterates all interfaces and counts the number of unique neighbour
+     ** processors and number of interfaces in total for this processor
+     ** and collects all interface cells of this rank.
+     **
+     ** @param interfaces The list of interfaces for the search
+     ** @return the CommunicationPattern
+     */
+    CommunicationPattern create_communication_pattern(
+        const gko::experimental::mpi::communicator &comm) const;
+
+
+    bool get_verbose() const { return verbose_; }
+
+    label get_local_nrows() const { return nrows_; }
+
+    label get_local_matrix_nnz() const { return local_matrix_nnz_; }
+
+        /** Create a view into host diag ptr  */
+    const scalar* get_diag() const {
+        return diag_;
+    }
+
+    gko::dim<2> get_size() const { return gko::dim<2>(nrows_, nrows_); }
+
+    const ExecutorHandler &get_exec_handler() const { return exec_; }
+};
+
+
+}  // namespace Foam
+#endif
diff --git a/MatrixWrapper/LDUMatrix/HostMatrixFreeFunctions.C b/MatrixWrapper/LDUMatrix/HostMatrixFreeFunctions.C
index 4ec1649e1..8a5d11e47 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrixFreeFunctions.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrixFreeFunctions.C
@@ -109,7 +109,7 @@ void init_local_sparsity(const label nrows, const label upper_nnz,
     // stored such that upper rows are monotonic ascending
     // upper - rows of lower triangular matrix
     // lower - columns of lower triangular matrix
-    label after_neighbours = (is_symmetric) ? upper_nnz : 2 * upper_nnz;
+    label after_neighbours = 2 * upper_nnz;
 
     // first pass order elements row wise
     // scan through all faces
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 54f509ca7..a90d2961e 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -38,7 +38,7 @@ private:
 
     const bool force_host_buffer_;
 
-    // const label ranks_per_gpu_;
+    const label ranks_per_gpu_;
 
     const word matrix_format_;
 
@@ -50,8 +50,8 @@ public:
           debug_(solverControls.lookupOrDefault<Switch>("debug", false)),
           force_host_buffer_(
               solverControls.lookupOrDefault("forceHostBuffer", false)),
-          // ranks_per_gpu_(
-          //     solverControls.lookupOrDefault<label>("ranksPerGPU", 1)),
+          ranks_per_gpu_(
+              solverControls.lookupOrDefault<label>("ranksPerGPU", 1)),
           matrix_format_(
               solverControls.lookupOrDefault<word>("matrixFormat", "Coo"))
     {
@@ -88,8 +88,8 @@ public:
             std::to_string(gko::experimental::mpi::is_gpu_aware()) +
             std::string("\n\tForces host buffer based communication: ") +
             std::to_string(force_host_buffer_) +
-            // std::string("\n\tCPU ranks per GPU: ") +
-            // std::to_string(ranks_per_gpu_) +
+            std::string("\n\tCPU ranks per GPU: ") +
+            std::to_string(ranks_per_gpu_) +
             std::string("\n\tMatrix format: ") + matrix_format_ +
             std::string("\n\tEnd OGL_INFO");
         MLOG_0(verbose_, msg)

From e933ad12d934886521d744c0129d0c8d5fab8724 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 24 May 2024 11:47:55 +0200
Subject: [PATCH 52/90] refactor start dedicated repartitioner class

---
 .../DistributedMatrixAdapter.H                |   9 +-
 DevicePersistent/Vector/Vector.H              |  23 +-
 .../CommunicationPattern.C                    | 109 ++---
 .../CommunicationPattern.H                    |  29 +-
 MatrixWrapper/Distributed/Distributed.H       | 392 +---------------
 MatrixWrapper/LDUMatrix/HostMatrix.H          | 442 +++++++++++++++---
 .../SparsityPattern/SparsityPattern.H         |  89 ++++
 lduLduBase/lduLduBase.H                       |   6 +
 8 files changed, 577 insertions(+), 522 deletions(-)
 create mode 100644 MatrixWrapper/SparsityPattern/SparsityPattern.H

diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 5e58ba2fd..84592ac9e 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -40,6 +40,8 @@ struct DistributedMatrixInitFunctor {
 
     const ExecutorHandler &exec_;
 
+    const Repartitioner & repartitioner_;
+
     const word field_name_;
 
     const label verbose_;
@@ -49,6 +51,7 @@ struct DistributedMatrixInitFunctor {
     const PersistentHostMatrixAdapter& host_A_;
 
     DistributedMatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
+                         const Repartitioner & repartitioner,
                          const label verbose,
                          const word field_name,
                          const PersistentHostMatrixAdapter& host_A,
@@ -56,6 +59,7 @@ struct DistributedMatrixInitFunctor {
                          )
         : db_(db),
           exec_(exec),
+          repartitioner_(repartitioner),
           field_name_(field_name),
           verbose_(verbose),
           solver_controls_(solver_controls),
@@ -72,7 +76,7 @@ struct DistributedMatrixInitFunctor {
     {
         return RepartDistMatrix<scalar, label, label>::create(
               solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
-              solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
+              repartitioner_,
               host_A_.get());
     }
 };
@@ -96,6 +100,7 @@ public:
     PersistentDistributedMatrix(
                   const objectRegistry &db,
                   const ExecutorHandler &exec,
+                  const Repartitioner& repartitioner,
                   const word field_name,
                   const PersistentHostMatrixAdapter& host_A,
                   const dictionary &controlDict,
@@ -106,7 +111,7 @@ public:
           field_name_(field_name),
           host_matrix_{
               field_name + "_distributed_matrix", db,
-              DistributedMatrixInitFunctor(db, exec, verbose_,
+              DistributedMatrixInitFunctor(db, exec, repartitioner, verbose_,
                                    field_name + "_distributed_matrix",
                                    host_A, 
                                    controlDict),
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 9796f711e..8a65b74a4 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -11,6 +11,7 @@
 #include "DevicePersistent/Base/Base.H"
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "DevicePersistent/Partition/Partition.H"
+#include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 #include "common/common.H"
 
 namespace Foam {
@@ -23,6 +24,8 @@ struct VectorInitFunctor {
 
     const word name_;
 
+    const Repartitioner & repartitioner_;
+
     const ExecutorHandler &exec_;
 
     const label verbose_;
@@ -34,11 +37,12 @@ struct VectorInitFunctor {
 
     const label size_;
 
-
     VectorInitFunctor(const ExecutorHandler &exec, const word name,
+        const Repartitioner & repartitioner,
                       const T *other, const label size, const label verbose,
                       const bool on_device = false)
         : name_(name),
+          repartitioner_(repartitioner),
           exec_(exec),
           on_device_(on_device),
           verbose_(verbose),
@@ -74,9 +78,16 @@ struct VectorInitFunctor {
         auto host_view = gko::array<T>::view(exec_.get_ref_exec(), local_size,
                                              const_cast<T *>(other_));
 
-        return gko::share(dist_vec::create(
-            exec, *exec_.get_communicator().get(),
-            vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)));
+        label repart_size = local_size;
+        auto local_coeffs = gko::array<scalar>(exec, repart_size);
+        FatalErrorInFunction << "Not implemented" << abort(FatalError);
+        return {};
+
+        // gather_to_owner(exec, comm, comm_pattern_, host_view,
+        //                 local_coeffs.get_data());
+
+        // return gko::share(dist_vec::create(
+        //     exec, *exec_.get_communicator().get(), local_coeffs));
     }
 };
 
@@ -103,7 +114,6 @@ class PersistentVector
     // updated even if was found in the object registry
     const bool update_;
 
-
 public:
     /* PersistentVector constructor using existing memory
      *
@@ -120,13 +130,14 @@ public:
      * @param ranks_per_gpu
      */
     PersistentVector(const T *memory, const label size, const word name,
+                     const Repartitioner & repartitioner,
                      const objectRegistry &db, const ExecutorHandler &exec,
                      const label verbose, const bool update,
                      const bool init_on_device)
         : PersistentBase<gko::experimental::distributed::Vector<T>,
                          VectorInitFunctor<T>>(
               name, db,
-              VectorInitFunctor<T>(exec, name, memory, size, verbose,
+              VectorInitFunctor<T>(exec, name, repartitioner, memory, size, verbose,
                                    init_on_device),
               update, verbose),
           db_(db),
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index 22be2a031..86bfaa9f2 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -29,7 +29,7 @@ SourceFiles
 #include "common/common.H"
 
 std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-           std::vector<label>, std::vector<label>>
+           std::vector<label>>
 compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
                          const gko::experimental::mpi::communicator &comm,
                          std::shared_ptr<const gko::Executor> exec, label size)
@@ -39,7 +39,7 @@ compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
 }
 
 std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-           std::vector<label>, std::vector<label>>
+           std::vector<label>>
 compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
                          const gko::experimental::mpi::communicator &comm,
                          std::shared_ptr<const gko::Executor> exec, label size,
@@ -93,21 +93,18 @@ compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
         comm.send(exec, &padding_after, 1, owner_rank, owner_rank);
     }
 
-    std::vector<label> target_recv_buffer(tot_recv_elements, 0);
-
-    return std::make_tuple(send_counts, recv_counts, send_offsets, recv_offsets,
-                           target_recv_buffer);
+    return std::make_tuple(send_counts, recv_counts, send_offsets,
+                           recv_offsets);
 }
 
-void gather_to_owner(
+std::vector<label> gather_to_owner(
     std::shared_ptr<const gko::Executor> exec,
-    gko::experimental::mpi::communicator &comm,
-    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-               std::vector<label>, std::vector<label>> &comm_pattern,
+    const gko::experimental::mpi::communicator &comm,
+    const std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+                     std::vector<label>> &comm_pattern,
     const scalar *send_buffer, scalar *recv_buffer)
 {
-    auto [send_counts, recv_counts, send_offsets, recv_offsets, recv_buffer_] =
-        comm_pattern;
+    auto [send_counts, recv_counts, send_offsets, recv_offsets] = comm_pattern;
     // TODO add some sanity checks for comm pattern
     // 1. length needs to be same as mpi ranks
     // 2. recv_buffer length should match ie at least length of recv_counts
@@ -125,31 +122,42 @@ void gather_to_owner(
     comm.all_to_all_v(exec, send_buffer, send_counts.data(),
                       send_offsets.data(), recv_buffer, recv_counts.data(),
                       recv_offsets.data());
+    return {};
 };
 
 std::vector<label> gather_to_owner(
     std::shared_ptr<const gko::Executor> exec,
-    gko::experimental::mpi::communicator &comm,
-    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-               std::vector<label>, std::vector<label>> &comm_pattern,
-    label size,
-    scalar *send_buffer, label offset = 0)
+    const gko::experimental::mpi::communicator &comm,
+    const std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+                     std::vector<label>> &comm_pattern,
+    label size, const label *data, label offset)
 {
     // TODO don't modify send_buffer
+    std::vector<label> send_buffer_copy(size);
     if (offset > 0) {
-        std::transform(send_buffer, send_buffer + size, send_buffer,
+        std::transform(data, data + size, send_buffer_copy.data(),
                        [&](label idx) { return idx + offset; });
     }
 
-    auto & [send_counts, recv_counts, send_offsets, recv_offsets, recv_buffer] =
-        comm_pattern;
+    const label *send_buffer = (offset > 0) ? send_buffer_copy.data() : data;
 
-    comm.all_to_all_v(exec, data, send_counts.data(), send_offsets.data(),
-                      recv_buffer.data(), recv_counts.data(),
-                      recv_offsets.data());
+    auto &[send_counts, recv_counts, send_offsets, recv_offsets] = comm_pattern;
 
-    return std::vector<label>{recv_buffer};
-};
+    // compute total recv buffer size
+    // TODO account for holes
+    auto total_recv_size = 0;
+    for (int i = 0; i < recv_counts.size(); i++) {
+        total_recv_size += recv_counts[i];
+    }
+
+    std::cout << " total_recv_ size " << total_recv_size << "\n";
+    std::vector<label> recv_buffer(total_recv_size);
+    comm.all_to_all_v(exec, send_buffer, send_counts.data(),
+                      send_offsets.data(), recv_buffer.data(),
+                      recv_counts.data(), recv_offsets.data());
+
+    return recv_buffer;
+}
 
 std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e)
 {
@@ -212,20 +220,22 @@ CommunicationPattern repartition_comm_pattern(
     label owner_rank = target_rank(rank);
     bool owner = rank == owner_rank;
 
-    // map for exchanging communication data where the index is the target rank
+    // map for exchanging communication data where the index is the target
+    // rank
     for (int i = 0; i < src_comm_pattern.target_ids.get_size(); i++) {
         label target_id = src_comm_pattern.target_ids.get_data()[i];
         // Step 1. Check if communication partner is non-local after
         // repartitioning. If it is non-local we keep it. Otherwise local
-        // communication can be discarded or merged. Here non-local means: the
-        // communication target proc id is != repartition proc id
+        // communication can be discarded or merged. Here non-local means:
+        // the communication target proc id is != repartition proc id
         //
         // [ 1 | 2 | 3 ] [ 4 | 5 | 6 ] where [] - bounds after repart
         //                                    | - bounds before repart
         // e.g. rank 3 has rank 2 and 4 as comm neighbours
         //      rank 2 can be discarded but rank 4 is non-local and needs
         //      to be communicated to rank 1 to be merged into the new
-        //      communicaton pattern. This needs an all to all communication.
+        //      communicaton pattern. This needs an all to all
+        //      communication.
 
         label repart_target_id = target_rank(target_id);
         label target_size = src_comm_pattern.target_sizes.get_const_data()[i];
@@ -233,9 +243,9 @@ CommunicationPattern repartition_comm_pattern(
 
         if (repart_target_id != owner_rank) {
             // communication pattern is non local, hence we keep it
-            // after repartitioning we now have to communicate with a different
-            // rank
-            // store new owner rank to which repart_target_* needs to be send to
+            // after repartitioning we now have to communicate with a
+            // different rank store new owner rank to which repart_target_*
+            // needs to be send to
             target_ids.push_back(repart_target_id);
             target_sizes.push_back(target_size);
             send_idxs.push_back(send_idx);
@@ -245,32 +255,25 @@ CommunicationPattern repartition_comm_pattern(
     // where to put elements from recv
     // communicate with all neighbours which are to be merged
     // how many elements the owner rank is to receive
-    auto [send_counts, recv_counts, send_offsets, recv_offsets,
-          target_recv_buffer] =
-        compute_send_recv_counts(ranks_per_gpu, owner_rank, comm, exec,
-                                 target_ids.size());
+    auto comm_pattern = compute_send_recv_counts(ranks_per_gpu, owner_rank,
+                                                 comm, exec, target_ids.size());
 
+    target_ids = gather_to_owner(exec, comm, comm_pattern, target_ids.size(),
+                                 target_ids.data());
 
-    comm.all_to_all_v(exec, target_ids.data(), send_counts.data(),
-                      send_offsets.data(), target_recv_buffer.data(),
-                      recv_counts.data(), recv_offsets.data());
-
-    target_ids = target_recv_buffer;
-
-    comm.all_to_all_v(exec, target_sizes.data(), send_counts.data(),
-                      send_offsets.data(), target_recv_buffer.data(),
-                      recv_counts.data(), recv_offsets.data());
-
-    target_sizes = target_recv_buffer;
+    target_sizes = gather_to_owner(exec, comm, comm_pattern,
+                                   target_sizes.size(), target_sizes.data());
 
     // communicate the send indices to owner rank
-    // send indices are packed into std::vector of gko::arrays thus we send them
-    // one by one
-    // additionally we already have then target_sizes and send_counts and
-    // recv_counts which can be reused to for sending the index vectors
+    // send indices are packed into std::vector of gko::arrays thus we send
+    // them one by one additionally we already have then target_sizes and
+    // send_counts and recv_counts which can be reused to for sending the
+    // index vectors
 
     // TODO this could be refactored to separtate function
     // this is also used distributed
+    auto recv_counts = std::get<1>(comm_pattern);
+    auto send_counts = std::get<0>(comm_pattern);
     if (owner) {
         // retrieve from all neighbours
         for (int i = 1; i < ranks_per_gpu; i++) {
@@ -319,8 +322,8 @@ CommunicationPattern repartition_comm_pattern(
 
     // Step 3. merge communication
     // we now might have communication partners multiple times, thus we can
-    // merge them sort repart_target_ids and use the sorting for target_sizes
-    // and repart_send_idxs
+    // merge them sort repart_target_ids and use the sorting for
+    // target_sizes and repart_send_idxs
     std::vector<label> merged_target_ids{};
     std::vector<label> merged_target_sizes{};
     std::vector<std::vector<label>> merged_send_idxs{};
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 836d38541..38ba6fcc1 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -31,10 +31,13 @@ SourceFiles
 
 // TODO move this file to top level since it is not really a matrix-wrapper
 
+using CommCounts =
+std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+           std::vector<label>>;
+
 
 // This function computes the send and recv counts vectors for repartitioning
-std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-           std::vector<label>, std::vector<label>>
+CommCounts
 compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
                          const gko::experimental::mpi::communicator &comm,
                          std::shared_ptr<const gko::Executor> exec, label size,
@@ -42,27 +45,24 @@ compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
                          label padding_after);
 
 // This function computes the send and recv counts vectors for repartitioning
-std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-           std::vector<label>, std::vector<label>>
+CommCounts
 compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
                          const gko::experimental::mpi::communicator &comm,
                          std::shared_ptr<const gko::Executor> exec, label size);
 
-void gather_to_owner(
+// TODO these are separate functions
+// create gather_coeffs, gather_idx, and a templated impl function
+std::vector<label> gather_to_owner(
     std::shared_ptr<const gko::Executor> exec,
-    gko::experimental::mpi::communicator &comm,
-    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-               std::vector<label>, std::vector<label>> &comm_pattern,
+    const gko::experimental::mpi::communicator &comm,
+    const CommCounts &comm_pattern,
     const scalar *send_buffer, scalar *recv_buffer);
 
 std::vector<label> gather_to_owner(
     std::shared_ptr<const gko::Executor> exec,
-    gko::experimental::mpi::communicator &comm,
-    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-               std::vector<label>, std::vector<label>> &comm_pattern,
-    const scalar *send_buffer, scalar *recv_buffer,
-    label offset=0
-);
+    const gko::experimental::mpi::communicator &comm,
+    const CommCounts &comm_pattern,
+    label size, const label *data, label offset = 0);
 
 struct CommunicationPattern {
     using comm_size_type = label;
@@ -83,6 +83,7 @@ struct CommunicationPattern {
 
 std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e);
 
+// TODO move to Repartitioner
 // TODO NOTE currently we have just one communication pattern for multiple proc
 // interfaces
 //  hence this will communicate vectors which are too long, FIXME potentially
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index f8e74a4a8..bf4745c13 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -165,7 +165,7 @@ public:
     std::shared_ptr<part_type> get_partition() { return this->partition_; }
 
     static std::shared_ptr<RepartDistMatrix> create(
-        word matrix_format, label ranks_per_gpu,
+        word matrix_format, const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A)
     {
         auto exec_handler = host_A.get()->get_exec_handler();
@@ -179,11 +179,10 @@ public:
         auto local_sparsity_ = host_A->compute_local_sparsity(exec);
         auto non_local_sparsity_ = host_A->compute_non_local_sparsity(exec);
 
-        auto orig_partition = gko::share(
-            gko::experimental::distributed::build_partition_from_local_size<
-                label, label>(exec, comm, local_sparsity_->dim[0]));
+        auto orig_partition = repartitioner.get_orig_partition();
 
         // create original communicator pattern
+        auto ranks_per_gpu = repartitioner.get_ranks_per_gpu();
         auto src_comm_pattern =
             host_A.get()->create_communication_pattern(comm);
         // create partiton here and pass to constructor
@@ -192,394 +191,24 @@ public:
 
         label rank{src_comm_pattern.comm.rank()};
 
-        auto target_rank = [ranks_per_gpu](label rank) {
-            return rank - (rank % ranks_per_gpu);
-        };
-
-        label owner_rank = target_rank(rank);
-        bool owner = rank == owner_rank;
-
-        auto repartition_sparsity =
-            [rank, owner, owner_rank, comm, ranks_per_gpu, src_comm_pattern,
-             exec, orig_partition, target_rank](
-                std::shared_ptr<SparsityPattern> src_local_pattern,
-                std::shared_ptr<SparsityPattern> src_non_local_pattern) {
-                // 1. obtain send recv sizes vector
-                // here we can reuse code from repartition_comm_pattern
-                //
-                // 2. initialize send and recv buffer
-                // if we keep interfaces separated row and col indices can be
-                //
-                // 3. ldu mapping needs to be update?
-                // 4. dim needs to be updated
-                // this is not implemented yet, but we can't fail here for
-                // debugging reasons
-                // FatalErrorInFunction << "Not implemented" <<
-                // abort(FatalError);
-
-
-                auto local_comm_pattern =
-                    compute_send_recv_counts(ranks_per_gpu, owner_rank, comm,
-                                             exec, src_local_pattern->size_);
-
-                auto gather_to_owner = [&](label offset, label *data,
-                                           label size, auto &comm_pattern) {
-                    if (offset > 0) {
-                        std::transform(data, data + size, data,
-                                       [&](label idx) { return idx + offset; });
-                    }
-
-                    auto [send_counts, recv_counts, send_offsets, recv_offsets,
-                          recv_buffer] = comm_pattern;
-
-                    comm.all_to_all_v(exec, data, send_counts.data(),
-                                      send_offsets.data(), recv_buffer.data(),
-                                      recv_counts.data(), recv_offsets.data());
-
-                    return std::vector<label>{recv_buffer};
-                };
-
-
-                label offset = orig_partition->get_range_bounds()[rank] -
-                               orig_partition->get_range_bounds()[owner_rank];
-
-                std::vector<label> merged_local_rows = gather_to_owner(
-                    offset, src_local_pattern->row_idxs.get_data(),
-                    src_local_pattern->row_idxs.get_size(), local_comm_pattern);
-
-                std::vector<label> merged_local_cols = gather_to_owner(
-                    offset, src_local_pattern->col_idxs.get_data(),
-                    src_local_pattern->row_idxs.get_size(), local_comm_pattern);
-
-                std::vector<label> merged_ldu_mapping = gather_to_owner(
-                    0, src_local_pattern->ldu_mapping.get_data(),
-                    src_local_pattern->row_idxs.get_size(), local_comm_pattern);
-
-                // after gathering on owner we need to make ldu_mapping_idx
-                // consecutive
-                {
-                    auto [send_counts, recv_counts, send_offsets, recv_offsets,
-                          recv_buffer] = local_comm_pattern;
-
-                    label ldu_offset = 0;
-                    auto *data = src_local_pattern->ldu_mapping.get_data();
-                    for (label i = 0; i < ranks_per_gpu; i++) {
-                        auto size = recv_counts[i];
-                        std::transform(
-                            data + ldu_offset, data + size, data + ldu_offset,
-                            [&](label idx) { return idx + ldu_offset; });
-                        ldu_offset += size;
-                    }
-                }
-
-
-                label rows = (owner) ? merged_local_rows.back() + 1 : 0;
-                gko::dim<2> merged_local_dim{rows, rows};
-
-                std::vector<label> merged_begin{0};
-                std::vector<label> merged_end{merged_local_rows.size()};
-                std::vector<label> merged_ranks{rank};
-
-                auto non_local_comm_pattern = compute_send_recv_counts(
-                    ranks_per_gpu, owner_rank, comm, exec,
-                    src_non_local_pattern->size_);
-
-                std::vector<label> merged_non_local_rows = gather_to_owner(
-                    offset, src_non_local_pattern->row_idxs.get_data(),
-                    src_non_local_pattern->row_idxs.get_size(),
-                    non_local_comm_pattern);
-
-                // the non local cols are in local idx of other side
-                // thus we need the new offset of the other side
-                for (label i = 0; i < src_non_local_pattern->rank.size(); i++) {
-                    auto comm_rank = src_non_local_pattern->rank[i];
-                    auto [begin, end] =
-                        src_non_local_pattern->interface_spans[i];
-                    label local_offset =
-                        orig_partition->get_range_bounds()[comm_rank] -
-                        orig_partition->get_range_bounds()[owner_rank];
-                    auto *data =
-                        src_non_local_pattern->col_idxs.get_data() + begin;
-                    auto size = end - begin;
-                    std::transform(data, data + size, data, [&](label idx) {
-                        return idx + local_offset;
-                    });
-                }
-
-                // we now dealed with the new offset thus we dont need an offset
-                // now
-                std::vector<label> merged_non_local_cols = gather_to_owner(
-                    0, src_non_local_pattern->col_idxs.get_data(),
-                    src_non_local_pattern->row_idxs.get_size(),
-                    non_local_comm_pattern);
-
-                std::vector<label> merged_non_local_ldu_mapping =
-                    gather_to_owner(
-                        0, src_non_local_pattern->ldu_mapping.get_data(),
-                        src_non_local_pattern->row_idxs.get_size(),
-                        non_local_comm_pattern);
-
-                auto [span_send_counts, span_recv_counts, span_send_offsets,
-                      span_recv_offsets, span_recv_buffer] =
-                    compute_send_recv_counts(
-                        ranks_per_gpu, owner_rank, comm, exec,
-                        src_non_local_pattern->interface_spans.size());
-
-                std::vector<label> spans_begin;
-                std::vector<label> spans_end;
-
-                for (auto elem : src_non_local_pattern->interface_spans) {
-                    spans_begin.push_back(elem.begin);
-                    spans_end.push_back(elem.end);
-                }
-
-                comm.all_to_all_v(
-                    exec, spans_begin.data(), span_send_counts.data(),
-                    span_send_offsets.data(), span_recv_buffer.data(),
-                    span_recv_counts.data(), span_recv_offsets.data());
-
-                std::vector<label> merged_non_local_begin{span_recv_buffer};
-
-                comm.all_to_all_v(
-                    exec, spans_end.data(), span_send_counts.data(),
-                    span_send_offsets.data(), span_recv_buffer.data(),
-                    span_recv_counts.data(), span_recv_offsets.data());
-                std::vector<label> merged_non_local_end{span_recv_buffer};
-
-                comm.all_to_all_v(
-                    exec, src_non_local_pattern->rank.data(),
-                    span_send_counts.data(), span_send_offsets.data(),
-                    span_recv_buffer.data(), span_recv_counts.data(),
-                    span_recv_offsets.data());
-                std::vector<label> merged_non_local_ranks{span_recv_buffer};
-
-
-                // fix the received merged_non_local_begin and ends
-                // we received begins and ends of the interfaces from
-                // the neighbour ranks. however the begin and ends are
-                // computed base on the non merged interfaces. Now we
-                // need to offset these values
-                // before merge
-                // rank 0 begin [ 0 10 ], end [10 20]
-                // rank 1 begin [ 0 10 ], end [10 20]
-                // merged begin [ 0 10 0 10 ], end [10 20 10 20]
-                // after fix
-                // merged begin [ 0 10 20 30 ], end [10 20 30 40]
-                for (label i = 1; i < ranks_per_gpu; i++) {
-                    auto recv_offset = span_recv_offsets[rank + i];
-                    auto recv_count = span_recv_counts[rank + i];
-                    auto range_offset = merged_non_local_end[recv_offset - 1];
-                    for (label j = 0; j < recv_count; j++) {
-                        merged_non_local_begin[recv_offset + j] += range_offset;
-                        merged_non_local_end[recv_offset + j] += range_offset;
-                    }
-                }
-
-
-                // TODO remove
-                //     if (rank == 0) {
-                // std::cout << __FILE__ << " on rank " << rank << "
-                // merged_begin"
-                //           << merged_begin
-                //           << " span_recv_counts " << span_recv_counts
-                //           << " span_recv_offsets " << span_recv_offsets
-                //           << " merged_non_local_begins " <<
-                //           merged_non_local_begin
-                //           << " merged_non_local_end " << merged_non_local_end
-                //           << " merged_end " << merged_end
-                //           << " merged_non_local_rows " <<
-                //           merged_non_local_rows
-                //           << " merged_ranks " << merged_non_local_ranks <<
-                //           "\n";
-                // }
-
-                // TODO shorten this
-                std::vector<label> gathered_non_local_rows;
-                std::vector<label> gathered_non_local_cols;
-                std::vector<label> gathered_non_local_mapping;
-                std::vector<label> gathered_ranks;
-                std::vector<label> gathered_begin;
-                std::vector<label> gathered_end;
-                label merged_ranks_size = merged_non_local_ranks.size();
-                // redistribute interfaces
-                for (label i = 0; i < merged_ranks_size; i++) {
-                    auto merged_target_rank = merged_non_local_ranks[i];
-                    label to_rank = target_rank(merged_target_rank);
-                    bool local = rank == to_rank;
-                    // these are the begin ends before merging they need to be
-                    // offseted;
-                    auto begin = merged_non_local_begin[i];
-                    auto end = merged_non_local_end[i];
-                    if (local) {
-                        // TODO depending on the interface simple non
-                        // transforming and interpolating interfaces like now
-                        // local processor interfaces could be merged
-                        label size_before = merged_local_rows.size();
-                        merged_local_rows.insert(
-                            merged_local_rows.end(),
-                            merged_non_local_rows.begin() + begin,
-                            merged_non_local_rows.begin() + end);
-                        merged_local_cols.insert(
-                            merged_local_cols.end(),
-                            merged_non_local_cols.begin() + begin,
-                            merged_non_local_cols.begin() + end);
-                        merged_ldu_mapping.insert(
-                            merged_ldu_mapping.end(),
-                            merged_non_local_ldu_mapping.data() + begin,
-                            merged_non_local_ldu_mapping.data() + end);
-                        // TODO store from rank ie from which rank it came
-                        merged_ranks.push_back(to_rank);
-
-                        label size_after = merged_local_rows.size();
-                        merged_begin.push_back(size_before);
-                        merged_end.push_back(size_after);
-                        // TODO remove
-                        // if (rank == 0) {
-                        //     std::cout << __FILE__ << " on rank " << rank
-                        //             << " begin " << begin << " end " << end
-                        //             << " insert merged_local_rows " <<
-                        //             merged_local_rows
-                        //             << "\n";
-                        //     }
-                    } else {
-                        label size_before = gathered_non_local_rows.size();
-                        gathered_non_local_rows.insert(
-                            gathered_non_local_rows.end(),
-                            merged_non_local_rows.begin() + begin,
-                            merged_non_local_rows.begin() + end);
-                        // FIXME
-                        // up until now non_local_cols are stil using
-                        // local_idx and we need to convert them to
-                        // consequtive compressed idxs
-                        gathered_non_local_cols.insert(
-                            gathered_non_local_cols.end(),
-                            merged_non_local_cols.begin() + begin,
-                            merged_non_local_cols.begin() + end);
-                        // TODO we don't do anything with non local mapping
-                        gathered_non_local_mapping.insert(
-                            gathered_non_local_mapping.end(),
-                            merged_non_local_cols.begin() + begin,
-                            merged_non_local_cols.begin() + end);
-                        label size_after = gathered_non_local_rows.size();
-                        // if (rank == 0) {
-                        // std::cout << __FILE__ << " on rank " << rank
-                        //         << " gathered_non_local_rows " <<
-                        //         gathered_non_local_rows
-                        //         << " begin " << begin << " end " << end
-                        //         << " non_local size_before " << size_before
-                        //         << " non_local size_after " << size_after
-                        //         << "\n";
-                        // }
-                        gathered_begin.push_back(size_before);
-                        gathered_end.push_back(size_after);
-                        gathered_ranks.push_back(to_rank);
-                    }
-                }
-
-                // for compressed cols we compute a mapping first
-                std::map<label, label> compressed_col_idxs{};
-                label unique_idx = 0;
-                for (auto idx : gathered_non_local_cols) {
-                    auto search = compressed_col_idxs.find(idx);
-                    if (search == compressed_col_idxs.end()) {
-                        std::cout << " insert " << idx << " " << unique_idx
-                                  << "\n";
-                        compressed_col_idxs[idx] = unique_idx;
-                        unique_idx += 1;
-                    };
-                }
-
-                for (int i = 0; i < gathered_non_local_cols.size(); i++) {
-                    label uncompressed = gathered_non_local_cols[i];
-                    label compressed = compressed_col_idxs[uncompressed];
-                    std::cout << " i " << i << " uncompressed " << uncompressed
-                              << " compressed " << compressed << "\n";
-                    gathered_non_local_cols[i] = compressed;
-                }
-
-                // add interfaces
-                // to add interfaces we go through the non_local_sparsity
-                // pattern and check if interface is still non_local
-                //
-                // iterate interfaces and send to owner.
-                // on owner decide to move to local or non_local
-                // for sending to owner we can also use the all_to_all_v
-                // approach we than have
-                //
-                // row [ 4 , 8,  12 | 16 , 17, 18 ] <- local row
-                // col [ 1 , 2, 3 | 1, 2, 3 ] <- just the interface ctr
-                // from repartition_comm_pattern we could get
-                //
-                // or we split local / non-local first
-                // for this we could store ranges when computing
-                // repartition_comm_pattern
-                //
-
-
-                if (owner) {
-                    auto new_local_spars_pattern =
-                        std::make_shared<SparsityPattern>(
-                            exec, merged_local_rows.size(), merged_local_dim,
-                            merged_local_rows, merged_local_cols,
-                            merged_ldu_mapping, merged_begin, merged_end,
-                            merged_ranks);
-
-                    // if (rank == 0) {
-                    // std::cout << __FILE__ << " on rank " << rank
-                    //           << merged_local_rows
-                    //           << " merged_begin " <<  merged_begin
-                    //           << " merged_end " <<  merged_end
-                    //           << "local dim ["
-                    //           << new_local_spars_pattern->dim[0] << " x "
-                    //           << new_local_spars_pattern->dim[1] << "] \n";
-                    //         }
-
-                    auto new_non_local_spars_pattern =
-                        std::make_shared<SparsityPattern>(
-                            exec, gathered_non_local_rows.size(),
-                            gko::dim<2>{merged_local_dim[0],
-                                        gathered_non_local_rows.size()},
-                            gathered_non_local_rows, gathered_non_local_cols,
-                            gathered_non_local_mapping, gathered_begin,
-                            gathered_end, gathered_ranks);
-
-                    return std::make_pair<std::shared_ptr<SparsityPattern>,
-                                          std::shared_ptr<SparsityPattern>>(
-                        std::move(new_local_spars_pattern),
-                        std::move(new_non_local_spars_pattern));
-                } else {
-                    auto new_local_spars_pattern =
-                        std::make_shared<SparsityPattern>(exec, 0);
-
-                    auto new_non_local_spars_pattern =
-                        std::make_shared<SparsityPattern>(exec, 0);
-
-                    new_local_spars_pattern->dim = merged_local_dim;
-                    new_non_local_spars_pattern->dim = merged_local_dim;
-
-                    return std::make_pair<std::shared_ptr<SparsityPattern>,
-                                          std::shared_ptr<SparsityPattern>>(
-                        std::move(new_local_spars_pattern),
-                        std::move(new_non_local_spars_pattern));
-                }
-            };
+        label owner_rank = repartitioner.get_owner_rank();
+        bool owner = repartitioner.get_owner();
 
         auto [repart_loc_sparsity, repart_non_loc_sparsity] =
-            repartition_sparsity(local_sparsity_, non_local_sparsity_);
+            repartitioner.repartition_sparsity(
+                local_sparsity_,
+                                               non_local_sparsity_);
 
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
-        // const label local_nnz = local_sparsity_->size_;
-        // const label non_local_nnz = non_local_sparsity_->size_;
-
         auto localized_partition = local_part_type::build_from_blocked_recv(
             exec, local_sparsity_->dim[0], dst_comm_pattern.send_idxs,
             dst_comm_pattern.target_ids, dst_comm_pattern.target_sizes);
 
 
-        std::cout << __FILE__ << " rank " << rank << " local sparsity dim"
-                  << local_sparsity_->dim[0] << "non_local sparsity dim"
+        std::cout << __FILE__ << " rank " << rank << " local sparsity dim "
+                  << local_sparsity_->dim[0] << "non_local sparsity dim "
                   << non_local_sparsity_->dim[0] << "x"
                   << non_local_sparsity_->dim[1] << " target_ids "
                   << dst_comm_pattern.target_ids << " target_sizes "
@@ -596,6 +225,7 @@ public:
         auto dist_A = generate_dist_mtx_with_inner_type<dist_mtx>(
             matrix_format, exec, sparse_comm, local_sparsity_,
             non_local_sparsity_);
+
         std::cout << __FILE__ << " rank " << rank
                   << " done generate dist_mtx\n";
 
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 6a2fa02b9..6f18cf04a 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -24,6 +24,8 @@ SourceFiles
 \*---------------------------------------------------------------------------*/
 #ifndef OGL_HostMatrix_INCLUDED_H
 #define OGL_HostMatrix_INCLUDED_H
+#include <map>
+
 #include <ginkgo/ginkgo.hpp>
 
 #include "fvCFD.H"
@@ -35,76 +37,392 @@ SourceFiles
 #include "DevicePersistent/IOGlobalIndex/gkoGlobalIndex.H"
 #include "MatrixWrapper/Combination/Combination.H"
 #include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
+#include "MatrixWrapper/SparsityPattern/SparsityPattern.H"
 #include "common/common.H"
 
 
 namespace Foam {
 
+/* @brief based on a comm_pattern consecutive ldu_mappings are computed
+ *
+ * After merging sparsity patterns during repartitioning the ldu mapping are
+ * not consecutive ie they could look like [0 1 2 3 | 0 1 2 3 | 0 1], where
+ * | is the former rank boundary based on the comm pattern we compute a new
+ * mapping as  [ 0 1 2 3 | 4 5 6 7| 8 9 ] where the following offsets [ 0 |
+ * 4 | 8 ] based on the recv_counts are used.
+ *
+ * */
+void make_ldu_mapping_consecutive(
+    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+               std::vector<label>>
+        comm_pattern,
+    std::vector<label> &ldu_mapping, label rank, label ranks_per_gpu);
+
+// TODO this should be persistent
+class Repartitioner {
+private:
+    using partition = gko::experimental::distributed::Partition<label, label>;
+    const label ranks_per_gpu_;
 
-struct SparsityPattern {
-    SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size)
-        : size_(size),
-          row_idxs{exec, static_cast<gko::size_type>(size)},
-          col_idxs{exec, static_cast<gko::size_type>(size)},
-          ldu_mapping{exec, static_cast<gko::size_type>(size)}
-    {}
-
-    SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size,
-                    gko::dim<2> dim, std::vector<label> &rows,
-                    std::vector<label> &cols, std::vector<label> &map,
-                    std::vector<label> &spans_begin,
-                    std::vector<label> &spans_end, std::vector<label> &ranks)
-        : size_(size),
-          dim(dim),
-          row_idxs{exec, static_cast<gko::size_type>(size)},
-          col_idxs{exec, static_cast<gko::size_type>(size)},
-          ldu_mapping{exec, static_cast<gko::size_type>(size)}
-    {
-        ASSERT_EQ(size, rows.size());
-        std::copy(rows.begin(), rows.end(), row_idxs.get_data());
-
-        ASSERT_EQ(size, cols.size());
-        std::copy(cols.begin(), cols.end(), col_idxs.get_data());
-
-        ASSERT_EQ(size, map.size());
-        std::copy(map.begin(), map.end(), ldu_mapping.get_data());
+    const ExecutorHandler exec_handler_;
 
-        ASSERT_EQ(spans_begin.size(), spans_end.size());
-        std::transform(spans_begin.begin(), spans_begin.end(),
-                       spans_end.begin(), std::back_inserter(interface_spans),
-                       [](const auto begin, const auto end) {
-                           return gko::span{static_cast<gko::size_type>(begin),
-                                            static_cast<gko::size_type>(end)};
-                       });
+    std::shared_ptr<const partition> orig_partition_;
 
-        ASSERT_EQ(spans_begin.size(), ranks.size());
-        rank = ranks;
-    }
 
-    // TODO rename the public members and remove trailing _
-    // number of nnz entries
-    const label size_;
+public:
+    Repartitioner(
+        label size,
+        label ranks_per_gpu,
+        const ExecutorHandler &exec_handler
+                )
+        : ranks_per_gpu_(ranks_per_gpu),
+          exec_handler_(exec_handler),
+          orig_partition_(
+            gko::share(
+            gko::experimental::distributed::build_partition_from_local_size<
+                label, label>(
+            exec_handler_.get_device_exec(),
+            *exec_handler_.get_communicator().get(),
+                    size))
+)
+        {};
+
+    /* returns the owner rank for a given rank */
+    label get_owner_rank(label rank) const
+    {
+        return rank - (rank % ranks_per_gpu_);
+    };
 
-    mutable gko::Array<label> row_idxs;
+    /* returns the owner rank for a given rank */
+    label get_owner_rank() const { return get_owner_rank(get_rank()); };
 
-    mutable gko::Array<label> col_idxs;
+    /* returns if current rank is an owner  */
+    bool get_owner() const { return get_rank() == get_owner_rank(get_rank()); };
 
-    // ldu_mapping[sorted_(csr)_position] =
-    // unsorted_(consecutive_ldu_)_position
-    mutable gko::Array<label> ldu_mapping;
+    /* shortcut to current rank */
+    label get_rank() const
+    {
+        return exec_handler_.get_communicator().get()->rank();
+    };
 
-    // dimensions of the resulting matrix
-    mutable gko::dim<2> dim;
+    label get_ranks_per_gpu() const { return ranks_per_gpu_; }
 
-    // A vector of spans indicating begin and end of each interface
-    // this is used to keep all col and row idx consecutive in memory
-    mutable std::vector<gko::span> interface_spans;
+    std::shared_ptr<const partition> get_orig_partition() const {return orig_partition_;};
 
-    // keep track of original (communication) rank of each interface or matrix
-    // block
-    mutable std::vector<label> rank;
+    std::pair<std::shared_ptr<SparsityPattern>,
+              std::shared_ptr<SparsityPattern>>
+    repartition_sparsity(
+        std::shared_ptr<SparsityPattern> src_local_pattern,
+        std::shared_ptr<SparsityPattern> src_non_local_pattern
+) const
+    {
+        // 1. obtain send recv sizes vector
+        // here we can reuse code from repartition_comm_pattern
+        //
+        // 2. initialize send and recv buffer
+        // if we keep interfaces separated row and col indices can be
+        //
+        // 3. ldu mapping needs to be update?
+        // 4. dim needs to be updated
+        // this is not implemented yet, but we can't fail here for
+        // debugging reasons
+        auto exec = exec_handler_.get_device_exec();
+        auto comm = *exec_handler_.get_communicator().get();
+        label rank = get_rank();
+        label owner_rank = get_owner_rank();
+        bool owner = get_owner();
+        label ranks_per_gpu = ranks_per_gpu_;
+
+        auto local_comm_pattern = compute_send_recv_counts(
+            ranks_per_gpu, owner_rank, comm, exec, src_local_pattern->size_);
+
+        label offset = orig_partition_->get_range_bounds()[rank] -
+                       orig_partition_->get_range_bounds()[owner_rank];
+
+        auto gather_closure = [exec, comm](auto &comm_pattern, auto &data,
+                                           label offset) {
+            return gather_to_owner(exec, comm, comm_pattern, data.get_size(),
+                                   data.get_data(), offset);
+        };
+
+        auto merged_local_rows = gather_closure(
+            local_comm_pattern, src_local_pattern->row_idxs, offset);
+
+        auto merged_local_cols = gather_closure(
+            local_comm_pattern, src_local_pattern->col_idxs, offset);
+
+        auto merged_ldu_mapping = gather_closure(
+            local_comm_pattern, src_local_pattern->ldu_mapping, 0);
+
+        make_ldu_mapping_consecutive(local_comm_pattern, merged_ldu_mapping,
+                                     rank, ranks_per_gpu);
+
+        label rows = (owner) ? merged_local_rows.back() + 1 : 0;
+        gko::dim<2> merged_local_dim{rows, rows};
+
+        std::vector<label> merged_begin{0};
+        std::vector<label> merged_end{label(merged_local_rows.size())};
+        std::vector<label> merged_ranks{rank};
+
+        auto non_local_comm_pattern =
+            compute_send_recv_counts(ranks_per_gpu, owner_rank, comm, exec,
+                                     src_non_local_pattern->size_);
+
+        auto merged_non_local_rows = gather_closure(
+            non_local_comm_pattern, src_non_local_pattern->row_idxs, offset);
+
+        // the non local cols are in local idx of other side
+        // thus we need the new offset of the other side
+        for (label i = 0; i < src_non_local_pattern->rank.size(); i++) {
+            auto comm_rank = src_non_local_pattern->rank[i];
+            auto [begin, end] = src_non_local_pattern->interface_spans[i];
+            label local_offset = orig_partition_->get_range_bounds()[comm_rank] -
+                                 orig_partition_->get_range_bounds()[owner_rank];
+            auto *data = src_non_local_pattern->col_idxs.get_data() + begin;
+            auto size = end - begin;
+            std::transform(data, data + size, data,
+                           [&](label idx) { return idx + local_offset; });
+        }
+
+        // we now dealed with the new offset thus we dont need an offset
+        // now
+        auto merged_non_local_cols = gather_closure(
+            non_local_comm_pattern, src_non_local_pattern->col_idxs, 0);
+
+        auto merged_non_local_ldu_mapping = gather_closure(
+            non_local_comm_pattern, src_non_local_pattern->row_idxs, 0);
+
+        auto span_comm_pattern = compute_send_recv_counts(
+            ranks_per_gpu, owner_rank, comm, exec,
+            src_non_local_pattern->interface_spans.size());
+
+        std::vector<label> spans_begin;
+        std::vector<label> spans_end;
+
+        for (auto elem : src_non_local_pattern->interface_spans) {
+            spans_begin.push_back(elem.begin);
+            spans_end.push_back(elem.end);
+        }
+
+        auto merged_non_local_begin =
+            gather_to_owner(exec, comm, span_comm_pattern, spans_begin.size(),
+                            spans_begin.data());
+
+        auto merged_non_local_end = gather_to_owner(
+            exec, comm, span_comm_pattern, spans_end.size(), spans_end.data());
+
+        auto merged_non_local_ranks = gather_to_owner(
+            exec, comm, span_comm_pattern, src_non_local_pattern->rank.size(),
+            src_non_local_pattern->rank.data());
+
+
+        // fix the received merged_non_local_begin and ends
+        // we received begins and ends of the interfaces from
+        // the neighbour ranks. however the begin and ends are
+        // computed base on the non merged interfaces. Now we
+        // need to offset these values
+        // before merge
+        // rank 0 begin [ 0 10 ], end [10 20]
+        // rank 1 begin [ 0 10 ], end [10 20]
+        // merged begin [ 0 10 0 10 ], end [10 20 10 20]
+        // after fix
+        // merged begin [ 0 10 20 30 ], end [10 20 30 40]
+        for (label i = 1; i < ranks_per_gpu; i++) {
+            auto recv_count = std::get<1>(span_comm_pattern)[rank + i];
+            auto recv_offset = std::get<3>(span_comm_pattern)[rank + i];
+            auto range_offset = merged_non_local_end[recv_offset - 1];
+            for (label j = 0; j < recv_count; j++) {
+                merged_non_local_begin[recv_offset + j] += range_offset;
+                merged_non_local_end[recv_offset + j] += range_offset;
+            }
+        }
+
+
+        // TODO remove
+        //     if (rank == 0) {
+        // std::cout << __FILE__ << " on rank " << rank << "
+        // merged_begin"
+        //           << merged_begin
+        //           << " span_recv_counts " << span_recv_counts
+        //           << " span_recv_offsets " << span_recv_offsets
+        //           << " merged_non_local_begins " <<
+        //           merged_non_local_begin
+        //           << " merged_non_local_end " << merged_non_local_end
+        //           << " merged_end " << merged_end
+        //           << " merged_non_local_rows " <<
+        //           merged_non_local_rows
+        //           << " merged_ranks " << merged_non_local_ranks <<
+        //           "\n";
+        // }
+
+        // TODO shorten this
+        std::vector<label> gathered_non_local_rows;
+        std::vector<label> gathered_non_local_cols;
+        std::vector<label> gathered_non_local_mapping;
+        std::vector<label> gathered_ranks;
+        std::vector<label> gathered_begin;
+        std::vector<label> gathered_end;
+        label merged_ranks_size = merged_non_local_ranks.size();
+        // redistribute interfaces
+        for (label i = 0; i < merged_ranks_size; i++) {
+            auto merged_target_rank = merged_non_local_ranks[i];
+            label to_rank = get_owner_rank(merged_target_rank);
+            bool local = rank == to_rank;
+            // these are the begin ends before merging they need to be
+            // offseted;
+            auto begin = merged_non_local_begin[i];
+            auto end = merged_non_local_end[i];
+            if (local) {
+                // TODO depending on the interface simple non
+                // transforming and interpolating interfaces like now
+                // local processor interfaces could be merged
+                label size_before = merged_local_rows.size();
+                merged_local_rows.insert(merged_local_rows.end(),
+                                         merged_non_local_rows.begin() + begin,
+                                         merged_non_local_rows.begin() + end);
+                merged_local_cols.insert(merged_local_cols.end(),
+                                         merged_non_local_cols.begin() + begin,
+                                         merged_non_local_cols.begin() + end);
+                merged_ldu_mapping.insert(
+                    merged_ldu_mapping.end(),
+                    merged_non_local_ldu_mapping.data() + begin,
+                    merged_non_local_ldu_mapping.data() + end);
+                // TODO store from rank ie from which rank it came
+                merged_ranks.push_back(to_rank);
+
+                label size_after = merged_local_rows.size();
+                merged_begin.push_back(size_before);
+                merged_end.push_back(size_after);
+                // TODO remove
+                // if (rank == 0) {
+                //     std::cout << __FILE__ << " on rank " << rank
+                //             << " begin " << begin << " end " << end
+                //             << " insert merged_local_rows " <<
+                //             merged_local_rows
+                //             << "\n";
+                //     }
+            } else {
+                label size_before = gathered_non_local_rows.size();
+                gathered_non_local_rows.insert(
+                    gathered_non_local_rows.end(),
+                    merged_non_local_rows.begin() + begin,
+                    merged_non_local_rows.begin() + end);
+                // FIXME
+                // up until now non_local_cols are stil using
+                // local_idx and we need to convert them to
+                // consequtive compressed idxs
+                gathered_non_local_cols.insert(
+                    gathered_non_local_cols.end(),
+                    merged_non_local_cols.begin() + begin,
+                    merged_non_local_cols.begin() + end);
+                // TODO we don't do anything with non local mapping
+                gathered_non_local_mapping.insert(
+                    gathered_non_local_mapping.end(),
+                    merged_non_local_cols.begin() + begin,
+                    merged_non_local_cols.begin() + end);
+                label size_after = gathered_non_local_rows.size();
+                // if (rank == 0) {
+                // std::cout << __FILE__ << " on rank " << rank
+                //         << " gathered_non_local_rows " <<
+                //         gathered_non_local_rows
+                //         << " begin " << begin << " end " << end
+                //         << " non_local size_before " << size_before
+                //         << " non_local size_after " << size_after
+                //         << "\n";
+                // }
+                gathered_begin.push_back(size_before);
+                gathered_end.push_back(size_after);
+                gathered_ranks.push_back(to_rank);
+            }
+        }
+
+        // for compressed cols we compute a mapping first
+        std::map<label, label> compressed_col_idxs{};
+        label unique_idx = 0;
+        for (auto idx : gathered_non_local_cols) {
+            auto search = compressed_col_idxs.find(idx);
+            if (search == compressed_col_idxs.end()) {
+                std::cout << " insert " << idx << " " << unique_idx << "\n";
+                compressed_col_idxs[idx] = unique_idx;
+                unique_idx += 1;
+            };
+        }
+
+        for (int i = 0; i < gathered_non_local_cols.size(); i++) {
+            label uncompressed = gathered_non_local_cols[i];
+            label compressed = compressed_col_idxs[uncompressed];
+            std::cout << " i " << i << " uncompressed " << uncompressed
+                      << " compressed " << compressed << "\n";
+            gathered_non_local_cols[i] = compressed;
+        }
+
+        // add interfaces
+        // to add interfaces we go through the non_local_sparsity
+        // pattern and check if interface is still non_local
+        //
+        // iterate interfaces and send to owner.
+        // on owner decide to move to local or non_local
+        // for sending to owner we can also use the all_to_all_v
+        // approach we than have
+        //
+        // row [ 4 , 8,  12 | 16 , 17, 18 ] <- local row
+        // col [ 1 , 2, 3 | 1, 2, 3 ] <- just the interface ctr
+        // from repartition_comm_pattern we could get
+        //
+        // or we split local / non-local first
+        // for this we could store ranges when computing
+        // repartition_comm_pattern
+        //
+
+
+        if (owner) {
+            auto new_local_spars_pattern = std::make_shared<SparsityPattern>(
+                exec, merged_local_rows.size(), merged_local_dim,
+                merged_local_rows, merged_local_cols, merged_ldu_mapping,
+                merged_begin, merged_end, merged_ranks);
+
+            // if (rank == 0) {
+            // std::cout << __FILE__ << " on rank " << rank
+            //           << merged_local_rows
+            //           << " merged_begin " <<  merged_begin
+            //           << " merged_end " <<  merged_end
+            //           << "local dim ["
+            //           << new_local_spars_pattern->dim[0] << " x "
+            //           << new_local_spars_pattern->dim[1] << "] \n";
+            //         }
+
+            auto new_non_local_spars_pattern =
+                std::make_shared<SparsityPattern>(
+                    exec, gathered_non_local_rows.size(),
+                    gko::dim<2>{merged_local_dim[0],
+                                gathered_non_local_rows.size()},
+                    gathered_non_local_rows, gathered_non_local_cols,
+                    gathered_non_local_mapping, gathered_begin, gathered_end,
+                    gathered_ranks);
+
+            return std::make_pair<std::shared_ptr<SparsityPattern>,
+                                  std::shared_ptr<SparsityPattern>>(
+                std::move(new_local_spars_pattern),
+                std::move(new_non_local_spars_pattern));
+        } else {
+            auto new_local_spars_pattern =
+                std::make_shared<SparsityPattern>(exec, 0);
+
+            auto new_non_local_spars_pattern =
+                std::make_shared<SparsityPattern>(exec, 0);
+
+            new_local_spars_pattern->dim = merged_local_dim;
+            new_non_local_spars_pattern->dim = merged_local_dim;
+
+            return std::make_pair<std::shared_ptr<SparsityPattern>,
+                                  std::shared_ptr<SparsityPattern>>(
+                std::move(new_local_spars_pattern),
+                std::move(new_non_local_spars_pattern));
+        }
+    }
 };
 
+
 // Free functions
 const lduInterfaceField *interface_getter(
     const lduInterfaceFieldPtrsList &interfaces, const label i);
@@ -440,29 +758,21 @@ public:
 
     label get_local_nrows() const { return nrows_; }
 
-    label get_upper_nnz() const { return  upper_nnz_;}
+    label get_upper_nnz() const { return upper_nnz_; }
 
     label get_local_matrix_nnz() const { return local_matrix_nnz_; }
 
     /** Create a view into host diag ptr  */
-    const scalar* get_diag() const {
-        return diag_;
-    }
+    const scalar *get_diag() const { return diag_; }
 
     /** Create a view into host lower ptr  */
-    const scalar* get_lower() const {
-        return lower_;
-    }
+    const scalar *get_lower() const { return lower_; }
 
     /** Create a view into host lower ptr  */
-    const scalar* get_upper() const {
-        return upper_;
-    }
+    const scalar *get_upper() const { return upper_; }
 
     /** Create a view into host lower ptr  */
-    bool symmetric() const {
-        return symmetric_;
-    }
+    bool symmetric() const { return symmetric_; }
 
     gko::dim<2> get_size() const { return gko::dim<2>(nrows_, nrows_); }
 
diff --git a/MatrixWrapper/SparsityPattern/SparsityPattern.H b/MatrixWrapper/SparsityPattern/SparsityPattern.H
new file mode 100644
index 000000000..8d5671c14
--- /dev/null
+++ b/MatrixWrapper/SparsityPattern/SparsityPattern.H
@@ -0,0 +1,89 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    HostMatrix.H
+
+\*---------------------------------------------------------------------------*/
+#pragma once
+
+struct SparsityPattern {
+    SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size)
+        : size_(size),
+          row_idxs{exec, static_cast<gko::size_type>(size)},
+          col_idxs{exec, static_cast<gko::size_type>(size)},
+          ldu_mapping{exec, static_cast<gko::size_type>(size)}
+    {}
+
+    SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size,
+                    gko::dim<2> dim, std::vector<label> &rows,
+                    std::vector<label> &cols, std::vector<label> &map,
+                    std::vector<label> &spans_begin,
+                    std::vector<label> &spans_end, std::vector<label> &ranks)
+        : size_(size),
+          dim(dim),
+          row_idxs{exec, static_cast<gko::size_type>(size)},
+          col_idxs{exec, static_cast<gko::size_type>(size)},
+          ldu_mapping{exec, static_cast<gko::size_type>(size)}
+    {
+        ASSERT_EQ(size, rows.size());
+        std::copy(rows.begin(), rows.end(), row_idxs.get_data());
+
+        ASSERT_EQ(size, cols.size());
+        std::copy(cols.begin(), cols.end(), col_idxs.get_data());
+
+        ASSERT_EQ(size, map.size());
+        std::copy(map.begin(), map.end(), ldu_mapping.get_data());
+
+        ASSERT_EQ(spans_begin.size(), spans_end.size());
+        std::transform(spans_begin.begin(), spans_begin.end(),
+                       spans_end.begin(), std::back_inserter(interface_spans),
+                       [](const auto begin, const auto end) {
+                           return gko::span{static_cast<gko::size_type>(begin),
+                                            static_cast<gko::size_type>(end)};
+                       });
+
+        ASSERT_EQ(spans_begin.size(), ranks.size());
+        rank = ranks;
+    }
+
+    // TODO rename the public members and remove trailing _
+    // number of nnz entries
+    const label size_;
+
+    mutable gko::Array<label> row_idxs;
+
+    mutable gko::Array<label> col_idxs;
+
+    // ldu_mapping[sorted_(csr)_position] =
+    // unsorted_(consecutive_ldu_)_position
+    mutable gko::Array<label> ldu_mapping;
+
+    // dimensions of the resulting matrix
+    mutable gko::dim<2> dim;
+
+    // A vector of spans indicating begin and end of each interface
+    // this is used to keep all col and row idx consecutive in memory
+    mutable std::vector<gko::span> interface_spans;
+
+    // keep track of original (communication) rank of each interface or matrix
+    // block
+    mutable std::vector<label> rank;
+};
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index a90d2961e..304200a6a 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -214,11 +214,15 @@ public:
                                            solverPerformance &solverPerf) const
     {
         auto ref_exec = exec_handler.get_ref_exec();
+        Repartitioner repartitioner {
+        psi.size(),
+        solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1), exec_handler};
 
         // create a distributed matrix from the host matrix
         // here also repartitioning is performed if required
         PersistentDistributedMatrix dist_A{db_,
                              exec_handler,
+                             repartitioner,
                              field_name_,
                              host_A_,
                              solver_controls_,
@@ -228,6 +232,7 @@ public:
             source.begin(),
             source.size(),
             field_name_ + "_rhs",
+            repartitioner,
             db_,
             exec_handler,
             verbose_,
@@ -239,6 +244,7 @@ public:
             psi.begin(),
             psi.size(),
             field_name_ + "_solution",
+            repartitioner,
             db_,
             exec_handler,
             verbose_,

From 13cea05f2b7431a3139de928d5f258d8e794bf5b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sat, 25 May 2024 22:21:42 +0200
Subject: [PATCH 53/90] add vector repartitioning, fix ldu consecutive

---
 DevicePersistent/Vector/Vector.H        |  21 +++--
 MatrixWrapper/Distributed/Distributed.H |  26 +++---
 MatrixWrapper/LDUMatrix/HostMatrix.C    |  62 +++++++++-----
 MatrixWrapper/LDUMatrix/HostMatrix.H    | 103 ++++++++++++++++--------
 4 files changed, 136 insertions(+), 76 deletions(-)

diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 8a65b74a4..42d15b217 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -12,6 +12,7 @@
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "DevicePersistent/Partition/Partition.H"
 #include "MatrixWrapper/LDUMatrix/HostMatrix.H"
+#include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
 #include "common/common.H"
 
 namespace Foam {
@@ -78,16 +79,22 @@ struct VectorInitFunctor {
         auto host_view = gko::array<T>::view(exec_.get_ref_exec(), local_size,
                                              const_cast<T *>(other_));
 
-        label repart_size = local_size;
+        label repart_size = repartitioner_.get_repart_size();
+        // this should happen on device
         auto local_coeffs = gko::array<scalar>(exec, repart_size);
-        FatalErrorInFunction << "Not implemented" << abort(FatalError);
-        return {};
 
-        // gather_to_owner(exec, comm, comm_pattern_, host_view,
-        //                 local_coeffs.get_data());
+        // FIXME this shouldn't be needed here also not during an
+        // update
+        auto comm_pattern =
+            compute_send_recv_counts(
+            exec_, repartitioner_.get_ranks_per_gpu(), local_size);
 
-        // return gko::share(dist_vec::create(
-        //     exec, *exec_.get_communicator().get(), local_coeffs));
+        gather_to_owner(exec_, comm_pattern, host_view.get_data(),
+                        local_coeffs.get_data());
+
+        return gko::share(dist_vec::create(
+            exec, *exec_.get_communicator().get(),
+            vec::create(exec, gko::dim<2>{repart_size, 1}, local_coeffs, 1)));
     }
 };
 
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index bf4745c13..6117575b5 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -189,14 +189,12 @@ public:
         auto dst_comm_pattern = repartition_comm_pattern(
             ranks_per_gpu, src_comm_pattern, orig_partition);
 
-        label rank{src_comm_pattern.comm.rank()};
-
+        label rank{src_comm_pattern.get_comm().rank()};
         label owner_rank = repartitioner.get_owner_rank();
         bool owner = repartitioner.get_owner();
 
         auto [repart_loc_sparsity, repart_non_loc_sparsity] =
-            repartitioner.repartition_sparsity(
-                local_sparsity_,
+            repartitioner.repartition_sparsity(local_sparsity_,
                                                non_local_sparsity_);
 
         local_sparsity_ = repart_loc_sparsity;
@@ -233,8 +231,10 @@ public:
         auto non_local_coeffs =
             gko::array<scalar>(exec, non_local_sparsity_->size_);
 
+
         local_coeffs.fill(0.0);
         non_local_coeffs.fill(0.0);
+        // FIXME make sure that we work on the device executor
 
         dist_A->read_distributed(
             device_matrix_data(exec, local_sparsity_->dim,
@@ -250,16 +250,16 @@ public:
         label local_matrix_nnz = host_A->get_local_matrix_nnz();
         // size + padding has to be local_matrix_nnz
         auto diag_comm_pattern = compute_send_recv_counts(
-            ranks_per_gpu, owner_rank, comm, exec, nrows, local_matrix_nnz,
+            exec_handler, ranks_per_gpu, nrows, local_matrix_nnz,
             local_matrix_nnz - nrows, 0);
 
         label upper_nnz = host_A->get_upper_nnz();
         auto upper_comm_pattern = compute_send_recv_counts(
-            ranks_per_gpu, owner_rank, comm, exec, upper_nnz, local_matrix_nnz,
-            0, local_matrix_nnz - upper_nnz);
+            exec_handler, ranks_per_gpu, upper_nnz,
+            local_matrix_nnz, 0, local_matrix_nnz - upper_nnz);
         auto lower_comm_pattern = compute_send_recv_counts(
-            ranks_per_gpu, owner_rank, comm, exec, upper_nnz, local_matrix_nnz,
-            upper_nnz, nrows);
+            exec_handler, ranks_per_gpu, upper_nnz,
+            local_matrix_nnz, upper_nnz, nrows);
 
         scalar *local_ptr;
 
@@ -273,17 +273,17 @@ public:
 
             local_ptr = const_cast<scalar *>(local->get_const_values());
         }
-        gather_to_owner(exec, comm, diag_comm_pattern, host_A->get_diag(),
+        gather_to_owner(exec_handler, diag_comm_pattern, host_A->get_diag(),
                         local_ptr);
-        gather_to_owner(exec, comm, upper_comm_pattern, host_A->get_upper(),
+        gather_to_owner(exec_handler, upper_comm_pattern, host_A->get_upper(),
                         local_ptr);
         if (host_A->symmetric()) {
             // TODO FIXME
             // if symmetric we can reuse already copied data
-            gather_to_owner(exec, comm, lower_comm_pattern, host_A->get_lower(),
+            gather_to_owner(exec_handler, lower_comm_pattern, host_A->get_lower(),
                             local_ptr);
         } else {
-            gather_to_owner(exec, comm, lower_comm_pattern, host_A->get_lower(),
+            gather_to_owner(exec_handler, lower_comm_pattern, host_A->get_lower(),
                             local_ptr);
         }
 
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index b5dded6e3..7cf86c40f 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -33,6 +33,25 @@ SourceFiles
 
 namespace Foam {
 
+void make_ldu_mapping_consecutive(
+    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+               std::vector<label>>
+        comm_pattern,
+    std::vector<label> &ldu_mapping, label rank, label ranks_per_gpu)
+{
+    auto [send_counts, recv_counts, send_offsets, recv_offsets] =
+        comm_pattern;
+
+    label ldu_offset = 0;
+    auto *data = ldu_mapping.data();
+
+    for (label i = 0; i < ranks_per_gpu; i++) {
+        auto size = recv_counts[i];
+        std::transform(data + ldu_offset, data + ldu_offset + size, data + ldu_offset,
+                       [&](label idx) { return idx + ldu_offset; });
+        ldu_offset += size;
+    }
+};
 
 HostMatrixWrapper::HostMatrixWrapper(
     const ExecutorHandler &exec, const objectRegistry &db, label nrows,
@@ -160,8 +179,7 @@ void neg_interface_iterator(const lduInterfaceFieldPtrsList &interfaces,
 
 
 CommunicationPattern HostMatrixWrapper::create_communication_pattern(
-    const gko::experimental::mpi::communicator& comm
-    ) const
+    const gko::experimental::mpi::communicator &comm) const
 {
     using comm_size_type = CommunicationPattern::comm_size_type;
     // temp map, mapping from neighbour rank interface cells
@@ -209,7 +227,7 @@ CommunicationPattern HostMatrixWrapper::create_communication_pattern(
         iter++;
     }
 
-    return CommunicationPattern{comm, target_ids, target_sizes, send_idxs};
+    return CommunicationPattern{exec_, target_ids, target_sizes, send_idxs};
 }
 
 
@@ -321,27 +339,27 @@ HostMatrixWrapper::collect_cells_on_non_local_interface(
     // vector of neighbour cell idx connected to interface
     std::vector<std::tuple<label, label, label, label>> non_local_idxs{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
-    interface_iterator<
-        processorFvPatch>(interfaces, [&](label, label interface_id,
-                                          label interface_size,
-                                          const processorLduInterface &,
-                                          const lduInterfaceField *iface) {
-        const auto &face_cells{iface->interface().faceCells()};
+    interface_iterator<processorFvPatch>(
+        interfaces,
+        [&](label, label interface_id, label interface_size,
+            const processorLduInterface &, const lduInterfaceField *iface) {
+            const auto &face_cells{iface->interface().faceCells()};
 
-        const processorLduInterface &pldui =
+            const processorLduInterface &pldui =
                 refCast<const processorLduInterface>(iface->interface());
-        const label neighbProcNo = pldui.neighbProcNo();
-        pldui.send(Pstream::commsTypes::blocking, face_cells);
+            const label neighbProcNo = pldui.neighbProcNo();
+            pldui.send(Pstream::commsTypes::blocking, face_cells);
 
-        auto otherSide_tmp = pldui.receive<label>(
+            auto otherSide_tmp = pldui.receive<label>(
                 Pstream::commsTypes::blocking, interface_size);
 
-        for (label cellI = 0; cellI < interface_size; cellI++) {
-            auto local_row = face_cells[cellI];
-            auto col = otherSide_tmp()[cellI];
-            non_local_idxs.push_back({interface_id, col, local_row, neighbProcNo});
-        }
-    });
+            for (label cellI = 0; cellI < interface_size; cellI++) {
+                auto local_row = face_cells[cellI];
+                auto col = otherSide_tmp()[cellI];
+                non_local_idxs.push_back(
+                    {interface_id, col, local_row, neighbProcNo});
+            }
+        });
 
     word msg = "done collecting neighbouring processor cell id";
     LOG_2(verbose_, msg)
@@ -355,8 +373,7 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
                                                     non_local_matrix_nnz_)};
     sparsity->dim = gko::dim<2>{nrows_, non_local_matrix_nnz_};
 
-    auto non_local_indices =
-        collect_cells_on_non_local_interface(interfaces_);
+    auto non_local_indices = collect_cells_on_non_local_interface(interfaces_);
     auto rows = sparsity->row_idxs.get_data();
     auto cols = sparsity->col_idxs.get_data();
     auto permute = sparsity->ldu_mapping.get_data();
@@ -375,7 +392,8 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
         cols[element_ctr] = col;
         permute[element_ctr] = element_ctr;
 
-        // a new interface started or the last element on last interface has been reached
+        // a new interface started or the last element on last interface has
+        // been reached
         bool last_element = element_ctr == non_local_indices.size() - 1;
         if (interface_idx > prev_interface_ctr || last_element) {
             // this check will be reached one element earlier if we reached
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 6f18cf04a..beb8dcf38 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -25,6 +25,7 @@ SourceFiles
 #ifndef OGL_HostMatrix_INCLUDED_H
 #define OGL_HostMatrix_INCLUDED_H
 #include <map>
+#include <numeric>
 
 #include <ginkgo/ginkgo.hpp>
 
@@ -62,35 +63,36 @@ void make_ldu_mapping_consecutive(
 class Repartitioner {
 private:
     using partition = gko::experimental::distributed::Partition<label, label>;
+
+    const label size_;  //! Size (n matrix rows ) before repartitioning
+
+    const label repart_size_;  //! Size after repartitioning
+
     const label ranks_per_gpu_;
 
-    const ExecutorHandler exec_handler_;
+    const ExecutorHandler &exec_handler_;
 
     std::shared_ptr<const partition> orig_partition_;
 
 
 public:
-    Repartitioner(
-        label size,
-        label ranks_per_gpu,
-        const ExecutorHandler &exec_handler
-                )
-        : ranks_per_gpu_(ranks_per_gpu),
+    Repartitioner(label size, label ranks_per_gpu,
+                  const ExecutorHandler &exec_handler)
+        : size_(size),
+          repart_size_(Repartitioner::compute_repart_size(size, ranks_per_gpu,
+                                                          exec_handler)),
+          ranks_per_gpu_(ranks_per_gpu),
           exec_handler_(exec_handler),
-          orig_partition_(
-            gko::share(
-            gko::experimental::distributed::build_partition_from_local_size<
-                label, label>(
-            exec_handler_.get_device_exec(),
-            *exec_handler_.get_communicator().get(),
-                    size))
-)
-        {};
+          orig_partition_(gko::share(
+              gko::experimental::distributed::build_partition_from_local_size<
+                  label, label>(exec_handler_.get_device_exec(),
+                                *exec_handler_.get_communicator().get(),
+                                size))) {};
 
     /* returns the owner rank for a given rank */
     label get_owner_rank(label rank) const
     {
-        return rank - (rank % ranks_per_gpu_);
+        return compute_owner_rank(rank, ranks_per_gpu_);
     };
 
     /* returns the owner rank for a given rank */
@@ -107,14 +109,36 @@ public:
 
     label get_ranks_per_gpu() const { return ranks_per_gpu_; }
 
-    std::shared_ptr<const partition> get_orig_partition() const {return orig_partition_;};
+    static label compute_repart_size(label local_size, label ranks_per_gpu,
+                                     const ExecutorHandler &exec_handler)
+    {
+        if (ranks_per_gpu == 1) {
+            return local_size;
+        }
+
+        CommCounts comm_pattern =
+            compute_send_recv_counts(exec_handler, ranks_per_gpu, 1);
+
+        std::vector<label> data{local_size};
+
+        auto local_sizes =
+            gather_to_owner(exec_handler, comm_pattern, 1, data.data());
+
+        return std::reduce(local_sizes.begin(), local_sizes.end());
+    }
+
+    label get_repart_size() const {return repart_size_;}
+
+    std::shared_ptr<const partition> get_orig_partition() const
+    {
+        return orig_partition_;
+    };
 
     std::pair<std::shared_ptr<SparsityPattern>,
               std::shared_ptr<SparsityPattern>>
     repartition_sparsity(
         std::shared_ptr<SparsityPattern> src_local_pattern,
-        std::shared_ptr<SparsityPattern> src_non_local_pattern
-) const
+        std::shared_ptr<SparsityPattern> src_non_local_pattern) const
     {
         // 1. obtain send recv sizes vector
         // here we can reuse code from repartition_comm_pattern
@@ -134,17 +158,19 @@ public:
         label ranks_per_gpu = ranks_per_gpu_;
 
         auto local_comm_pattern = compute_send_recv_counts(
-            ranks_per_gpu, owner_rank, comm, exec, src_local_pattern->size_);
+            exec_handler_, ranks_per_gpu, src_local_pattern->size_);
 
         label offset = orig_partition_->get_range_bounds()[rank] -
                        orig_partition_->get_range_bounds()[owner_rank];
 
-        auto gather_closure = [exec, comm](auto &comm_pattern, auto &data,
-                                           label offset) {
-            return gather_to_owner(exec, comm, comm_pattern, data.get_size(),
+        auto &exec_handler = exec_handler_;
+        auto gather_closure = [exec_handler](auto &comm_pattern, auto &data,
+                                             label offset) {
+            return gather_to_owner(exec_handler, comm_pattern, data.get_size(),
                                    data.get_data(), offset);
         };
 
+        std::cout << __FILE__ << " rank " << rank << "before gathered_local\n";
         auto merged_local_rows = gather_closure(
             local_comm_pattern, src_local_pattern->row_idxs, offset);
 
@@ -154,8 +180,10 @@ public:
         auto merged_ldu_mapping = gather_closure(
             local_comm_pattern, src_local_pattern->ldu_mapping, 0);
 
+        std::cout << __FILE__ << " rank " << rank << " gathered_local\n";
         make_ldu_mapping_consecutive(local_comm_pattern, merged_ldu_mapping,
                                      rank, ranks_per_gpu);
+        std::cout << __FILE__ << " rank " << rank << " done ldu mapping consecutive\n";
 
         label rows = (owner) ? merged_local_rows.back() + 1 : 0;
         gko::dim<2> merged_local_dim{rows, rows};
@@ -164,25 +192,27 @@ public:
         std::vector<label> merged_end{label(merged_local_rows.size())};
         std::vector<label> merged_ranks{rank};
 
-        auto non_local_comm_pattern =
-            compute_send_recv_counts(ranks_per_gpu, owner_rank, comm, exec,
-                                     src_non_local_pattern->size_);
+        auto non_local_comm_pattern = compute_send_recv_counts(
+            exec_handler, ranks_per_gpu, src_non_local_pattern->size_);
 
         auto merged_non_local_rows = gather_closure(
             non_local_comm_pattern, src_non_local_pattern->row_idxs, offset);
+        std::cout << __FILE__ << " rank " << rank << " gathered_non_local\n";
 
         // the non local cols are in local idx of other side
         // thus we need the new offset of the other side
         for (label i = 0; i < src_non_local_pattern->rank.size(); i++) {
             auto comm_rank = src_non_local_pattern->rank[i];
             auto [begin, end] = src_non_local_pattern->interface_spans[i];
-            label local_offset = orig_partition_->get_range_bounds()[comm_rank] -
-                                 orig_partition_->get_range_bounds()[owner_rank];
+            label local_offset =
+                orig_partition_->get_range_bounds()[comm_rank] -
+                orig_partition_->get_range_bounds()[owner_rank];
             auto *data = src_non_local_pattern->col_idxs.get_data() + begin;
             auto size = end - begin;
             std::transform(data, data + size, data,
                            [&](label idx) { return idx + local_offset; });
         }
+        std::cout << __FILE__ << " rank " << rank << " done offset non_local\n";
 
         // we now dealed with the new offset thus we dont need an offset
         // now
@@ -193,9 +223,11 @@ public:
             non_local_comm_pattern, src_non_local_pattern->row_idxs, 0);
 
         auto span_comm_pattern = compute_send_recv_counts(
-            ranks_per_gpu, owner_rank, comm, exec,
+            exec_handler, ranks_per_gpu,
             src_non_local_pattern->interface_spans.size());
 
+        std::cout << __FILE__ << " rank " << rank << " done merged non_local\n";
+
         std::vector<label> spans_begin;
         std::vector<label> spans_end;
 
@@ -205,16 +237,19 @@ public:
         }
 
         auto merged_non_local_begin =
-            gather_to_owner(exec, comm, span_comm_pattern, spans_begin.size(),
+            gather_to_owner(exec_handler, span_comm_pattern, spans_begin.size(),
                             spans_begin.data());
 
-        auto merged_non_local_end = gather_to_owner(
-            exec, comm, span_comm_pattern, spans_end.size(), spans_end.data());
+        auto merged_non_local_end =
+            gather_to_owner(exec_handler, span_comm_pattern, spans_end.size(),
+                            spans_end.data());
 
         auto merged_non_local_ranks = gather_to_owner(
-            exec, comm, span_comm_pattern, src_non_local_pattern->rank.size(),
+            exec_handler, span_comm_pattern, src_non_local_pattern->rank.size(),
             src_non_local_pattern->rank.data());
 
+        std::cout << __FILE__ << " rank " << rank << " done merged non_local2\n";
+
 
         // fix the received merged_non_local_begin and ends
         // we received begins and ends of the interfaces from

From 7249d4e293da4db5551ee8e504b1e9e397dcd71b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 26 May 2024 20:58:38 +0200
Subject: [PATCH 54/90] add a lot of output, trying to random failures

---
 .../DistributedMatrixAdapter.H                |   3 +-
 .../ExecutorHandler/ExecutorHandler.H         |   5 +
 DevicePersistent/Vector/Vector.H              |  41 +++--
 MatrixWrapper/Combination/Combination.H       |   3 +
 .../CommunicationPattern.C                    |  60 ++++---
 .../CommunicationPattern.C.bck                |  22 +--
 .../CommunicationPattern.H                    |  61 +++++---
 .../CommunicationPattern.H.bck                |   4 +-
 MatrixWrapper/Distributed/Distributed.H       | 148 ++++++++++++------
 MatrixWrapper/LDUMatrix/HostMatrix.C          |  24 ++-
 MatrixWrapper/LDUMatrix/HostMatrix.H          |  73 +++++----
 StoppingCriterion/StoppingCriterion.C         |   1 +
 lduLduBase/lduLduBase.H                       |  81 ++++------
 13 files changed, 313 insertions(+), 213 deletions(-)

diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 84592ac9e..7db5d767e 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -30,6 +30,7 @@ SourceFiles
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "DevicePersistent/Partition/Partition.H"
 #include "MatrixWrapper/Distributed/Distributed.H"
+#include "DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H"
 
 #include "fvCFD.H"
 
@@ -40,7 +41,7 @@ struct DistributedMatrixInitFunctor {
 
     const ExecutorHandler &exec_;
 
-    const Repartitioner & repartitioner_;
+    const Repartitioner &repartitioner_;
 
     const word field_name_;
 
diff --git a/DevicePersistent/ExecutorHandler/ExecutorHandler.H b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
index 87a6dec49..59138eabe 100644
--- a/DevicePersistent/ExecutorHandler/ExecutorHandler.H
+++ b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
@@ -114,7 +114,12 @@ private:
 
     const word device_executor_name_;
 
+
+
 public:
+
+    ExecutorHandler(const ExecutorHandler&) = delete;
+
     ExecutorHandler(const objectRegistry &db, const dictionary &solverControls,
                     const word field_name)
         : PersistentBase<gko::Executor, ExecutorInitFunctor>(
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 42d15b217..05c4d8011 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -25,7 +25,7 @@ struct VectorInitFunctor {
 
     const word name_;
 
-    const Repartitioner & repartitioner_;
+    const Repartitioner &repartitioner_;
 
     const ExecutorHandler &exec_;
 
@@ -69,19 +69,19 @@ struct VectorInitFunctor {
         auto exec =
             (on_device_) ? exec_.get_device_exec() : exec_.get_ref_exec();
         auto comm = exec_.get_gko_mpi_host_comm();
+        auto rank = comm->rank();
 
         const auto local_size = size_;
+        label repart_size = repartitioner_.get_repart_size();
         word msg{"initialising vector " + name_ + " of size " +
-                 std::to_string(local_size)};
+                 std::to_string(repart_size)};
         msg += (on_device_) ? " on device" : " on host";
         LOG_1(verbose_, msg)
 
+        std::cout << __FILE__  << __LINE__ << " on rank " << rank << " before host view \n";
         auto host_view = gko::array<T>::view(exec_.get_ref_exec(), local_size,
                                              const_cast<T *>(other_));
-
-        label repart_size = repartitioner_.get_repart_size();
-        // this should happen on device
-        auto local_coeffs = gko::array<scalar>(exec, repart_size);
+        std::cout << __FILE__ << " on rank " << rank << " done host view \n";
 
         // FIXME this shouldn't be needed here also not during an
         // update
@@ -89,12 +89,20 @@ struct VectorInitFunctor {
             compute_send_recv_counts(
             exec_, repartitioner_.get_ranks_per_gpu(), local_size);
 
-        gather_to_owner(exec_, comm_pattern, host_view.get_data(),
+        // this should happen on device
+        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank << " before local coeffs  \n" << std::endl;
+        auto local_coeffs = gko::array<scalar>(exec, repart_size);
+        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank << " done local coeffs  \n";
+
+        communicate_values(exec_, comm_pattern, host_view.get_data(),
                         local_coeffs.get_data());
+        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank << "  gather_to_owner done \n";
 
-        return gko::share(dist_vec::create(
-            exec, *exec_.get_communicator().get(),
+        auto ret =  gko::share(dist_vec::create(
+            exec, *comm.get(),
             vec::create(exec, gko::dim<2>{repart_size, 1}, local_coeffs, 1)));
+        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank << "  done dist vector create \n";
+        return ret;
     }
 };
 
@@ -159,13 +167,14 @@ public:
      **/
     void copy_back(std::shared_ptr<const part_type> partition)
     {
-        auto ref_exec = exec_.get_ref_exec();
-        auto host_buffer_view = gko::array<T>::view(
-            exec_.get_device_exec(), size_, get_vector()->get_local_values());
-
-        auto to_view =
-            gko::array<T>::view(ref_exec, size_, const_cast<T *>(memory_));
-        to_view = host_buffer_view;
+        // TODO FIXME
+        // auto ref_exec = exec_.get_ref_exec();
+        // auto host_buffer_view = gko::array<T>::view(
+        //     exec_.get_device_exec(), size_, get_vector()->get_local_values());
+
+        // auto to_view =
+        //     gko::array<T>::view(ref_exec, size_, const_cast<T *>(memory_));
+        // to_view = host_buffer_view;
     }
 
     /** Writes the content of the distributed vector to disk
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index c5b29f9f6..e5d528e54 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -34,6 +34,9 @@ public:
     void read(const device_matrix_data &data) override
     {
         std::cout << __FILE__ << " read combination \n";
+        // TODO FIXME if no matrix is given because of repartitioning
+        // which might remove non_local matrix
+        // a zero sized dummy matrix needs to be created
         auto exec = this->get_executor();
         auto data_exec = data.get_executor();
 
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index 86bfaa9f2..cd9b1278c 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -28,27 +28,34 @@ SourceFiles
 #include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
 #include "common/common.H"
 
+
+label compute_owner_rank(label rank, label ranks_per_gpu)
+{
+    return rank - (rank % ranks_per_gpu);
+};
+
 std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
            std::vector<label>>
-compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
-                         const gko::experimental::mpi::communicator &comm,
-                         std::shared_ptr<const gko::Executor> exec, label size)
+compute_send_recv_counts(const ExecutorHandler &exec_handler,
+                         label ranks_per_gpu, label size)
 {
-    return compute_send_recv_counts(ranks_per_gpu, owner_rank, comm, exec, size,
-                                    size, 0, 0);
+    return compute_send_recv_counts(exec_handler, ranks_per_gpu, size, size, 0,
+                                    0);
 }
 
 std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
            std::vector<label>>
-compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
-                         const gko::experimental::mpi::communicator &comm,
-                         std::shared_ptr<const gko::Executor> exec, label size,
-                         label total_size, label padding_before,
-                         label padding_after)
+compute_send_recv_counts(const ExecutorHandler &exec_handler,
+                         label ranks_per_gpu, label size, label total_size,
+                         label padding_before, label padding_after)
 {
+    auto exec = exec_handler.get_device_exec();
+    auto comm = *exec_handler.get_communicator().get();
+
     ASSERT_EQ(total_size, size + padding_before + padding_after);
     label total_ranks{comm.size()};
     label rank{comm.rank()};
+    label owner_rank = compute_owner_rank(rank, ranks_per_gpu);
     std::vector<label> send_counts(total_ranks, 0);
     std::vector<label> recv_counts(total_ranks, 0);
     std::vector<label> send_offsets(total_ranks, 0);
@@ -97,13 +104,14 @@ compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
                            recv_offsets);
 }
 
-std::vector<label> gather_to_owner(
-    std::shared_ptr<const gko::Executor> exec,
-    const gko::experimental::mpi::communicator &comm,
+void communicate_values (
+    const ExecutorHandler &exec_handler,
     const std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
                      std::vector<label>> &comm_pattern,
     const scalar *send_buffer, scalar *recv_buffer)
 {
+    auto exec = exec_handler.get_device_exec();
+    auto comm = *exec_handler.get_communicator().get();
     auto [send_counts, recv_counts, send_offsets, recv_offsets] = comm_pattern;
     // TODO add some sanity checks for comm pattern
     // 1. length needs to be same as mpi ranks
@@ -122,16 +130,16 @@ std::vector<label> gather_to_owner(
     comm.all_to_all_v(exec, send_buffer, send_counts.data(),
                       send_offsets.data(), recv_buffer, recv_counts.data(),
                       recv_offsets.data());
-    return {};
 };
 
 std::vector<label> gather_to_owner(
-    std::shared_ptr<const gko::Executor> exec,
-    const gko::experimental::mpi::communicator &comm,
+    const ExecutorHandler &exec_handler,
     const std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
                      std::vector<label>> &comm_pattern,
     label size, const label *data, label offset)
 {
+    auto exec = exec_handler.get_ref_exec();
+    auto comm = *exec_handler.get_communicator().get();
     // TODO don't modify send_buffer
     std::vector<label> send_buffer_copy(size);
     if (offset > 0) {
@@ -150,8 +158,8 @@ std::vector<label> gather_to_owner(
         total_recv_size += recv_counts[i];
     }
 
-    std::cout << " total_recv_ size " << total_recv_size << "\n";
     std::vector<label> recv_buffer(total_recv_size);
+    auto rank = comm.rank();
     comm.all_to_all_v(exec, send_buffer, send_counts.data(),
                       send_offsets.data(), recv_buffer.data(),
                       recv_counts.data(), recv_offsets.data());
@@ -161,7 +169,7 @@ std::vector<label> gather_to_owner(
 
 std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e)
 {
-    out << "CommunicationPattern: for rank: " << e.comm.rank();
+    out << "CommunicationPattern: for rank: " << e.get_comm().rank();
     out << " {";
     out << "\ntarget_ids: " << e.target_ids;
     out << "\ntarget_sizes: " << e.target_sizes;
@@ -202,12 +210,14 @@ CommunicationPattern repartition_comm_pattern(
         return CommunicationPattern(src_comm_pattern);
     }
 
+    auto & exec_handler = src_comm_pattern.exec_handler;
     auto exec = src_comm_pattern.target_ids.get_executor();
+    auto comm = src_comm_pattern.get_comm();
+
     auto target_rank = [ranks_per_gpu](label rank) {
         return rank - (rank % ranks_per_gpu);
     };
 
-    auto comm = src_comm_pattern.comm;
 
     // find ranks which are now local
     // the target_ids, target_sizes etc need to be communicated
@@ -216,7 +226,7 @@ CommunicationPattern repartition_comm_pattern(
     std::vector<label> target_ids{};
     std::vector<label> target_sizes{};
     std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
-    label rank{src_comm_pattern.comm.rank()};
+    label rank{src_comm_pattern.get_comm().rank()};
     label owner_rank = target_rank(rank);
     bool owner = rank == owner_rank;
 
@@ -255,13 +265,13 @@ CommunicationPattern repartition_comm_pattern(
     // where to put elements from recv
     // communicate with all neighbours which are to be merged
     // how many elements the owner rank is to receive
-    auto comm_pattern = compute_send_recv_counts(ranks_per_gpu, owner_rank,
-                                                 comm, exec, target_ids.size());
+    auto comm_pattern = compute_send_recv_counts(exec_handler, ranks_per_gpu,
+                                                 target_ids.size());
 
-    target_ids = gather_to_owner(exec, comm, comm_pattern, target_ids.size(),
+    target_ids = gather_to_owner(exec_handler, comm_pattern, target_ids.size(),
                                  target_ids.data());
 
-    target_sizes = gather_to_owner(exec, comm, comm_pattern,
+    target_sizes = gather_to_owner(exec_handler, comm_pattern,
                                    target_sizes.size(), target_sizes.data());
 
     // communicate the send indices to owner rank
@@ -371,7 +381,7 @@ CommunicationPattern repartition_comm_pattern(
     }
 
     return CommunicationPattern{
-        src_comm_pattern.comm,
+        exec_handler,
         gko::array<comm_size_type>{exec, merged_target_ids.begin(),
                                    merged_target_ids.end()},
         gko::array<comm_size_type>{exec, merged_target_sizes.begin(),
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C.bck b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C.bck
index 0b14cb92e..fd16b054d 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C.bck
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C.bck
@@ -33,9 +33,7 @@ std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
 compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
                          const gko::experimental::mpi::communicator &comm,
                          std::shared_ptr<const gko::Executor> exec,
-                         label padding_before,
-                         label padding_after
-)
+                         label padding_before, label padding_after)
 {
     label total_ranks{comm.size()};
     label rank{comm.rank()};
@@ -214,11 +212,8 @@ CommunicationPattern repartition_comm_pattern(
     // this is also used distributed
     if (owner) {
         // retrieve from all neighbours
-        std::cout << __FILE__ << " on rank "
-                  << rank << " owner "
-                  << ranks_per_gpu
-                  << " recv_counts " << recv_counts
-                  << " \n";
+        std::cout << __FILE__ << " on rank " << rank << " owner "
+                  << ranks_per_gpu << " recv_counts " << recv_counts << " \n";
         for (int i = 1; i < ranks_per_gpu; i++) {
             // retrieve all send_idxs
 
@@ -226,17 +221,14 @@ CommunicationPattern repartition_comm_pattern(
                 auto target_size = target_sizes[j + 1];
                 auto target_id = target_ids[j + 1];
                 std::vector<label> recv_buffer(target_size);
-                std::cout << __FILE__ << " on rank " << rank
-                          << " recv from  i " << rank + i
-                          <<  " interface j " << j
-                    << " target_size " << target_size
-                    << " target_id " << target_id
-                    << "\n";
+                std::cout << __FILE__ << " on rank " << rank << " recv from  i "
+                          << rank + i << " interface j " << j << " target_size "
+                          << target_size << " target_id " << target_id << "\n";
 
                 comm.recv(exec, recv_buffer.data(), target_size, rank + i,
                           rank);
                 std::cout << __FILE__ << " on rank " << rank
-                    << " after recv \n";
+                          << " after recv \n";
 
 
                 // the new offset is
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 38ba6fcc1..7bcdfc24a 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -27,47 +27,51 @@ SourceFiles
 
 #pragma once
 #include <ginkgo/ginkgo.hpp>
+
 #include "fvCFD.H"
 
+#include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
+
 // TODO move this file to top level since it is not really a matrix-wrapper
 
-using CommCounts =
-std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-           std::vector<label>>;
+using CommCounts = std::tuple<std::vector<label>, std::vector<label>,
+                              std::vector<label>, std::vector<label>>;
 
 
 // This function computes the send and recv counts vectors for repartitioning
-CommCounts
-compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
-                         const gko::experimental::mpi::communicator &comm,
-                         std::shared_ptr<const gko::Executor> exec, label size,
-                         label total_size, label padding_before,
-                         label padding_after);
+CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
+                                    label ranks_per_gpu, label size,
+                                    label total_size, label padding_before,
+                                    label padding_after);
 
 // This function computes the send and recv counts vectors for repartitioning
-CommCounts
-compute_send_recv_counts(label ranks_per_gpu, label owner_rank,
-                         const gko::experimental::mpi::communicator &comm,
-                         std::shared_ptr<const gko::Executor> exec, label size);
+CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
+                                    label ranks_per_gpu, label size);
 
-// TODO these are separate functions
+/* Given a rank id and ranks_per_gpus this function computes the corresponding
+ * owner rank  */
+label compute_owner_rank(label rank, label ranks_per_gpu);
+
+// TODO these are separate functions, the scalar version generally operates
+// on the device where as labels are only communicated on the host side
 // create gather_coeffs, gather_idx, and a templated impl function
-std::vector<label> gather_to_owner(
-    std::shared_ptr<const gko::Executor> exec,
-    const gko::experimental::mpi::communicator &comm,
-    const CommCounts &comm_pattern,
-    const scalar *send_buffer, scalar *recv_buffer);
-
-std::vector<label> gather_to_owner(
-    std::shared_ptr<const gko::Executor> exec,
-    const gko::experimental::mpi::communicator &comm,
-    const CommCounts &comm_pattern,
-    label size, const label *data, label offset = 0);
+/* Given a comm_pattern this function executes the communication
+**
+*/
+void communicate_values (const ExecutorHandler &exec_handler,
+                                   const CommCounts &comm_pattern,
+                                   const scalar *send_buffer,
+                                   scalar *recv_buffer);
+
+/* Returns a vector of length total received elements + padding */
+std::vector<label> gather_to_owner(const ExecutorHandler &exec_handler,
+                                   const CommCounts &comm_pattern, label size,
+                                   const label *data, label offset = 0);
 
 struct CommunicationPattern {
     using comm_size_type = label;
 
-    gko::experimental::mpi::communicator comm;
+    const ExecutorHandler &exec_handler;
 
     // an array storing to which rank to communicate
     gko::array<comm_size_type> target_ids;
@@ -79,6 +83,11 @@ struct CommunicationPattern {
     // send_idx stores the index_set ie which cells
     // are owned by the interface and the corresponding target rank
     std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
+
+    const gko::experimental::mpi::communicator &get_comm() const
+    {
+        return *exec_handler.get_communicator().get();
+    }
 };
 
 std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e);
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H.bck b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H.bck
index e46aa555a..d6b422ed3 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H.bck
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H.bck
@@ -41,9 +41,7 @@ std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
 compute_send_recv_counts(label ranks_per_gpu, label owner_rank, label size,
                          const gko::experimental::mpi::communicator &comm,
                          std::shared_ptr<const gko::Executor> exec,
-                         label padding_before=0,
-                         label padding_after=0
-);
+                         label padding_before = 0, label padding_after = 0);
 
 struct CommunicationPattern {
     using comm_size_type = label;
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 6117575b5..17a708520 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -168,7 +168,7 @@ public:
         word matrix_format, const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A)
     {
-        auto exec_handler = host_A.get()->get_exec_handler();
+        auto& exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_device_exec();
         auto comm = *exec_handler.get_communicator().get();
 
@@ -200,15 +200,29 @@ public:
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
+        std::cout << __FILE__ << " rank " << rank
+                  << " build_localized_partition "
+                  << " dim " << local_sparsity_->dim[0]
+                  << " send idxs size " << dst_comm_pattern.send_idxs.size()
+                  << " target ids " << dst_comm_pattern.target_ids
+                  << " target sizes " << dst_comm_pattern.target_sizes
+                  << "\n";
+
         auto localized_partition = local_part_type::build_from_blocked_recv(
             exec, local_sparsity_->dim[0], dst_comm_pattern.send_idxs,
             dst_comm_pattern.target_ids, dst_comm_pattern.target_sizes);
 
-
-        std::cout << __FILE__ << " rank " << rank << " local sparsity dim "
-                  << local_sparsity_->dim[0] << "non_local sparsity dim "
+        std::cout << __FILE__ << " rank " << rank
+                  << " local sparsity size "
+                << local_sparsity_->size_
+                  << " local sparsity dim ["
+                  << local_sparsity_->dim[0]
+                  << "x"
+                  << local_sparsity_->dim[1]
+                  << "] non_local sparsity size " << non_local_sparsity_->size_
+                  << " non local sparsity dim ["
                   << non_local_sparsity_->dim[0] << "x"
-                  << non_local_sparsity_->dim[1] << " target_ids "
+                  << non_local_sparsity_->dim[1] << "] target_ids "
                   << dst_comm_pattern.target_ids << " target_sizes "
                   << dst_comm_pattern.target_sizes << " target_send_idxs.size "
                   << dst_comm_pattern.send_idxs.size()
@@ -232,10 +246,12 @@ public:
             gko::array<scalar>(exec, non_local_sparsity_->size_);
 
 
-        local_coeffs.fill(0.0);
-        non_local_coeffs.fill(0.0);
-        // FIXME make sure that we work on the device executor
+        if (owner) {
+            local_coeffs.fill(0.0);
+            non_local_coeffs.fill(0.0);
+        }
 
+        // FIXME make sure that we work on the device executor
         dist_A->read_distributed(
             device_matrix_data(exec, local_sparsity_->dim,
                                local_sparsity_->row_idxs,
@@ -255,13 +271,14 @@ public:
 
         label upper_nnz = host_A->get_upper_nnz();
         auto upper_comm_pattern = compute_send_recv_counts(
-            exec_handler, ranks_per_gpu, upper_nnz,
-            local_matrix_nnz, 0, local_matrix_nnz - upper_nnz);
-        auto lower_comm_pattern = compute_send_recv_counts(
-            exec_handler, ranks_per_gpu, upper_nnz,
-            local_matrix_nnz, upper_nnz, nrows);
+            exec_handler, ranks_per_gpu, upper_nnz, local_matrix_nnz, 0,
+            local_matrix_nnz - upper_nnz);
+        auto lower_comm_pattern =
+            compute_send_recv_counts(exec_handler, ranks_per_gpu, upper_nnz,
+                                     local_matrix_nnz, upper_nnz, nrows);
 
         scalar *local_ptr;
+        label local_interfaces_count = 0;
 
         if (owner) {
             using Coo = gko::matrix::Coo<scalar, label>;
@@ -270,58 +287,95 @@ public:
                                  dist_A->get_local_matrix())
                                  ->get_combination()
                                  ->get_operators()[0]);
+            local_interfaces_count =
+                gko::as<CombinationMatrix<scalar, label, Coo>>(
+                    dist_A->get_local_matrix())
+                    ->get_combination()
+                    ->get_operators()
+                    .size();
 
             local_ptr = const_cast<scalar *>(local->get_const_values());
         }
-        gather_to_owner(exec_handler, diag_comm_pattern, host_A->get_diag(),
+        communicate_values(exec_handler, diag_comm_pattern, host_A->get_diag(),
                         local_ptr);
-        gather_to_owner(exec_handler, upper_comm_pattern, host_A->get_upper(),
+        communicate_values(exec_handler, upper_comm_pattern, host_A->get_upper(),
                         local_ptr);
         if (host_A->symmetric()) {
             // TODO FIXME
             // if symmetric we can reuse already copied data
-            gather_to_owner(exec_handler, lower_comm_pattern, host_A->get_lower(),
-                            local_ptr);
+            communicate_values(exec_handler, lower_comm_pattern,
+                            host_A->get_lower(), local_ptr);
         } else {
-            gather_to_owner(exec_handler, lower_comm_pattern, host_A->get_lower(),
-                            local_ptr);
+            communicate_values(exec_handler, lower_comm_pattern,
+                            host_A->get_lower(), local_ptr);
         }
 
 
         // next update local and non_local interfaces
+        // TODO FIXME
+        for (int i = 1; i < local_interfaces_count; i++) {
+            using Coo = gko::matrix::Coo<scalar, label>;
+            std::shared_ptr<const Coo> local =
+                gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
+                                 dist_A->get_local_matrix())
+                                 ->get_combination()
+                                 ->get_operators()[i]);
+            local_ptr = const_cast<scalar *>(local->get_const_values());
 
-
-        // TODO make sure this doesn't copy
-        // create a non owning dense matrix of local_values
-        using dim_type = gko::dim<2>::dimension_type;
-        auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
-            exec,
-            gko::dim<2>{static_cast<dim_type>(repart_loc_sparsity->size_), 1},
-            gko::array<scalar>::view(exec, repart_loc_sparsity->size_,
-                                     local_ptr),
-            1));
-        if (rank == 0) {
-            std::cout << __FILE__ << " on rank " << rank
-                      << " created row collection of size  "
-                      << repart_loc_sparsity->size_ << " ldu_mapping "
-                      << local_sparsity_->ldu_mapping << "\n";
+            for (int j = 0; j < 10; j++) {
+                local_ptr[j] = 1.0;
+            }
         }
 
-        // TODO this needs to copy ldu_mapping to the device
-        auto dense_vec = row_collection->clone();
-        dense_vec->row_gather(&local_sparsity_->ldu_mapping,
-                              row_collection.get());
 
-        return std::make_shared<RepartDistMatrix>(
-            exec, comm, host_A->get_size(), dist_A->get_size(),
+        if (owner) {
+            // TODO make sure this doesn't copy
+            // create a non owning dense matrix of local_values
+            using dim_type = gko::dim<2>::dimension_type;
+            auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
+                exec,
+                gko::dim<2>{static_cast<dim_type>(repart_loc_sparsity->size_),
+                            1},
+                gko::array<scalar>::view(exec, repart_loc_sparsity->size_,
+                                         local_ptr),
+                1));
+            if (rank == 0) {
+                std::cout << __FILE__ << " on rank " << rank
+                          << " created row collection of size  "
+                          << repart_loc_sparsity->size_ << " ldu_mapping "
+                          << local_sparsity_->ldu_mapping << "\n";
+            }
+
+            // TODO this needs to copy ldu_mapping to the device
+            auto dense_vec = row_collection->clone();
+            dense_vec->row_gather(&local_sparsity_->ldu_mapping,
+                                  row_collection.get());
+            if (rank == 0) {
+                std::cout << __FILE__ << " on rank " << rank
+                    << " done row gather\n";
+            }
+        }
+
+        std::cout << __FILE__ << " on rank " << rank
+            << " make_shared "
+            << " dist_A size " << dist_A->get_size() <<
+            "\n";
+        auto ret = std::make_shared<RepartDistMatrix>(
+            exec, comm, repartitioner.get_repart_dim() , dist_A->get_size(),
             std::move(dist_A));
+            if (rank == 0) {
+                std::cout << __FILE__ << " on rank " << rank
+                    << "done make_shared\n";
+            }
+
+        return ret;
     };
 
     // TODO matrix_format can be data member
     void update(word matrix_format,
                 std::shared_ptr<const HostMatrixWrapper> host_A)
     {
-        auto exec_handler = host_A.get()->get_exec_handler();
+        auto& exec_handler = host_A.get()->get_exec_handler();
         auto device_exec = exec_handler.get_device_exec();
 
         scalar *local_value_ptr{};
@@ -398,9 +452,9 @@ public:
           dist_mtx_(std::move(dist_mtx))
     {
         this->set_size(global_size);
-        partition_ = gko::share(
-            gko::experimental::distributed::build_partition_from_local_size<
-                label, label>(exec, comm, local_size[0]));
+        // partition_ = gko::share(
+        //     gko::experimental::distributed::build_partition_from_local_size<
+        //         label, label>(exec, comm, local_size[0]));
     }
 
     void write(const word field_name_, const objectRegistry &db_) const
@@ -418,13 +472,13 @@ public:
                     dist_matrix->get_local_matrix())
                     ->get_combination()
                     ->get_operators();
+            export_mtx<Coo>(word(field_name_ + "_local"), local_interfaces,
+                            db_);
             auto non_local_interfaces =
                 gko::as<CombinationMatrix<scalar, label, Coo>>(
                     dist_matrix->get_non_local_matrix())
                     ->get_combination()
                     ->get_operators();
-            export_mtx<Coo>(word(field_name_ + "_local"), local_interfaces,
-                            db_);
             export_mtx<Coo>(field_name_ + "_non_local", non_local_interfaces,
                             db_);
         }
@@ -482,6 +536,8 @@ protected:
     // of these are always true in this example).
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
+
+        std::cout << __FILE__ << __LINE__ << "call apply\n";
         this->dist_mtx_->apply(b, x);
     }
 
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index 7cf86c40f..1a12cd687 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -39,20 +39,38 @@ void make_ldu_mapping_consecutive(
         comm_pattern,
     std::vector<label> &ldu_mapping, label rank, label ranks_per_gpu)
 {
-    auto [send_counts, recv_counts, send_offsets, recv_offsets] =
-        comm_pattern;
+    auto [send_counts, recv_counts, send_offsets, recv_offsets] = comm_pattern;
 
     label ldu_offset = 0;
     auto *data = ldu_mapping.data();
 
     for (label i = 0; i < ranks_per_gpu; i++) {
         auto size = recv_counts[i];
-        std::transform(data + ldu_offset, data + ldu_offset + size, data + ldu_offset,
+        std::transform(data + ldu_offset, data + ldu_offset + size,
+                       data + ldu_offset,
                        [&](label idx) { return idx + ldu_offset; });
         ldu_offset += size;
     }
 };
 
+void make_begin_and_ends_consecutive(
+    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+               std::vector<label>>
+        comm_pattern,
+    std::vector<label> &begin, std::vector<label> &end, label rank,
+    label ranks_per_gpu)
+{
+    for (label i = 1; i < ranks_per_gpu; i++) {
+        auto recv_count = std::get<1>(comm_pattern)[rank + i];
+        auto recv_offset = std::get<3>(comm_pattern)[rank + i];
+        auto range_offset = end[recv_offset - 1];
+        for (label j = 0; j < recv_count; j++) {
+            begin[recv_offset + j] += range_offset;
+            end[recv_offset + j] += range_offset;
+        }
+    }
+}
+
 HostMatrixWrapper::HostMatrixWrapper(
     const ExecutorHandler &exec, const objectRegistry &db, label nrows,
     label upper_nnz, bool symmetric, const scalar *diag, const scalar *upper,
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index beb8dcf38..ea237b594 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -59,6 +59,28 @@ void make_ldu_mapping_consecutive(
         comm_pattern,
     std::vector<label> &ldu_mapping, label rank, label ranks_per_gpu);
 
+
+/** fix the received merged_non_local_begin and ends
+ *  we received begins and ends of the interfaces from
+ *  the neighbour ranks. however the begin and ends are
+ *  computed based on the unmerged interfaces. Now we
+ *  need to offset these values. For example
+ *  before communication and merge:
+ *  rank 0 begin [ 0 10 ], end [10 20]
+ *  rank 1 begin [ 0 10 ], end [10 20]
+ *  after communication
+ * on owner merged begin [ 0 10 0 10 ], end [10 20 10 20]
+ *    after fix
+ *   merged begin [ 0 10 20 30 ], end [10 20 30 40]
+ */
+void make_begin_and_ends_consecutive(
+    std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
+               std::vector<label>>
+        comm_pattern,
+    std::vector<label> &begin, std::vector<label> &end, label rank,
+    label ranks_per_gpu);
+
+
 // TODO this should be persistent
 class Repartitioner {
 private:
@@ -70,7 +92,7 @@ private:
 
     const label ranks_per_gpu_;
 
-    const ExecutorHandler &exec_handler_;
+    const ExecutorHandler & exec_handler_;
 
     std::shared_ptr<const partition> orig_partition_;
 
@@ -127,7 +149,9 @@ public:
         return std::reduce(local_sizes.begin(), local_sizes.end());
     }
 
-    label get_repart_size() const {return repart_size_;}
+    label get_repart_size() const { return repart_size_; }
+
+    gko::dim<2> get_repart_dim() const { return gko::dim<2>{repart_size_,repart_size_}; }
 
     std::shared_ptr<const partition> get_orig_partition() const
     {
@@ -164,7 +188,7 @@ public:
                        orig_partition_->get_range_bounds()[owner_rank];
 
         auto &exec_handler = exec_handler_;
-        auto gather_closure = [exec_handler](auto &comm_pattern, auto &data,
+        auto gather_closure = [&](auto &comm_pattern, auto &data,
                                              label offset) {
             return gather_to_owner(exec_handler, comm_pattern, data.get_size(),
                                    data.get_data(), offset);
@@ -180,10 +204,10 @@ public:
         auto merged_ldu_mapping = gather_closure(
             local_comm_pattern, src_local_pattern->ldu_mapping, 0);
 
-        std::cout << __FILE__ << " rank " << rank << " gathered_local\n";
-        make_ldu_mapping_consecutive(local_comm_pattern, merged_ldu_mapping,
-                                     rank, ranks_per_gpu);
-        std::cout << __FILE__ << " rank " << rank << " done ldu mapping consecutive\n";
+        if (owner) {
+            make_ldu_mapping_consecutive(local_comm_pattern, merged_ldu_mapping,
+                                         rank, ranks_per_gpu);
+        }
 
         label rows = (owner) ? merged_local_rows.back() + 1 : 0;
         gko::dim<2> merged_local_dim{rows, rows};
@@ -248,28 +272,11 @@ public:
             exec_handler, span_comm_pattern, src_non_local_pattern->rank.size(),
             src_non_local_pattern->rank.data());
 
-        std::cout << __FILE__ << " rank " << rank << " done merged non_local2\n";
-
-
-        // fix the received merged_non_local_begin and ends
-        // we received begins and ends of the interfaces from
-        // the neighbour ranks. however the begin and ends are
-        // computed base on the non merged interfaces. Now we
-        // need to offset these values
-        // before merge
-        // rank 0 begin [ 0 10 ], end [10 20]
-        // rank 1 begin [ 0 10 ], end [10 20]
-        // merged begin [ 0 10 0 10 ], end [10 20 10 20]
-        // after fix
-        // merged begin [ 0 10 20 30 ], end [10 20 30 40]
-        for (label i = 1; i < ranks_per_gpu; i++) {
-            auto recv_count = std::get<1>(span_comm_pattern)[rank + i];
-            auto recv_offset = std::get<3>(span_comm_pattern)[rank + i];
-            auto range_offset = merged_non_local_end[recv_offset - 1];
-            for (label j = 0; j < recv_count; j++) {
-                merged_non_local_begin[recv_offset + j] += range_offset;
-                merged_non_local_end[recv_offset + j] += range_offset;
-            }
+
+        if (owner) {
+            make_begin_and_ends_consecutive(
+                span_comm_pattern, merged_non_local_begin, merged_non_local_end,
+                rank, ranks_per_gpu);
         }
 
 
@@ -370,6 +377,8 @@ public:
                 gathered_ranks.push_back(to_rank);
             }
         }
+        std::cout << __FILE__ << " rank " << rank
+                  << " done merged non_local4\n";
 
         // for compressed cols we compute a mapping first
         std::map<label, label> compressed_col_idxs{};
@@ -382,6 +391,8 @@ public:
                 unique_idx += 1;
             };
         }
+        std::cout << __FILE__ << " rank " << rank
+                  << " done merged non_local5\n";
 
         for (int i = 0; i < gathered_non_local_cols.size(); i++) {
             label uncompressed = gathered_non_local_cols[i];
@@ -390,6 +401,8 @@ public:
                       << " compressed " << compressed << "\n";
             gathered_non_local_cols[i] = compressed;
         }
+        std::cout << __FILE__ << " rank " << rank
+                  << " done merged non_local6\n";
 
         // add interfaces
         // to add interfaces we go through the non_local_sparsity
@@ -605,7 +618,7 @@ private:
     using vec = gko::matrix::Dense<scalar>;
     using idx_array = gko::array<label>;
 
-    const ExecutorHandler exec_;
+    const ExecutorHandler &exec_;
 
     const DeviceIdGuardHandler device_id_guard_;
 
diff --git a/StoppingCriterion/StoppingCriterion.C b/StoppingCriterion/StoppingCriterion.C
index 8d3b3f082..9357866b0 100644
--- a/StoppingCriterion/StoppingCriterion.C
+++ b/StoppingCriterion/StoppingCriterion.C
@@ -74,6 +74,7 @@ bool StoppingCriterion::OpenFOAMDistStoppingCriterion::check_impl(
     gko::array<gko::stopping_status> *stop_status, bool *one_changed,
     const Criterion::Updater &updater)
 {
+    std::cout << __FILE__ << ":" << __LINE__ << "Stopping check \n";
     // Dont check residual norm before minIter is reached
     if (*(parameters_.iter) > 0 &&
         *(parameters_.iter) < parameters_.openfoam_minIter) {
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 304200a6a..33bf70d64 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -4,8 +4,8 @@
 
 #pragma once
 
-#include "DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H"
 #include "DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H"
+#include "DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H"
 #include "DevicePersistent/Vector/Vector.H"
 #include "Preconditioner/Preconditioner.H"
 #include "common/common.H"
@@ -135,7 +135,7 @@ private:
 
     const dictionary &solver_controls_;
 
-    const word & field_name_;
+    const word &field_name_;
 
     const label verbose_;
 
@@ -153,30 +153,20 @@ public:
                const dictionary &solverControls)
         : OGL_Info(fieldName, solverControls, matrix.mesh().thisDb()),
           MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
-                         interfaceIntCoeffs, interfaces, solverControls),
+                             interfaceIntCoeffs, interfaces, solverControls),
           SolverFactory{solverControls, matrix.mesh().thisDb(), fieldName},
           Preconditioner(fieldName, matrix.mesh().thisDb(), solverControls,
-                solverControls.lookupOrDefault<label>("verbose", 0)     
-                  ),
+                         solverControls.lookupOrDefault<label>("verbose", 0)),
           db_(matrix.mesh().thisDb()),
           matrix_(matrix),
           solver_controls_(solverControls),
           field_name_(fieldName),
           verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
-          exec_handler_{
-              db_, solver_controls_, fieldName
-          },
+          exec_handler_{db_, solver_controls_, fieldName},
           host_A_{
-              db_,
-              exec_handler_,
-              solver_controls_,
-              fieldName,
-              verbose_,
-              matrix_,
-              interfaceBouCoeffs,
-              interfaceIntCoeffs,
-              interfaces
-          }
+              db_,       exec_handler_, solver_controls_,   fieldName,
+              verbose_,  matrix_,       interfaceBouCoeffs, interfaceIntCoeffs,
+              interfaces}
     {}
 
 
@@ -207,26 +197,21 @@ public:
     {}
 
 
-    solverPerformance solve_multi_gpu_impl(
-            ExecutorHandler &exec_handler,
-            scalarField &psi,
+    solverPerformance solve_multi_gpu_impl(scalarField &psi,
                                            const scalarField &source,
                                            solverPerformance &solverPerf) const
     {
-        auto ref_exec = exec_handler.get_ref_exec();
-        Repartitioner repartitioner {
-        psi.size(),
-        solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1), exec_handler};
+        auto ref_exec = exec_handler_.get_ref_exec();
+        Repartitioner repartitioner{
+            psi.size(),
+            solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
+            exec_handler_};
 
         // create a distributed matrix from the host matrix
         // here also repartitioning is performed if required
-        PersistentDistributedMatrix dist_A{db_,
-                             exec_handler,
-                             repartitioner,
-                             field_name_,
-                             host_A_,
-                             solver_controls_,
-                             verbose_};
+        PersistentDistributedMatrix dist_A{
+            db_,     exec_handler_,    repartitioner, field_name_,
+            host_A_, solver_controls_, verbose_};
 
         PersistentVector<scalar> dist_b{
             source.begin(),
@@ -234,7 +219,7 @@ public:
             field_name_ + "_rhs",
             repartitioner,
             db_,
-            exec_handler,
+            exec_handler_,
             verbose_,
             solver_controls_.lookupOrDefault<Switch>("updateRHS", true),
             false  // whether data for init is on device
@@ -246,7 +231,7 @@ public:
             field_name_ + "_solution",
             repartitioner,
             db_,
-            exec_handler,
+            exec_handler_,
             verbose_,
             solver_controls_.lookupOrDefault<Switch>("updateInitGuess", false),
             false  // whether data for init is on device
@@ -268,10 +253,9 @@ public:
                 dist_b.get_vector()->scale(dense_scaling.get());)
         }
 
-        TIME_WITH_FIELDNAME(
-            verbose_, init_precond, field_name_,
-            auto precond = this->init_preconditioner(
-                dist_A_v, exec_handler.get_device_exec());)
+        TIME_WITH_FIELDNAME(verbose_, init_precond, field_name_,
+                            auto precond = this->init_preconditioner(
+                                dist_A_v, exec_handler_.get_device_exec());)
 
         bool write(solver_controls_.lookupOrDefault<Switch>("export", false));
         if (write && db_.time().writeTime()) {
@@ -283,8 +267,8 @@ public:
 
         LOG_1(verbose_, "create solver")
         auto solver_gen = this->create_dist_solver(
-            exec_handler.get_device_exec(), dist_A_v, dist_x_v,
-            dist_b_v, verbose_, write, precond);
+            exec_handler_.get_device_exec(), dist_A_v, dist_x_v, dist_b_v,
+            verbose_, write, precond);
 
         TIME_WITH_FIELDNAME(verbose_, generate_solver, field_name_,
                             auto solver = solver_gen->generate(dist_A_v);)
@@ -305,12 +289,14 @@ public:
         auto time_for_res_norm_eval = this->get_res_norm_time();
         auto time_per_iter =
             delta_t_solve / max(this->get_number_of_iterations(), 1);
-        scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
-        exec_handler.get_gko_mpi_host_comm()->broadcast(
-            exec_handler.get_ref_exec(), &prev_rel_res_cost, 1, 0);
+        scalar prev_rel_res_cost =
+            time_per_iter / (time_for_res_norm_eval + SMALL);
+        exec_handler_.get_gko_mpi_host_comm()->broadcast(
+            exec_handler_.get_ref_exec(), &prev_rel_res_cost, 1, 0);
         this->set_prev_rel_res_cost(prev_rel_res_cost);
-        auto time_per_iter_and_dof =
-            time_per_iter * 1000.0 / partition->get_size();
+        auto time_per_iter_and_dof = 0;
+        // TOOD FIXME
+        // time_per_iter * 1000.0 / partition->get_size();
         word msg =
             "\nStatistics:\n\tTime per iteration: " +
             std::to_string(time_per_iter) +
@@ -333,17 +319,16 @@ public:
             FatalErrorInFunction << "Only parallel runs are supported for OGL"
                                  << exit(FatalError);
         }
-        auto exec_handler = host_A_.get()->get_exec_handler();
         // --- Setup class containing solver performance data
         solverPerformance solverPerf(
             lduMatrix::preconditioner::getName(solver_controls_) +
-                exec_handler.get_exec_name() + typeName,
+                exec_handler_.get_exec_name() + typeName,
             field_name_);
 
         // Solve system
         TIME_WITH_FIELDNAME(
             verbose_, solve_multi_gpu, field_name_,
-            auto res = solve_multi_gpu_impl(exec_handler, psi, source, solverPerf);)
+            auto res = solve_multi_gpu_impl(psi, source, solverPerf);)
         return res;
 
         return solverPerf;

From dc2619a51140e81b9aa3c0ed19f1a5025bec191a Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 29 May 2024 08:30:35 +0200
Subject: [PATCH 55/90] fix zero combination matrix issue, refactor
 repartitioner, fix repartition communication pattern

---
 DevicePersistent/Vector/Vector.H              |   1 +
 MatrixWrapper/Combination/Combination.H       |   8 +-
 .../CommunicationPattern.C                    | 217 +-----
 .../CommunicationPattern.H                    |  47 +-
 MatrixWrapper/Distributed/Distributed.H       |   7 +-
 MatrixWrapper/LDUMatrix/HostMatrix.H          | 388 -----------
 Repartitioner/Repartitioner.H                 | 636 ++++++++++++++++++
 common/common.H                               |  11 -
 8 files changed, 680 insertions(+), 635 deletions(-)
 create mode 100644 Repartitioner/Repartitioner.H

diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 05c4d8011..80513a5eb 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -13,6 +13,7 @@
 #include "DevicePersistent/Partition/Partition.H"
 #include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 #include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
+#include "Repartitioner/Repartitioner.H"
 #include "common/common.H"
 
 namespace Foam {
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index e5d528e54..2b4fb14d9 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -139,7 +139,9 @@ protected:
     // are always true in this example).
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
-        this->comb_->apply(b, x);
+        if (interface_ranges_.size() > 0) {
+            this->comb_->apply(b, x);
+        }
     }
 
     // There is also a version of the apply function which does the operation
@@ -149,7 +151,9 @@ protected:
     void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
                     const gko::LinOp *beta, gko::LinOp *x) const override
     {
-        this->comb_->apply(alpha, b, beta, x);
+        if (interface_ranges_.size() > 0) {
+            this->comb_->apply(alpha, b, beta, x);
+        }
     }
 
 
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index cd9b1278c..019a1279a 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -140,7 +140,6 @@ std::vector<label> gather_to_owner(
 {
     auto exec = exec_handler.get_ref_exec();
     auto comm = *exec_handler.get_communicator().get();
-    // TODO don't modify send_buffer
     std::vector<label> send_buffer_copy(size);
     if (offset > 0) {
         std::transform(data, data + size, send_buffer_copy.data(),
@@ -160,9 +159,13 @@ std::vector<label> gather_to_owner(
 
     std::vector<label> recv_buffer(total_recv_size);
     auto rank = comm.rank();
+    std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+        << "before alltoallv\n";
     comm.all_to_all_v(exec, send_buffer, send_counts.data(),
                       send_offsets.data(), recv_buffer.data(),
                       recv_counts.data(), recv_offsets.data());
+    std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+        << "after alltoallv\n";
 
     return recv_buffer;
 }
@@ -176,215 +179,3 @@ std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e)
     out << "}\n";
     return out;
 }
-
-template <typename T, typename Compare>
-std::vector<label> sort_permutation(const std::vector<T> &vec, Compare compare)
-{
-    std::vector<label> p(vec.size());
-    std::iota(p.begin(), p.end(), 0);
-    std::stable_sort(p.begin(), p.end(), [&](std::size_t i, std::size_t j) {
-        return compare(vec[i], vec[j]);
-    });
-    return p;
-}
-
-template <typename T>
-std::vector<T> apply_permutation(const std::vector<T> vec,
-                                 const std::vector<label> &p)
-{
-    std::vector<T> sorted_vec(vec.size());
-    std::transform(p.begin(), p.end(), sorted_vec.begin(),
-                   [&](label i) { return vec[i]; });
-    return sorted_vec;
-}
-
-CommunicationPattern repartition_comm_pattern(
-    label ranks_per_gpu, CommunicationPattern &src_comm_pattern,
-    std::shared_ptr<
-        const gko::experimental::distributed::Partition<label, label>>
-        partition)
-{
-    using comm_size_type = label;
-
-    if (ranks_per_gpu == 1) {
-        return CommunicationPattern(src_comm_pattern);
-    }
-
-    auto & exec_handler = src_comm_pattern.exec_handler;
-    auto exec = src_comm_pattern.target_ids.get_executor();
-    auto comm = src_comm_pattern.get_comm();
-
-    auto target_rank = [ranks_per_gpu](label rank) {
-        return rank - (rank % ranks_per_gpu);
-    };
-
-
-    // find ranks which are now local
-    // the target_ids, target_sizes etc need to be communicated
-    // with new owner. thus we use maps to store them where the index
-    // is the new owner.
-    std::vector<label> target_ids{};
-    std::vector<label> target_sizes{};
-    std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
-    label rank{src_comm_pattern.get_comm().rank()};
-    label owner_rank = target_rank(rank);
-    bool owner = rank == owner_rank;
-
-    // map for exchanging communication data where the index is the target
-    // rank
-    for (int i = 0; i < src_comm_pattern.target_ids.get_size(); i++) {
-        label target_id = src_comm_pattern.target_ids.get_data()[i];
-        // Step 1. Check if communication partner is non-local after
-        // repartitioning. If it is non-local we keep it. Otherwise local
-        // communication can be discarded or merged. Here non-local means:
-        // the communication target proc id is != repartition proc id
-        //
-        // [ 1 | 2 | 3 ] [ 4 | 5 | 6 ] where [] - bounds after repart
-        //                                    | - bounds before repart
-        // e.g. rank 3 has rank 2 and 4 as comm neighbours
-        //      rank 2 can be discarded but rank 4 is non-local and needs
-        //      to be communicated to rank 1 to be merged into the new
-        //      communicaton pattern. This needs an all to all
-        //      communication.
-
-        label repart_target_id = target_rank(target_id);
-        label target_size = src_comm_pattern.target_sizes.get_const_data()[i];
-        auto send_idx = src_comm_pattern.send_idxs[i];
-
-        if (repart_target_id != owner_rank) {
-            // communication pattern is non local, hence we keep it
-            // after repartitioning we now have to communicate with a
-            // different rank store new owner rank to which repart_target_*
-            // needs to be send to
-            target_ids.push_back(repart_target_id);
-            target_sizes.push_back(target_size);
-            send_idxs.push_back(send_idx);
-        }
-    }
-
-    // where to put elements from recv
-    // communicate with all neighbours which are to be merged
-    // how many elements the owner rank is to receive
-    auto comm_pattern = compute_send_recv_counts(exec_handler, ranks_per_gpu,
-                                                 target_ids.size());
-
-    target_ids = gather_to_owner(exec_handler, comm_pattern, target_ids.size(),
-                                 target_ids.data());
-
-    target_sizes = gather_to_owner(exec_handler, comm_pattern,
-                                   target_sizes.size(), target_sizes.data());
-
-    // communicate the send indices to owner rank
-    // send indices are packed into std::vector of gko::arrays thus we send
-    // them one by one additionally we already have then target_sizes and
-    // send_counts and recv_counts which can be reused to for sending the
-    // index vectors
-
-    // TODO this could be refactored to separtate function
-    // this is also used distributed
-    auto recv_counts = std::get<1>(comm_pattern);
-    auto send_counts = std::get<0>(comm_pattern);
-    if (owner) {
-        // retrieve from all neighbours
-        for (int i = 1; i < ranks_per_gpu; i++) {
-            // retrieve all send_idxs
-            for (int j = 0; j < recv_counts[rank + i]; j++) {
-                auto target_size = target_sizes[j + 1];
-                auto target_id = target_ids[j + 1];
-                std::vector<label> recv_buffer(target_size);
-
-                comm.recv(exec, recv_buffer.data(), target_size, rank + i,
-                          rank);
-
-                // the new offset is
-                auto offset = partition->get_range_bounds()[rank + i] -
-                              partition->get_range_bounds()[rank];
-
-                std::transform(recv_buffer.begin(), recv_buffer.end(),
-                               recv_buffer.begin(),
-                               [&](label idx) { return idx + offset; });
-
-                send_idxs.push_back({gko::array<label>{
-                                         exec,
-                                         recv_buffer.begin(),
-                                         recv_buffer.end(),
-                                     },
-                                     target_id});
-            }
-        }
-    } else {
-        // send all comm pairs to new owner
-        auto send_count = send_counts[owner_rank];
-        for (int i = 0; i < send_count; i++) {
-            auto send_buffer = send_idxs[i].first;
-            comm.send(exec, send_buffer.get_const_data(),
-                      send_buffer.get_size(), owner_rank, owner_rank);
-        }
-    }
-
-
-    // clear communcation neighbors on non owning rank
-    if (!owner) {
-        target_ids.clear();
-        target_sizes.clear();
-        send_idxs.clear();
-    }
-
-    // Step 3. merge communication
-    // we now might have communication partners multiple times, thus we can
-    // merge them sort repart_target_ids and use the sorting for
-    // target_sizes and repart_send_idxs
-    std::vector<label> merged_target_ids{};
-    std::vector<label> merged_target_sizes{};
-    std::vector<std::vector<label>> merged_send_idxs{};
-    if (owner) {
-        auto p = sort_permutation(target_ids,
-                                  [](label a, label b) { return a < b; });
-
-        target_sizes = apply_permutation(target_sizes, p);
-        target_ids = apply_permutation(target_ids, p);
-        send_idxs = apply_permutation(send_idxs, p);
-
-        // Step 4.
-        // Merge communication pattern with corresponding neighbours
-        if (target_ids.size() > 1) {
-            merged_target_ids.push_back(target_ids[0]);
-            merged_target_sizes.push_back(target_sizes[0]);
-            merged_send_idxs.emplace_back(std::vector<label>(
-                send_idxs[0].first.get_data(),
-                send_idxs[0].first.get_data() + send_idxs[0].first.get_size()));
-        }
-
-        for (int i = 1; i < target_ids.size(); i++) {
-            if (target_ids[i] == merged_target_ids.back()) {
-                merged_target_sizes.back() += target_sizes[i];
-                merged_send_idxs.back().insert(
-                    merged_send_idxs.back().end(),
-                    send_idxs[i].first.get_data(),
-                    send_idxs[i].first.get_data() +
-                        send_idxs[i].first.get_size());
-            } else {
-                merged_target_ids.push_back(target_ids[i]);
-            }
-        }
-    }
-
-    // recompute send_idxs
-    send_idxs.clear();
-
-    for (int i = 0; i < merged_target_ids.size(); i++) {
-        label target_id = merged_target_ids[i];
-        send_idxs.emplace_back(std::pair<gko::array<label>, comm_size_type>{
-            gko::array<label>{exec, merged_send_idxs[i].begin(),
-                              merged_send_idxs[i].end()},
-            target_id});
-    }
-
-    return CommunicationPattern{
-        exec_handler,
-        gko::array<comm_size_type>{exec, merged_target_ids.begin(),
-                                   merged_target_ids.end()},
-        gko::array<comm_size_type>{exec, merged_target_sizes.begin(),
-                                   merged_target_sizes.end()},
-        send_idxs};
-}
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 7bcdfc24a..89ae65504 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -38,13 +38,24 @@ using CommCounts = std::tuple<std::vector<label>, std::vector<label>,
                               std::vector<label>, std::vector<label>>;
 
 
-// This function computes the send and recv counts vectors for repartitioning
+/* @brief  This function computes the send and recv counts vectors for
+ *repartitioning to an owner rank
+ *
+ * @parameter size how many elements to send to owner
+ * @parameter total_size size of the block of memory on the receiver site
+ * @parameter padding_before unused elements before inserting size elements
+ * @parameter padding_after unused elements after insesting size elements
+ * */
 CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
                                     label ranks_per_gpu, label size,
                                     label total_size, label padding_before,
                                     label padding_after);
 
-// This function computes the send and recv counts vectors for repartitioning
+/* @brief  This function computes the send and recv counts vectors for
+ *repartitioning to an owner rank
+ *
+ * @parameter size how many elements to send to owner
+ * */
 CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
                                     label ranks_per_gpu, label size);
 
@@ -58,10 +69,9 @@ label compute_owner_rank(label rank, label ranks_per_gpu);
 /* Given a comm_pattern this function executes the communication
 **
 */
-void communicate_values (const ExecutorHandler &exec_handler,
-                                   const CommCounts &comm_pattern,
-                                   const scalar *send_buffer,
-                                   scalar *recv_buffer);
+void communicate_values(const ExecutorHandler &exec_handler,
+                        const CommCounts &comm_pattern,
+                        const scalar *send_buffer, scalar *recv_buffer);
 
 /* Returns a vector of length total received elements + padding */
 std::vector<label> gather_to_owner(const ExecutorHandler &exec_handler,
@@ -84,6 +94,19 @@ struct CommunicationPattern {
     // are owned by the interface and the corresponding target rank
     std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
 
+    CommunicationPattern(
+        const ExecutorHandler &exec, gko::array<comm_size_type> ids,
+        gko::array<comm_size_type> sizes,
+        std::vector<std::pair<gko::array<label>, comm_size_type>> idxs)
+        : exec_handler(exec),
+          target_ids(ids),
+          target_sizes(sizes),
+          send_idxs(idxs)
+    {
+        ASSERT_EQ(target_ids.get_size(), target_sizes.get_size());
+        //ASSERT_EQ(target_ids.get_size(), send_idxs.size());
+    };
+
     const gko::experimental::mpi::communicator &get_comm() const
     {
         return *exec_handler.get_communicator().get();
@@ -91,15 +114,3 @@ struct CommunicationPattern {
 };
 
 std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e);
-
-// TODO move to Repartitioner
-// TODO NOTE currently we have just one communication pattern for multiple proc
-// interfaces
-//  hence this will communicate vectors which are too long, FIXME potentially
-//  have the wrong idxs and will communictate the same remote buffer several
-//  times through the combination matrix.
-CommunicationPattern repartition_comm_pattern(
-    label ranks_per_gpu, CommunicationPattern &src_comm_pattern,
-    std::shared_ptr<
-        const gko::experimental::distributed::Partition<label, label>>
-        partition);
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 17a708520..53c5136e7 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -32,6 +32,7 @@ SourceFiles
 
 #include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
 #include "MatrixWrapper/LDUMatrix/HostMatrix.H"
+#include "Repartitioner/Repartitioner.H"
 
 
 template <typename DistMtxType>
@@ -186,12 +187,12 @@ public:
         auto src_comm_pattern =
             host_A.get()->create_communication_pattern(comm);
         // create partiton here and pass to constructor
-        auto dst_comm_pattern = repartition_comm_pattern(
-            ranks_per_gpu, src_comm_pattern, orig_partition);
+        auto dst_comm_pattern = repartitioner.repartition_comm_pattern(
+            src_comm_pattern, orig_partition);
 
         label rank{src_comm_pattern.get_comm().rank()};
         label owner_rank = repartitioner.get_owner_rank();
-        bool owner = repartitioner.get_owner();
+        bool owner = repartitioner.is_owner();
 
         auto [repart_loc_sparsity, repart_non_loc_sparsity] =
             repartitioner.repartition_sparsity(local_sparsity_,
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index ea237b594..14be35418 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -81,394 +81,6 @@ void make_begin_and_ends_consecutive(
     label ranks_per_gpu);
 
 
-// TODO this should be persistent
-class Repartitioner {
-private:
-    using partition = gko::experimental::distributed::Partition<label, label>;
-
-    const label size_;  //! Size (n matrix rows ) before repartitioning
-
-    const label repart_size_;  //! Size after repartitioning
-
-    const label ranks_per_gpu_;
-
-    const ExecutorHandler & exec_handler_;
-
-    std::shared_ptr<const partition> orig_partition_;
-
-
-public:
-    Repartitioner(label size, label ranks_per_gpu,
-                  const ExecutorHandler &exec_handler)
-        : size_(size),
-          repart_size_(Repartitioner::compute_repart_size(size, ranks_per_gpu,
-                                                          exec_handler)),
-          ranks_per_gpu_(ranks_per_gpu),
-          exec_handler_(exec_handler),
-          orig_partition_(gko::share(
-              gko::experimental::distributed::build_partition_from_local_size<
-                  label, label>(exec_handler_.get_device_exec(),
-                                *exec_handler_.get_communicator().get(),
-                                size))) {};
-
-    /* returns the owner rank for a given rank */
-    label get_owner_rank(label rank) const
-    {
-        return compute_owner_rank(rank, ranks_per_gpu_);
-    };
-
-    /* returns the owner rank for a given rank */
-    label get_owner_rank() const { return get_owner_rank(get_rank()); };
-
-    /* returns if current rank is an owner  */
-    bool get_owner() const { return get_rank() == get_owner_rank(get_rank()); };
-
-    /* shortcut to current rank */
-    label get_rank() const
-    {
-        return exec_handler_.get_communicator().get()->rank();
-    };
-
-    label get_ranks_per_gpu() const { return ranks_per_gpu_; }
-
-    static label compute_repart_size(label local_size, label ranks_per_gpu,
-                                     const ExecutorHandler &exec_handler)
-    {
-        if (ranks_per_gpu == 1) {
-            return local_size;
-        }
-
-        CommCounts comm_pattern =
-            compute_send_recv_counts(exec_handler, ranks_per_gpu, 1);
-
-        std::vector<label> data{local_size};
-
-        auto local_sizes =
-            gather_to_owner(exec_handler, comm_pattern, 1, data.data());
-
-        return std::reduce(local_sizes.begin(), local_sizes.end());
-    }
-
-    label get_repart_size() const { return repart_size_; }
-
-    gko::dim<2> get_repart_dim() const { return gko::dim<2>{repart_size_,repart_size_}; }
-
-    std::shared_ptr<const partition> get_orig_partition() const
-    {
-        return orig_partition_;
-    };
-
-    std::pair<std::shared_ptr<SparsityPattern>,
-              std::shared_ptr<SparsityPattern>>
-    repartition_sparsity(
-        std::shared_ptr<SparsityPattern> src_local_pattern,
-        std::shared_ptr<SparsityPattern> src_non_local_pattern) const
-    {
-        // 1. obtain send recv sizes vector
-        // here we can reuse code from repartition_comm_pattern
-        //
-        // 2. initialize send and recv buffer
-        // if we keep interfaces separated row and col indices can be
-        //
-        // 3. ldu mapping needs to be update?
-        // 4. dim needs to be updated
-        // this is not implemented yet, but we can't fail here for
-        // debugging reasons
-        auto exec = exec_handler_.get_device_exec();
-        auto comm = *exec_handler_.get_communicator().get();
-        label rank = get_rank();
-        label owner_rank = get_owner_rank();
-        bool owner = get_owner();
-        label ranks_per_gpu = ranks_per_gpu_;
-
-        auto local_comm_pattern = compute_send_recv_counts(
-            exec_handler_, ranks_per_gpu, src_local_pattern->size_);
-
-        label offset = orig_partition_->get_range_bounds()[rank] -
-                       orig_partition_->get_range_bounds()[owner_rank];
-
-        auto &exec_handler = exec_handler_;
-        auto gather_closure = [&](auto &comm_pattern, auto &data,
-                                             label offset) {
-            return gather_to_owner(exec_handler, comm_pattern, data.get_size(),
-                                   data.get_data(), offset);
-        };
-
-        std::cout << __FILE__ << " rank " << rank << "before gathered_local\n";
-        auto merged_local_rows = gather_closure(
-            local_comm_pattern, src_local_pattern->row_idxs, offset);
-
-        auto merged_local_cols = gather_closure(
-            local_comm_pattern, src_local_pattern->col_idxs, offset);
-
-        auto merged_ldu_mapping = gather_closure(
-            local_comm_pattern, src_local_pattern->ldu_mapping, 0);
-
-        if (owner) {
-            make_ldu_mapping_consecutive(local_comm_pattern, merged_ldu_mapping,
-                                         rank, ranks_per_gpu);
-        }
-
-        label rows = (owner) ? merged_local_rows.back() + 1 : 0;
-        gko::dim<2> merged_local_dim{rows, rows};
-
-        std::vector<label> merged_begin{0};
-        std::vector<label> merged_end{label(merged_local_rows.size())};
-        std::vector<label> merged_ranks{rank};
-
-        auto non_local_comm_pattern = compute_send_recv_counts(
-            exec_handler, ranks_per_gpu, src_non_local_pattern->size_);
-
-        auto merged_non_local_rows = gather_closure(
-            non_local_comm_pattern, src_non_local_pattern->row_idxs, offset);
-        std::cout << __FILE__ << " rank " << rank << " gathered_non_local\n";
-
-        // the non local cols are in local idx of other side
-        // thus we need the new offset of the other side
-        for (label i = 0; i < src_non_local_pattern->rank.size(); i++) {
-            auto comm_rank = src_non_local_pattern->rank[i];
-            auto [begin, end] = src_non_local_pattern->interface_spans[i];
-            label local_offset =
-                orig_partition_->get_range_bounds()[comm_rank] -
-                orig_partition_->get_range_bounds()[owner_rank];
-            auto *data = src_non_local_pattern->col_idxs.get_data() + begin;
-            auto size = end - begin;
-            std::transform(data, data + size, data,
-                           [&](label idx) { return idx + local_offset; });
-        }
-        std::cout << __FILE__ << " rank " << rank << " done offset non_local\n";
-
-        // we now dealed with the new offset thus we dont need an offset
-        // now
-        auto merged_non_local_cols = gather_closure(
-            non_local_comm_pattern, src_non_local_pattern->col_idxs, 0);
-
-        auto merged_non_local_ldu_mapping = gather_closure(
-            non_local_comm_pattern, src_non_local_pattern->row_idxs, 0);
-
-        auto span_comm_pattern = compute_send_recv_counts(
-            exec_handler, ranks_per_gpu,
-            src_non_local_pattern->interface_spans.size());
-
-        std::cout << __FILE__ << " rank " << rank << " done merged non_local\n";
-
-        std::vector<label> spans_begin;
-        std::vector<label> spans_end;
-
-        for (auto elem : src_non_local_pattern->interface_spans) {
-            spans_begin.push_back(elem.begin);
-            spans_end.push_back(elem.end);
-        }
-
-        auto merged_non_local_begin =
-            gather_to_owner(exec_handler, span_comm_pattern, spans_begin.size(),
-                            spans_begin.data());
-
-        auto merged_non_local_end =
-            gather_to_owner(exec_handler, span_comm_pattern, spans_end.size(),
-                            spans_end.data());
-
-        auto merged_non_local_ranks = gather_to_owner(
-            exec_handler, span_comm_pattern, src_non_local_pattern->rank.size(),
-            src_non_local_pattern->rank.data());
-
-
-        if (owner) {
-            make_begin_and_ends_consecutive(
-                span_comm_pattern, merged_non_local_begin, merged_non_local_end,
-                rank, ranks_per_gpu);
-        }
-
-
-        // TODO remove
-        //     if (rank == 0) {
-        // std::cout << __FILE__ << " on rank " << rank << "
-        // merged_begin"
-        //           << merged_begin
-        //           << " span_recv_counts " << span_recv_counts
-        //           << " span_recv_offsets " << span_recv_offsets
-        //           << " merged_non_local_begins " <<
-        //           merged_non_local_begin
-        //           << " merged_non_local_end " << merged_non_local_end
-        //           << " merged_end " << merged_end
-        //           << " merged_non_local_rows " <<
-        //           merged_non_local_rows
-        //           << " merged_ranks " << merged_non_local_ranks <<
-        //           "\n";
-        // }
-
-        // TODO shorten this
-        std::vector<label> gathered_non_local_rows;
-        std::vector<label> gathered_non_local_cols;
-        std::vector<label> gathered_non_local_mapping;
-        std::vector<label> gathered_ranks;
-        std::vector<label> gathered_begin;
-        std::vector<label> gathered_end;
-        label merged_ranks_size = merged_non_local_ranks.size();
-        // redistribute interfaces
-        for (label i = 0; i < merged_ranks_size; i++) {
-            auto merged_target_rank = merged_non_local_ranks[i];
-            label to_rank = get_owner_rank(merged_target_rank);
-            bool local = rank == to_rank;
-            // these are the begin ends before merging they need to be
-            // offseted;
-            auto begin = merged_non_local_begin[i];
-            auto end = merged_non_local_end[i];
-            if (local) {
-                // TODO depending on the interface simple non
-                // transforming and interpolating interfaces like now
-                // local processor interfaces could be merged
-                label size_before = merged_local_rows.size();
-                merged_local_rows.insert(merged_local_rows.end(),
-                                         merged_non_local_rows.begin() + begin,
-                                         merged_non_local_rows.begin() + end);
-                merged_local_cols.insert(merged_local_cols.end(),
-                                         merged_non_local_cols.begin() + begin,
-                                         merged_non_local_cols.begin() + end);
-                merged_ldu_mapping.insert(
-                    merged_ldu_mapping.end(),
-                    merged_non_local_ldu_mapping.data() + begin,
-                    merged_non_local_ldu_mapping.data() + end);
-                // TODO store from rank ie from which rank it came
-                merged_ranks.push_back(to_rank);
-
-                label size_after = merged_local_rows.size();
-                merged_begin.push_back(size_before);
-                merged_end.push_back(size_after);
-                // TODO remove
-                // if (rank == 0) {
-                //     std::cout << __FILE__ << " on rank " << rank
-                //             << " begin " << begin << " end " << end
-                //             << " insert merged_local_rows " <<
-                //             merged_local_rows
-                //             << "\n";
-                //     }
-            } else {
-                label size_before = gathered_non_local_rows.size();
-                gathered_non_local_rows.insert(
-                    gathered_non_local_rows.end(),
-                    merged_non_local_rows.begin() + begin,
-                    merged_non_local_rows.begin() + end);
-                // FIXME
-                // up until now non_local_cols are stil using
-                // local_idx and we need to convert them to
-                // consequtive compressed idxs
-                gathered_non_local_cols.insert(
-                    gathered_non_local_cols.end(),
-                    merged_non_local_cols.begin() + begin,
-                    merged_non_local_cols.begin() + end);
-                // TODO we don't do anything with non local mapping
-                gathered_non_local_mapping.insert(
-                    gathered_non_local_mapping.end(),
-                    merged_non_local_cols.begin() + begin,
-                    merged_non_local_cols.begin() + end);
-                label size_after = gathered_non_local_rows.size();
-                // if (rank == 0) {
-                // std::cout << __FILE__ << " on rank " << rank
-                //         << " gathered_non_local_rows " <<
-                //         gathered_non_local_rows
-                //         << " begin " << begin << " end " << end
-                //         << " non_local size_before " << size_before
-                //         << " non_local size_after " << size_after
-                //         << "\n";
-                // }
-                gathered_begin.push_back(size_before);
-                gathered_end.push_back(size_after);
-                gathered_ranks.push_back(to_rank);
-            }
-        }
-        std::cout << __FILE__ << " rank " << rank
-                  << " done merged non_local4\n";
-
-        // for compressed cols we compute a mapping first
-        std::map<label, label> compressed_col_idxs{};
-        label unique_idx = 0;
-        for (auto idx : gathered_non_local_cols) {
-            auto search = compressed_col_idxs.find(idx);
-            if (search == compressed_col_idxs.end()) {
-                std::cout << " insert " << idx << " " << unique_idx << "\n";
-                compressed_col_idxs[idx] = unique_idx;
-                unique_idx += 1;
-            };
-        }
-        std::cout << __FILE__ << " rank " << rank
-                  << " done merged non_local5\n";
-
-        for (int i = 0; i < gathered_non_local_cols.size(); i++) {
-            label uncompressed = gathered_non_local_cols[i];
-            label compressed = compressed_col_idxs[uncompressed];
-            std::cout << " i " << i << " uncompressed " << uncompressed
-                      << " compressed " << compressed << "\n";
-            gathered_non_local_cols[i] = compressed;
-        }
-        std::cout << __FILE__ << " rank " << rank
-                  << " done merged non_local6\n";
-
-        // add interfaces
-        // to add interfaces we go through the non_local_sparsity
-        // pattern and check if interface is still non_local
-        //
-        // iterate interfaces and send to owner.
-        // on owner decide to move to local or non_local
-        // for sending to owner we can also use the all_to_all_v
-        // approach we than have
-        //
-        // row [ 4 , 8,  12 | 16 , 17, 18 ] <- local row
-        // col [ 1 , 2, 3 | 1, 2, 3 ] <- just the interface ctr
-        // from repartition_comm_pattern we could get
-        //
-        // or we split local / non-local first
-        // for this we could store ranges when computing
-        // repartition_comm_pattern
-        //
-
-
-        if (owner) {
-            auto new_local_spars_pattern = std::make_shared<SparsityPattern>(
-                exec, merged_local_rows.size(), merged_local_dim,
-                merged_local_rows, merged_local_cols, merged_ldu_mapping,
-                merged_begin, merged_end, merged_ranks);
-
-            // if (rank == 0) {
-            // std::cout << __FILE__ << " on rank " << rank
-            //           << merged_local_rows
-            //           << " merged_begin " <<  merged_begin
-            //           << " merged_end " <<  merged_end
-            //           << "local dim ["
-            //           << new_local_spars_pattern->dim[0] << " x "
-            //           << new_local_spars_pattern->dim[1] << "] \n";
-            //         }
-
-            auto new_non_local_spars_pattern =
-                std::make_shared<SparsityPattern>(
-                    exec, gathered_non_local_rows.size(),
-                    gko::dim<2>{merged_local_dim[0],
-                                gathered_non_local_rows.size()},
-                    gathered_non_local_rows, gathered_non_local_cols,
-                    gathered_non_local_mapping, gathered_begin, gathered_end,
-                    gathered_ranks);
-
-            return std::make_pair<std::shared_ptr<SparsityPattern>,
-                                  std::shared_ptr<SparsityPattern>>(
-                std::move(new_local_spars_pattern),
-                std::move(new_non_local_spars_pattern));
-        } else {
-            auto new_local_spars_pattern =
-                std::make_shared<SparsityPattern>(exec, 0);
-
-            auto new_non_local_spars_pattern =
-                std::make_shared<SparsityPattern>(exec, 0);
-
-            new_local_spars_pattern->dim = merged_local_dim;
-            new_non_local_spars_pattern->dim = merged_local_dim;
-
-            return std::make_pair<std::shared_ptr<SparsityPattern>,
-                                  std::shared_ptr<SparsityPattern>>(
-                std::move(new_local_spars_pattern),
-                std::move(new_non_local_spars_pattern));
-        }
-    }
-};
 
 
 // Free functions
diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
new file mode 100644
index 000000000..d0e3267e0
--- /dev/null
+++ b/Repartitioner/Repartitioner.H
@@ -0,0 +1,636 @@
+/*---------------------------------------------------------------------------*\
+License
+    This file is part of OGL.
+
+    OGL is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OGL is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OGL.  If not, see <http://www.gnu.org/licenses/>.
+
+
+Author: Gregor Olenik <go@hpsim.de>
+
+SourceFiles
+    HostMatrix.H
+
+\*---------------------------------------------------------------------------*/
+#pragma once
+
+#include <ginkgo/ginkgo.hpp>
+
+#include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
+#include "MatrixWrapper/LDUMatrix/HostMatrix.H"
+
+template <typename T>
+std::vector<T> apply_permutation(const std::vector<T> vec,
+                                 const std::vector<label> &p)
+{
+    std::vector<T> sorted_vec(vec.size());
+    std::transform(p.begin(), p.end(), sorted_vec.begin(),
+                   [&](label i) { return vec[i]; });
+    return sorted_vec;
+}
+
+template <typename T, typename Compare>
+std::vector<label> sort_permutation(const std::vector<T> &vec, Compare compare)
+{
+    std::vector<label> p(vec.size());
+    std::iota(p.begin(), p.end(), 0);
+    std::stable_sort(p.begin(), p.end(), [&](std::size_t i, std::size_t j) {
+        return compare(vec[i], vec[j]);
+    });
+    return p;
+}
+
+// TODO this should be persistent
+class Repartitioner {
+private:
+    using partition = gko::experimental::distributed::Partition<label, label>;
+
+    const label size_;  //! Size (n matrix rows ) before repartitioning
+
+    const label repart_size_;  //! Size after repartitioning
+
+    const label ranks_per_gpu_;
+
+    const ExecutorHandler &exec_handler_;
+
+    std::shared_ptr<const partition> orig_partition_;
+
+
+public:
+    Repartitioner(label size, label ranks_per_gpu,
+                  const ExecutorHandler &exec_handler)
+        : size_(size),
+          repart_size_(Repartitioner::compute_repart_size(size, ranks_per_gpu,
+                                                          exec_handler)),
+          ranks_per_gpu_(ranks_per_gpu),
+          exec_handler_(exec_handler),
+          orig_partition_(gko::share(
+              gko::experimental::distributed::build_partition_from_local_size<
+                  label, label>(exec_handler_.get_device_exec(),
+                                *exec_handler_.get_communicator().get(),
+                                size))) {};
+
+    /* returns the owner rank for a given rank */
+    label get_owner_rank(label rank) const
+    {
+        return compute_owner_rank(rank, ranks_per_gpu_);
+    };
+
+    /* returns the owner rank for a given rank */
+    label get_owner_rank() const { return get_owner_rank(get_rank()); };
+
+    /* returns if current rank is an owner  */
+    bool is_owner() const { return get_rank() == get_owner_rank(get_rank()); };
+
+    /* @brief check if the given rank gets local after repartitioning
+     *
+     * */
+    bool reparts_to_local(label rank) const
+    {
+        return get_owner_rank() == compute_owner_rank(rank, ranks_per_gpu_);
+    };
+
+    /* shortcut to current rank */
+    label get_rank() const
+    {
+        return exec_handler_.get_communicator().get()->rank();
+    };
+
+    label get_ranks_per_gpu() const { return ranks_per_gpu_; }
+
+    static label compute_repart_size(label local_size, label ranks_per_gpu,
+                                     const ExecutorHandler &exec_handler)
+    {
+        if (ranks_per_gpu == 1) {
+            return local_size;
+        }
+
+        CommCounts comm_pattern =
+            compute_send_recv_counts(exec_handler, ranks_per_gpu, 1);
+
+        std::vector<label> data{local_size};
+
+        auto local_sizes =
+            gather_to_owner(exec_handler, comm_pattern, 1, data.data());
+
+        return std::reduce(local_sizes.begin(), local_sizes.end());
+    }
+
+    label get_repart_size() const { return repart_size_; }
+
+    gko::dim<2> get_repart_dim() const
+    {
+        return gko::dim<2>{repart_size_, repart_size_};
+    }
+
+    std::shared_ptr<const partition> get_orig_partition() const
+    {
+        return orig_partition_;
+    };
+
+    std::pair<std::shared_ptr<SparsityPattern>,
+              std::shared_ptr<SparsityPattern>>
+    repartition_sparsity(
+        std::shared_ptr<SparsityPattern> src_local_pattern,
+        std::shared_ptr<SparsityPattern> src_non_local_pattern) const
+    {
+        // 1. obtain send recv sizes vector
+        // here we can reuse code from repartition_comm_pattern
+        //
+        // 2. initialize send and recv buffer
+        // if we keep interfaces separated row and col indices can be
+        //
+        // 3. ldu mapping needs to be update?
+        // 4. dim needs to be updated
+        // this is not implemented yet, but we can't fail here for
+        // debugging reasons
+        auto exec = exec_handler_.get_device_exec();
+        auto comm = *exec_handler_.get_communicator().get();
+        label rank = get_rank();
+        label owner_rank = get_owner_rank();
+        label ranks_per_gpu = ranks_per_gpu_;
+
+        auto local_comm_pattern = compute_send_recv_counts(
+            exec_handler_, ranks_per_gpu, src_local_pattern->size_);
+
+        label offset = orig_partition_->get_range_bounds()[rank] -
+                       orig_partition_->get_range_bounds()[owner_rank];
+
+        auto &exec_handler = exec_handler_;
+        auto gather_closure = [&](auto &comm_pattern, auto &data,
+                                  label offset) {
+            return gather_to_owner(exec_handler, comm_pattern, data.get_size(),
+                                   data.get_data(), offset);
+        };
+
+        std::cout << __FILE__ << " rank " << rank << "before gathered_local\n";
+        auto merged_local_rows = gather_closure(
+            local_comm_pattern, src_local_pattern->row_idxs, offset);
+
+        auto merged_local_cols = gather_closure(
+            local_comm_pattern, src_local_pattern->col_idxs, offset);
+
+        auto merged_ldu_mapping = gather_closure(
+            local_comm_pattern, src_local_pattern->ldu_mapping, 0);
+
+        if (is_owner()) {
+            make_ldu_mapping_consecutive(local_comm_pattern, merged_ldu_mapping,
+                                         rank, ranks_per_gpu);
+        }
+
+        label rows = (is_owner()) ? merged_local_rows.back() + 1 : 0;
+        gko::dim<2> merged_local_dim{rows, rows};
+
+        std::vector<label> merged_begin{0};
+        std::vector<label> merged_end{label(merged_local_rows.size())};
+        std::vector<label> merged_ranks{rank};
+
+        auto non_local_comm_pattern = compute_send_recv_counts(
+            exec_handler, ranks_per_gpu, src_non_local_pattern->size_);
+
+        auto merged_non_local_rows = gather_closure(
+            non_local_comm_pattern, src_non_local_pattern->row_idxs, offset);
+        std::cout << __FILE__ << " rank " << rank << " gathered_non_local\n";
+
+        // the non local cols are in local idx of other side
+        // thus we need the new offset of the other side
+        for (label i = 0; i < src_non_local_pattern->rank.size(); i++) {
+            auto comm_rank = src_non_local_pattern->rank[i];
+            auto [begin, end] = src_non_local_pattern->interface_spans[i];
+            label local_offset =
+                orig_partition_->get_range_bounds()[comm_rank] -
+                orig_partition_->get_range_bounds()[owner_rank];
+            auto *data = src_non_local_pattern->col_idxs.get_data() + begin;
+            auto size = end - begin;
+            std::transform(data, data + size, data,
+                           [&](label idx) { return idx + local_offset; });
+        }
+        std::cout << __FILE__ << " rank " << rank << " done offset non_local\n";
+
+        // we now dealed with the new offset thus we dont need an offset
+        // now
+        auto merged_non_local_cols = gather_closure(
+            non_local_comm_pattern, src_non_local_pattern->col_idxs, 0);
+
+        auto merged_non_local_ldu_mapping = gather_closure(
+            non_local_comm_pattern, src_non_local_pattern->row_idxs, 0);
+
+        auto span_comm_pattern = compute_send_recv_counts(
+            exec_handler, ranks_per_gpu,
+            src_non_local_pattern->interface_spans.size());
+
+        std::cout << __FILE__ << " rank " << rank << " done merged non_local\n";
+
+        std::vector<label> spans_begin;
+        std::vector<label> spans_end;
+
+        for (auto elem : src_non_local_pattern->interface_spans) {
+            spans_begin.push_back(elem.begin);
+            spans_end.push_back(elem.end);
+        }
+
+        auto merged_non_local_begin =
+            gather_to_owner(exec_handler, span_comm_pattern, spans_begin.size(),
+                            spans_begin.data());
+
+        auto merged_non_local_end =
+            gather_to_owner(exec_handler, span_comm_pattern, spans_end.size(),
+                            spans_end.data());
+
+        auto merged_non_local_ranks = gather_to_owner(
+            exec_handler, span_comm_pattern, src_non_local_pattern->rank.size(),
+            src_non_local_pattern->rank.data());
+
+
+        if (is_owner()) {
+            make_begin_and_ends_consecutive(
+                span_comm_pattern, merged_non_local_begin, merged_non_local_end,
+                rank, ranks_per_gpu);
+        }
+
+
+        // TODO remove
+        //     if (rank == 0) {
+        // std::cout << __FILE__ << " on rank " << rank << "
+        // merged_begin"
+        //           << merged_begin
+        //           << " span_recv_counts " << span_recv_counts
+        //           << " span_recv_offsets " << span_recv_offsets
+        //           << " merged_non_local_begins " <<
+        //           merged_non_local_begin
+        //           << " merged_non_local_end " << merged_non_local_end
+        //           << " merged_end " << merged_end
+        //           << " merged_non_local_rows " <<
+        //           merged_non_local_rows
+        //           << " merged_ranks " << merged_non_local_ranks <<
+        //           "\n";
+        // }
+
+        // TODO shorten this
+        std::vector<label> gathered_non_local_rows;
+        std::vector<label> gathered_non_local_cols;
+        std::vector<label> gathered_non_local_mapping;
+        std::vector<label> gathered_ranks;
+        std::vector<label> gathered_begin;
+        std::vector<label> gathered_end;
+        label merged_ranks_size = merged_non_local_ranks.size();
+        // redistribute interfaces
+        for (label i = 0; i < merged_ranks_size; i++) {
+            auto merged_target_rank = merged_non_local_ranks[i];
+            label to_rank = get_owner_rank(merged_target_rank);
+            bool local = rank == to_rank;
+            // these are the begin ends before merging they need to be
+            // offseted;
+            auto begin = merged_non_local_begin[i];
+            auto end = merged_non_local_end[i];
+            if (local) {
+                // TODO depending on the interface simple non
+                // transforming and interpolating interfaces like now
+                // local processor interfaces could be merged
+                label size_before = merged_local_rows.size();
+                merged_local_rows.insert(merged_local_rows.end(),
+                                         merged_non_local_rows.begin() + begin,
+                                         merged_non_local_rows.begin() + end);
+                merged_local_cols.insert(merged_local_cols.end(),
+                                         merged_non_local_cols.begin() + begin,
+                                         merged_non_local_cols.begin() + end);
+                merged_ldu_mapping.insert(
+                    merged_ldu_mapping.end(),
+                    merged_non_local_ldu_mapping.data() + begin,
+                    merged_non_local_ldu_mapping.data() + end);
+                // TODO store from rank ie from which rank it came
+                merged_ranks.push_back(to_rank);
+
+                label size_after = merged_local_rows.size();
+                merged_begin.push_back(size_before);
+                merged_end.push_back(size_after);
+                // TODO remove
+                // if (rank == 0) {
+                //     std::cout << __FILE__ << " on rank " << rank
+                //             << " begin " << begin << " end " << end
+                //             << " insert merged_local_rows " <<
+                //             merged_local_rows
+                //             << "\n";
+                //     }
+            } else {
+                label size_before = gathered_non_local_rows.size();
+                gathered_non_local_rows.insert(
+                    gathered_non_local_rows.end(),
+                    merged_non_local_rows.begin() + begin,
+                    merged_non_local_rows.begin() + end);
+                // FIXME
+                // up until now non_local_cols are stil using
+                // local_idx and we need to convert them to
+                // consequtive compressed idxs
+                gathered_non_local_cols.insert(
+                    gathered_non_local_cols.end(),
+                    merged_non_local_cols.begin() + begin,
+                    merged_non_local_cols.begin() + end);
+                // TODO we don't do anything with non local mapping
+                gathered_non_local_mapping.insert(
+                    gathered_non_local_mapping.end(),
+                    merged_non_local_cols.begin() + begin,
+                    merged_non_local_cols.begin() + end);
+                label size_after = gathered_non_local_rows.size();
+                // if (rank == 0) {
+                // std::cout << __FILE__ << " on rank " << rank
+                //         << " gathered_non_local_rows " <<
+                //         gathered_non_local_rows
+                //         << " begin " << begin << " end " << end
+                //         << " non_local size_before " << size_before
+                //         << " non_local size_after " << size_after
+                //         << "\n";
+                // }
+                gathered_begin.push_back(size_before);
+                gathered_end.push_back(size_after);
+                gathered_ranks.push_back(to_rank);
+            }
+        }
+        std::cout << __FILE__ << " rank " << rank
+                  << " done merged non_local4\n";
+
+        // for compressed cols we compute a mapping first
+        std::map<label, label> compressed_col_idxs{};
+        label unique_idx = 0;
+        for (auto idx : gathered_non_local_cols) {
+            auto search = compressed_col_idxs.find(idx);
+            if (search == compressed_col_idxs.end()) {
+                std::cout << " insert " << idx << " " << unique_idx << "\n";
+                compressed_col_idxs[idx] = unique_idx;
+                unique_idx += 1;
+            };
+        }
+        std::cout << __FILE__ << " rank " << rank
+                  << " done merged non_local5\n";
+
+        for (int i = 0; i < gathered_non_local_cols.size(); i++) {
+            label uncompressed = gathered_non_local_cols[i];
+            label compressed = compressed_col_idxs[uncompressed];
+            std::cout << " i " << i << " uncompressed " << uncompressed
+                      << " compressed " << compressed << "\n";
+            gathered_non_local_cols[i] = compressed;
+        }
+        std::cout << __FILE__ << " rank " << rank
+                  << " done merged non_local6\n";
+
+        // add interfaces
+        // to add interfaces we go through the non_local_sparsity
+        // pattern and check if interface is still non_local
+        //
+        // iterate interfaces and send to owner.
+        // on owner decide to move to local or non_local
+        // for sending to owner we can also use the all_to_all_v
+        // approach we than have
+        //
+        // row [ 4 , 8,  12 | 16 , 17, 18 ] <- local row
+        // col [ 1 , 2, 3 | 1, 2, 3 ] <- just the interface ctr
+        // from repartition_comm_pattern we could get
+        //
+        // or we split local / non-local first
+        // for this we could store ranges when computing
+        // repartition_comm_pattern
+        //
+
+
+        if (is_owner()) {
+            auto new_local_spars_pattern = std::make_shared<SparsityPattern>(
+                exec, merged_local_rows.size(), merged_local_dim,
+                merged_local_rows, merged_local_cols, merged_ldu_mapping,
+                merged_begin, merged_end, merged_ranks);
+
+            // if (rank == 0) {
+            // std::cout << __FILE__ << " on rank " << rank
+            //           << merged_local_rows
+            //           << " merged_begin " <<  merged_begin
+            //           << " merged_end " <<  merged_end
+            //           << "local dim ["
+            //           << new_local_spars_pattern->dim[0] << " x "
+            //           << new_local_spars_pattern->dim[1] << "] \n";
+            //         }
+
+            auto new_non_local_spars_pattern =
+                std::make_shared<SparsityPattern>(
+                    exec, gathered_non_local_rows.size(),
+                    gko::dim<2>{merged_local_dim[0],
+                                gathered_non_local_rows.size()},
+                    gathered_non_local_rows, gathered_non_local_cols,
+                    gathered_non_local_mapping, gathered_begin, gathered_end,
+                    gathered_ranks);
+
+            return std::make_pair<std::shared_ptr<SparsityPattern>,
+                                  std::shared_ptr<SparsityPattern>>(
+                std::move(new_local_spars_pattern),
+                std::move(new_non_local_spars_pattern));
+        } else {
+            auto new_local_spars_pattern =
+                std::make_shared<SparsityPattern>(exec, 0);
+
+            auto new_non_local_spars_pattern =
+                std::make_shared<SparsityPattern>(exec, 0);
+
+            new_local_spars_pattern->dim = merged_local_dim;
+            new_non_local_spars_pattern->dim = merged_local_dim;
+
+            return std::make_pair<std::shared_ptr<SparsityPattern>,
+                                  std::shared_ptr<SparsityPattern>>(
+                std::move(new_local_spars_pattern),
+                std::move(new_non_local_spars_pattern));
+        }
+    }
+
+    CommunicationPattern repartition_comm_pattern(
+        CommunicationPattern &src_comm_pattern,
+        std::shared_ptr<
+            const gko::experimental::distributed::Partition<label, label>>
+            partition) const
+    {
+        using comm_size_type = label;
+
+        if (ranks_per_gpu_ == 1) {
+            return CommunicationPattern(src_comm_pattern);
+        }
+
+        auto &exec_handler = src_comm_pattern.exec_handler;
+        auto exec = src_comm_pattern.target_ids.get_executor();
+        auto comm = src_comm_pattern.get_comm();
+
+        // Step 1. Check if communication partner is non-local after
+        // repartitioning. If it is non-local we keep it. Otherwise local
+        // communication can be discarded. Here non-local means:
+        // the communication target proc id is != repartition proc id
+        std::vector<label> target_ids{};
+        std::vector<label> target_sizes{};
+        std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
+        label communication_partner = src_comm_pattern.target_ids.get_size();
+        for (int i = 0; i < communication_partner; i++) {
+            label target_id = src_comm_pattern.target_ids.get_data()[i];
+            if (!reparts_to_local(target_id)) {
+                // communication pattern is non local, hence we keep it
+                // after repartitioning we now have to communicate with a
+                // different rank store new owner rank to which repart_target_*
+                // needs to be send to
+                target_ids.push_back(get_owner_rank(target_id));
+                target_sizes.push_back(
+                    src_comm_pattern.target_sizes.get_const_data()[i]);
+                send_idxs.push_back(src_comm_pattern.send_idxs[i]);
+            }
+        }
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << get_rank()
+                  << " target_ids.size() " << target_ids.size() << "\n";
+
+        // send all remaining non local ids and sizes to the new
+        // owner rank
+        auto comm_pattern = compute_send_recv_counts(
+            exec_handler, ranks_per_gpu_, target_ids.size());
+
+        auto gathered_target_ids = gather_to_owner(
+            exec_handler, comm_pattern, target_ids.size(), target_ids.data());
+
+        auto gathered_target_sizes =
+            gather_to_owner(exec_handler, comm_pattern, target_sizes.size(),
+                            target_sizes.data());
+
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << get_rank()
+                  << " gathered_target_ids " << gathered_target_ids << "\n";
+
+        // next the send_ixs need to be updated we send them piecewise since
+        // the send_idxs are a vector of gko::arrays
+        label rank = get_rank();
+        if (is_owner()) {
+            auto recv_counts = std::get<1>(comm_pattern);
+            label owner_recv_counts = recv_counts[rank];
+            // retrieved from i-th neighbor
+            for (int i = 1; i < ranks_per_gpu_; i++) {
+                // how many gko::arrays to with send_indexes to receive
+                // from i-th neighbor
+                label recv_count = recv_counts[rank + i];
+                for (int j = 0; j < recv_count; j++) {
+                    auto target_size =
+                        gathered_target_sizes[j + owner_recv_counts];
+                    std::vector<label> recv_buffer(target_size);
+
+                    comm.recv(exec, recv_buffer.data(), target_size, rank + i,
+                              rank);
+
+                    // the new offset is
+                    auto offset = partition->get_range_bounds()[rank + i] -
+                                  partition->get_range_bounds()[rank];
+
+                    std::transform(recv_buffer.begin(), recv_buffer.end(),
+                                   recv_buffer.begin(),
+                                   [&](label idx) { return idx + offset; });
+
+                    auto target_id = gathered_target_ids[j + owner_recv_counts];
+                    send_idxs.push_back({gko::array<label>{
+                                             exec,
+                                             recv_buffer.begin(),
+                                             recv_buffer.end(),
+                                         },
+                                         target_id});
+                }
+            }
+        } else {
+            label owner = get_owner_rank();
+            auto send_count = std::get<0>(comm_pattern)[owner];
+            for (int i = 0; i < send_count; i++) {
+                auto send_buffer = send_idxs[i].first;
+                comm.send(exec, send_buffer.get_const_data(),
+                          send_buffer.get_size(), owner, owner);
+            }
+        }
+
+        // clear communcation neighbors on non owning rank
+        if (!is_owner()) {
+            // TODO NOTE should it be gathered_target_ids and is it needed?
+            target_ids.clear();
+            target_sizes.clear();
+            send_idxs.clear();
+
+            target_ids.emplace_back(get_rank());
+            target_sizes.emplace_back(0);
+
+            return CommunicationPattern{
+                exec_handler,
+                gko::array<comm_size_type>{exec, target_ids.begin(),
+                                           target_ids.end()},
+                gko::array<comm_size_type>{exec, target_sizes.begin(),
+                                           target_sizes.end()},
+
+                send_idxs};
+        }
+
+        // merge communication
+        // we now might have communication partners multiple times, thus we can
+        // merge them sort repart_target_ids and use the sorting for
+        // target_sizes and repart_send_idxs
+        std::vector<label> merged_target_ids{};
+        std::vector<label> merged_target_sizes{};
+        std::vector<std::vector<label>> merged_send_idxs{};
+        auto p = sort_permutation(gathered_target_ids,
+                                  [](label a, label b) { return a < b; });
+
+        target_sizes = apply_permutation(gathered_target_sizes, p);
+        target_ids = apply_permutation(gathered_target_ids, p);
+        send_idxs = apply_permutation(send_idxs, p);
+
+        // Step 4.
+        // Merge communication pattern with corresponding neighbours
+        if (target_ids.size() > 1) {
+            merged_target_ids.push_back(target_ids[0]);
+            merged_target_sizes.push_back(target_sizes[0]);
+            merged_send_idxs.emplace_back(std::vector<label>(
+                send_idxs[0].first.get_data(),
+                send_idxs[0].first.get_data() + send_idxs[0].first.get_size()));
+        }
+
+        for (int i = 1; i < target_ids.size(); i++) {
+            // communicates with same target rank
+            // thus we have only have to adapt the number
+            // of elemts and the send_ixs
+            auto *send_idx_begin = send_idxs[i].first.get_data();
+            auto *send_idx_end =
+                send_idxs[i].first.get_data() + send_idxs[i].first.get_size();
+            if (target_ids[i] == merged_target_ids.back()) {
+                merged_target_sizes.back() += target_sizes[i];
+                merged_send_idxs.back().insert(merged_send_idxs.back().end(),
+                                               send_idx_begin, send_idx_end);
+            } else {
+                merged_target_ids.push_back(target_ids[i]);
+                merged_target_sizes.push_back(target_sizes[i]);
+                merged_send_idxs.emplace_back(
+                    std::vector<label>(send_idx_begin, send_idx_end));
+            }
+        }
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << get_rank()
+                  << " gathered_target_ids " << merged_target_ids << "\n";
+
+        // recompute send_idxs
+        send_idxs.clear();
+
+        for (int i = 0; i < merged_target_ids.size(); i++) {
+            label target_id = merged_target_ids[i];
+            send_idxs.emplace_back(std::pair<gko::array<label>, comm_size_type>{
+                gko::array<label>{exec, merged_send_idxs[i].begin(),
+                                  merged_send_idxs[i].end()},
+                target_id});
+        }
+
+        return CommunicationPattern{
+            exec_handler,
+            gko::array<comm_size_type>{exec, merged_target_ids.begin(),
+                                       merged_target_ids.end()},
+            gko::array<comm_size_type>{exec, merged_target_sizes.begin(),
+                                       merged_target_sizes.end()},
+            send_idxs};
+    }
+};
diff --git a/common/common.H b/common/common.H
index e46c78ada..5daea8abb 100644
--- a/common/common.H
+++ b/common/common.H
@@ -107,17 +107,6 @@ std::ostream &operator<<(
     const std::shared_ptr<gko::matrix::Dense<scalar>> array_in);
 
 
-// template <typename T, typename Compare>
-// std::vector<std::size_t> sort_permutation(
-//     const std::vector<T>& vec,
-//     Compare compare);
-
-// template <typename T>
-// std::vector<T> apply_permutation(
-//     const std::vector<T>& vec,
-//     const std::vector<std::size_t>& p);
-
-
 // TODO move to somewhere else
 void export_system(const word fieldName, const gko::matrix::Csr<scalar> *A,
                    const gko::matrix::Dense<scalar> *x,

From ee636f54153372dca518e673a64595f11e3916c8 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 29 May 2024 08:37:20 +0200
Subject: [PATCH 56/90] Replace std::reduce by std::accumulate

---
 Repartitioner/Repartitioner.H | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index d0e3267e0..c244468db 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -123,7 +123,7 @@ public:
         auto local_sizes =
             gather_to_owner(exec_handler, comm_pattern, 1, data.data());
 
-        return std::reduce(local_sizes.begin(), local_sizes.end());
+        return std::accumulate(local_sizes.begin(), local_sizes.end(),0);
     }
 
     label get_repart_size() const { return repart_size_; }

From dbd0a045ca342873bcfbc0859d47675768e0f503 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 29 May 2024 09:31:51 +0200
Subject: [PATCH 57/90] Add more const qualifiers

---
 .../DistributedMatrixAdapter.H                         |  6 +++---
 MatrixWrapper/Distributed/Distributed.H                |  2 +-
 Preconditioner/Preconditioner.H                        | 10 +++++-----
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 7db5d767e..65d5c2820 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -68,7 +68,7 @@ struct DistributedMatrixInitFunctor {
     {}
 
     void update(
-        std::shared_ptr<RepartDistMatrix<scalar, label, label>> persistent_device_matrix) const
+        std::shared_ptr< RepartDistMatrix<scalar, label, label>> persistent_device_matrix) const
     {
         // persistent_device_matrix->update();
     }
@@ -94,7 +94,7 @@ private:
 
     const word field_name_;
 
-    mutable PersistentBase<RepartDistMatrix<scalar, label, label>, DistributedMatrixInitFunctor> host_matrix_;
+    mutable PersistentBase< RepartDistMatrix<scalar, label, label>, DistributedMatrixInitFunctor> host_matrix_;
 
 
 public:
@@ -120,7 +120,7 @@ public:
               verbose_}
     {}
 
-    std::shared_ptr<RepartDistMatrix<scalar, label, label>> get() const
+    std::shared_ptr<const RepartDistMatrix<scalar, label, label>> get() const
     {
         return host_matrix_.get_persistent_object();
     }
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 53c5136e7..125b785bb 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -163,7 +163,7 @@ public:
     }
 
 
-    std::shared_ptr<part_type> get_partition() { return this->partition_; }
+    std::shared_ptr<const part_type> get_partition() const { return this->partition_; }
 
     static std::shared_ptr<RepartDistMatrix> create(
         word matrix_format, const Repartitioner &repartitioner,
diff --git a/Preconditioner/Preconditioner.H b/Preconditioner/Preconditioner.H
index 350382d4f..ca75213a1 100644
--- a/Preconditioner/Preconditioner.H
+++ b/Preconditioner/Preconditioner.H
@@ -46,7 +46,7 @@ public:
     template <typename PrecondFactory>
     std::shared_ptr<gko::LinOp> wrap_schwarz(
         std::shared_ptr<const gko::LinOp> gkomatrix,
-        std::shared_ptr<gko::Executor> device_exec,
+        std::shared_ptr<const gko::Executor> device_exec,
         std::unique_ptr<PrecondFactory> precond) const
     {
         if (Pstream::parRun()) {
@@ -65,7 +65,7 @@ public:
     template <typename PrecondFactory, typename Factorization>
     std::shared_ptr<gko::LinOp> wrap_schwarz(
         std::shared_ptr<const gko::LinOp> gkomatrix,
-        std::shared_ptr<gko::Executor> device_exec,
+        std::shared_ptr<const gko::Executor> device_exec,
         std::unique_ptr<PrecondFactory> precond,
         std::shared_ptr<Factorization> factorization) const
     {
@@ -82,7 +82,7 @@ public:
     std::shared_ptr<gko::LinOp> init_preconditioner_impl(
         const word name, const dictionary &controls,
         std::shared_ptr<const gko::LinOp> gkomatrix,
-        std::shared_ptr<gko::Executor> device_exec) const
+        std::shared_ptr<const gko::Executor> device_exec) const
     {
         bool skip_sorting =
             controls.lookupOrDefault<Switch>("skipSorting", true);
@@ -351,8 +351,8 @@ public:
     }
 
     std::shared_ptr<gko::LinOp> init_preconditioner(
-        std::shared_ptr<gko::LinOp> gkomatrix,
-        std::shared_ptr<gko::Executor> device_exec) const
+        std::shared_ptr<const gko::LinOp> gkomatrix,
+        std::shared_ptr<const gko::Executor> device_exec) const
     {
         const word precond_store_name = "Cached_preconditinoner";
         const fileName path = precond_store_name;

From d0373e8e32020c846f9a708952eab81b79078a65 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 29 May 2024 09:45:25 +0200
Subject: [PATCH 58/90] Pass executor around shared_ptr<const Exec>

---
 BaseWrapper/lduBase/GKOlduBase.H                   | 2 +-
 DevicePersistent/DeviceIdGuard/DeviceIdGuard.H     | 6 +++---
 DevicePersistent/ExecutorHandler/ExecutorHandler.H | 4 ++--
 MatrixWrapper/Distributed/Distributed.H            | 2 +-
 Solver/BiCGStab/GKOBiCGStab.H                      | 4 ++--
 Solver/CG/GKOCG.H                                  | 4 ++--
 Solver/GMRES/GKOGMRES.H                            | 4 ++--
 Solver/IR/GKOIR.H                                  | 4 ++--
 Solver/Multigrid/GKOMultigrid.H                    | 4 ++--
 StoppingCriterion/StoppingCriterion.H              | 2 +-
 10 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/BaseWrapper/lduBase/GKOlduBase.H b/BaseWrapper/lduBase/GKOlduBase.H
index f6a0f178f..9fb5070eb 100644
--- a/BaseWrapper/lduBase/GKOlduBase.H
+++ b/BaseWrapper/lduBase/GKOlduBase.H
@@ -22,7 +22,7 @@ namespace Foam {
 
 #define CREATE_SOLVER_METHODS(Solver)                                          \
     std::unique_ptr<Solver::Factory, std::default_delete<Solver::Factory>>     \
-    create_dist_solver(std::shared_ptr<gko::Executor> exec,                    \
+    create_dist_solver(std::shared_ptr<const gko::Executor> exec,                    \
                        std::shared_ptr<const gko::LinOp> gkomatrix,            \
                        std::shared_ptr<const dist_vec> x,                      \
                        std::shared_ptr<const dist_vec> b, const label verbose, \
diff --git a/DevicePersistent/DeviceIdGuard/DeviceIdGuard.H b/DevicePersistent/DeviceIdGuard/DeviceIdGuard.H
index 4089f4b1b..12a3e3be5 100644
--- a/DevicePersistent/DeviceIdGuard/DeviceIdGuard.H
+++ b/DevicePersistent/DeviceIdGuard/DeviceIdGuard.H
@@ -13,10 +13,10 @@
 namespace Foam {
 
 struct DeviceIdGuardInitFunctor {
-    const std::shared_ptr<gko::Executor> executor_;
+    std::shared_ptr<const gko::Executor> executor_;
 
 
-    DeviceIdGuardInitFunctor(const std::shared_ptr<gko::Executor> executor)
+    DeviceIdGuardInitFunctor(std::shared_ptr<const gko::Executor> executor)
         : executor_(executor)
     {}
 
@@ -34,7 +34,7 @@ class DeviceIdGuardHandler : public PersistentBase<gko::scoped_device_id_guard,
                                                    DeviceIdGuardInitFunctor> {
 public:
     DeviceIdGuardHandler(const objectRegistry &db, const word field_name,
-                         const std::shared_ptr<gko::Executor> executor)
+                         const std::shared_ptr<const gko::Executor> executor)
         : PersistentBase<gko::scoped_device_id_guard, DeviceIdGuardInitFunctor>(
               "scoped_device_id_guard_" + field_name, db,
               DeviceIdGuardInitFunctor(executor), true, 0)
diff --git a/DevicePersistent/ExecutorHandler/ExecutorHandler.H b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
index 59138eabe..e8749d3bb 100644
--- a/DevicePersistent/ExecutorHandler/ExecutorHandler.H
+++ b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
@@ -150,12 +150,12 @@ public:
     }
 
 
-    const std::shared_ptr<gko::Executor> get_device_exec() const
+    std::shared_ptr<const gko::Executor> get_device_exec() const
     {
         return this->get_persistent_object();
     }
 
-    const std::shared_ptr<gko::Executor> get_ref_exec() const
+    std::shared_ptr<const gko::Executor> get_ref_exec() const
     {
         return get_device_exec()->get_master();
     }
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 125b785bb..ab462e52b 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -37,7 +37,7 @@ SourceFiles
 
 template <typename DistMtxType>
 std::unique_ptr<DistMtxType> generate_dist_mtx_with_inner_type(
-    word matrix_format, std::shared_ptr<gko::Executor> exec,
+    word matrix_format, std::shared_ptr<const gko::Executor> exec,
     std::shared_ptr<const gko::experimental::distributed::sparse_communicator>
         comm,
     std::shared_ptr<const SparsityPattern> local_sparsity,
diff --git a/Solver/BiCGStab/GKOBiCGStab.H b/Solver/BiCGStab/GKOBiCGStab.H
index 8ecfa3649..da0714fe5 100644
--- a/Solver/BiCGStab/GKOBiCGStab.H
+++ b/Solver/BiCGStab/GKOBiCGStab.H
@@ -46,7 +46,7 @@ public:
     CREATE_SOLVER_METHODS(bicgstab)
 
     std::unique_ptr<bicgstab::Factory, std::default_delete<bicgstab::Factory>>
-    create_default(std::shared_ptr<gko::Executor> exec) const
+    create_default(std::shared_ptr<const gko::Executor> exec) const
     {
         auto bicgstab = gko::solver::Bicgstab<scalar>::build()
                             .with_criteria(stoppingCriterionVec_)
@@ -55,7 +55,7 @@ public:
     };
 
     std::unique_ptr<bicgstab::Factory, std::default_delete<bicgstab::Factory>>
-    create_precond(std::shared_ptr<gko::Executor> exec,
+    create_precond(std::shared_ptr<const gko::Executor> exec,
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto bicgstab = bicgstab::build()
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 646480a4c..8a1b792af 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -38,14 +38,14 @@ public:
     CREATE_SOLVER_METHODS(cg)
 
     std::unique_ptr<cg::Factory, std::default_delete<cg::Factory>>
-    create_default(std::shared_ptr<gko::Executor> exec) const
+    create_default(std::shared_ptr<const gko::Executor> exec) const
     {
         auto cg = cg::build().with_criteria(stoppingCriterionVec_).on(exec);
         return cg;
     };
 
     std::unique_ptr<cg::Factory, std::default_delete<cg::Factory>>
-    create_precond(std::shared_ptr<gko::Executor> exec,
+    create_precond(std::shared_ptr<const gko::Executor> exec,
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto cg = cg::build()
diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 29d418131..794a79b43 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -43,7 +43,7 @@ public:
     CREATE_SOLVER_METHODS(gmres)
 
     std::unique_ptr<gmres::Factory, std::default_delete<gmres::Factory>>
-    create_default(std::shared_ptr<gko::Executor> exec) const
+    create_default(std::shared_ptr<const gko::Executor> exec) const
     {
         auto gmres =
             gmres::build().with_criteria(stoppingCriterionVec_).on(exec);
@@ -51,7 +51,7 @@ public:
     };
 
     std::unique_ptr<gmres::Factory, std::default_delete<gmres::Factory>>
-    create_precond(std::shared_ptr<gko::Executor> exec,
+    create_precond(std::shared_ptr<const gko::Executor> exec,
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto gmres = gmres::build()
diff --git a/Solver/IR/GKOIR.H b/Solver/IR/GKOIR.H
index 7720aef32..61cc79869 100644
--- a/Solver/IR/GKOIR.H
+++ b/Solver/IR/GKOIR.H
@@ -54,7 +54,7 @@ public:
 
     std::unique_ptr<ir::Factory,
                     std::default_delete<gko::solver::Ir<scalar>::Factory>>
-    create_default(std::shared_ptr<gko::Executor> exec,
+    create_default(std::shared_ptr<const gko::Executor> exec,
                    std::shared_ptr<mtx> gkomatrix, std::shared_ptr<vec> x,
                    std::shared_ptr<vec> b, const label verbose) const
     {
@@ -77,7 +77,7 @@ public:
 
     std::unique_ptr<gko::solver::Ir<scalar>::Factory,
                     std::default_delete<gko::solver::Ir<scalar>::Factory>>
-    create_scalar_jacobi(std::shared_ptr<gko::Executor> exec,
+    create_scalar_jacobi(std::shared_ptr<const gko::Executor> exec,
                          std::shared_ptr<mtx> gkomatrix, std::shared_ptr<vec> x,
                          std::shared_ptr<vec> b, const label verbose) const
     {
diff --git a/Solver/Multigrid/GKOMultigrid.H b/Solver/Multigrid/GKOMultigrid.H
index 03e384074..c8100a87d 100644
--- a/Solver/Multigrid/GKOMultigrid.H
+++ b/Solver/Multigrid/GKOMultigrid.H
@@ -86,7 +86,7 @@ public:
                                                                 label(10))){};
 
     std::unique_ptr<mg::Factory, std::default_delete<mg::Factory>>
-    create_dist_solver(std::shared_ptr<gko::Executor> exec,
+    create_dist_solver(std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<gko::LinOp> gkomatrix,
                        std::shared_ptr<dist_vec> x, std::shared_ptr<dist_vec> b,
                        const label verbose, const bool export_res,
@@ -143,7 +143,7 @@ public:
 
     // Create solver
     std::unique_ptr<mg::Factory, std::default_delete<mg::Factory>>
-    create_solver(std::shared_ptr<gko::Executor> exec,
+    create_solver(std::shared_ptr<const gko::Executor> exec,
                   std::shared_ptr<mtx> gkomatrix, std::shared_ptr<vec> x,
                   std::shared_ptr<vec> b, const label verbose,
                   const bool export_res,
diff --git a/StoppingCriterion/StoppingCriterion.H b/StoppingCriterion/StoppingCriterion.H
index aa7af9738..f3db2b536 100644
--- a/StoppingCriterion/StoppingCriterion.H
+++ b/StoppingCriterion/StoppingCriterion.H
@@ -190,7 +190,7 @@ public:
     }
 
     std::shared_ptr<const gko::stop::CriterionFactory>
-    build_dist_stopping_criterion(std::shared_ptr<gko::Executor> device_exec,
+    build_dist_stopping_criterion(std::shared_ptr<const gko::Executor> device_exec,
                                   std::shared_ptr<const gko::LinOp> gkomatrix,
                                   std::shared_ptr<const dist_vec> x,
                                   std::shared_ptr<const dist_vec> b,

From 228d856a7c9ba5aad29ebeb0bafbb0a68f1f8af6 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 30 May 2024 08:07:24 +0200
Subject: [PATCH 59/90] fix length issue for reordering, prepare update
 refactor

---
 .../DistributedMatrixAdapter.H                |   5 +-
 MatrixWrapper/Combination/Combination.H       |  37 +++---
 MatrixWrapper/Distributed/Distributed.H       | 125 +++++++++---------
 3 files changed, 87 insertions(+), 80 deletions(-)

diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 65d5c2820..eb9ceee43 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -70,7 +70,10 @@ struct DistributedMatrixInitFunctor {
     void update(
         std::shared_ptr< RepartDistMatrix<scalar, label, label>> persistent_device_matrix) const
     {
-        // persistent_device_matrix->update();
+         persistent_device_matrix->update(
+              solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
+              repartitioner_,
+           host_A_.get());
     }
 
     std::shared_ptr<RepartDistMatrix<scalar, label, label>> init() const
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index 2b4fb14d9..2d99f9772 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -44,34 +44,37 @@ public:
             auto length = interface.length();
             auto offset = interface.begin;
             auto op = gko::share(InnerMatrixType::create(exec));
-            auto rows = gko::array<IndexType>(
-                    exec,
-                    make_const_array_view(data_exec, length,
-                                          data.get_const_row_idxs() + offset)
-                        .copy_to_array());
-            std::cout << __FILE__ << " read combination  interface"
-                      << " offset: " <<  offset << " rows " << rows << " \n";
+            // auto rows = gko::array<IndexType>(
+            //         exec,
+            //         make_const_array_view(data_exec, length,
+            //                               data.get_const_row_idxs() + offset)
+            //             .copy_to_array());
+
+            // std::cout << __FILE__ << " read combination  interface"
+            //           << " data.size() " << data.get_num_stored_elements()
+            //           << " offset: " <<  offset
+            //           << " length: " <<  length
+            //           << " comb_->get_size() " << comb_->get_size()
+            //           << " rows " << rows
+            //           << " \n";
 
 
             device_matrix_data in(
                 exec, comb_->get_size(),
                 gko::array<IndexType>(
                     exec,
-                    make_const_array_view(data_exec, length,
-                                          data.get_const_row_idxs() + offset)
-                        .copy_to_array()),
+                    data.get_const_row_idxs() + offset,
+                    data.get_const_row_idxs() + offset + length),
                 gko::array<IndexType>(
                     exec,
-                    make_const_array_view(data_exec, length,
-                                          data.get_const_col_idxs() + offset)
-                        .copy_to_array()),
+                    data.get_const_col_idxs() + offset,
+                    data.get_const_col_idxs() + offset + length),
                 gko::array<ValueType>(
                     exec,
-                    make_const_array_view(data_exec, length,
-                                          data.get_const_values() + offset)
-                        .copy_to_array()));
+                    data.get_const_values() + offset,
+                    data.get_const_values() + offset + length));
 
-            op->read(std::move(in));
+            op->read(in);
             this->comb_->add_operators(
                 gko::initialize<gko::matrix::Dense<ValueType>>({1}, exec), op);
         }
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index ab462e52b..379c9dbc4 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -163,13 +163,16 @@ public:
     }
 
 
-    std::shared_ptr<const part_type> get_partition() const { return this->partition_; }
+    std::shared_ptr<const part_type> get_partition() const
+    {
+        return this->partition_;
+    }
 
     static std::shared_ptr<RepartDistMatrix> create(
         word matrix_format, const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A)
     {
-        auto& exec_handler = host_A.get()->get_exec_handler();
+        auto &exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_device_exec();
         auto comm = *exec_handler.get_communicator().get();
 
@@ -203,27 +206,21 @@ public:
 
         std::cout << __FILE__ << " rank " << rank
                   << " build_localized_partition "
-                  << " dim " << local_sparsity_->dim[0]
-                  << " send idxs size " << dst_comm_pattern.send_idxs.size()
-                  << " target ids " << dst_comm_pattern.target_ids
-                  << " target sizes " << dst_comm_pattern.target_sizes
-                  << "\n";
+                  << " dim " << local_sparsity_->dim[0] << " send idxs size "
+                  << dst_comm_pattern.send_idxs.size() << " target ids "
+                  << dst_comm_pattern.target_ids << " target sizes "
+                  << dst_comm_pattern.target_sizes << "\n";
 
         auto localized_partition = local_part_type::build_from_blocked_recv(
             exec, local_sparsity_->dim[0], dst_comm_pattern.send_idxs,
             dst_comm_pattern.target_ids, dst_comm_pattern.target_sizes);
 
-        std::cout << __FILE__ << " rank " << rank
-                  << " local sparsity size "
-                << local_sparsity_->size_
-                  << " local sparsity dim ["
-                  << local_sparsity_->dim[0]
-                  << "x"
-                  << local_sparsity_->dim[1]
+        std::cout << __FILE__ << " rank " << rank << " local sparsity size "
+                  << local_sparsity_->size_ << " local sparsity dim ["
+                  << local_sparsity_->dim[0] << "x" << local_sparsity_->dim[1]
                   << "] non_local sparsity size " << non_local_sparsity_->size_
-                  << " non local sparsity dim ["
-                  << non_local_sparsity_->dim[0] << "x"
-                  << non_local_sparsity_->dim[1] << "] target_ids "
+                  << " non local sparsity dim [" << non_local_sparsity_->dim[0]
+                  << "x" << non_local_sparsity_->dim[1] << "] target_ids "
                   << dst_comm_pattern.target_ids << " target_sizes "
                   << dst_comm_pattern.target_sizes << " target_send_idxs.size "
                   << dst_comm_pattern.send_idxs.size()
@@ -281,6 +278,7 @@ public:
         scalar *local_ptr;
         label local_interfaces_count = 0;
 
+        // update values
         if (owner) {
             using Coo = gko::matrix::Coo<scalar, label>;
             std::shared_ptr<const Coo> local =
@@ -298,85 +296,89 @@ public:
             local_ptr = const_cast<scalar *>(local->get_const_values());
         }
         communicate_values(exec_handler, diag_comm_pattern, host_A->get_diag(),
-                        local_ptr);
-        communicate_values(exec_handler, upper_comm_pattern, host_A->get_upper(),
-                        local_ptr);
+                           local_ptr);
+        communicate_values(exec_handler, upper_comm_pattern,
+                           host_A->get_upper(), local_ptr);
         if (host_A->symmetric()) {
             // TODO FIXME
             // if symmetric we can reuse already copied data
             communicate_values(exec_handler, lower_comm_pattern,
-                            host_A->get_lower(), local_ptr);
+                               host_A->get_lower(), local_ptr);
         } else {
             communicate_values(exec_handler, lower_comm_pattern,
-                            host_A->get_lower(), local_ptr);
+                               host_A->get_lower(), local_ptr);
         }
 
+        if (owner) {
+            // next update local and non_local interfaces
+            // TODO FIXME
+            for (int i = 1; i < local_interfaces_count; i++) {
+                using Coo = gko::matrix::Coo<scalar, label>;
+                std::shared_ptr<const Coo> local =
+                    gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
+                                     dist_A->get_local_matrix())
+                                     ->get_combination()
+                                     ->get_operators()[i]);
+                local_ptr = const_cast<scalar *>(local->get_const_values());
+
+                // for (int j = 0; j < 10; j++) {
+                //     local_ptr[j] = 1.0;
+                // }
+            }
+        }
 
-        // next update local and non_local interfaces
-        // TODO FIXME
-        for (int i = 1; i < local_interfaces_count; i++) {
+        // reorder updated values
+        if (owner) {
+            // NOTE local sparsity size includes the interfaces
             using Coo = gko::matrix::Coo<scalar, label>;
+            using dim_type = gko::dim<2>::dimension_type;
             std::shared_ptr<const Coo> local =
                 gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
                                  dist_A->get_local_matrix())
                                  ->get_combination()
-                                 ->get_operators()[i]);
+                                 ->get_operators()[0]);
+            auto local_elements = local->get_num_stored_elements();
             local_ptr = const_cast<scalar *>(local->get_const_values());
-
-            for (int j = 0; j < 10; j++) {
-                local_ptr[j] = 1.0;
-            }
-        }
-
-
-        if (owner) {
             // TODO make sure this doesn't copy
             // create a non owning dense matrix of local_values
-            using dim_type = gko::dim<2>::dimension_type;
+
             auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
-                exec,
-                gko::dim<2>{static_cast<dim_type>(repart_loc_sparsity->size_),
-                            1},
-                gko::array<scalar>::view(exec, repart_loc_sparsity->size_,
-                                         local_ptr),
-                1));
-            if (rank == 0) {
-                std::cout << __FILE__ << " on rank " << rank
-                          << " created row collection of size  "
-                          << repart_loc_sparsity->size_ << " ldu_mapping "
-                          << local_sparsity_->ldu_mapping << "\n";
-            }
+                exec, gko::dim<2>{static_cast<dim_type>(local_elements), 1},
+                gko::array<scalar>::view(exec, local_elements, local_ptr), 1));
+
+            auto mapping_view = gko::array<label>::view(
+                 exec, local_elements, local_sparsity_->ldu_mapping.get_data());
+
 
             // TODO this needs to copy ldu_mapping to the device
             auto dense_vec = row_collection->clone();
-            dense_vec->row_gather(&local_sparsity_->ldu_mapping,
+            dense_vec->row_gather(&mapping_view,
                                   row_collection.get());
             if (rank == 0) {
                 std::cout << __FILE__ << " on rank " << rank
-                    << " done row gather\n";
+                          << " done row gather\n";
             }
         }
 
-        std::cout << __FILE__ << " on rank " << rank
-            << " make_shared "
-            << " dist_A size " << dist_A->get_size() <<
-            "\n";
+        std::cout << __FILE__ << " on rank " << rank << " make_shared "
+                  << " dist_A size " << dist_A->get_size() << "\n";
         auto ret = std::make_shared<RepartDistMatrix>(
-            exec, comm, repartitioner.get_repart_dim() , dist_A->get_size(),
+            exec, comm, repartitioner.get_repart_dim(), dist_A->get_size(),
             std::move(dist_A));
-            if (rank == 0) {
-                std::cout << __FILE__ << " on rank " << rank
-                    << "done make_shared\n";
-            }
+        if (rank == 0) {
+            std::cout << __FILE__ << " on rank " << rank
+                      << "done make_shared\n";
+        }
 
         return ret;
     };
 
     // TODO matrix_format can be data member
-    void update(word matrix_format,
-                std::shared_ptr<const HostMatrixWrapper> host_A)
+    void update(
+        word matrix_format, const Repartitioner &repartitioner,
+        std::shared_ptr<const HostMatrixWrapper> host_A)
     {
-        auto& exec_handler = host_A.get()->get_exec_handler();
+        auto &exec_handler = host_A.get()->get_exec_handler();
         auto device_exec = exec_handler.get_device_exec();
 
         scalar *local_value_ptr{};
@@ -537,7 +539,6 @@ protected:
     // of these are always true in this example).
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
-
         std::cout << __FILE__ << __LINE__ << "call apply\n";
         this->dist_mtx_->apply(b, x);
     }

From 9363ba1ce179bf600b7caac5a3ed1e320570e4e4 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 30 May 2024 08:28:51 +0200
Subject: [PATCH 60/90] move update to separate function

---
 MatrixWrapper/Distributed/Distributed.H | 127 ++++++++----------------
 1 file changed, 41 insertions(+), 86 deletions(-)

diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 379c9dbc4..f587c48f5 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -260,6 +260,46 @@ public:
             sparse_comm);
 
 
+
+        std::cout << __FILE__ << " on rank " << rank << " make_shared "
+                  << " dist_A size " << dist_A->get_size() << "\n";
+        auto ret = std::make_shared<RepartDistMatrix>(
+            exec, comm, repartitioner.get_repart_dim(), dist_A->get_size(),
+            std::move(dist_A));
+        if (rank == 0) {
+            std::cout << __FILE__ << " on rank " << rank
+                      << "done make_shared\n";
+        }
+        update_impl(matrix_format, repartitioner, host_A, ret, local_sparsity_);
+
+
+        return ret;
+    };
+
+    void update(
+        word matrix_format, const Repartitioner &repartitioner,
+        std::shared_ptr<const HostMatrixWrapper> host_A)
+        {
+
+        };
+
+    // TODO matrix_format can be data member
+    static void update_impl(
+        word matrix_format, const Repartitioner &repartitioner,
+        std::shared_ptr<const HostMatrixWrapper> host_A,
+        std::shared_ptr<RepartDistMatrix> dist_A,
+        std::shared_ptr<const SparsityPattern> local_sparsity
+)
+    {
+        auto &exec_handler = host_A.get()->get_exec_handler();
+        auto exec = exec_handler.get_ref_exec();
+        auto device_exec = exec_handler.get_device_exec();
+        auto ranks_per_gpu = repartitioner.get_ranks_per_gpu();
+        label rank{repartitioner.get_rank()};
+        label owner_rank = repartitioner.get_owner_rank();
+        bool owner = repartitioner.is_owner();
+
+
         label nrows = host_A->get_local_nrows();
         label local_matrix_nnz = host_A->get_local_matrix_nnz();
         // size + padding has to be local_matrix_nnz
@@ -347,7 +387,7 @@ public:
                 gko::array<scalar>::view(exec, local_elements, local_ptr), 1));
 
             auto mapping_view = gko::array<label>::view(
-                 exec, local_elements, local_sparsity_->ldu_mapping.get_data());
+                 exec, local_elements, local_sparsity->ldu_mapping.get_data());
 
 
             // TODO this needs to copy ldu_mapping to the device
@@ -359,91 +399,6 @@ public:
                           << " done row gather\n";
             }
         }
-
-        std::cout << __FILE__ << " on rank " << rank << " make_shared "
-                  << " dist_A size " << dist_A->get_size() << "\n";
-        auto ret = std::make_shared<RepartDistMatrix>(
-            exec, comm, repartitioner.get_repart_dim(), dist_A->get_size(),
-            std::move(dist_A));
-        if (rank == 0) {
-            std::cout << __FILE__ << " on rank " << rank
-                      << "done make_shared\n";
-        }
-
-        return ret;
-    };
-
-    // TODO matrix_format can be data member
-    void update(
-        word matrix_format, const Repartitioner &repartitioner,
-        std::shared_ptr<const HostMatrixWrapper> host_A)
-    {
-        auto &exec_handler = host_A.get()->get_exec_handler();
-        auto device_exec = exec_handler.get_device_exec();
-
-        scalar *local_value_ptr{};
-        scalar *non_local_value_ptr{};
-
-        FatalErrorInFunction << "Not implemented" << abort(FatalError);
-        // TODO local matrix is now of type Combination<Csr>
-        // Combination should get an update(gko::array<> coeffs function)
-        //
-        // if (matrix_format == "Csr") {
-        //     local_value_ptr =
-        //         (gko::matrix::Csr<scalar, label>
-        //         *)(dist_mtx_->get_local_matrix())
-        //             ->get_values();
-
-        //     non_local_value_ptr = (gko::matrix::Csr<scalar, label>
-        //     *)dist_mtx_
-        //                               ->get_non_local_matrix()
-        //                               ->get_values();
-        // }
-        // if (matrix_format == "Ell") {
-        //     local_value_ptr =
-        //         (gko::matrix::Ell<scalar, label>
-        //         *)dist_mtx_->get_local_matrix()
-        //             ->get_values();
-
-        //     non_local_value_ptr = (gko::matrix::Ell<scalar, label>
-        //     *)dist_mtx_
-        //                               ->get_non_local_matrix()
-        //                               ->get_values();
-        // }
-        // if (matrix_format == "Coo") {
-        //     local_value_ptr =
-        //         (gko::matrix::Coo<scalar, label>
-        //         *)dist_mtx_->get_local_matrix()
-        //             ->get_values();
-
-        //     non_local_value_ptr = (gko::matrix::Coo<scalar, label>
-        //     *)dist_mtx_
-        //                               ->get_non_local_matrix()
-        //                               ->get_values();
-        // }
-
-        const label local_nnz = local_sparsity_->size_;
-        const label non_local_nnz = non_local_sparsity_->size_;
-        auto local_coeffs = gko::array<scalar>(device_exec, local_nnz);
-        auto non_local_coeffs = gko::array<scalar>(device_exec, non_local_nnz);
-
-        host_A.get()->compute_local_coeffs(local_sparsity_, local_coeffs);
-        host_A.get()->compute_non_local_coeffs(non_local_sparsity_,
-                                               non_local_coeffs);
-
-        auto value_view =
-            val_array::view(device_exec, local_nnz, local_value_ptr);
-
-        auto non_local_view =
-            val_array::view(device_exec, non_local_nnz, non_local_value_ptr);
-
-        value_view = local_coeffs;
-        non_local_view = non_local_coeffs;
-
-        // On update dist_mtx already exists
-        // and only local and non local coeffs need to be updated
-        //
-        // repartition_coeffs(local_device_data, non_local_device_data)
     };
 
     RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,

From 0c4b04e820cdc2850638bb2c248563d6f4f90daa Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 2 Jun 2024 13:57:32 +0200
Subject: [PATCH 61/90] update interface values

---
 BaseWrapper/lduBase/GKOlduBase.H              |   4 +-
 DevicePersistent/Base/Base.H                  |   2 +-
 .../DistributedMatrixAdapter.H                |  62 ++-
 .../ExecutorHandler/ExecutorHandler.H         |  11 +-
 .../HostMatrixAdapter/HostMatrixAdapter.H     |  32 +-
 DevicePersistent/Vector/Vector.H              |  38 +-
 MatrixWrapper/Combination/Combination.H       |  13 +-
 .../CommunicationPattern.H                    |   2 +-
 MatrixWrapper/Distributed/Distributed.H       | 181 ++++++---
 MatrixWrapper/LDUMatrix/HostMatrix.C          |  32 +-
 MatrixWrapper/LDUMatrix/HostMatrix.H          |  11 +-
 MatrixWrapper/LduMatrix/GKOACG/GKOACG.H       |   6 +-
 .../SparsityPattern/SparsityPattern.H         |  59 ++-
 Repartitioner/Repartitioner.H                 | 377 ++++++++----------
 Solver/BiCGStab/GKOBiCGStab.H                 |   6 +-
 Solver/CG/GKOCG.H                             |   6 +-
 Solver/GMRES/GKOGMRES.H                       |   6 +-
 Solver/IR/GKOIR.H                             |   6 +-
 Solver/Multigrid/GKOMultigrid.H               |   6 +-
 StoppingCriterion/StoppingCriterion.H         |  16 +-
 common/common.H                               |   6 +-
 21 files changed, 485 insertions(+), 397 deletions(-)

diff --git a/BaseWrapper/lduBase/GKOlduBase.H b/BaseWrapper/lduBase/GKOlduBase.H
index 9fb5070eb..8039ed702 100644
--- a/BaseWrapper/lduBase/GKOlduBase.H
+++ b/BaseWrapper/lduBase/GKOlduBase.H
@@ -22,7 +22,7 @@ namespace Foam {
 
 #define CREATE_SOLVER_METHODS(Solver)                                          \
     std::unique_ptr<Solver::Factory, std::default_delete<Solver::Factory>>     \
-    create_dist_solver(std::shared_ptr<const gko::Executor> exec,                    \
+    create_dist_solver(std::shared_ptr<const gko::Executor> exec,              \
                        std::shared_ptr<const gko::LinOp> gkomatrix,            \
                        std::shared_ptr<const dist_vec> x,                      \
                        std::shared_ptr<const dist_vec> b, const label verbose, \
@@ -51,7 +51,7 @@ public:
                      const dictionary &solverControls)
         : lduLduBase<lduMatrix, SolverFactory>(
               fieldName, matrix, interfaceBouCoeffs, interfaceIntCoeffs,
-              interfaces, solverControls){};
+              interfaces, solverControls) {};
 
     virtual solverPerformance solve_impl(word typeName, scalarField &psi,
                                          const scalarField &source,
diff --git a/DevicePersistent/Base/Base.H b/DevicePersistent/Base/Base.H
index 8a5897a5f..8aa62ce01 100644
--- a/DevicePersistent/Base/Base.H
+++ b/DevicePersistent/Base/Base.H
@@ -27,7 +27,7 @@ public:
 
     //- Construct from IOobject and a PtrList
     DevicePersistentBase(const IOobject &io, std::shared_ptr<T> in_ptr)
-        : regIOobject(io), ptr_(in_ptr){};
+        : regIOobject(io), ptr_(in_ptr) {};
 
     // get a ptr to the underlying data
     std::shared_ptr<T> get_ptr() { return ptr_; };
diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index eb9ceee43..903c8a0a2 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -28,9 +28,9 @@ SourceFiles
 
 #include "DevicePersistent/Array/Array.H"
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
+#include "DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H"
 #include "DevicePersistent/Partition/Partition.H"
 #include "MatrixWrapper/Distributed/Distributed.H"
-#include "DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H"
 
 #include "fvCFD.H"
 
@@ -47,17 +47,16 @@ struct DistributedMatrixInitFunctor {
 
     const label verbose_;
 
-    const dictionary& solver_controls_;
+    const dictionary &solver_controls_;
 
-    const PersistentHostMatrixAdapter& host_A_;
+    const PersistentHostMatrixAdapter &host_A_;
 
-    DistributedMatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
-                         const Repartitioner & repartitioner,
-                         const label verbose,
-                         const word field_name,
-                         const PersistentHostMatrixAdapter& host_A,
-                         const dictionary& solver_controls
-                         )
+    DistributedMatrixInitFunctor(const objectRegistry &db,
+                                 const ExecutorHandler &exec,
+                                 const Repartitioner &repartitioner,
+                                 const label verbose, const word field_name,
+                                 const PersistentHostMatrixAdapter &host_A,
+                                 const dictionary &solver_controls)
         : db_(db),
           exec_(exec),
           repartitioner_(repartitioner),
@@ -67,21 +66,19 @@ struct DistributedMatrixInitFunctor {
           host_A_(host_A)
     {}
 
-    void update(
-        std::shared_ptr< RepartDistMatrix<scalar, label, label>> persistent_device_matrix) const
+    void update(std::shared_ptr<RepartDistMatrix<scalar, label, label>>
+                    persistent_device_matrix) const
     {
-         persistent_device_matrix->update(
-              solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
-              repartitioner_,
-           host_A_.get());
+        persistent_device_matrix->update(
+            solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
+            repartitioner_, host_A_.get());
     }
 
     std::shared_ptr<RepartDistMatrix<scalar, label, label>> init() const
     {
         return RepartDistMatrix<scalar, label, label>::create(
-              solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
-              repartitioner_,
-              host_A_.get());
+            solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
+            repartitioner_, host_A_.get());
     }
 };
 
@@ -97,28 +94,27 @@ private:
 
     const word field_name_;
 
-    mutable PersistentBase< RepartDistMatrix<scalar, label, label>, DistributedMatrixInitFunctor> host_matrix_;
+    mutable PersistentBase<RepartDistMatrix<scalar, label, label>,
+                           DistributedMatrixInitFunctor>
+        host_matrix_;
 
 
 public:
-    PersistentDistributedMatrix(
-                  const objectRegistry &db,
-                  const ExecutorHandler &exec,
-                  const Repartitioner& repartitioner,
-                  const word field_name,
-                  const PersistentHostMatrixAdapter& host_A,
-                  const dictionary &controlDict,
-                  const label verbose
-                  )
+    PersistentDistributedMatrix(const objectRegistry &db,
+                                const ExecutorHandler &exec,
+                                const Repartitioner &repartitioner,
+                                const word field_name,
+                                const PersistentHostMatrixAdapter &host_A,
+                                const dictionary &controlDict,
+                                const label verbose)
         : db_(db),
           verbose_(verbose),
           field_name_(field_name),
           host_matrix_{
               field_name + "_distributed_matrix", db,
               DistributedMatrixInitFunctor(db, exec, repartitioner, verbose_,
-                                   field_name + "_distributed_matrix",
-                                   host_A, 
-                                   controlDict),
+                                           field_name + "_distributed_matrix",
+                                           host_A, controlDict),
               controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
               verbose_}
     {}
@@ -128,7 +124,7 @@ public:
         return host_matrix_.get_persistent_object();
     }
 
-    // void write() 
+    // void write()
     // {
     //     this->get()->write();
     // }
diff --git a/DevicePersistent/ExecutorHandler/ExecutorHandler.H b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
index e8749d3bb..8b323eba1 100644
--- a/DevicePersistent/ExecutorHandler/ExecutorHandler.H
+++ b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
@@ -13,7 +13,6 @@
 namespace Foam {
 
 
-
 struct ExecutorInitFunctor {
     mutable std::shared_ptr<gko::experimental::mpi::communicator> comm_;
 
@@ -32,7 +31,7 @@ struct ExecutorInitFunctor {
                     ? std::make_shared<gko::experimental::mpi::communicator>(
                           MPI_COMM_WORLD, force_host_buffer)
                     : NULL),
-          device_id_( comm_->rank() / gpus_per_rank ),
+          device_id_(comm_->rank() / gpus_per_rank),
           executor_name_(executor_name),
           field_name_(field_name),
           verbose_(verbose)
@@ -115,10 +114,8 @@ private:
     const word device_executor_name_;
 
 
-
 public:
-
-    ExecutorHandler(const ExecutorHandler&) = delete;
+    ExecutorHandler(const ExecutorHandler &) = delete;
 
     ExecutorHandler(const objectRegistry &db, const dictionary &solverControls,
                     const word field_name)
@@ -169,8 +166,8 @@ public:
             MPI_COMM_WORLD, gko_force_host_buffer_);
     }
 
-    std::shared_ptr<gko::experimental::mpi::communicator>
-    get_communicator() const
+    std::shared_ptr<gko::experimental::mpi::communicator> get_communicator()
+        const
     {
         return this->device_comm_;
     }
diff --git a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
index 265e78e47..cc06bdd99 100644
--- a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
+++ b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
@@ -56,8 +56,7 @@ struct lduMatrixInitFunctor {
     const dictionary &solver_controls_;
 
     lduMatrixInitFunctor(const objectRegistry &db, const ExecutorHandler &exec,
-                         const label verbose,
-                         const word field_name,
+                         const label verbose, const word field_name,
                          const lduMatrix &matrix,
                          const FieldField<Field, scalar> &interfaceBouCoeffs,
                          const FieldField<Field, scalar> &interfaceIntCoeffs,
@@ -88,10 +87,9 @@ struct lduMatrixInitFunctor {
         return HostMatrixWrapper::create(
             exec_, db_, matrix_.diag().size(), matrix_.upper().size(),
             matrix_.symmetric(), matrix_.diag().begin(),
-            matrix_.upper().begin(), matrix_.lower().begin(),
-            matrix_.lduAddr(), interfaceBouCoeffs_,
-            interfaceIntCoeffs_, interfaces_, solver_controls_, field_name_,
-            verbose_);
+            matrix_.upper().begin(), matrix_.lower().begin(), matrix_.lduAddr(),
+            interfaceBouCoeffs_, interfaceIntCoeffs_, interfaces_,
+            solver_controls_, field_name_, verbose_);
     }
 };
 
@@ -107,26 +105,26 @@ private:
 
     const word field_name_;
 
-    mutable PersistentBase<HostMatrixWrapper, lduMatrixInitFunctor> host_matrix_;
+    mutable PersistentBase<HostMatrixWrapper, lduMatrixInitFunctor>
+        host_matrix_;
 
 
 public:
-    template<typename matrixType>
-    PersistentHostMatrixAdapter(const objectRegistry &db, const ExecutorHandler &exec,
-                  const dictionary &controlDict, const word field_name,
-                  const label verbose,
-                  const matrixType& matrix,
-                  const FieldField<Field, scalar> &interfaceBouCoeffs,
-                  const FieldField<Field, scalar> &interfaceIntCoeffs,
-                  const lduInterfaceFieldPtrsList &interfaces)
+    template <typename matrixType>
+    PersistentHostMatrixAdapter(
+        const objectRegistry &db, const ExecutorHandler &exec,
+        const dictionary &controlDict, const word field_name,
+        const label verbose, const matrixType &matrix,
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        const FieldField<Field, scalar> &interfaceIntCoeffs,
+        const lduInterfaceFieldPtrsList &interfaces)
         : db_(db),
           verbose_(verbose),
           field_name_(field_name),
           host_matrix_{
               field_name_ + "_host_matrix", db,
               lduMatrixInitFunctor(db, exec, verbose_,
-                                   field_name + "_host_matrix",
-                                   matrix,
+                                   field_name + "_host_matrix", matrix,
                                    interfaceBouCoeffs, interfaceIntCoeffs,
                                    interfaces, controlDict),
               controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 80513a5eb..02c55237c 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -11,8 +11,8 @@
 #include "DevicePersistent/Base/Base.H"
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "DevicePersistent/Partition/Partition.H"
-#include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 #include "MatrixWrapper/CommunicationPattern/CommunicationPattern.H"
+#include "MatrixWrapper/LDUMatrix/HostMatrix.H"
 #include "Repartitioner/Repartitioner.H"
 #include "common/common.H"
 
@@ -40,8 +40,8 @@ struct VectorInitFunctor {
     const label size_;
 
     VectorInitFunctor(const ExecutorHandler &exec, const word name,
-        const Repartitioner & repartitioner,
-                      const T *other, const label size, const label verbose,
+                      const Repartitioner &repartitioner, const T *other,
+                      const label size, const label verbose,
                       const bool on_device = false)
         : name_(name),
           repartitioner_(repartitioner),
@@ -79,30 +79,35 @@ struct VectorInitFunctor {
         msg += (on_device_) ? " on device" : " on host";
         LOG_1(verbose_, msg)
 
-        std::cout << __FILE__  << __LINE__ << " on rank " << rank << " before host view \n";
+        std::cout << __FILE__ << __LINE__ << " on rank " << rank
+                  << " before host view \n";
         auto host_view = gko::array<T>::view(exec_.get_ref_exec(), local_size,
                                              const_cast<T *>(other_));
         std::cout << __FILE__ << " on rank " << rank << " done host view \n";
 
         // FIXME this shouldn't be needed here also not during an
         // update
-        auto comm_pattern =
-            compute_send_recv_counts(
+        auto comm_pattern = compute_send_recv_counts(
             exec_, repartitioner_.get_ranks_per_gpu(), local_size);
 
         // this should happen on device
-        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank << " before local coeffs  \n" << std::endl;
+        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank
+                  << " before local coeffs  \n"
+                  << std::endl;
         auto local_coeffs = gko::array<scalar>(exec, repart_size);
-        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank << " done local coeffs  \n";
+        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank
+                  << " done local coeffs  \n";
 
         communicate_values(exec_, comm_pattern, host_view.get_data(),
-                        local_coeffs.get_data());
-        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank << "  gather_to_owner done \n";
+                           local_coeffs.get_data());
+        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank
+                  << "  gather_to_owner done \n";
 
-        auto ret =  gko::share(dist_vec::create(
+        auto ret = gko::share(dist_vec::create(
             exec, *comm.get(),
             vec::create(exec, gko::dim<2>{repart_size, 1}, local_coeffs, 1)));
-        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank << "  done dist vector create \n";
+        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank
+                  << "  done dist vector create \n";
         return ret;
     }
 };
@@ -146,15 +151,15 @@ public:
      * @param ranks_per_gpu
      */
     PersistentVector(const T *memory, const label size, const word name,
-                     const Repartitioner & repartitioner,
+                     const Repartitioner &repartitioner,
                      const objectRegistry &db, const ExecutorHandler &exec,
                      const label verbose, const bool update,
                      const bool init_on_device)
         : PersistentBase<gko::experimental::distributed::Vector<T>,
                          VectorInitFunctor<T>>(
               name, db,
-              VectorInitFunctor<T>(exec, name, repartitioner, memory, size, verbose,
-                                   init_on_device),
+              VectorInitFunctor<T>(exec, name, repartitioner, memory, size,
+                                   verbose, init_on_device),
               update, verbose),
           db_(db),
           name_(name),
@@ -171,7 +176,8 @@ public:
         // TODO FIXME
         // auto ref_exec = exec_.get_ref_exec();
         // auto host_buffer_view = gko::array<T>::view(
-        //     exec_.get_device_exec(), size_, get_vector()->get_local_values());
+        //     exec_.get_device_exec(), size_,
+        //     get_vector()->get_local_values());
 
         // auto to_view =
         //     gko::array<T>::view(ref_exec, size_, const_cast<T *>(memory_));
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index 2d99f9772..555fe32fe 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -62,16 +62,13 @@ public:
             device_matrix_data in(
                 exec, comb_->get_size(),
                 gko::array<IndexType>(
-                    exec,
-                    data.get_const_row_idxs() + offset,
+                    exec, data.get_const_row_idxs() + offset,
                     data.get_const_row_idxs() + offset + length),
                 gko::array<IndexType>(
-                    exec,
-                    data.get_const_col_idxs() + offset,
+                    exec, data.get_const_col_idxs() + offset,
                     data.get_const_col_idxs() + offset + length),
                 gko::array<ValueType>(
-                    exec,
-                    data.get_const_values() + offset,
+                    exec, data.get_const_values() + offset,
                     data.get_const_values() + offset + length));
 
             op->read(in);
@@ -94,8 +91,8 @@ public:
      */
     CombinationMatrix &operator=(const CombinationMatrix &other)
     {
-    FatalErrorInFunction << "Copying CombinationMatrix is not allowed"
-                         << abort(FatalError);
+        FatalErrorInFunction << "Copying CombinationMatrix is not allowed"
+                             << abort(FatalError);
     }
 
     /**
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 89ae65504..5fcf512f2 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -104,7 +104,7 @@ struct CommunicationPattern {
           send_idxs(idxs)
     {
         ASSERT_EQ(target_ids.get_size(), target_sizes.get_size());
-        //ASSERT_EQ(target_ids.get_size(), send_idxs.size());
+        // ASSERT_EQ(target_ids.get_size(), send_idxs.size());
     };
 
     const gko::experimental::mpi::communicator &get_comm() const
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index f587c48f5..720e0b810 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -197,10 +197,15 @@ public:
         label owner_rank = repartitioner.get_owner_rank();
         bool owner = repartitioner.is_owner();
 
-        auto [repart_loc_sparsity, repart_non_loc_sparsity] =
+        auto [repart_loc_sparsity, repart_non_loc_sparsity, local_interfaces] =
             repartitioner.repartition_sparsity(local_sparsity_,
                                                non_local_sparsity_);
 
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                << " repart_non_loc_sparsity.rank"
+                <<  repart_non_loc_sparsity->rank
+                << "\n";
+
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
@@ -232,9 +237,9 @@ public:
         auto sparse_comm =
             sparse_communicator::create(comm, localized_partition);
 
-        auto dist_A = generate_dist_mtx_with_inner_type<dist_mtx>(
+        auto dist_A = gko::share(generate_dist_mtx_with_inner_type<dist_mtx>(
             matrix_format, exec, sparse_comm, local_sparsity_,
-            non_local_sparsity_);
+            non_local_sparsity_));
 
         std::cout << __FILE__ << " rank " << rank
                   << " done generate dist_mtx\n";
@@ -260,36 +265,33 @@ public:
             sparse_comm);
 
 
+        update_impl(matrix_format, repartitioner, host_A, dist_A,
+                    local_sparsity_, non_local_sparsity_, src_comm_pattern,
+                    local_interfaces);
 
-        std::cout << __FILE__ << " on rank " << rank << " make_shared "
-                  << " dist_A size " << dist_A->get_size() << "\n";
         auto ret = std::make_shared<RepartDistMatrix>(
             exec, comm, repartitioner.get_repart_dim(), dist_A->get_size(),
             std::move(dist_A));
-        if (rank == 0) {
-            std::cout << __FILE__ << " on rank " << rank
-                      << "done make_shared\n";
-        }
-        update_impl(matrix_format, repartitioner, host_A, ret, local_sparsity_);
-
 
         return ret;
     };
 
-    void update(
-        word matrix_format, const Repartitioner &repartitioner,
-        std::shared_ptr<const HostMatrixWrapper> host_A)
-        {
+    void update(word matrix_format, const Repartitioner &repartitioner,
+                std::shared_ptr<const HostMatrixWrapper> host_A) {
 
-        };
+    };
 
     // TODO matrix_format can be data member
     static void update_impl(
         word matrix_format, const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A,
-        std::shared_ptr<RepartDistMatrix> dist_A,
-        std::shared_ptr<const SparsityPattern> local_sparsity
-)
+        std::shared_ptr<
+            gko::experimental::distributed::Matrix<scalar, label, label>>
+            dist_A,
+        std::shared_ptr<const SparsityPattern> local_sparsity,
+        std::shared_ptr<const SparsityPattern> non_local_sparsity,
+        CommunicationPattern &src_comm_pattern,
+        std::vector<std::pair<bool,label>> local_interfaces)
     {
         auto &exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_ref_exec();
@@ -298,7 +300,8 @@ public:
         label rank{repartitioner.get_rank()};
         label owner_rank = repartitioner.get_owner_rank();
         bool owner = repartitioner.is_owner();
-
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << "done coeffs\n";
 
         label nrows = host_A->get_local_nrows();
         label local_matrix_nnz = host_A->get_local_matrix_nnz();
@@ -316,29 +319,34 @@ public:
                                      local_matrix_nnz, upper_nnz, nrows);
 
         scalar *local_ptr;
-        label local_interfaces_count = 0;
 
-        // update values
+        // update main values
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << " is owner " << owner << "done coeffs\n";
         if (owner) {
             using Coo = gko::matrix::Coo<scalar, label>;
+            auto local_mtx = dist_A->get_local_matrix();
+            std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                      << "done coeffs\n";
+
             std::shared_ptr<const Coo> local =
                 gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
                                  dist_A->get_local_matrix())
                                  ->get_combination()
                                  ->get_operators()[0]);
-            local_interfaces_count =
-                gko::as<CombinationMatrix<scalar, label, Coo>>(
-                    dist_A->get_local_matrix())
-                    ->get_combination()
-                    ->get_operators()
-                    .size();
-
             local_ptr = const_cast<scalar *>(local->get_const_values());
         }
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << "done coeffs\n";
         communicate_values(exec_handler, diag_comm_pattern, host_A->get_diag(),
                            local_ptr);
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << "done coeffs\n";
         communicate_values(exec_handler, upper_comm_pattern,
                            host_A->get_upper(), local_ptr);
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << "done coeffs\n";
+
         if (host_A->symmetric()) {
             // TODO FIXME
             // if symmetric we can reuse already copied data
@@ -348,24 +356,102 @@ public:
             communicate_values(exec_handler, lower_comm_pattern,
                                host_A->get_lower(), local_ptr);
         }
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << "done coeffs\n";
 
+
+        auto comm = *exec_handler.get_communicator().get();
         if (owner) {
-            // next update local and non_local interfaces
-            // TODO FIXME
-            for (int i = 1; i < local_interfaces_count; i++) {
-                using Coo = gko::matrix::Coo<scalar, label>;
-                std::shared_ptr<const Coo> local =
-                    gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
-                                     dist_A->get_local_matrix())
-                                     ->get_combination()
-                                     ->get_operators()[i]);
-                local_ptr = const_cast<scalar *>(local->get_const_values());
-
-                // for (int j = 0; j < 10; j++) {
-                //     local_ptr[j] = 1.0;
-                // }
+            using Coo = gko::matrix::Coo<scalar, label>;
+            std::shared_ptr<const Coo> mtx;
+            label loc_ctr{1};
+            label nloc_ctr{0};
+            label host_interface_ctr{0};
+            label tag = 0;
+            label comm_rank, comm_size;
+            scalar *recv_buffer_ptr;
+            label remain_host_interfaces = host_A->get_interface_size();
+            for (auto [is_local,comm_rank] : local_interfaces) {
+                label &ctr = (is_local) ? loc_ctr : nloc_ctr;
+                if (is_local) {
+                    mtx = gko::as<Coo>(
+                        gko::as<CombinationMatrix<scalar, label, Coo>>(
+                            dist_A->get_local_matrix())
+                            ->get_combination()
+                            ->get_operators()[ctr]);
+                    comm_size = local_sparsity->interface_spans[ctr].length();
+                } else {
+                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                            << " non local update\n";
+                    mtx = gko::as<Coo>(
+                        gko::as<CombinationMatrix<scalar, label, Coo>>(
+                            dist_A->get_non_local_matrix())
+                            ->get_combination()
+                            ->get_operators()[ctr]);
+                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                            << " non local update\n";
+                    comm_size =
+                        non_local_sparsity->interface_spans[ctr].length();
+                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                            << " non local update\n";
+                    // FIXME this is the communication target not the origin
+                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                            << " comm_rank " << comm_rank << " comm_size " << comm_size
+                            << " non local update\n";
+                }
+                recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
+
+                // TODO this test is not sufficient
+                // we can determine the number of local data interfaces from the
+                // original matrix since the local interfaces are stored first
+                //
+                //
+                // thus the first n interfaces are from this rank regardless of
+                // the locality
+                // only communicate if data is on on other rank, ie interface orignaly from other
+                // rank
+                if (comm_rank != rank) {
+                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                            << " recv from "
+                            << comm_rank << " size " << comm_size << "\n";
+                    comm.recv(exec, recv_buffer_ptr, comm_size, comm_rank, tag);
+                } else {
+                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                            << " remain_host_interfaces "
+                            << remain_host_interfaces << " owner rank update\n";
+                    // if data is already on this rank
+                    auto data_view = gko::array<scalar>::const_view(
+                        exec, comm_size,
+                        host_A->get_interface_data(host_interface_ctr));
+
+                    // TODO FIXME this needs target executor
+                    auto target_view = gko::array<scalar>::view(
+                        exec, comm_size, recv_buffer_ptr);
+
+                    target_view = data_view;
+
+                    host_interface_ctr++;
+                    remain_host_interfaces--;
+                }
+                ctr++;
+            }
+        } else {
+            // the non owner has send all its interfaces to owner
+            // thus all values need to be communicated to the owner as well
+            label num_interfaces = src_comm_pattern.target_ids.get_size();
+            label tag = 0;
+            for (int i = 0; i < num_interfaces; i++) {
+                label comm_size =
+                    src_comm_pattern.target_sizes.get_const_data()[i];
+                const scalar *send_buffer_ptr = host_A->get_interface_data(i);
+                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                            << " send to "
+                            << owner_rank << " size " << comm_size << "\n";
+                comm.send(exec, send_buffer_ptr, comm_size, owner_rank, tag);
             }
         }
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << "done coeffs\n";
 
         // reorder updated values
         if (owner) {
@@ -387,13 +473,12 @@ public:
                 gko::array<scalar>::view(exec, local_elements, local_ptr), 1));
 
             auto mapping_view = gko::array<label>::view(
-                 exec, local_elements, local_sparsity->ldu_mapping.get_data());
+                exec, local_elements, local_sparsity->ldu_mapping.get_data());
 
 
             // TODO this needs to copy ldu_mapping to the device
             auto dense_vec = row_collection->clone();
-            dense_vec->row_gather(&mapping_view,
-                                  row_collection.get());
+            dense_vec->row_gather(&mapping_view, row_collection.get());
             if (rank == 0) {
                 std::cout << __FILE__ << " on rank " << rank
                           << " done row gather\n";
@@ -404,7 +489,7 @@ public:
     RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
                      communicator comm, gko::dim<2> local_size,
                      gko::dim<2> global_size,
-                     std::unique_ptr<dist_mtx> dist_mtx)
+                     std::shared_ptr<dist_mtx> dist_mtx)
         : gko::experimental::EnableDistributedLinOp<RepartDistMatrix>(exec),
           gko::experimental::distributed::DistributedBase(comm),
           dist_mtx_(std::move(dist_mtx))
@@ -512,7 +597,7 @@ protected:
 
 
 private:
-    std::unique_ptr<dist_mtx> dist_mtx_;
+    std::shared_ptr<dist_mtx> dist_mtx_;
 
     std::shared_ptr<part_type> partition_;
 
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index 1a12cd687..dbcbad4bf 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -100,7 +100,15 @@ HostMatrixWrapper::HostMatrixWrapper(
       interfaces_(interfaces),
       interfaceBouCoeffs_(interfaceBouCoeffs),
       non_local_matrix_nnz_{count_interface_nnz(interfaces, true)}
-{}
+{
+    for (label i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+        interface_ptr_.push_back(interfaceBouCoeffs[i].begin());
+    }
+}
 
 
 label HostMatrixWrapper::count_interface_nnz(
@@ -387,14 +395,11 @@ HostMatrixWrapper::collect_cells_on_non_local_interface(
 std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
     std::shared_ptr<const gko::Executor> exec) const
 {
-    auto sparsity{std::make_shared<SparsityPattern>(exec->get_master(),
-                                                    non_local_matrix_nnz_)};
-    sparsity->dim = gko::dim<2>{nrows_, non_local_matrix_nnz_};
+    auto dim = gko::dim<2>{nrows_, non_local_matrix_nnz_};
+
+    SparsityPatternVector sparsity;
 
     auto non_local_indices = collect_cells_on_non_local_interface(interfaces_);
-    auto rows = sparsity->row_idxs.get_data();
-    auto cols = sparsity->col_idxs.get_data();
-    auto permute = sparsity->ldu_mapping.get_data();
     label prev_interface_ctr{0};
     label end{0};
     label start{0};
@@ -406,9 +411,9 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
     // TODO currently we set permute eventhough this is not required
     // anymore, remove permute from non_local_interfaces
     for (auto [interface_idx, col, row, rank] : non_local_indices) {
-        rows[element_ctr] = row;
-        cols[element_ctr] = col;
-        permute[element_ctr] = element_ctr;
+        sparsity.rows.push_back(row);
+        sparsity.cols.push_back(col);
+        sparsity.mapping.push_back(element_ctr);
 
         // a new interface started or the last element on last interface has
         // been reached
@@ -418,8 +423,9 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
             // the end of the non_local_indices thus we need to increment
             // the elment_ctr once more
             end = (last_element) ? element_ctr + 1 : element_ctr;
-            sparsity->interface_spans.emplace_back(start, end);
-            sparsity->rank.emplace_back(prev_rank);
+            sparsity.begin.emplace_back(start);
+            sparsity.end.emplace_back(end);
+            sparsity.ranks.emplace_back(prev_rank);
             start = end;
             prev_interface_ctr = interface_idx;
         }
@@ -427,7 +433,7 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
         element_ctr++;
     }
 
-    return sparsity;
+    return std::make_shared<SparsityPattern>(exec, dim, sparsity);
 }
 
 std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_local_sparsity(
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 14be35418..03a4ca2b6 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -81,8 +81,6 @@ void make_begin_and_ends_consecutive(
     label ranks_per_gpu);
 
 
-
-
 // Free functions
 const lduInterfaceField *interface_getter(
     const lduInterfaceFieldPtrsList &interfaces, const label i);
@@ -289,6 +287,8 @@ private:
 
     const FieldField<Field, scalar> &interfaceBouCoeffs_;
 
+    mutable std::vector<const scalar*> interface_ptr_;
+
     /* Iterates all interfaces and collects the coefficients into a vector
     **
     ** @param local whether local or non local coefficients should be collected
@@ -413,6 +413,13 @@ public:
     CommunicationPattern create_communication_pattern(
         const gko::experimental::mpi::communicator &comm) const;
 
+    label get_interface_size() const {
+        return interface_ptr_.size();
+    } const;
+
+    const scalar* get_interface_data(label i) const {
+        return interface_ptr_[i];
+    } const;
 
     bool get_verbose() const { return verbose_; }
 
diff --git a/MatrixWrapper/LduMatrix/GKOACG/GKOACG.H b/MatrixWrapper/LduMatrix/GKOACG/GKOACG.H
index 175ca0cc7..2db153532 100644
--- a/MatrixWrapper/LduMatrix/GKOACG/GKOACG.H
+++ b/MatrixWrapper/LduMatrix/GKOACG/GKOACG.H
@@ -27,7 +27,7 @@ public:
           db_(db),
           sysMatrixName_(sysMatrixName),
           preconditioner_(solverControls_.lookupOrDefault("preconditioner",
-                                                          word("none"))){};
+                                                          word("none"))) {};
 
     std::unique_ptr<gko::solver::Cg<double>::Factory,
                     std::default_delete<gko::solver::Cg<double>::Factory>>
@@ -92,10 +92,10 @@ public:
            const LduMatrix<vector, scalar, scalar> &matrix,
            const dictionary &solverControls)
         : GKOLduBaseSolver<vector, scalar, GKOACGFactory>(fieldName, matrix,
-                                                          solverControls){};
+                                                          solverControls) {};
 
     //- Destructor
-    virtual ~GKOACG(){};
+    virtual ~GKOACG() {};
 
 
     // Member Functions
diff --git a/MatrixWrapper/SparsityPattern/SparsityPattern.H b/MatrixWrapper/SparsityPattern/SparsityPattern.H
index 8d5671c14..cc4a71b0e 100644
--- a/MatrixWrapper/SparsityPattern/SparsityPattern.H
+++ b/MatrixWrapper/SparsityPattern/SparsityPattern.H
@@ -24,19 +24,48 @@ SourceFiles
 \*---------------------------------------------------------------------------*/
 #pragma once
 
+/* Vector variant of the SparsityPattern
+**
+*/
+struct SparsityPatternVector {
+    std::vector<label> rows;
+    std::vector<label> cols;
+    std::vector<label> mapping;
+    std::vector<label> begin;
+    std::vector<label> end;
+    std::vector<label> ranks;
+};
+
 struct SparsityPattern {
-    SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size)
+    SparsityPattern(std::shared_ptr<const gko::Executor> exec)
+        : size_(0),
+          row_idxs{exec, static_cast<gko::size_type>(size_)},
+          col_idxs{exec, static_cast<gko::size_type>(size_)},
+          ldu_mapping{exec, static_cast<gko::size_type>(size_)},
+          dim(gko::dim<2>{0, 0}),
+          interface_spans(std::vector<gko::span>{}),
+          rank(std::vector<label>{})
+    {}
+
+    SparsityPattern(std::shared_ptr<const gko::Executor> exec, label size)
         : size_(size),
-          row_idxs{exec, static_cast<gko::size_type>(size)},
-          col_idxs{exec, static_cast<gko::size_type>(size)},
-          ldu_mapping{exec, static_cast<gko::size_type>(size)}
+          row_idxs{exec, static_cast<gko::size_type>(size_)},
+          col_idxs{exec, static_cast<gko::size_type>(size_)},
+          ldu_mapping{exec, static_cast<gko::size_type>(size_)}
     {}
 
+    SparsityPattern(std::shared_ptr<const gko::Executor> exec, gko::dim<2> dim,
+                    const SparsityPatternVector &in)
+        : SparsityPattern(exec, in.rows.size(), dim, in.rows, in.cols,
+                          in.mapping, in.begin, in.end, in.ranks) {}
+
     SparsityPattern(std::shared_ptr<const gko::Executor> exec, const label size,
-                    gko::dim<2> dim, std::vector<label> &rows,
-                    std::vector<label> &cols, std::vector<label> &map,
-                    std::vector<label> &spans_begin,
-                    std::vector<label> &spans_end, std::vector<label> &ranks)
+                    gko::dim<2> dim, const std::vector<label> &rows,
+                    const std::vector<label> &cols,
+                    const std::vector<label> &map,
+                    const std::vector<label> &spans_begin,
+                    const std::vector<label> &spans_end,
+                    const std::vector<label> &ranks)
         : size_(size),
           dim(dim),
           row_idxs{exec, static_cast<gko::size_type>(size)},
@@ -83,7 +112,19 @@ struct SparsityPattern {
     // this is used to keep all col and row idx consecutive in memory
     mutable std::vector<gko::span> interface_spans;
 
-    // keep track of original (communication) rank of each interface or matrix
+    // keep track of communication rank id of each interface or matrix
     // block
     mutable std::vector<label> rank;
 };
+
+// struct SparsityDataMap {
+
+//     SparsityDataMap (
+//         const SparsityPattern& pattern_in,
+//     {}
+
+//     const SparsityPattern& pattern;
+
+//     const vector<scalar*> data;
+
+// };
diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index c244468db..331166646 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -1,4 +1,4 @@
-/*---------------------------------------------------------------------------*\
+/*---------------------------------------------------------------------------* \
 License
     This file is part of OGL.
 
@@ -123,7 +123,7 @@ public:
         auto local_sizes =
             gather_to_owner(exec_handler, comm_pattern, 1, data.data());
 
-        return std::accumulate(local_sizes.begin(), local_sizes.end(),0);
+        return std::accumulate(local_sizes.begin(), local_sizes.end(), 0);
     }
 
     label get_repart_size() const { return repart_size_; }
@@ -138,8 +138,75 @@ public:
         return orig_partition_;
     };
 
-    std::pair<std::shared_ptr<SparsityPattern>,
-              std::shared_ptr<SparsityPattern>>
+
+    /* @brief given received interfaces this function sorts into local and non
+     *local
+     * returns: SparsityPatternVector storing new non_local sparsity pattern,
+     *  and a pair of locality information, where the first bool stores if the
+     *  interface performs a local communication and the second bool if the
+     *  the interface was originally from this rank
+     */
+    std::pair<SparsityPatternVector, std::vector<bool>>
+    build_non_local_interfaces(SparsityPatternVector &loc,
+                               const SparsityPatternVector &non_loc) const
+    {
+        std::vector<label> rows, cols, ldu_mapping, ranks, begins, ends;
+        std::vector<bool> is_local;
+        label merged_ranks_size = non_loc.ranks.size();
+
+        for (label i = 0; i < merged_ranks_size; i++) {
+            // these are the begin ends before merging they need to be
+            // offseted;
+            auto begin = non_loc.begin[i];
+            auto end = non_loc.end[i];
+            bool local = reparts_to_local(non_loc.ranks[i]);
+            is_local.push_back(local);
+            if (local) {
+                // TODO depending on the interface simple non
+                // transforming and interpolating interfaces like now
+                // local processor interfaces could be merged
+                loc.begin.push_back(loc.rows.size());
+                loc.rows.insert(loc.rows.end(), non_loc.rows.begin() + begin,
+                                non_loc.rows.begin() + end);
+                loc.cols.insert(loc.cols.end(), non_loc.cols.begin() + begin,
+                                non_loc.cols.begin() + end);
+                loc.mapping.insert(loc.mapping.end(),
+                                   non_loc.mapping.data() + begin,
+                                   non_loc.mapping.data() + end);
+                // TODO store from rank ie from which rank it came
+                // this is currently unused
+                loc.end.push_back(loc.rows.size());
+                loc.ranks.push_back(non_loc.ranks[i]);
+            } else {
+                begins.push_back(rows.size());
+                rows.insert(rows.end(), non_loc.rows.begin() + begin,
+                            non_loc.rows.begin() + end);
+                cols.insert(cols.end(), non_loc.cols.begin() + begin,
+                            non_loc.cols.begin() + end);
+                // NOTE we don't do anything with non local mapping
+                ldu_mapping.insert(ldu_mapping.end(),
+                                   non_loc.cols.begin() + begin,
+                                   non_loc.cols.begin() + end);
+                ends.push_back(rows.size());
+                ranks.push_back(get_owner_rank(non_loc.ranks[i]));
+            }
+        }
+        return std::make_pair(
+            SparsityPatternVector{rows, cols, ldu_mapping, begins, ends, ranks},
+            is_local);
+    }
+
+
+    /*
+    ** returns the the new sparsity patterns (local and non-local) and a vector
+    ** tracking the interfaces. This contains a pair where the first entry
+    *(bool)
+    ** signals whether this is a new local interface (no communication), and the
+    ** second entry (label) tracks the original rank of the interface
+    */
+    std::tuple<std::shared_ptr<SparsityPattern>,
+               std::shared_ptr<SparsityPattern>,
+               std::vector<std::pair<bool, label>>>
     repartition_sparsity(
         std::shared_ptr<SparsityPattern> src_local_pattern,
         std::shared_ptr<SparsityPattern> src_non_local_pattern) const
@@ -173,37 +240,44 @@ public:
                                    data.get_data(), offset);
         };
 
-        std::cout << __FILE__ << " rank " << rank << "before gathered_local\n";
-        auto merged_local_rows = gather_closure(
-            local_comm_pattern, src_local_pattern->row_idxs, offset);
-
-        auto merged_local_cols = gather_closure(
-            local_comm_pattern, src_local_pattern->col_idxs, offset);
+        SparsityPatternVector merged_local{
+            gather_closure(local_comm_pattern, src_local_pattern->row_idxs,
+                           offset),
+            gather_closure(local_comm_pattern, src_local_pattern->col_idxs,
+                           offset),
+            gather_closure(local_comm_pattern, src_local_pattern->ldu_mapping,
+                           0),
+            std::vector<label>{0},
+            std::vector<label>{0},
+            std::vector{rank}};
+        merged_local.end[0] = merged_local.rows.size();
 
-        auto merged_ldu_mapping = gather_closure(
-            local_comm_pattern, src_local_pattern->ldu_mapping, 0);
 
         if (is_owner()) {
-            make_ldu_mapping_consecutive(local_comm_pattern, merged_ldu_mapping,
-                                         rank, ranks_per_gpu);
+            make_ldu_mapping_consecutive(
+                local_comm_pattern, merged_local.mapping, rank, ranks_per_gpu);
         }
 
-        label rows = (is_owner()) ? merged_local_rows.back() + 1 : 0;
+        label rows = (is_owner()) ? merged_local.rows.back() + 1 : 0;
         gko::dim<2> merged_local_dim{rows, rows};
 
-        std::vector<label> merged_begin{0};
-        std::vector<label> merged_end{label(merged_local_rows.size())};
-        std::vector<label> merged_ranks{rank};
-
         auto non_local_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, src_non_local_pattern->size_);
 
-        auto merged_non_local_rows = gather_closure(
-            non_local_comm_pattern, src_non_local_pattern->row_idxs, offset);
-        std::cout << __FILE__ << " rank " << rank << " gathered_non_local\n";
+        std::vector<label> spans_begin;
+        std::vector<label> spans_end;
+        auto span_comm_pattern = compute_send_recv_counts(
+            exec_handler, ranks_per_gpu,
+            src_non_local_pattern->interface_spans.size());
+        for (auto elem : src_non_local_pattern->interface_spans) {
+            spans_begin.push_back(elem.begin);
+            spans_end.push_back(elem.end);
+        }
 
         // the non local cols are in local idx of other side
         // thus we need the new offset of the other side
+        // NOTE TODO this modifies src_non_local_pattern better do the offsets
+        // on the owner side after gathering
         for (label i = 0; i < src_non_local_pattern->rank.size(); i++) {
             auto comm_rank = src_non_local_pattern->rank[i];
             auto [begin, end] = src_non_local_pattern->interface_spans[i];
@@ -215,237 +289,122 @@ public:
             std::transform(data, data + size, data,
                            [&](label idx) { return idx + local_offset; });
         }
-        std::cout << __FILE__ << " rank " << rank << " done offset non_local\n";
-
-        // we now dealed with the new offset thus we dont need an offset
-        // now
-        auto merged_non_local_cols = gather_closure(
-            non_local_comm_pattern, src_non_local_pattern->col_idxs, 0);
-
-        auto merged_non_local_ldu_mapping = gather_closure(
-            non_local_comm_pattern, src_non_local_pattern->row_idxs, 0);
-
-        auto span_comm_pattern = compute_send_recv_counts(
-            exec_handler, ranks_per_gpu,
-            src_non_local_pattern->interface_spans.size());
-
-        std::cout << __FILE__ << " rank " << rank << " done merged non_local\n";
 
-        std::vector<label> spans_begin;
-        std::vector<label> spans_end;
-
-        for (auto elem : src_non_local_pattern->interface_spans) {
-            spans_begin.push_back(elem.begin);
-            spans_end.push_back(elem.end);
-        }
-
-        auto merged_non_local_begin =
+        SparsityPatternVector merged_non_local{
+            gather_closure(non_local_comm_pattern,
+                           src_non_local_pattern->row_idxs, offset),
+            gather_closure(non_local_comm_pattern,
+                           src_non_local_pattern->col_idxs, 0),
+            gather_closure(non_local_comm_pattern,
+                           src_non_local_pattern->row_idxs, 0),
             gather_to_owner(exec_handler, span_comm_pattern, spans_begin.size(),
-                            spans_begin.data());
-
-        auto merged_non_local_end =
+                            spans_begin.data()),
             gather_to_owner(exec_handler, span_comm_pattern, spans_end.size(),
-                            spans_end.data());
-
-        auto merged_non_local_ranks = gather_to_owner(
-            exec_handler, span_comm_pattern, src_non_local_pattern->rank.size(),
-            src_non_local_pattern->rank.data());
+                            spans_end.data()),
+            gather_to_owner(exec_handler, span_comm_pattern,
+                            src_non_local_pattern->rank.size(),
+                            src_non_local_pattern->rank.data())};
 
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << " merged_non_local.ranks" << merged_non_local.ranks
+                  << "\n";
 
         if (is_owner()) {
             make_begin_and_ends_consecutive(
-                span_comm_pattern, merged_non_local_begin, merged_non_local_end,
+                span_comm_pattern, merged_non_local.begin, merged_non_local.end,
                 rank, ranks_per_gpu);
         }
 
+        auto [gathered_non_local, is_local] =
+            build_non_local_interfaces(merged_local, merged_non_local);
 
-        // TODO remove
-        //     if (rank == 0) {
-        // std::cout << __FILE__ << " on rank " << rank << "
-        // merged_begin"
-        //           << merged_begin
-        //           << " span_recv_counts " << span_recv_counts
-        //           << " span_recv_offsets " << span_recv_offsets
-        //           << " merged_non_local_begins " <<
-        //           merged_non_local_begin
-        //           << " merged_non_local_end " << merged_non_local_end
-        //           << " merged_end " << merged_end
-        //           << " merged_non_local_rows " <<
-        //           merged_non_local_rows
-        //           << " merged_ranks " << merged_non_local_ranks <<
-        //           "\n";
-        // }
-
-        // TODO shorten this
-        std::vector<label> gathered_non_local_rows;
-        std::vector<label> gathered_non_local_cols;
-        std::vector<label> gathered_non_local_mapping;
-        std::vector<label> gathered_ranks;
-        std::vector<label> gathered_begin;
-        std::vector<label> gathered_end;
-        label merged_ranks_size = merged_non_local_ranks.size();
-        // redistribute interfaces
-        for (label i = 0; i < merged_ranks_size; i++) {
-            auto merged_target_rank = merged_non_local_ranks[i];
-            label to_rank = get_owner_rank(merged_target_rank);
-            bool local = rank == to_rank;
-            // these are the begin ends before merging they need to be
-            // offseted;
-            auto begin = merged_non_local_begin[i];
-            auto end = merged_non_local_end[i];
-            if (local) {
-                // TODO depending on the interface simple non
-                // transforming and interpolating interfaces like now
-                // local processor interfaces could be merged
-                label size_before = merged_local_rows.size();
-                merged_local_rows.insert(merged_local_rows.end(),
-                                         merged_non_local_rows.begin() + begin,
-                                         merged_non_local_rows.begin() + end);
-                merged_local_cols.insert(merged_local_cols.end(),
-                                         merged_non_local_cols.begin() + begin,
-                                         merged_non_local_cols.begin() + end);
-                merged_ldu_mapping.insert(
-                    merged_ldu_mapping.end(),
-                    merged_non_local_ldu_mapping.data() + begin,
-                    merged_non_local_ldu_mapping.data() + end);
-                // TODO store from rank ie from which rank it came
-                merged_ranks.push_back(to_rank);
-
-                label size_after = merged_local_rows.size();
-                merged_begin.push_back(size_before);
-                merged_end.push_back(size_after);
-                // TODO remove
-                // if (rank == 0) {
-                //     std::cout << __FILE__ << " on rank " << rank
-                //             << " begin " << begin << " end " << end
-                //             << " insert merged_local_rows " <<
-                //             merged_local_rows
-                //             << "\n";
-                //     }
-            } else {
-                label size_before = gathered_non_local_rows.size();
-                gathered_non_local_rows.insert(
-                    gathered_non_local_rows.end(),
-                    merged_non_local_rows.begin() + begin,
-                    merged_non_local_rows.begin() + end);
-                // FIXME
-                // up until now non_local_cols are stil using
-                // local_idx and we need to convert them to
-                // consequtive compressed idxs
-                gathered_non_local_cols.insert(
-                    gathered_non_local_cols.end(),
-                    merged_non_local_cols.begin() + begin,
-                    merged_non_local_cols.begin() + end);
-                // TODO we don't do anything with non local mapping
-                gathered_non_local_mapping.insert(
-                    gathered_non_local_mapping.end(),
-                    merged_non_local_cols.begin() + begin,
-                    merged_non_local_cols.begin() + end);
-                label size_after = gathered_non_local_rows.size();
-                // if (rank == 0) {
-                // std::cout << __FILE__ << " on rank " << rank
-                //         << " gathered_non_local_rows " <<
-                //         gathered_non_local_rows
-                //         << " begin " << begin << " end " << end
-                //         << " non_local size_before " << size_before
-                //         << " non_local size_after " << size_after
-                //         << "\n";
-                // }
-                gathered_begin.push_back(size_before);
-                gathered_end.push_back(size_after);
-                gathered_ranks.push_back(to_rank);
+        // build vector with locality information
+        std::vector<std::pair<bool, label>> locality;
+
+        auto recv_counts = std::get<1>(span_comm_pattern);
+        label ctr{0};
+        for (int i = 0; i < ranks_per_gpu; i++) {
+            label comm_rank = rank + i;
+            label num_elems = recv_counts[comm_rank];
+            for (int j = 0; j < num_elems; j++) {
+                locality.emplace_back(is_local[ctr], comm_rank);
+                ctr++;
             }
         }
-        std::cout << __FILE__ << " rank " << rank
-                  << " done merged non_local4\n";
+        // All interfaces that are in is_local need to be accounted for
+        ASSERT_EQ(ctr, is_local.size());
 
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << " gathered_non_local.ranks" << gathered_non_local.ranks
+                  << "\n";
         // for compressed cols we compute a mapping first
         std::map<label, label> compressed_col_idxs{};
         label unique_idx = 0;
-        for (auto idx : gathered_non_local_cols) {
+        for (auto idx : gathered_non_local.cols) {
             auto search = compressed_col_idxs.find(idx);
             if (search == compressed_col_idxs.end()) {
-                std::cout << " insert " << idx << " " << unique_idx << "\n";
                 compressed_col_idxs[idx] = unique_idx;
                 unique_idx += 1;
             };
         }
-        std::cout << __FILE__ << " rank " << rank
-                  << " done merged non_local5\n";
 
-        for (int i = 0; i < gathered_non_local_cols.size(); i++) {
-            label uncompressed = gathered_non_local_cols[i];
+        for (int i = 0; i < gathered_non_local.cols.size(); i++) {
+            label uncompressed = gathered_non_local.cols[i];
             label compressed = compressed_col_idxs[uncompressed];
-            std::cout << " i " << i << " uncompressed " << uncompressed
-                      << " compressed " << compressed << "\n";
-            gathered_non_local_cols[i] = compressed;
+            gathered_non_local.cols[i] = compressed;
         }
-        std::cout << __FILE__ << " rank " << rank
-                  << " done merged non_local6\n";
-
-        // add interfaces
-        // to add interfaces we go through the non_local_sparsity
-        // pattern and check if interface is still non_local
-        //
-        // iterate interfaces and send to owner.
-        // on owner decide to move to local or non_local
-        // for sending to owner we can also use the all_to_all_v
-        // approach we than have
-        //
-        // row [ 4 , 8,  12 | 16 , 17, 18 ] <- local row
-        // col [ 1 , 2, 3 | 1, 2, 3 ] <- just the interface ctr
-        // from repartition_comm_pattern we could get
-        //
-        // or we split local / non-local first
-        // for this we could store ranges when computing
-        // repartition_comm_pattern
-        //
-
 
         if (is_owner()) {
             auto new_local_spars_pattern = std::make_shared<SparsityPattern>(
-                exec, merged_local_rows.size(), merged_local_dim,
-                merged_local_rows, merged_local_cols, merged_ldu_mapping,
-                merged_begin, merged_end, merged_ranks);
-
-            // if (rank == 0) {
-            // std::cout << __FILE__ << " on rank " << rank
-            //           << merged_local_rows
-            //           << " merged_begin " <<  merged_begin
-            //           << " merged_end " <<  merged_end
-            //           << "local dim ["
-            //           << new_local_spars_pattern->dim[0] << " x "
-            //           << new_local_spars_pattern->dim[1] << "] \n";
-            //         }
+                exec, merged_local_dim, merged_local);
 
             auto new_non_local_spars_pattern =
                 std::make_shared<SparsityPattern>(
-                    exec, gathered_non_local_rows.size(),
+                    exec,
                     gko::dim<2>{merged_local_dim[0],
-                                gathered_non_local_rows.size()},
-                    gathered_non_local_rows, gathered_non_local_cols,
-                    gathered_non_local_mapping, gathered_begin, gathered_end,
-                    gathered_ranks);
+                                gathered_non_local.rows.size()},
+                    gathered_non_local);
 
-            return std::make_pair<std::shared_ptr<SparsityPattern>,
-                                  std::shared_ptr<SparsityPattern>>(
+            return std::make_tuple<std::shared_ptr<SparsityPattern>,
+                                   std::shared_ptr<SparsityPattern>,
+                                   std::vector<std::pair<bool, label>>>(
                 std::move(new_local_spars_pattern),
-                std::move(new_non_local_spars_pattern));
+                std::move(new_non_local_spars_pattern), std::move(locality));
         } else {
             auto new_local_spars_pattern =
-                std::make_shared<SparsityPattern>(exec, 0);
+                std::make_shared<SparsityPattern>(exec);
 
             auto new_non_local_spars_pattern =
-                std::make_shared<SparsityPattern>(exec, 0);
+                std::make_shared<SparsityPattern>(exec);
 
             new_local_spars_pattern->dim = merged_local_dim;
             new_non_local_spars_pattern->dim = merged_local_dim;
 
-            return std::make_pair<std::shared_ptr<SparsityPattern>,
-                                  std::shared_ptr<SparsityPattern>>(
+            return std::make_tuple<std::shared_ptr<SparsityPattern>,
+                                   std::shared_ptr<SparsityPattern>,
+                                   std::vector<std::pair<bool, label>>>(
                 std::move(new_local_spars_pattern),
-                std::move(new_non_local_spars_pattern));
+                std::move(new_non_local_spars_pattern),
+                std::vector<std::pair<bool, label>>{});
         }
+        // TODO Some old documentation see what is still needed
+        // add interfaces
+        // to add interfaces we go through the non_local_sparsity
+        // pattern and check if interface is still non_local
+        //
+        // iterate interfaces and send to owner.
+        // on owner decide to move to local or non_local
+        // for sending to owner we can also use the all_to_all_v
+        // approach we than have
+        //
+        // row [ 4 , 8,  12 | 16 , 17, 18 ] <- local row
+        // col [ 1 , 2, 3 | 1, 2, 3 ] <- just the interface ctr
+        // from repartition_comm_pattern we could get
+        //
+        // or we split local / non-local first
+        // for this we could store ranges when computing
+        // repartition_comm_pattern
+        //
     }
 
     CommunicationPattern repartition_comm_pattern(
@@ -485,8 +444,6 @@ public:
                 send_idxs.push_back(src_comm_pattern.send_idxs[i]);
             }
         }
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << get_rank()
-                  << " target_ids.size() " << target_ids.size() << "\n";
 
         // send all remaining non local ids and sizes to the new
         // owner rank
diff --git a/Solver/BiCGStab/GKOBiCGStab.H b/Solver/BiCGStab/GKOBiCGStab.H
index da0714fe5..8dc0e5b6d 100644
--- a/Solver/BiCGStab/GKOBiCGStab.H
+++ b/Solver/BiCGStab/GKOBiCGStab.H
@@ -41,7 +41,7 @@ public:
         : solverControls_(solverControls),
           db_(db),
           sysMatrixName_(sysMatrixName),
-          stoppingCriterion_(solverControls){};
+          stoppingCriterion_(solverControls) {};
 
     CREATE_SOLVER_METHODS(bicgstab)
 
@@ -137,10 +137,10 @@ public:
                 const lduInterfaceFieldPtrsList &interfaces,
                 const dictionary &solverControls)
         : GKOlduBaseSolver(fieldName, matrix, interfaceBouCoeffs,
-                           interfaceIntCoeffs, interfaces, solverControls){};
+                           interfaceIntCoeffs, interfaces, solverControls) {};
 
     //- Destructor
-    virtual ~GKOBiCGStab(){};
+    virtual ~GKOBiCGStab() {};
 
 
     // Member Functions
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 8a1b792af..b7b233547 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -33,7 +33,7 @@ public:
         : solverControls_(solverControls),
           db_(db),
           sysMatrixName_(sysMatrixName),
-          stoppingCriterion_(solverControls){};
+          stoppingCriterion_(solverControls) {};
 
     CREATE_SOLVER_METHODS(cg)
 
@@ -130,10 +130,10 @@ public:
           const lduInterfaceFieldPtrsList &interfaces,
           const dictionary &solverControls)
         : GKOlduBaseSolver(fieldName, matrix, interfaceBouCoeffs,
-                           interfaceIntCoeffs, interfaces, solverControls){};
+                           interfaceIntCoeffs, interfaces, solverControls) {};
 
     //- Destructor
-    virtual ~GKOCG(){};
+    virtual ~GKOCG() {};
 
 
     // Member Functions
diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 794a79b43..218da11d5 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -38,7 +38,7 @@ public:
         : solverControls_(solverControls),
           db_(db),
           sysMatrixName_(sysMatrixName),
-          stoppingCriterion_(solverControls){};
+          stoppingCriterion_(solverControls) {};
 
     CREATE_SOLVER_METHODS(gmres)
 
@@ -137,10 +137,10 @@ public:
              const lduInterfaceFieldPtrsList &interfaces,
              const dictionary &solverControls)
         : GKOlduBaseSolver(fieldName, matrix, interfaceBouCoeffs,
-                           interfaceIntCoeffs, interfaces, solverControls){};
+                           interfaceIntCoeffs, interfaces, solverControls) {};
 
     //- Destructor
-    virtual ~GKOGMRES(){};
+    virtual ~GKOGMRES() {};
 
 
     // Member Functions
diff --git a/Solver/IR/GKOIR.H b/Solver/IR/GKOIR.H
index 61cc79869..fa645b1b6 100644
--- a/Solver/IR/GKOIR.H
+++ b/Solver/IR/GKOIR.H
@@ -48,7 +48,7 @@ public:
           db_(db),
           sysMatrixName_(sysMatrixName),
           outerStoppingCriterion_(solverControls),
-          innerStoppingCriterion_(StoppingCriterion(innerSolverControls_)){};
+          innerStoppingCriterion_(StoppingCriterion(innerSolverControls_)) {};
 
     CREATE_SOLVER_METHODS(ir)
 
@@ -161,10 +161,10 @@ public:
           const lduInterfaceFieldPtrsList &interfaces,
           const dictionary &solverControls)
         : GKOlduBaseSolver(fieldName, matrix, interfaceBouCoeffs,
-                           interfaceIntCoeffs, interfaces, solverControls){};
+                           interfaceIntCoeffs, interfaces, solverControls) {};
 
     //- Destructor
-    virtual ~GKOIR(){};
+    virtual ~GKOIR() {};
 
 
     // Member Functions
diff --git a/Solver/Multigrid/GKOMultigrid.H b/Solver/Multigrid/GKOMultigrid.H
index c8100a87d..6128b702a 100644
--- a/Solver/Multigrid/GKOMultigrid.H
+++ b/Solver/Multigrid/GKOMultigrid.H
@@ -83,7 +83,7 @@ public:
           max_levels_(
               innerSolverControls_.lookupOrDefault("maxLevels", label(9))),
           min_coarse_rows_(innerSolverControls_.lookupOrDefault("minCoarseRows",
-                                                                label(10))){};
+                                                                label(10))) {};
 
     std::unique_ptr<mg::Factory, std::default_delete<mg::Factory>>
     create_dist_solver(std::shared_ptr<const gko::Executor> exec,
@@ -258,10 +258,10 @@ public:
                  const lduInterfaceFieldPtrsList &interfaces,
                  const dictionary &solverControls)
         : GKOlduBaseSolver(fieldName, matrix, interfaceBouCoeffs,
-                           interfaceIntCoeffs, interfaces, solverControls){};
+                           interfaceIntCoeffs, interfaces, solverControls) {};
 
     //- Destructor
-    virtual ~GKOMultigrid(){};
+    virtual ~GKOMultigrid() {};
 
 
     // Member Functions
diff --git a/StoppingCriterion/StoppingCriterion.H b/StoppingCriterion/StoppingCriterion.H
index f3db2b536..e9350ed88 100644
--- a/StoppingCriterion/StoppingCriterion.H
+++ b/StoppingCriterion/StoppingCriterion.H
@@ -91,8 +91,7 @@ class StoppingCriterion {
          * or rewritten as || r - ( b - x* ) || + || (b - x*) ||
          *  */
         scalar compute_normfactor_dist(
-            std::shared_ptr<const gko::Executor> device_exec,
-            const dist_vec* r,
+            std::shared_ptr<const gko::Executor> device_exec, const dist_vec *r,
             std::shared_ptr<const gko::LinOp> gkomatrix,
             std::shared_ptr<const dist_vec> x,
             std::shared_ptr<const dist_vec> b) const;
@@ -190,13 +189,12 @@ public:
     }
 
     std::shared_ptr<const gko::stop::CriterionFactory>
-    build_dist_stopping_criterion(std::shared_ptr<const gko::Executor> device_exec,
-                                  std::shared_ptr<const gko::LinOp> gkomatrix,
-                                  std::shared_ptr<const dist_vec> x,
-                                  std::shared_ptr<const dist_vec> b,
-                                  label verbose, bool export_res,
-                                  label prev_solve_iters,
-                                  scalar prev_rel_cost) const
+    build_dist_stopping_criterion(
+        std::shared_ptr<const gko::Executor> device_exec,
+        std::shared_ptr<const gko::LinOp> gkomatrix,
+        std::shared_ptr<const dist_vec> x, std::shared_ptr<const dist_vec> b,
+        label verbose, bool export_res, label prev_solve_iters,
+        scalar prev_rel_cost) const
     {
         label minIter = minIter_;
         label frequency = frequency_;
diff --git a/common/common.H b/common/common.H
index 5daea8abb..0ef47b020 100644
--- a/common/common.H
+++ b/common/common.H
@@ -90,7 +90,7 @@ namespace Foam {
 
 #define UNUSED(x) (void)(x)
 
-#define ASSERT_EQ(_val1, _val2)                                            \
+#define ASSERT_EQ(_val1, _val2)                                                \
     if (_val1 != _val2) {                                                      \
         throw ::gko::ValueMismatch(__FILE__, __LINE__, __func__, _val1, _val2, \
                                    "expected equal values");                   \
@@ -98,9 +98,9 @@ namespace Foam {
 
 #define OGL_NOT_IMPLEMENTED GKO_NOT_IMPLEMENTED
 
-std::ostream& operator<<(std::ostream& out, const std::vector<label>& e);
+std::ostream &operator<<(std::ostream &out, const std::vector<label> &e);
 
-std::ostream& operator<<(std::ostream& out, const gko::array<label>& e); 
+std::ostream &operator<<(std::ostream &out, const gko::array<label> &e);
 
 std::ostream &operator<<(
     std::ostream &os,

From d0cd47e02e6658f39d1753602550b83e4db82660 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 2 Jun 2024 14:26:09 +0200
Subject: [PATCH 62/90] scale interface values with -1

---
 MatrixWrapper/Distributed/Distributed.H | 54 +++++++------------------
 1 file changed, 15 insertions(+), 39 deletions(-)

diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 720e0b810..3782f85d6 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -202,9 +202,8 @@ public:
                                                non_local_sparsity_);
 
         std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                << " repart_non_loc_sparsity.rank"
-                <<  repart_non_loc_sparsity->rank
-                << "\n";
+                  << " repart_non_loc_sparsity.rank"
+                  << repart_non_loc_sparsity->rank << "\n";
 
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
@@ -291,7 +290,7 @@ public:
         std::shared_ptr<const SparsityPattern> local_sparsity,
         std::shared_ptr<const SparsityPattern> non_local_sparsity,
         CommunicationPattern &src_comm_pattern,
-        std::vector<std::pair<bool,label>> local_interfaces)
+        std::vector<std::pair<bool, label>> local_interfaces)
     {
         auto &exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_ref_exec();
@@ -371,7 +370,7 @@ public:
             label comm_rank, comm_size;
             scalar *recv_buffer_ptr;
             label remain_host_interfaces = host_A->get_interface_size();
-            for (auto [is_local,comm_rank] : local_interfaces) {
+            for (auto [is_local, comm_rank] : local_interfaces) {
                 label &ctr = (is_local) ? loc_ctr : nloc_ctr;
                 if (is_local) {
                     mtx = gko::as<Coo>(
@@ -381,44 +380,19 @@ public:
                             ->get_operators()[ctr]);
                     comm_size = local_sparsity->interface_spans[ctr].length();
                 } else {
-                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                            << " non local update\n";
                     mtx = gko::as<Coo>(
                         gko::as<CombinationMatrix<scalar, label, Coo>>(
                             dist_A->get_non_local_matrix())
                             ->get_combination()
                             ->get_operators()[ctr]);
-                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                            << " non local update\n";
                     comm_size =
                         non_local_sparsity->interface_spans[ctr].length();
-                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                            << " non local update\n";
-                    // FIXME this is the communication target not the origin
-                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                            << " comm_rank " << comm_rank << " comm_size " << comm_size
-                            << " non local update\n";
                 }
                 recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
 
-                // TODO this test is not sufficient
-                // we can determine the number of local data interfaces from the
-                // original matrix since the local interfaces are stored first
-                //
-                //
-                // thus the first n interfaces are from this rank regardless of
-                // the locality
-                // only communicate if data is on on other rank, ie interface orignaly from other
-                // rank
                 if (comm_rank != rank) {
-                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                            << " recv from "
-                            << comm_rank << " size " << comm_size << "\n";
                     comm.recv(exec, recv_buffer_ptr, comm_size, comm_rank, tag);
                 } else {
-                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                            << " remain_host_interfaces "
-                            << remain_host_interfaces << " owner rank update\n";
                     // if data is already on this rank
                     auto data_view = gko::array<scalar>::const_view(
                         exec, comm_size,
@@ -433,7 +407,18 @@ public:
                     host_interface_ctr++;
                     remain_host_interfaces--;
                 }
+
                 ctr++;
+                // interface values need to be multiplied by -1
+                //
+                using vec = gko::matrix::Dense<ValueType>;
+                auto neg_one = gko::initialize<vec>({-1.0}, exec);
+                auto interface_dense = vec::create(
+                    exec, gko::dim<2>{comm_size, 1},
+                    gko::array<scalar>::view(exec, comm_size, recv_buffer_ptr),
+                    1);
+
+                interface_dense->scale(neg_one);
             }
         } else {
             // the non owner has send all its interfaces to owner
@@ -444,14 +429,9 @@ public:
                 label comm_size =
                     src_comm_pattern.target_sizes.get_const_data()[i];
                 const scalar *send_buffer_ptr = host_A->get_interface_data(i);
-                    std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                            << " send to "
-                            << owner_rank << " size " << comm_size << "\n";
                 comm.send(exec, send_buffer_ptr, comm_size, owner_rank, tag);
             }
         }
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << "done coeffs\n";
 
         // reorder updated values
         if (owner) {
@@ -479,10 +459,6 @@ public:
             // TODO this needs to copy ldu_mapping to the device
             auto dense_vec = row_collection->clone();
             dense_vec->row_gather(&mapping_view, row_collection.get());
-            if (rank == 0) {
-                std::cout << __FILE__ << " on rank " << rank
-                          << " done row gather\n";
-            }
         }
     };
 

From b1ed3432a88ce7629bbff80770229899406e0ca2 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 3 Jun 2024 10:57:39 +0200
Subject: [PATCH 63/90] a seemingly working cpu version

---
 DevicePersistent/Vector/Vector.H              | 18 ++--------
 MatrixWrapper/Combination/Combination.H       | 17 ++-------
 .../CommunicationPattern.C                    |  5 ---
 MatrixWrapper/Distributed/Distributed.H       | 36 ++++++-------------
 Repartitioner/Repartitioner.H                 | 31 ++++++----------
 StoppingCriterion/StoppingCriterion.C         | 16 ++++++++-
 6 files changed, 41 insertions(+), 82 deletions(-)

diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 02c55237c..16ada0d70 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -79,11 +79,8 @@ struct VectorInitFunctor {
         msg += (on_device_) ? " on device" : " on host";
         LOG_1(verbose_, msg)
 
-        std::cout << __FILE__ << __LINE__ << " on rank " << rank
-                  << " before host view \n";
-        auto host_view = gko::array<T>::view(exec_.get_ref_exec(), local_size,
-                                             const_cast<T *>(other_));
-        std::cout << __FILE__ << " on rank " << rank << " done host view \n";
+        auto host_view =
+            gko::array<T>::const_view(exec_.get_ref_exec(), local_size, other_);
 
         // FIXME this shouldn't be needed here also not during an
         // update
@@ -91,23 +88,14 @@ struct VectorInitFunctor {
             exec_, repartitioner_.get_ranks_per_gpu(), local_size);
 
         // this should happen on device
-        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank
-                  << " before local coeffs  \n"
-                  << std::endl;
         auto local_coeffs = gko::array<scalar>(exec, repart_size);
-        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank
-                  << " done local coeffs  \n";
 
-        communicate_values(exec_, comm_pattern, host_view.get_data(),
+        communicate_values(exec_, comm_pattern, host_view.get_const_data(),
                            local_coeffs.get_data());
-        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank
-                  << "  gather_to_owner done \n";
 
         auto ret = gko::share(dist_vec::create(
             exec, *comm.get(),
             vec::create(exec, gko::dim<2>{repart_size, 1}, local_coeffs, 1)));
-        std::cout << __FILE__ << ":" << __LINE__ << " on rank " << rank
-                  << "  done dist vector create \n";
         return ret;
     }
 };
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index 555fe32fe..93f551afd 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -33,7 +33,6 @@ public:
 
     void read(const device_matrix_data &data) override
     {
-        std::cout << __FILE__ << " read combination \n";
         // TODO FIXME if no matrix is given because of repartitioning
         // which might remove non_local matrix
         // a zero sized dummy matrix needs to be created
@@ -44,20 +43,6 @@ public:
             auto length = interface.length();
             auto offset = interface.begin;
             auto op = gko::share(InnerMatrixType::create(exec));
-            // auto rows = gko::array<IndexType>(
-            //         exec,
-            //         make_const_array_view(data_exec, length,
-            //                               data.get_const_row_idxs() + offset)
-            //             .copy_to_array());
-
-            // std::cout << __FILE__ << " read combination  interface"
-            //           << " data.size() " << data.get_num_stored_elements()
-            //           << " offset: " <<  offset
-            //           << " length: " <<  length
-            //           << " comb_->get_size() " << comb_->get_size()
-            //           << " rows " << rows
-            //           << " \n";
-
 
             device_matrix_data in(
                 exec, comb_->get_size(),
@@ -140,6 +125,7 @@ protected:
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
         if (interface_ranges_.size() > 0) {
+            std::cout << __FILE__ << ":" << __LINE__ << "apply\n";
             this->comb_->apply(b, x);
         }
     }
@@ -152,6 +138,7 @@ protected:
                     const gko::LinOp *beta, gko::LinOp *x) const override
     {
         if (interface_ranges_.size() > 0) {
+        std::cout << __FILE__ << ":" << __LINE__ << "apply 2\n";
             this->comb_->apply(alpha, b, beta, x);
         }
     }
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index 019a1279a..20c8c510d 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -159,14 +159,9 @@ std::vector<label> gather_to_owner(
 
     std::vector<label> recv_buffer(total_recv_size);
     auto rank = comm.rank();
-    std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-        << "before alltoallv\n";
     comm.all_to_all_v(exec, send_buffer, send_counts.data(),
                       send_offsets.data(), recv_buffer.data(),
                       recv_counts.data(), recv_offsets.data());
-    std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-        << "after alltoallv\n";
-
     return recv_buffer;
 }
 
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 3782f85d6..52d04c53b 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -197,6 +197,9 @@ public:
         label owner_rank = repartitioner.get_owner_rank();
         bool owner = repartitioner.is_owner();
 
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << " before repart sparsity \n";
+
         auto [repart_loc_sparsity, repart_non_loc_sparsity, local_interfaces] =
             repartitioner.repartition_sparsity(local_sparsity_,
                                                non_local_sparsity_);
@@ -208,12 +211,12 @@ public:
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
-        std::cout << __FILE__ << " rank " << rank
-                  << " build_localized_partition "
-                  << " dim " << local_sparsity_->dim[0] << " send idxs size "
-                  << dst_comm_pattern.send_idxs.size() << " target ids "
-                  << dst_comm_pattern.target_ids << " target sizes "
-                  << dst_comm_pattern.target_sizes << "\n";
+        // std::cout << __FILE__ << " rank " << rank
+        //           << " build_localized_partition "
+        //           << " dim " << local_sparsity_->dim[0] << " send idxs size "
+        //           << dst_comm_pattern.send_idxs.size() << " target ids "
+        //           << dst_comm_pattern.target_ids << " target sizes "
+        //           << dst_comm_pattern.target_sizes << "\n";
 
         auto localized_partition = local_part_type::build_from_blocked_recv(
             exec, local_sparsity_->dim[0], dst_comm_pattern.send_idxs,
@@ -240,8 +243,8 @@ public:
             matrix_format, exec, sparse_comm, local_sparsity_,
             non_local_sparsity_));
 
-        std::cout << __FILE__ << " rank " << rank
-                  << " done generate dist_mtx\n";
+        // std::cout << __FILE__ << " rank " << rank
+        //           << " done generate dist_mtx\n";
 
         auto local_coeffs = gko::array<scalar>(exec, local_sparsity_->size_);
         auto non_local_coeffs =
@@ -299,9 +302,6 @@ public:
         label rank{repartitioner.get_rank()};
         label owner_rank = repartitioner.get_owner_rank();
         bool owner = repartitioner.is_owner();
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << "done coeffs\n";
-
         label nrows = host_A->get_local_nrows();
         label local_matrix_nnz = host_A->get_local_matrix_nnz();
         // size + padding has to be local_matrix_nnz
@@ -320,13 +320,9 @@ public:
         scalar *local_ptr;
 
         // update main values
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << " is owner " << owner << "done coeffs\n";
         if (owner) {
             using Coo = gko::matrix::Coo<scalar, label>;
             auto local_mtx = dist_A->get_local_matrix();
-            std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                      << "done coeffs\n";
 
             std::shared_ptr<const Coo> local =
                 gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
@@ -335,16 +331,10 @@ public:
                                  ->get_operators()[0]);
             local_ptr = const_cast<scalar *>(local->get_const_values());
         }
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << "done coeffs\n";
         communicate_values(exec_handler, diag_comm_pattern, host_A->get_diag(),
                            local_ptr);
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << "done coeffs\n";
         communicate_values(exec_handler, upper_comm_pattern,
                            host_A->get_upper(), local_ptr);
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << "done coeffs\n";
 
         if (host_A->symmetric()) {
             // TODO FIXME
@@ -355,9 +345,6 @@ public:
             communicate_values(exec_handler, lower_comm_pattern,
                                host_A->get_lower(), local_ptr);
         }
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << "done coeffs\n";
-
 
         auto comm = *exec_handler.get_communicator().get();
         if (owner) {
@@ -555,7 +542,6 @@ protected:
     // of these are always true in this example).
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
-        std::cout << __FILE__ << __LINE__ << "call apply\n";
         this->dist_mtx_->apply(b, x);
     }
 
diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index 331166646..25b0525ff 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -252,7 +252,6 @@ public:
             std::vector{rank}};
         merged_local.end[0] = merged_local.rows.size();
 
-
         if (is_owner()) {
             make_ldu_mapping_consecutive(
                 local_comm_pattern, merged_local.mapping, rank, ranks_per_gpu);
@@ -305,10 +304,6 @@ public:
                             src_non_local_pattern->rank.size(),
                             src_non_local_pattern->rank.data())};
 
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << " merged_non_local.ranks" << merged_non_local.ranks
-                  << "\n";
-
         if (is_owner()) {
             make_begin_and_ends_consecutive(
                 span_comm_pattern, merged_non_local.begin, merged_non_local.end,
@@ -320,23 +315,22 @@ public:
 
         // build vector with locality information
         std::vector<std::pair<bool, label>> locality;
-
-        auto recv_counts = std::get<1>(span_comm_pattern);
         label ctr{0};
-        for (int i = 0; i < ranks_per_gpu; i++) {
-            label comm_rank = rank + i;
-            label num_elems = recv_counts[comm_rank];
-            for (int j = 0; j < num_elems; j++) {
-                locality.emplace_back(is_local[ctr], comm_rank);
-                ctr++;
+
+        if (is_owner()) {
+            auto recv_counts = std::get<1>(span_comm_pattern);
+            for (int i = 0; i < ranks_per_gpu; i++) {
+                label comm_rank = rank + i;
+                label num_elems = recv_counts[comm_rank];
+                for (int j = 0; j < num_elems; j++) {
+                    locality.emplace_back(is_local[ctr], comm_rank);
+                    ctr++;
+                }
             }
         }
         // All interfaces that are in is_local need to be accounted for
         ASSERT_EQ(ctr, is_local.size());
 
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << " gathered_non_local.ranks" << gathered_non_local.ranks
-                  << "\n";
         // for compressed cols we compute a mapping first
         std::map<label, label> compressed_col_idxs{};
         label unique_idx = 0;
@@ -457,9 +451,6 @@ public:
             gather_to_owner(exec_handler, comm_pattern, target_sizes.size(),
                             target_sizes.data());
 
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << get_rank()
-                  << " gathered_target_ids " << gathered_target_ids << "\n";
-
         // next the send_ixs need to be updated we send them piecewise since
         // the send_idxs are a vector of gko::arrays
         label rank = get_rank();
@@ -568,8 +559,6 @@ public:
                     std::vector<label>(send_idx_begin, send_idx_end));
             }
         }
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << get_rank()
-                  << " gathered_target_ids " << merged_target_ids << "\n";
 
         // recompute send_idxs
         send_idxs.clear();
diff --git a/StoppingCriterion/StoppingCriterion.C b/StoppingCriterion/StoppingCriterion.C
index 9357866b0..ee1fe4ef6 100644
--- a/StoppingCriterion/StoppingCriterion.C
+++ b/StoppingCriterion/StoppingCriterion.C
@@ -20,13 +20,19 @@ void StoppingCriterion::OpenFOAMDistStoppingCriterion::compute_Axref_dist(
 
     auto xAvg_host = gko::initialize<gko::matrix::Dense<scalar>>(
         1, {0}, device_exec->get_master());
+
     xAvg->move_to(xAvg_host);
     auto xAvg_vec = gko::share(dist_vec::create(
         device_exec, x->get_communicator(), gko::dim<2>{global_size, 1},
         gko::dim<2>{local_size, 1}));
     xAvg_vec->fill(xAvg_host->at(0));
 
+    ASSERT_EQ(xAvg_host->at(0), xAvg_host->at(0));
+
+    std::cout << __FILE__ << ":" << __LINE__
+              << "Axref apply\n";
     gkomatrix->apply(xAvg_vec.get(), res.get());
+    std::cout << __FILE__ << ":" << __LINE__ << "done Axref apply\n";
 }
 
 scalar
@@ -41,6 +47,10 @@ StoppingCriterion::OpenFOAMDistStoppingCriterion::compute_normfactor_dist(
     gko::dim<2> local_size = x->get_local_vector()->get_size();
     gko::dim<2> global_size = x->get_size();
 
+    std::cout << __FILE__ << ":" << __LINE__
+              << " setup Axref local_size" << local_size
+              << " global size " << global_size
+              << "before Axref apply\n";
     auto Axref = gko::share(
         dist_vec::create(device_exec, comm, global_size, local_size));
     Axref->fill(0.0);
@@ -74,7 +84,6 @@ bool StoppingCriterion::OpenFOAMDistStoppingCriterion::check_impl(
     gko::array<gko::stopping_status> *stop_status, bool *one_changed,
     const Criterion::Updater &updater)
 {
-    std::cout << __FILE__ << ":" << __LINE__ << "Stopping check \n";
     // Dont check residual norm before minIter is reached
     if (*(parameters_.iter) > 0 &&
         *(parameters_.iter) < parameters_.openfoam_minIter) {
@@ -98,6 +107,11 @@ bool StoppingCriterion::OpenFOAMDistStoppingCriterion::check_impl(
     norm1_host->copy_from(norm1.get());
     scalar residual_norm = norm1_host->at(0);
 
+    if (residual_norm != residual_norm){
+            FatalErrorInFunction << " Problem with residual norm detected: " << residual_norm
+                                 << exit(FatalError);
+    }
+
     bool result = false;
 
     // Store initial residual

From b5ea6cbac1043a09458a7a1ed9ac2f7c791440a4 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 3 Jun 2024 14:34:11 +0200
Subject: [PATCH 64/90] fixup!

---
 MatrixWrapper/LDUMatrix/HostMatrix.H | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 03a4ca2b6..24d3df1c7 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -415,11 +415,11 @@ public:
 
     label get_interface_size() const {
         return interface_ptr_.size();
-    } const;
+    };
 
     const scalar* get_interface_data(label i) const {
         return interface_ptr_[i];
-    } const;
+    };
 
     bool get_verbose() const { return verbose_; }
 

From 539224239e9d34f47dc335bd333ea3ca940703fa Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 3 Jun 2024 20:52:55 +0200
Subject: [PATCH 65/90] quickfix single comm partner case

---
 Repartitioner/Repartitioner.H | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index 25b0525ff..28e560fca 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -439,6 +439,8 @@ public:
             }
         }
 
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " <<  comm.rank() << " target_ids " << target_ids << "\n"; 
+
         // send all remaining non local ids and sizes to the new
         // owner rank
         auto comm_pattern = compute_send_recv_counts(
@@ -451,6 +453,8 @@ public:
             gather_to_owner(exec_handler, comm_pattern, target_sizes.size(),
                             target_sizes.data());
 
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " <<  comm.rank() << " gathered_target_ids " << gathered_target_ids << "\n"; 
+
         // next the send_ixs need to be updated we send them piecewise since
         // the send_idxs are a vector of gko::arrays
         label rank = get_rank();
@@ -533,13 +537,13 @@ public:
 
         // Step 4.
         // Merge communication pattern with corresponding neighbours
-        if (target_ids.size() > 1) {
+        // if (target_ids.size() > 0) {
             merged_target_ids.push_back(target_ids[0]);
             merged_target_sizes.push_back(target_sizes[0]);
             merged_send_idxs.emplace_back(std::vector<label>(
                 send_idxs[0].first.get_data(),
                 send_idxs[0].first.get_data() + send_idxs[0].first.get_size()));
-        }
+        // }
 
         for (int i = 1; i < target_ids.size(); i++) {
             // communicates with same target rank

From b77cfe8ae3554dc0196f5e0f73238c18ce227f89 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 4 Jun 2024 10:55:17 +0200
Subject: [PATCH 66/90] call update, use shared_ptr for CommPattern

---
 MatrixWrapper/Distributed/Distributed.H | 62 ++++++++++++++++---------
 MatrixWrapper/LDUMatrix/HostMatrix.C    |  4 +-
 MatrixWrapper/LDUMatrix/HostMatrix.H    |  2 +-
 Repartitioner/Repartitioner.H           | 28 +++++------
 4 files changed, 56 insertions(+), 40 deletions(-)

diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 52d04c53b..5fb952c21 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -193,7 +193,7 @@ public:
         auto dst_comm_pattern = repartitioner.repartition_comm_pattern(
             src_comm_pattern, orig_partition);
 
-        label rank{src_comm_pattern.get_comm().rank()};
+        label rank{src_comm_pattern->get_comm().rank()};
         label owner_rank = repartitioner.get_owner_rank();
         bool owner = repartitioner.is_owner();
 
@@ -219,8 +219,8 @@ public:
         //           << dst_comm_pattern.target_sizes << "\n";
 
         auto localized_partition = local_part_type::build_from_blocked_recv(
-            exec, local_sparsity_->dim[0], dst_comm_pattern.send_idxs,
-            dst_comm_pattern.target_ids, dst_comm_pattern.target_sizes);
+            exec, local_sparsity_->dim[0], dst_comm_pattern->send_idxs,
+            dst_comm_pattern->target_ids, dst_comm_pattern->target_sizes);
 
         std::cout << __FILE__ << " rank " << rank << " local sparsity size "
                   << local_sparsity_->size_ << " local sparsity dim ["
@@ -228,9 +228,9 @@ public:
                   << "] non_local sparsity size " << non_local_sparsity_->size_
                   << " non local sparsity dim [" << non_local_sparsity_->dim[0]
                   << "x" << non_local_sparsity_->dim[1] << "] target_ids "
-                  << dst_comm_pattern.target_ids << " target_sizes "
-                  << dst_comm_pattern.target_sizes << " target_send_idxs.size "
-                  << dst_comm_pattern.send_idxs.size()
+                  << dst_comm_pattern->target_ids << " target_sizes "
+                  << dst_comm_pattern->target_sizes << " target_send_idxs.size "
+                  << dst_comm_pattern->send_idxs.size()
                   << " non_local_sparsity.size " << non_local_sparsity_->size_
                   << " get_recv_indices "
                   << localized_partition->get_recv_indices().get_num_elems()
@@ -243,9 +243,6 @@ public:
             matrix_format, exec, sparse_comm, local_sparsity_,
             non_local_sparsity_));
 
-        // std::cout << __FILE__ << " rank " << rank
-        //           << " done generate dist_mtx\n";
-
         auto local_coeffs = gko::array<scalar>(exec, local_sparsity_->size_);
         auto non_local_coeffs =
             gko::array<scalar>(exec, non_local_sparsity_->size_);
@@ -273,14 +270,18 @@ public:
 
         auto ret = std::make_shared<RepartDistMatrix>(
             exec, comm, repartitioner.get_repart_dim(), dist_A->get_size(),
-            std::move(dist_A));
+            std::move(dist_A), local_sparsity_, non_local_sparsity_,
+            src_comm_pattern, local_interfaces);
 
         return ret;
     };
 
     void update(word matrix_format, const Repartitioner &repartitioner,
-                std::shared_ptr<const HostMatrixWrapper> host_A) {
-
+                std::shared_ptr<const HostMatrixWrapper> host_A)
+    {
+        update_impl(matrix_format, repartitioner, host_A, dist_mtx_,
+                    local_sparsity_, non_local_sparsity_, src_comm_pattern_,
+                    local_interfaces_);
     };
 
     // TODO matrix_format can be data member
@@ -292,7 +293,7 @@ public:
             dist_A,
         std::shared_ptr<const SparsityPattern> local_sparsity,
         std::shared_ptr<const SparsityPattern> non_local_sparsity,
-        CommunicationPattern &src_comm_pattern,
+        std::shared_ptr<const CommunicationPattern> src_comm_pattern,
         std::vector<std::pair<bool, label>> local_interfaces)
     {
         auto &exec_handler = host_A.get()->get_exec_handler();
@@ -304,6 +305,8 @@ public:
         bool owner = repartitioner.is_owner();
         label nrows = host_A->get_local_nrows();
         label local_matrix_nnz = host_A->get_local_matrix_nnz();
+        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
+                  << " call update_impl \n";
         // size + padding has to be local_matrix_nnz
         auto diag_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, nrows, local_matrix_nnz,
@@ -410,11 +413,11 @@ public:
         } else {
             // the non owner has send all its interfaces to owner
             // thus all values need to be communicated to the owner as well
-            label num_interfaces = src_comm_pattern.target_ids.get_size();
+            label num_interfaces = src_comm_pattern->target_ids.get_size();
             label tag = 0;
             for (int i = 0; i < num_interfaces; i++) {
                 label comm_size =
-                    src_comm_pattern.target_sizes.get_const_data()[i];
+                    src_comm_pattern->target_sizes.get_const_data()[i];
                 const scalar *send_buffer_ptr = host_A->get_interface_data(i);
                 comm.send(exec, send_buffer_ptr, comm_size, owner_rank, tag);
             }
@@ -449,13 +452,21 @@ public:
         }
     };
 
-    RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
-                     communicator comm, gko::dim<2> local_size,
-                     gko::dim<2> global_size,
-                     std::shared_ptr<dist_mtx> dist_mtx)
+    RepartDistMatrix(
+        std::shared_ptr<const gko::Executor> exec, communicator comm,
+        gko::dim<2> local_size, gko::dim<2> global_size,
+        std::shared_ptr<dist_mtx> dist_mtx,
+        std::shared_ptr<const SparsityPattern> local_sparsity,
+        std::shared_ptr<const SparsityPattern> non_local_sparsity,
+        std::shared_ptr<const CommunicationPattern> src_comm_pattern,
+        std::vector<std::pair<bool, label>> &local_interfaces)
         : gko::experimental::EnableDistributedLinOp<RepartDistMatrix>(exec),
           gko::experimental::distributed::DistributedBase(comm),
-          dist_mtx_(std::move(dist_mtx))
+          dist_mtx_(std::move(dist_mtx)),
+          local_sparsity_(local_sparsity),
+          non_local_sparsity_(non_local_sparsity),
+          src_comm_pattern_(src_comm_pattern),
+          local_interfaces_(local_interfaces)
     {
         this->set_size(global_size);
         // partition_ = gko::share(
@@ -525,6 +536,7 @@ public:
         }
     }
 
+    // Needed for distributed/polymorphic_object.hpp
     RepartDistMatrix(std::shared_ptr<const gko::Executor> exec,
                      communicator comm)
         : gko::experimental::EnableDistributedLinOp<RepartDistMatrix>(exec),
@@ -561,9 +573,13 @@ protected:
 private:
     std::shared_ptr<dist_mtx> dist_mtx_;
 
-    std::shared_ptr<part_type> partition_;
+    std::shared_ptr<const part_type> partition_;
+
+    std::shared_ptr<const SparsityPattern> local_sparsity_;
+
+    std::shared_ptr<const SparsityPattern> non_local_sparsity_;
 
-    std::shared_ptr<SparsityPattern> local_sparsity_;
+    std::shared_ptr<const CommunicationPattern> src_comm_pattern_;
 
-    std::shared_ptr<SparsityPattern> non_local_sparsity_;
+    std::vector<std::pair<bool, label>> local_interfaces_;
 };
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index dbcbad4bf..8fad1a278 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -204,7 +204,7 @@ void neg_interface_iterator(const lduInterfaceFieldPtrsList &interfaces,
 }
 
 
-CommunicationPattern HostMatrixWrapper::create_communication_pattern(
+std::shared_ptr<CommunicationPattern> HostMatrixWrapper::create_communication_pattern(
     const gko::experimental::mpi::communicator &comm) const
 {
     using comm_size_type = CommunicationPattern::comm_size_type;
@@ -253,7 +253,7 @@ CommunicationPattern HostMatrixWrapper::create_communication_pattern(
         iter++;
     }
 
-    return CommunicationPattern{exec_, target_ids, target_sizes, send_idxs};
+    return std::make_shared<CommunicationPattern>(exec_, target_ids, target_sizes, send_idxs);
 }
 
 
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 24d3df1c7..f5fa49725 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -410,7 +410,7 @@ public:
      ** @param interfaces The list of interfaces for the search
      ** @return the CommunicationPattern
      */
-    CommunicationPattern create_communication_pattern(
+    std::shared_ptr<CommunicationPattern> create_communication_pattern(
         const gko::experimental::mpi::communicator &comm) const;
 
     label get_interface_size() const {
diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index 28e560fca..87ef0448a 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -401,8 +401,8 @@ public:
         //
     }
 
-    CommunicationPattern repartition_comm_pattern(
-        CommunicationPattern &src_comm_pattern,
+    std::shared_ptr<const CommunicationPattern> repartition_comm_pattern(
+        std::shared_ptr<const CommunicationPattern> src_comm_pattern,
         std::shared_ptr<
             const gko::experimental::distributed::Partition<label, label>>
             partition) const
@@ -410,12 +410,12 @@ public:
         using comm_size_type = label;
 
         if (ranks_per_gpu_ == 1) {
-            return CommunicationPattern(src_comm_pattern);
+            return src_comm_pattern;
         }
 
-        auto &exec_handler = src_comm_pattern.exec_handler;
-        auto exec = src_comm_pattern.target_ids.get_executor();
-        auto comm = src_comm_pattern.get_comm();
+        auto &exec_handler = src_comm_pattern->exec_handler;
+        auto exec = src_comm_pattern->target_ids.get_executor();
+        auto comm = src_comm_pattern->get_comm();
 
         // Step 1. Check if communication partner is non-local after
         // repartitioning. If it is non-local we keep it. Otherwise local
@@ -424,9 +424,9 @@ public:
         std::vector<label> target_ids{};
         std::vector<label> target_sizes{};
         std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
-        label communication_partner = src_comm_pattern.target_ids.get_size();
+        label communication_partner = src_comm_pattern->target_ids.get_size();
         for (int i = 0; i < communication_partner; i++) {
-            label target_id = src_comm_pattern.target_ids.get_data()[i];
+            label target_id = src_comm_pattern->target_ids.get_const_data()[i];
             if (!reparts_to_local(target_id)) {
                 // communication pattern is non local, hence we keep it
                 // after repartitioning we now have to communicate with a
@@ -434,8 +434,8 @@ public:
                 // needs to be send to
                 target_ids.push_back(get_owner_rank(target_id));
                 target_sizes.push_back(
-                    src_comm_pattern.target_sizes.get_const_data()[i]);
-                send_idxs.push_back(src_comm_pattern.send_idxs[i]);
+                    src_comm_pattern->target_sizes.get_const_data()[i]);
+                send_idxs.push_back(src_comm_pattern->send_idxs[i]);
             }
         }
 
@@ -511,14 +511,14 @@ public:
             target_ids.emplace_back(get_rank());
             target_sizes.emplace_back(0);
 
-            return CommunicationPattern{
+            return std::make_shared<CommunicationPattern>(
                 exec_handler,
                 gko::array<comm_size_type>{exec, target_ids.begin(),
                                            target_ids.end()},
                 gko::array<comm_size_type>{exec, target_sizes.begin(),
                                            target_sizes.end()},
 
-                send_idxs};
+                send_idxs);
         }
 
         // merge communication
@@ -575,12 +575,12 @@ public:
                 target_id});
         }
 
-        return CommunicationPattern{
+        return std::make_shared<CommunicationPattern>(
             exec_handler,
             gko::array<comm_size_type>{exec, merged_target_ids.begin(),
                                        merged_target_ids.end()},
             gko::array<comm_size_type>{exec, merged_target_sizes.begin(),
                                        merged_target_sizes.end()},
-            send_idxs};
+            send_idxs);
     }
 };

From e41a3562a6fcc5456a92f0e993fc9559384b92cc Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 4 Jun 2024 11:47:08 +0200
Subject: [PATCH 67/90] add a early return if no comm partners are left

---
 Repartitioner/Repartitioner.H | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index 87ef0448a..0e46bb15d 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -439,8 +439,6 @@ public:
             }
         }
 
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " <<  comm.rank() << " target_ids " << target_ids << "\n"; 
-
         // send all remaining non local ids and sizes to the new
         // owner rank
         auto comm_pattern = compute_send_recv_counts(
@@ -453,8 +451,6 @@ public:
             gather_to_owner(exec_handler, comm_pattern, target_sizes.size(),
                             target_sizes.data());
 
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " <<  comm.rank() << " gathered_target_ids " << gathered_target_ids << "\n"; 
-
         // next the send_ixs need to be updated we send them piecewise since
         // the send_idxs are a vector of gko::arrays
         label rank = get_rank();
@@ -507,10 +503,10 @@ public:
             target_ids.clear();
             target_sizes.clear();
             send_idxs.clear();
+        }
 
-            target_ids.emplace_back(get_rank());
-            target_sizes.emplace_back(0);
-
+        // early return if no communication partners are left
+        if (target_sizes.size() == 0) {
             return std::make_shared<CommunicationPattern>(
                 exec_handler,
                 gko::array<comm_size_type>{exec, target_ids.begin(),
@@ -537,13 +533,11 @@ public:
 
         // Step 4.
         // Merge communication pattern with corresponding neighbours
-        // if (target_ids.size() > 0) {
-            merged_target_ids.push_back(target_ids[0]);
-            merged_target_sizes.push_back(target_sizes[0]);
-            merged_send_idxs.emplace_back(std::vector<label>(
-                send_idxs[0].first.get_data(),
-                send_idxs[0].first.get_data() + send_idxs[0].first.get_size()));
-        // }
+        merged_target_ids.push_back(target_ids[0]);
+        merged_target_sizes.push_back(target_sizes[0]);
+        merged_send_idxs.emplace_back(std::vector<label>(
+            send_idxs[0].first.get_data(),
+            send_idxs[0].first.get_data() + send_idxs[0].first.get_size()));
 
         for (int i = 1; i < target_ids.size(); i++) {
             // communicates with same target rank

From 7b30e2d97dcb9113e3af16ee4e79d66e4c542943 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 4 Jun 2024 11:47:35 +0200
Subject: [PATCH 68/90] clean up

---
 MatrixWrapper/Distributed/Distributed.H | 35 +++++++++----------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 5fb952c21..3d93e20db 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -197,17 +197,10 @@ public:
         label owner_rank = repartitioner.get_owner_rank();
         bool owner = repartitioner.is_owner();
 
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << " before repart sparsity \n";
-
         auto [repart_loc_sparsity, repart_non_loc_sparsity, local_interfaces] =
             repartitioner.repartition_sparsity(local_sparsity_,
                                                non_local_sparsity_);
 
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << " repart_non_loc_sparsity.rank"
-                  << repart_non_loc_sparsity->rank << "\n";
-
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
@@ -222,19 +215,19 @@ public:
             exec, local_sparsity_->dim[0], dst_comm_pattern->send_idxs,
             dst_comm_pattern->target_ids, dst_comm_pattern->target_sizes);
 
-        std::cout << __FILE__ << " rank " << rank << " local sparsity size "
-                  << local_sparsity_->size_ << " local sparsity dim ["
-                  << local_sparsity_->dim[0] << "x" << local_sparsity_->dim[1]
-                  << "] non_local sparsity size " << non_local_sparsity_->size_
-                  << " non local sparsity dim [" << non_local_sparsity_->dim[0]
-                  << "x" << non_local_sparsity_->dim[1] << "] target_ids "
-                  << dst_comm_pattern->target_ids << " target_sizes "
-                  << dst_comm_pattern->target_sizes << " target_send_idxs.size "
-                  << dst_comm_pattern->send_idxs.size()
-                  << " non_local_sparsity.size " << non_local_sparsity_->size_
-                  << " get_recv_indices "
-                  << localized_partition->get_recv_indices().get_num_elems()
-                  << " \n";
+        // std::cout << __FILE__ << " rank " << rank << " local sparsity size "
+        //           << local_sparsity_->size_ << " local sparsity dim ["
+        //           << local_sparsity_->dim[0] << "x" << local_sparsity_->dim[1]
+        //           << "] non_local sparsity size " << non_local_sparsity_->size_
+        //           << " non local sparsity dim [" << non_local_sparsity_->dim[0]
+        //           << "x" << non_local_sparsity_->dim[1] << "] target_ids "
+        //           << dst_comm_pattern->target_ids << " target_sizes "
+        //           << dst_comm_pattern->target_sizes << " target_send_idxs.size "
+        //           << dst_comm_pattern->send_idxs.size()
+        //           << " non_local_sparsity.size " << non_local_sparsity_->size_
+        //           << " get_recv_indices "
+        //           << localized_partition->get_recv_indices().get_num_elems()
+        //           << " \n";
 
         auto sparse_comm =
             sparse_communicator::create(comm, localized_partition);
@@ -305,8 +298,6 @@ public:
         bool owner = repartitioner.is_owner();
         label nrows = host_A->get_local_nrows();
         label local_matrix_nnz = host_A->get_local_matrix_nnz();
-        std::cout << __FILE__ << ":" << __LINE__ << "rank " << rank
-                  << " call update_impl \n";
         // size + padding has to be local_matrix_nnz
         auto diag_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, nrows, local_matrix_nnz,

From 0e8843a42852b8f468c5dd6757a6dbb4607d36da Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 5 Jun 2024 12:50:20 +0200
Subject: [PATCH 69/90] implement copy back, some clean up, modified non local
 row compress

---
 DevicePersistent/Vector/Vector.H              | 30 ++++++----
 MatrixWrapper/Combination/Combination.H       |  2 -
 .../CommunicationPattern.C                    | 55 ++++++++++++++++---
 .../CommunicationPattern.H                    | 15 +++++
 MatrixWrapper/Distributed/Distributed.H       | 23 +++++---
 Repartitioner/Repartitioner.H                 | 22 +++-----
 lduLduBase/lduLduBase.H                       |  3 +-
 7 files changed, 104 insertions(+), 46 deletions(-)

diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 16ada0d70..5258353e4 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -123,7 +123,10 @@ class PersistentVector
     // updated even if was found in the object registry
     const bool update_;
 
+    const Repartitioner & repartitioner_;
+
 public:
+
     /* PersistentVector constructor using existing memory
      *
      * @param memory ptr to memory on host from which the gko array is
@@ -154,22 +157,27 @@ public:
           memory_(memory),
           size_(size),
           exec_(exec),
-          update_(update)
+          update_(update),
+          repartitioner_(repartitioner)
     {}
 
     /** Copies the content of the distributed vector back to the original source
      **/
-    void copy_back(std::shared_ptr<const part_type> partition)
+    void copy_back()
     {
-        // TODO FIXME
-        // auto ref_exec = exec_.get_ref_exec();
-        // auto host_buffer_view = gko::array<T>::view(
-        //     exec_.get_device_exec(), size_,
-        //     get_vector()->get_local_values());
-
-        // auto to_view =
-        //     gko::array<T>::view(ref_exec, size_, const_cast<T *>(memory_));
-        // to_view = host_buffer_view;
+        auto comm_pattern = compute_scatter_counts(
+            exec_, repartitioner_.get_ranks_per_gpu(), size_);
+
+        // std::cout << __FILE__ << ":" << __LINE__ << " rank " << repartitioner_.get_rank()
+        //     << " comm_pattern send " << std::get<0>(comm_pattern)
+        //     << " comm_pattern recv " << std::get<1>(comm_pattern)
+        //     << " comm_pattern send_offs " << std::get<2>(comm_pattern)
+        //     << " comm_pattern recv_offs " << std::get<3>(comm_pattern)
+        //     << "\n";
+
+        communicate_values(exec_, comm_pattern,
+                           get_vector()->get_local_values(),
+                           const_cast<T *>(memory_));
     }
 
     /** Writes the content of the distributed vector to disk
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index 93f551afd..e2c8b8c58 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -125,7 +125,6 @@ protected:
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
         if (interface_ranges_.size() > 0) {
-            std::cout << __FILE__ << ":" << __LINE__ << "apply\n";
             this->comb_->apply(b, x);
         }
     }
@@ -138,7 +137,6 @@ protected:
                     const gko::LinOp *beta, gko::LinOp *x) const override
     {
         if (interface_ranges_.size() > 0) {
-        std::cout << __FILE__ << ":" << __LINE__ << "apply 2\n";
             this->comb_->apply(alpha, b, beta, x);
         }
     }
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index 20c8c510d..2eda49241 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -34,8 +34,7 @@ label compute_owner_rank(label rank, label ranks_per_gpu)
     return rank - (rank % ranks_per_gpu);
 };
 
-std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-           std::vector<label>>
+CommCounts
 compute_send_recv_counts(const ExecutorHandler &exec_handler,
                          label ranks_per_gpu, label size)
 {
@@ -43,8 +42,49 @@ compute_send_recv_counts(const ExecutorHandler &exec_handler,
                                     0);
 }
 
-std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-           std::vector<label>>
+CommCounts
+compute_scatter_counts(const ExecutorHandler &exec_handler,
+                         label ranks_per_gpu, label size)
+{
+    auto exec = exec_handler.get_device_exec();
+    auto comm = *exec_handler.get_communicator().get();
+
+    label total_ranks{comm.size()};
+    label rank{comm.rank()};
+    label owner_rank = compute_owner_rank(rank, ranks_per_gpu);
+    std::vector<label> send_counts(total_ranks, 0);
+    std::vector<label> recv_counts(total_ranks, 0);
+    std::vector<label> send_offsets(total_ranks, 0);
+    std::vector<label> recv_offsets(total_ranks, 0);
+
+    label tot_recv_elements{0};
+    label comm_elements_buffer{0};
+    if (rank == owner_rank) {
+        // send and recv to it self
+        recv_offsets[owner_rank] = 0;
+        send_counts[owner_rank] = size;
+        recv_counts[owner_rank] = size;
+        // the start of the next rank data
+        tot_recv_elements = size;
+
+        for (int i = 1; i < ranks_per_gpu; i++) {
+            // receive the recv counts
+            comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
+            send_counts[rank + i] = comm_elements_buffer;
+            send_offsets[rank + i] = tot_recv_elements;
+            tot_recv_elements += comm_elements_buffer;
+        }
+    } else {
+        // send how many elements to communicate
+        comm.send(exec, &size, 1, owner_rank, owner_rank);
+        recv_counts[owner_rank] = size;
+    }
+
+    return std::make_tuple(send_counts, recv_counts, send_offsets,
+                           recv_offsets);
+}
+
+CommCounts
 compute_send_recv_counts(const ExecutorHandler &exec_handler,
                          label ranks_per_gpu, label size, label total_size,
                          label padding_before, label padding_after)
@@ -104,10 +144,10 @@ compute_send_recv_counts(const ExecutorHandler &exec_handler,
                            recv_offsets);
 }
 
+
 void communicate_values (
     const ExecutorHandler &exec_handler,
-    const std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-                     std::vector<label>> &comm_pattern,
+    const CommCounts &comm_pattern,
     const scalar *send_buffer, scalar *recv_buffer)
 {
     auto exec = exec_handler.get_device_exec();
@@ -134,8 +174,7 @@ void communicate_values (
 
 std::vector<label> gather_to_owner(
     const ExecutorHandler &exec_handler,
-    const std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
-                     std::vector<label>> &comm_pattern,
+    const CommCounts &comm_pattern,
     label size, const label *data, label offset)
 {
     auto exec = exec_handler.get_ref_exec();
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 5fcf512f2..10a3c38ad 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -37,7 +37,14 @@ SourceFiles
 using CommCounts = std::tuple<std::vector<label>, std::vector<label>,
                               std::vector<label>, std::vector<label>>;
 
+CommCounts
+invert_comm_pattern(CommCounts in_pattern);
 
+CommCounts
+compute_scatter_counts(const ExecutorHandler &exec_handler,
+                         label ranks_per_gpu, label size);
+
+// TODO rename function to compute_gather_to_owner_comm
 /* @brief  This function computes the send and recv counts vectors for
  *repartitioning to an owner rank
  *
@@ -59,6 +66,14 @@ CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
 CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
                                     label ranks_per_gpu, label size);
 
+/* @brief  This function computes the send and recv counts vectors for
+ *repartitioning to an owner rank
+ *
+ * @parameter size how many elements to send to owner
+ * */
+CommCounts compute_scatter_from_owner_comm(const ExecutorHandler &exec_handler,
+                                    label ranks_per_gpu, label size);
+
 /* Given a rank id and ranks_per_gpus this function computes the corresponding
  * owner rank  */
 label compute_owner_rank(label rank, label ranks_per_gpu);
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 3d93e20db..92c996b71 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -157,17 +157,14 @@ public:
             gko::experimental::EnableDistributedLinOp<
                 RepartDistMatrix>::operator=(std::move(other));
             this->dist_mtx_ = std::move(other.dist_mtx_);
-            this->set_size(this->dist_mtx_->get_size());
+            this->local_sparsity_ =std::move(other.local_sparsity_);
+            this->non_local_sparsity_ =std::move(other.non_local_sparsity_);
+            this->src_comm_pattern_ =std::move(other.src_comm_pattern_);
+            this->local_interfaces_ =std::move(other.local_interfaces_);
         }
         return *this;
     }
 
-
-    std::shared_ptr<const part_type> get_partition() const
-    {
-        return this->partition_;
-    }
-
     static std::shared_ptr<RepartDistMatrix> create(
         word matrix_format, const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A)
@@ -272,6 +269,7 @@ public:
     void update(word matrix_format, const Repartitioner &repartitioner,
                 std::shared_ptr<const HostMatrixWrapper> host_A)
     {
+
         update_impl(matrix_format, repartitioner, host_A, dist_mtx_,
                     local_sparsity_, non_local_sparsity_, src_comm_pattern_,
                     local_interfaces_);
@@ -299,10 +297,15 @@ public:
         label nrows = host_A->get_local_nrows();
         label local_matrix_nnz = host_A->get_local_matrix_nnz();
         // size + padding has to be local_matrix_nnz
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << "update_impl  0\n";
         auto diag_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, nrows, local_matrix_nnz,
             local_matrix_nnz - nrows, 0);
 
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << "update_impl  1\n";
+
         label upper_nnz = host_A->get_upper_nnz();
         auto upper_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, upper_nnz, local_matrix_nnz, 0,
@@ -311,6 +314,9 @@ public:
             compute_send_recv_counts(exec_handler, ranks_per_gpu, upper_nnz,
                                      local_matrix_nnz, upper_nnz, nrows);
 
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << "update_impl  2\n";
+
         scalar *local_ptr;
 
         // update main values
@@ -391,7 +397,6 @@ public:
 
                 ctr++;
                 // interface values need to be multiplied by -1
-                //
                 using vec = gko::matrix::Dense<ValueType>;
                 auto neg_one = gko::initialize<vec>({-1.0}, exec);
                 auto interface_dense = vec::create(
@@ -564,7 +569,7 @@ protected:
 private:
     std::shared_ptr<dist_mtx> dist_mtx_;
 
-    std::shared_ptr<const part_type> partition_;
+    // std::shared_ptr<const part_type> partition_;
 
     std::shared_ptr<const SparsityPattern> local_sparsity_;
 
diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index 0e46bb15d..9001cbbc9 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -331,21 +331,15 @@ public:
         // All interfaces that are in is_local need to be accounted for
         ASSERT_EQ(ctr, is_local.size());
 
-        // for compressed cols we compute a mapping first
-        std::map<label, label> compressed_col_idxs{};
-        label unique_idx = 0;
-        for (auto idx : gathered_non_local.cols) {
-            auto search = compressed_col_idxs.find(idx);
-            if (search == compressed_col_idxs.end()) {
-                compressed_col_idxs[idx] = unique_idx;
-                unique_idx += 1;
-            };
-        }
-
+        // TODO FIXME the correct way would be to check the communication
+        // pattern where a particular face is in the send idxs
+        // since we have already the row id of the other side
+        // it should be doable. Or alternatively we know that we
+        // keep an interface together thus we can just count the idx up to the size.
+        // But we have to make sure that the interfaces are in the same order
+        // on both communication sides.
         for (int i = 0; i < gathered_non_local.cols.size(); i++) {
-            label uncompressed = gathered_non_local.cols[i];
-            label compressed = compressed_col_idxs[uncompressed];
-            gathered_non_local.cols[i] = compressed;
+            gathered_non_local.cols[i] = i;
         }
 
         if (is_owner()) {
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 33bf70d64..8023fb487 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -276,9 +276,8 @@ public:
         TIME_WITH_FIELDNAME(verbose_, solve, field_name_,
                             solver->apply(dist_b_v, dist_x_v);)
 
-        auto partition = dist_A.get()->get_partition();
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, field_name_,
-                            dist_x.copy_back(partition);)
+                            dist_x.copy_back();)
         auto bandwidth_copy_back = sizeof(scalar) * psi.size() /
                                    (delta_t_copy_x_back + SMALL) / 1000.0;
 

From 3892b5d8c10609a0805ee6b53878c6282a5fd4e6 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 6 Jun 2024 08:54:57 +0200
Subject: [PATCH 70/90] dont store executor handler as member in device
 persistent class

---
 .../DistributedMatrixAdapter.H                |   6 +
 MatrixWrapper/Distributed/Distributed.H       |  48 +++++---
 MatrixWrapper/LDUMatrix/HostMatrix.C          | 116 ++----------------
 MatrixWrapper/LDUMatrix/HostMatrix.H          |  19 +--
 Repartitioner/Repartitioner.H                 |  67 +++++-----
 lduLduBase/lduLduBase.H                       |   4 +-
 6 files changed, 82 insertions(+), 178 deletions(-)

diff --git a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
index 903c8a0a2..ea11f21f0 100644
--- a/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
+++ b/DevicePersistent/DistributedMatrixAdapter/DistributedMatrixAdapter.H
@@ -70,13 +70,19 @@ struct DistributedMatrixInitFunctor {
                     persistent_device_matrix) const
     {
         persistent_device_matrix->update(
+            exec_,
             solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
             repartitioner_, host_A_.get());
+        // persistent_device_matrix->operator=(
+        //  *RepartDistMatrix<scalar, label, label>::create(
+        //     solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
+        //     repartitioner_, host_A_.get()).get());
     }
 
     std::shared_ptr<RepartDistMatrix<scalar, label, label>> init() const
     {
         return RepartDistMatrix<scalar, label, label>::create(
+          exec_,
             solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"),
             repartitioner_, host_A_.get());
     }
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 92c996b71..5a164432a 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -154,6 +154,8 @@ public:
     RepartDistMatrix &operator=(RepartDistMatrix &&other)
     {
         if (&other != this) {
+            FatalErrorInFunction << "Not implemented"
+                                << abort(FatalError);
             gko::experimental::EnableDistributedLinOp<
                 RepartDistMatrix>::operator=(std::move(other));
             this->dist_mtx_ = std::move(other.dist_mtx_);
@@ -166,10 +168,10 @@ public:
     }
 
     static std::shared_ptr<RepartDistMatrix> create(
+        const ExecutorHandler &exec_handler,
         word matrix_format, const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A)
     {
-        auto &exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_device_exec();
         auto comm = *exec_handler.get_communicator().get();
 
@@ -185,17 +187,17 @@ public:
         // create original communicator pattern
         auto ranks_per_gpu = repartitioner.get_ranks_per_gpu();
         auto src_comm_pattern =
-            host_A.get()->create_communication_pattern(comm);
+            host_A.get()->create_communication_pattern(exec_handler);
         // create partiton here and pass to constructor
         auto dst_comm_pattern = repartitioner.repartition_comm_pattern(
-            src_comm_pattern, orig_partition);
+            exec_handler, src_comm_pattern, orig_partition);
 
         label rank{src_comm_pattern->get_comm().rank()};
-        label owner_rank = repartitioner.get_owner_rank();
-        bool owner = repartitioner.is_owner();
+        label owner_rank = repartitioner.get_owner_rank(exec_handler);
+        bool owner = repartitioner.is_owner(exec_handler);
 
         auto [repart_loc_sparsity, repart_non_loc_sparsity, local_interfaces] =
-            repartitioner.repartition_sparsity(local_sparsity_,
+            repartitioner.repartition_sparsity(exec_handler, local_sparsity_,
                                                non_local_sparsity_);
 
         local_sparsity_ = repart_loc_sparsity;
@@ -254,7 +256,7 @@ public:
             sparse_comm);
 
 
-        update_impl(matrix_format, repartitioner, host_A, dist_A,
+        update_impl(exec_handler, matrix_format, repartitioner, host_A, dist_A,
                     local_sparsity_, non_local_sparsity_, src_comm_pattern,
                     local_interfaces);
 
@@ -266,18 +268,31 @@ public:
         return ret;
     };
 
-    void update(word matrix_format, const Repartitioner &repartitioner,
+    void update(
+        const ExecutorHandler &exec_handler,
+        word matrix_format, const Repartitioner &repartitioner,
                 std::shared_ptr<const HostMatrixWrapper> host_A)
     {
 
-        update_impl(matrix_format, repartitioner, host_A, dist_mtx_,
+        auto exec = exec_handler.get_ref_exec();
+        auto device_exec = exec_handler.get_device_exec();
+        auto ranks_per_gpu = repartitioner.get_ranks_per_gpu();
+        label rank{repartitioner.get_rank(exec_handler)};
+        label owner_rank = repartitioner.get_owner_rank(exec_handler);
+        bool owner = repartitioner.is_owner(exec_handler);
+        label nrows = host_A->get_local_nrows();
+        label local_matrix_nnz = host_A->get_local_matrix_nnz();
+
+        update_impl(exec_handler, matrix_format, repartitioner, host_A, dist_mtx_,
                     local_sparsity_, non_local_sparsity_, src_comm_pattern_,
                     local_interfaces_);
     };
 
     // TODO matrix_format can be data member
     static void update_impl(
-        word matrix_format, const Repartitioner &repartitioner,
+        const ExecutorHandler &exec_handler,
+        word matrix_format,
+        const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A,
         std::shared_ptr<
             gko::experimental::distributed::Matrix<scalar, label, label>>
@@ -287,13 +302,12 @@ public:
         std::shared_ptr<const CommunicationPattern> src_comm_pattern,
         std::vector<std::pair<bool, label>> local_interfaces)
     {
-        auto &exec_handler = host_A.get()->get_exec_handler();
         auto exec = exec_handler.get_ref_exec();
         auto device_exec = exec_handler.get_device_exec();
         auto ranks_per_gpu = repartitioner.get_ranks_per_gpu();
-        label rank{repartitioner.get_rank()};
-        label owner_rank = repartitioner.get_owner_rank();
-        bool owner = repartitioner.is_owner();
+        label rank{repartitioner.get_rank(exec_handler)};
+        label owner_rank = repartitioner.get_owner_rank(exec_handler);
+        bool owner = repartitioner.is_owner(exec_handler);
         label nrows = host_A->get_local_nrows();
         label local_matrix_nnz = host_A->get_local_matrix_nnz();
         // size + padding has to be local_matrix_nnz
@@ -346,6 +360,7 @@ public:
                                host_A->get_lower(), local_ptr);
         }
 
+        // copy interface values
         auto comm = *exec_handler.get_communicator().get();
         if (owner) {
             using Coo = gko::matrix::Coo<scalar, label>;
@@ -465,9 +480,6 @@ public:
           local_interfaces_(local_interfaces)
     {
         this->set_size(global_size);
-        // partition_ = gko::share(
-        //     gko::experimental::distributed::build_partition_from_local_size<
-        //         label, label>(exec, comm, local_size[0]));
     }
 
     void write(const word field_name_, const objectRegistry &db_) const
@@ -569,8 +581,6 @@ protected:
 private:
     std::shared_ptr<dist_mtx> dist_mtx_;
 
-    // std::shared_ptr<const part_type> partition_;
-
     std::shared_ptr<const SparsityPattern> local_sparsity_;
 
     std::shared_ptr<const SparsityPattern> non_local_sparsity_;
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index 8fad1a278..807845c7d 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -79,8 +79,7 @@ HostMatrixWrapper::HostMatrixWrapper(
     const FieldField<Field, scalar> &interfaceIntCoeffs,
     const lduInterfaceFieldPtrsList &interfaces,
     const dictionary &solverControls, const word &fieldName, label verbose)
-    : exec_{exec},
-      device_id_guard_{db, fieldName, exec_.get_device_exec()},
+    : device_id_guard_{db, fieldName, exec.get_device_exec()},
       verbose_(verbose),
       field_name_(fieldName),
       reorder_on_copy_(
@@ -205,9 +204,11 @@ void neg_interface_iterator(const lduInterfaceFieldPtrsList &interfaces,
 
 
 std::shared_ptr<CommunicationPattern> HostMatrixWrapper::create_communication_pattern(
-    const gko::experimental::mpi::communicator &comm) const
+    const ExecutorHandler &exec_handler) const
 {
     using comm_size_type = CommunicationPattern::comm_size_type;
+    auto exec = exec_handler.get_ref_exec();
+    auto comm = *exec_handler.get_communicator().get();
     // temp map, mapping from neighbour rank interface cells
     std::map<label, std::vector<label>> interface_cell_map{};
 
@@ -235,7 +236,6 @@ std::shared_ptr<CommunicationPattern> HostMatrixWrapper::create_communication_pa
     // create index_sets
     std::vector<std::pair<gko::array<label>, comm_size_type>> send_idxs;
     for (auto [proc, interface_cells] : interface_cell_map) {
-        auto exec = exec_.get_ref_exec();
         send_idxs.push_back(std::pair<gko::array<label>, comm_size_type>(
             gko::array<label>(exec, interface_cells.begin(),
                               interface_cells.end()),
@@ -243,8 +243,8 @@ std::shared_ptr<CommunicationPattern> HostMatrixWrapper::create_communication_pa
     }
 
     // convert to gko::array
-    gko::array<comm_size_type> target_ids{exec_.get_ref_exec(), n_procs};
-    gko::array<comm_size_type> target_sizes{exec_.get_ref_exec(), n_procs};
+    gko::array<comm_size_type> target_ids{exec, n_procs};
+    gko::array<comm_size_type> target_sizes{exec, n_procs};
 
     label iter = 0;
     for (const auto &[proc, interface_cells] : interface_cell_map) {
@@ -253,7 +253,7 @@ std::shared_ptr<CommunicationPattern> HostMatrixWrapper::create_communication_pa
         iter++;
     }
 
-    return std::make_shared<CommunicationPattern>(exec_, target_ids, target_sizes, send_idxs);
+    return std::make_shared<CommunicationPattern>(exec_handler, target_ids, target_sizes, send_idxs);
 }
 
 
@@ -538,107 +538,5 @@ void HostMatrixWrapper::update(
     // TODO implement a regenerating update
 }
 
-void HostMatrixWrapper::compute_local_coeffs(
-    std::shared_ptr<const SparsityPattern> sparsity,
-    gko::array<scalar> &local_coeffs) const
-{
-    auto ref_exec = exec_.get_ref_exec();
-    auto permute = sparsity->ldu_mapping.get_data();
-
-    //     // TODO
-    //     // - this should be moved to separate function to avoid making this
-    //     // too long
-    //
-    //     using dim_type = gko::dim<2>::dimension_type;
-    //
-    //     auto target_exec =
-    //         (reorder_on_copy_) ? ref_exec : exec_.get_device_exec();
-    //
-    //     // copy upper
-    //     auto u_host_view =
-    //         gko::array<scalar>::const_view(ref_exec, upper_nnz_, upper_);
-    //     auto u_device_view = gko::array<scalar>::view(target_exec,
-    //     upper_nnz_,
-    //                                                   local_coeffs.get_data());
-    //     u_device_view = u_host_view;
-    //
-    //     // for the non-symmetric case copy lower to the device
-    //     if (!symmetric_) {
-    //         auto l_host_view =
-    //             gko::array<scalar>::const_view(ref_exec, upper_nnz_, lower_);
-    //         auto l_device_view = gko::array<scalar>::view(
-    //             target_exec, upper_nnz_,
-    //             &local_coeffs.get_data()[upper_nnz_]);
-    //         l_device_view = l_host_view;
-    //     }
-    //
-    //     // copy diag
-    //     auto diag_host_view =
-    //         gko::array<scalar>::const_view(ref_exec, nrows_, diag_);
-    //     const label diag_start = (symmetric_) ? upper_nnz_ : 2 * upper_nnz_;
-    //     auto diag_device_view = gko::array<scalar>::view(
-    //         target_exec, nrows_, &local_coeffs.get_data()[diag_start]);
-    //     diag_device_view = diag_host_view;
-    //
-    //     // copy local interfaces
-    //     if (local_interface_nnz_) {
-    //         const label interface_start = diag_start + nrows_;
-    //         auto couple_coeffs =
-    //             collect_interface_coeffs(interfaces, interfaceBouCoeffs,
-    //             true);
-    //         auto interface_host_view = gko::array<scalar>::view(
-    //             ref_exec, local_interface_nnz_, couple_coeffs.data());
-    //         auto interface_device_view = gko::array<scalar>::view(
-    //             target_exec, local_interface_nnz_,
-    //             &local_coeffs.get_data()[interface_start]);
-    //         interface_device_view = interface_host_view;
-    //     }
-    //
-    //     // permute and update local_coeffs
-    //     auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
-    //         target_exec,
-    //         gko::dim<2>{static_cast<dim_type>(local_matrix_w_interfaces_nnz_),
-    //                     1}));
-    //
-    //     auto dense_vec = vec::create(
-    //         target_exec,
-    //         gko::dim<2>{static_cast<dim_type>(local_matrix_w_interfaces_nnz_),
-    //                     1},
-    //         gko::array<scalar>::view(target_exec,
-    //                                  local_matrix_w_interfaces_nnz_,
-    //                                  local_coeffs.get_data()),
-    //         1);
-    //
-    //     // TODO this needs to copy ldu_mapping to the device
-    //     dense_vec->row_gather(&local_sparsity_pattern_.ldu_mapping_,
-    //                           row_collection.get());
-    //
-    //     dense_vec->copy_from(row_collection);
-}
-
-
-void HostMatrixWrapper::compute_non_local_coeffs(
-    std::shared_ptr<const SparsityPattern>, gko::array<scalar> &coeffs) const
-{
-    auto ref_exec = exec_.get_ref_exec();
-    auto interface_coeffs =
-        collect_interface_coeffs(interfaces_, interfaceBouCoeffs_, false);
-
-    // copy interfaces
-    auto tmp_contiguous_iface =
-        gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
-    auto contiguous_iface = tmp_contiguous_iface.get_data();
-
-    for (label element_ctr = 0; element_ctr < non_local_matrix_nnz_;
-         element_ctr++) {
-        contiguous_iface[element_ctr] = interface_coeffs[element_ctr];
-    }
-
-    // copy to persistent
-    auto i_device_view = gko::array<scalar>::view(
-        ref_exec, non_local_matrix_nnz_, coeffs.get_data());
-
-    i_device_view = tmp_contiguous_iface;
-}
 
 }  // namespace Foam
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index f5fa49725..46deea0c3 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -228,8 +228,6 @@ private:
     using vec = gko::matrix::Dense<scalar>;
     using idx_array = gko::array<label>;
 
-    const ExecutorHandler &exec_;
-
     const DeviceIdGuardHandler device_id_guard_;
 
     const label verbose_;
@@ -379,20 +377,6 @@ public:
     std::shared_ptr<SparsityPattern> compute_local_sparsity(
         std::shared_ptr<const gko::Executor> exec) const;
 
-    /** Copies the LDU matrix coefficients to local_coeffs without changing or
-     ** reinstantiating the sparsity pattern.
-     ** This uses the local_sparsity_.ldu_mapping to permute the data already
-     ** on the host or device to be in row major order.
-     **/
-    void compute_local_coeffs(std::shared_ptr<const SparsityPattern> sparsity,
-                              gko::array<scalar> &target_coeffs) const;
-
-    /** Copies the interface matrix coefficients to non_local_coeffs without
-    ** changing or reinstantiating the sparsity pattern.
-    **/
-    void compute_non_local_coeffs(
-        std::shared_ptr<const SparsityPattern> sparsity,
-        gko::array<scalar> &target_coeffs) const;
 
     /** Based on OpenFOAMs interfaces this function computes two
      ** consecutive index arrays (non_local_sparsisty_.row_idxs and col_idxs) in
@@ -411,7 +395,7 @@ public:
      ** @return the CommunicationPattern
      */
     std::shared_ptr<CommunicationPattern> create_communication_pattern(
-        const gko::experimental::mpi::communicator &comm) const;
+        const ExecutorHandler &exec) const;
 
     label get_interface_size() const {
         return interface_ptr_.size();
@@ -443,7 +427,6 @@ public:
 
     gko::dim<2> get_size() const { return gko::dim<2>(nrows_, nrows_); }
 
-    const ExecutorHandler &get_exec_handler() const { return exec_; }
 };
 
 
diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index 9001cbbc9..6c8c8b2d8 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -61,8 +61,6 @@ private:
 
     const label ranks_per_gpu_;
 
-    const ExecutorHandler &exec_handler_;
-
     std::shared_ptr<const partition> orig_partition_;
 
 
@@ -73,11 +71,10 @@ public:
           repart_size_(Repartitioner::compute_repart_size(size, ranks_per_gpu,
                                                           exec_handler)),
           ranks_per_gpu_(ranks_per_gpu),
-          exec_handler_(exec_handler),
           orig_partition_(gko::share(
               gko::experimental::distributed::build_partition_from_local_size<
-                  label, label>(exec_handler_.get_device_exec(),
-                                *exec_handler_.get_communicator().get(),
+                  label, label>(exec_handler.get_device_exec(),
+                                *exec_handler.get_communicator().get(),
                                 size))) {};
 
     /* returns the owner rank for a given rank */
@@ -87,23 +84,29 @@ public:
     };
 
     /* returns the owner rank for a given rank */
-    label get_owner_rank() const { return get_owner_rank(get_rank()); };
+    label get_owner_rank(
+ const ExecutorHandler& exec_handler
+) const { return get_owner_rank(get_rank(exec_handler)); };
 
     /* returns if current rank is an owner  */
-    bool is_owner() const { return get_rank() == get_owner_rank(get_rank()); };
+    bool is_owner(
+ const ExecutorHandler& exec_handler
+) const { return get_rank(exec_handler) == get_owner_rank(get_rank(exec_handler)); };
 
     /* @brief check if the given rank gets local after repartitioning
      *
      * */
-    bool reparts_to_local(label rank) const
+    bool reparts_to_local(
+ const ExecutorHandler& exec_handler,
+        label rank) const
     {
-        return get_owner_rank() == compute_owner_rank(rank, ranks_per_gpu_);
+        return get_owner_rank(exec_handler) == compute_owner_rank(rank, ranks_per_gpu_);
     };
 
     /* shortcut to current rank */
-    label get_rank() const
+    label get_rank(const ExecutorHandler& exec_handler) const
     {
-        return exec_handler_.get_communicator().get()->rank();
+        return exec_handler.get_communicator().get()->rank();
     };
 
     label get_ranks_per_gpu() const { return ranks_per_gpu_; }
@@ -147,7 +150,9 @@ public:
      *  the interface was originally from this rank
      */
     std::pair<SparsityPatternVector, std::vector<bool>>
-    build_non_local_interfaces(SparsityPatternVector &loc,
+    build_non_local_interfaces(
+        const ExecutorHandler& exec_handler,
+        SparsityPatternVector &loc,
                                const SparsityPatternVector &non_loc) const
     {
         std::vector<label> rows, cols, ldu_mapping, ranks, begins, ends;
@@ -159,7 +164,7 @@ public:
             // offseted;
             auto begin = non_loc.begin[i];
             auto end = non_loc.end[i];
-            bool local = reparts_to_local(non_loc.ranks[i]);
+            bool local = reparts_to_local(exec_handler, non_loc.ranks[i]);
             is_local.push_back(local);
             if (local) {
                 // TODO depending on the interface simple non
@@ -208,6 +213,7 @@ public:
                std::shared_ptr<SparsityPattern>,
                std::vector<std::pair<bool, label>>>
     repartition_sparsity(
+        const ExecutorHandler& exec_handler,
         std::shared_ptr<SparsityPattern> src_local_pattern,
         std::shared_ptr<SparsityPattern> src_non_local_pattern) const
     {
@@ -221,19 +227,18 @@ public:
         // 4. dim needs to be updated
         // this is not implemented yet, but we can't fail here for
         // debugging reasons
-        auto exec = exec_handler_.get_device_exec();
-        auto comm = *exec_handler_.get_communicator().get();
-        label rank = get_rank();
-        label owner_rank = get_owner_rank();
+        auto exec = exec_handler.get_device_exec();
+        auto comm = *exec_handler.get_communicator().get();
+        label rank = get_rank(exec_handler);
+        label owner_rank = get_owner_rank(exec_handler);
         label ranks_per_gpu = ranks_per_gpu_;
 
         auto local_comm_pattern = compute_send_recv_counts(
-            exec_handler_, ranks_per_gpu, src_local_pattern->size_);
+            exec_handler, ranks_per_gpu, src_local_pattern->size_);
 
         label offset = orig_partition_->get_range_bounds()[rank] -
                        orig_partition_->get_range_bounds()[owner_rank];
 
-        auto &exec_handler = exec_handler_;
         auto gather_closure = [&](auto &comm_pattern, auto &data,
                                   label offset) {
             return gather_to_owner(exec_handler, comm_pattern, data.get_size(),
@@ -252,12 +257,12 @@ public:
             std::vector{rank}};
         merged_local.end[0] = merged_local.rows.size();
 
-        if (is_owner()) {
+        if (is_owner(exec_handler)) {
             make_ldu_mapping_consecutive(
                 local_comm_pattern, merged_local.mapping, rank, ranks_per_gpu);
         }
 
-        label rows = (is_owner()) ? merged_local.rows.back() + 1 : 0;
+        label rows = (is_owner(exec_handler)) ? merged_local.rows.back() + 1 : 0;
         gko::dim<2> merged_local_dim{rows, rows};
 
         auto non_local_comm_pattern = compute_send_recv_counts(
@@ -304,20 +309,20 @@ public:
                             src_non_local_pattern->rank.size(),
                             src_non_local_pattern->rank.data())};
 
-        if (is_owner()) {
+        if (is_owner(exec_handler)) {
             make_begin_and_ends_consecutive(
                 span_comm_pattern, merged_non_local.begin, merged_non_local.end,
                 rank, ranks_per_gpu);
         }
 
         auto [gathered_non_local, is_local] =
-            build_non_local_interfaces(merged_local, merged_non_local);
+            build_non_local_interfaces(exec_handler, merged_local, merged_non_local);
 
         // build vector with locality information
         std::vector<std::pair<bool, label>> locality;
         label ctr{0};
 
-        if (is_owner()) {
+        if (is_owner(exec_handler)) {
             auto recv_counts = std::get<1>(span_comm_pattern);
             for (int i = 0; i < ranks_per_gpu; i++) {
                 label comm_rank = rank + i;
@@ -342,7 +347,7 @@ public:
             gathered_non_local.cols[i] = i;
         }
 
-        if (is_owner()) {
+        if (is_owner(exec_handler)) {
             auto new_local_spars_pattern = std::make_shared<SparsityPattern>(
                 exec, merged_local_dim, merged_local);
 
@@ -396,6 +401,7 @@ public:
     }
 
     std::shared_ptr<const CommunicationPattern> repartition_comm_pattern(
+        const ExecutorHandler& exec_handler,
         std::shared_ptr<const CommunicationPattern> src_comm_pattern,
         std::shared_ptr<
             const gko::experimental::distributed::Partition<label, label>>
@@ -407,7 +413,6 @@ public:
             return src_comm_pattern;
         }
 
-        auto &exec_handler = src_comm_pattern->exec_handler;
         auto exec = src_comm_pattern->target_ids.get_executor();
         auto comm = src_comm_pattern->get_comm();
 
@@ -421,7 +426,7 @@ public:
         label communication_partner = src_comm_pattern->target_ids.get_size();
         for (int i = 0; i < communication_partner; i++) {
             label target_id = src_comm_pattern->target_ids.get_const_data()[i];
-            if (!reparts_to_local(target_id)) {
+            if (!reparts_to_local(exec_handler, target_id)) {
                 // communication pattern is non local, hence we keep it
                 // after repartitioning we now have to communicate with a
                 // different rank store new owner rank to which repart_target_*
@@ -447,8 +452,8 @@ public:
 
         // next the send_ixs need to be updated we send them piecewise since
         // the send_idxs are a vector of gko::arrays
-        label rank = get_rank();
-        if (is_owner()) {
+        label rank = get_rank(exec_handler);
+        if (is_owner(exec_handler)) {
             auto recv_counts = std::get<1>(comm_pattern);
             label owner_recv_counts = recv_counts[rank];
             // retrieved from i-th neighbor
@@ -482,7 +487,7 @@ public:
                 }
             }
         } else {
-            label owner = get_owner_rank();
+            label owner = get_owner_rank(exec_handler);
             auto send_count = std::get<0>(comm_pattern)[owner];
             for (int i = 0; i < send_count; i++) {
                 auto send_buffer = send_idxs[i].first;
@@ -492,7 +497,7 @@ public:
         }
 
         // clear communcation neighbors on non owning rank
-        if (!is_owner()) {
+        if (!is_owner(exec_handler)) {
             // TODO NOTE should it be gathered_target_ids and is it needed?
             target_ids.clear();
             target_sizes.clear();
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 8023fb487..51d18413b 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -162,6 +162,8 @@ public:
           solver_controls_(solverControls),
           field_name_(fieldName),
           verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+          // FIXME NOTE the underlying exec_handler is invalid in the next call but the
+          // exec isnt
           exec_handler_{db_, solver_controls_, fieldName},
           host_A_{
               db_,       exec_handler_, solver_controls_,   fieldName,
@@ -201,7 +203,6 @@ public:
                                            const scalarField &source,
                                            solverPerformance &solverPerf) const
     {
-        auto ref_exec = exec_handler_.get_ref_exec();
         Repartitioner repartitioner{
             psi.size(),
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
@@ -243,6 +244,7 @@ public:
 
         scalar scaling =
             solver_controls_.lookupOrDefault<scalar>("scaling", 1.0);
+        auto ref_exec = exec_handler_.get_ref_exec();
         if (scaling != 1) {
             auto dense_scaling =
                 gko::share(gko::initialize<gko::matrix::Dense<scalar>>(

From 1147e527e631b72699156855e151eeba641b4dfc Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 6 Jun 2024 10:36:34 +0200
Subject: [PATCH 71/90] clean up, use swap to refresh host_A

---
 .../HostMatrixAdapter/HostMatrixAdapter.H            | 12 ++++++++----
 .../CommunicationPattern/CommunicationPattern.C      |  2 +-
 MatrixWrapper/LDUMatrix/HostMatrix.C                 | 10 ----------
 MatrixWrapper/LDUMatrix/HostMatrix.H                 |  4 ----
 4 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
index cc06bdd99..a7c799a04 100644
--- a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
+++ b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
@@ -76,10 +76,14 @@ struct lduMatrixInitFunctor {
     void update(
         std::shared_ptr<HostMatrixWrapper> persistent_device_matrix) const
     {
-        persistent_device_matrix->update(
-            matrix_.diag().begin(), matrix_.upper().begin(),
-            matrix_.lower().begin(), interfaceBouCoeffs_, interfaceIntCoeffs_,
-            interfaces_);
+        auto ret = HostMatrixWrapper::create(
+            exec_, db_, matrix_.diag().size(), matrix_.upper().size(),
+            matrix_.symmetric(), matrix_.diag().begin(),
+            matrix_.upper().begin(), matrix_.lower().begin(), matrix_.lduAddr(),
+            interfaceBouCoeffs_, interfaceIntCoeffs_, interfaces_,
+            solver_controls_, field_name_, verbose_);
+
+        persistent_device_matrix.swap(ret);
     }
 
     std::shared_ptr<HostMatrixWrapper> init() const
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index 2eda49241..4fce7e2a5 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -165,7 +165,7 @@ void communicate_values (
     //
     // send_buffer should be on the host
     // recv_buffer should be on the device
-    auto rank = comm.rank();
+    // auto rank = comm.rank();
 
     comm.all_to_all_v(exec, send_buffer, send_counts.data(),
                       send_offsets.data(), recv_buffer, recv_counts.data(),
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index 807845c7d..f646eb395 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -529,14 +529,4 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_local_sparsity(
     return sparsity;
 }
 
-void HostMatrixWrapper::update(
-    const scalar *diag, const scalar *upper, const scalar *lower,
-    const FieldField<Field, scalar> &interfaceBouCoeffs,
-    const FieldField<Field, scalar> &interfaceIntCoeffs,
-    const lduInterfaceFieldPtrsList &interfaces)
-{
-    // TODO implement a regenerating update
-}
-
-
 }  // namespace Foam
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 46deea0c3..96cf37ac5 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -364,10 +364,6 @@ public:
             fieldName, verbose);
     };
 
-    void update(const scalar *diag, const scalar *upper, const scalar *lower,
-                const FieldField<Field, scalar> &interfaceBouCoeffs,
-                const FieldField<Field, scalar> &interfaceIntCoeffs,
-                const lduInterfaceFieldPtrsList &interfaces);
 
     /** Based on OpenFOAMs ldu matrix format this function computes two
      ** consecutive index arrays (local_sparsisty_.row_idxs and col_idxs) in row

From 8b2cfd2d33117735a4ef85efdfe8d196e583cc4d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 6 Jun 2024 11:20:53 +0200
Subject: [PATCH 72/90] Use explicit update_from instead of swap

---
 .../HostMatrixAdapter/HostMatrixAdapter.H     |  3 +-
 MatrixWrapper/Distributed/Distributed.H       |  7 -----
 MatrixWrapper/LDUMatrix/HostMatrix.H          | 30 +++++++++++++++++--
 lduLduBase/lduLduBase.H                       |  2 --
 4 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
index a7c799a04..af2c44165 100644
--- a/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
+++ b/DevicePersistent/HostMatrixAdapter/HostMatrixAdapter.H
@@ -83,7 +83,8 @@ struct lduMatrixInitFunctor {
             interfaceBouCoeffs_, interfaceIntCoeffs_, interfaces_,
             solver_controls_, field_name_, verbose_);
 
-        persistent_device_matrix.swap(ret);
+        persistent_device_matrix->update_from(ret);
+
     }
 
     std::shared_ptr<HostMatrixWrapper> init() const
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 5a164432a..d33c41d67 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -311,14 +311,10 @@ public:
         label nrows = host_A->get_local_nrows();
         label local_matrix_nnz = host_A->get_local_matrix_nnz();
         // size + padding has to be local_matrix_nnz
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-                  << "update_impl  0\n";
         auto diag_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, nrows, local_matrix_nnz,
             local_matrix_nnz - nrows, 0);
 
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-                  << "update_impl  1\n";
 
         label upper_nnz = host_A->get_upper_nnz();
         auto upper_comm_pattern = compute_send_recv_counts(
@@ -328,9 +324,6 @@ public:
             compute_send_recv_counts(exec_handler, ranks_per_gpu, upper_nnz,
                                      local_matrix_nnz, upper_nnz, nrows);
 
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-                  << "update_impl  2\n";
-
         scalar *local_ptr;
 
         // update main values
diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.H b/MatrixWrapper/LDUMatrix/HostMatrix.H
index 96cf37ac5..667691c93 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.H
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.H
@@ -240,11 +240,11 @@ private:
 
     const lduAddressing &addr_;
 
-    const scalar *diag_;
+    mutable const scalar *diag_;
 
-    const scalar *upper_;
+    mutable const scalar *upper_;
 
-    const scalar *lower_;
+    mutable const scalar *lower_;
 
     // multiply the complete system by this factor, ie sAx=sb
     // NOTE this needed to avoid negative diagonal matrix entries, but
@@ -364,6 +364,23 @@ public:
             fieldName, verbose);
     };
 
+    void update_from(std::shared_ptr<HostMatrixWrapper> in) {
+        diag_ = in->get_diag();
+        upper_ = in->get_upper();
+        lower_ = in->get_lower();
+
+        interface_ptr_.clear();
+        auto interfaces = in->get_interfaces();
+        for (label i = 0; i < interfaces.size(); i++) {
+            if (interface_getter(interfaces, i) == nullptr) {
+                continue;
+            }
+            const auto iface{interface_getter(interfaces, i)};
+            interface_ptr_.push_back(in->get_interfaceBouCoeffs()[i].begin());
+        }
+
+    }
+
 
     /** Based on OpenFOAMs ldu matrix format this function computes two
      ** consecutive index arrays (local_sparsisty_.row_idxs and col_idxs) in row
@@ -401,6 +418,13 @@ public:
         return interface_ptr_[i];
     };
 
+    const FieldField<Field, scalar> & get_interfaceBouCoeffs()
+        const {return interfaceBouCoeffs_;};
+
+    const lduInterfaceFieldPtrsList &get_interfaces() const {
+        return interfaces_;
+    };
+
     bool get_verbose() const { return verbose_; }
 
     label get_local_nrows() const { return nrows_; }
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 51d18413b..bd47fdea9 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -162,8 +162,6 @@ public:
           solver_controls_(solverControls),
           field_name_(fieldName),
           verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
-          // FIXME NOTE the underlying exec_handler is invalid in the next call but the
-          // exec isnt
           exec_handler_{db_, solver_controls_, fieldName},
           host_A_{
               db_,       exec_handler_, solver_controls_,   fieldName,

From 4c860e65d6770a41325f24a74b541c4fd692b26c Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 6 Jun 2024 11:34:34 +0200
Subject: [PATCH 73/90] clean up

---
 StoppingCriterion/StoppingCriterion.C | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/StoppingCriterion/StoppingCriterion.C b/StoppingCriterion/StoppingCriterion.C
index ee1fe4ef6..f9f2bdd24 100644
--- a/StoppingCriterion/StoppingCriterion.C
+++ b/StoppingCriterion/StoppingCriterion.C
@@ -29,10 +29,7 @@ void StoppingCriterion::OpenFOAMDistStoppingCriterion::compute_Axref_dist(
 
     ASSERT_EQ(xAvg_host->at(0), xAvg_host->at(0));
 
-    std::cout << __FILE__ << ":" << __LINE__
-              << "Axref apply\n";
     gkomatrix->apply(xAvg_vec.get(), res.get());
-    std::cout << __FILE__ << ":" << __LINE__ << "done Axref apply\n";
 }
 
 scalar
@@ -47,10 +44,6 @@ StoppingCriterion::OpenFOAMDistStoppingCriterion::compute_normfactor_dist(
     gko::dim<2> local_size = x->get_local_vector()->get_size();
     gko::dim<2> global_size = x->get_size();
 
-    std::cout << __FILE__ << ":" << __LINE__
-              << " setup Axref local_size" << local_size
-              << " global size " << global_size
-              << "before Axref apply\n";
     auto Axref = gko::share(
         dist_vec::create(device_exec, comm, global_size, local_size));
     Axref->fill(0.0);

From 84827e02c1d097c9c3fa99e03d52db0fa05de828 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 7 Jun 2024 09:39:25 +0200
Subject: [PATCH 74/90] add DistributedMatrix copy, use device exec for
 communication

---
 MatrixWrapper/Distributed/Distributed.H | 30 +++++++++++++------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index d33c41d67..5fc90ac21 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -141,9 +141,20 @@ public:
      */
     RepartDistMatrix &operator=(const RepartDistMatrix &other)
     {
-        FatalErrorInFunction << "Copying the RepartDistMatrix is disallowed "
-                                "for performance reasons"
-                             << abort(FatalError);
+        if (&other != this) {
+            std::cout << __FILE__ << __LINE__ << "WARN issued a copy of RepartDistMatrix\n";
+        // FatalErrorInFunction << "Copying the RepartDistMatrix is disallowed "
+        //                         "for performance reasons"
+        //                      << abort(FatalError);
+            gko::experimental::EnableDistributedLinOp<
+                RepartDistMatrix>::operator=(std::move(other));
+            this->dist_mtx_ = other.dist_mtx_;
+            this->local_sparsity_ = other.local_sparsity_;
+            this->non_local_sparsity_ = other.non_local_sparsity_;
+            this->src_comm_pattern_ =other.src_comm_pattern_;
+            this->local_interfaces_ =other.local_interfaces_;
+        }
+        return *this;
     }
 
     /**
@@ -172,7 +183,7 @@ public:
         word matrix_format, const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A)
     {
-        auto exec = exec_handler.get_device_exec();
+        auto exec = exec_handler.get_ref_exec();
         auto comm = *exec_handler.get_communicator().get();
 
         // repartition things here first by creating device_matrix_data on
@@ -231,6 +242,7 @@ public:
         auto sparse_comm =
             sparse_communicator::create(comm, localized_partition);
 
+        auto device_exec = exec_handler.get_device_exec();
         auto dist_A = gko::share(generate_dist_mtx_with_inner_type<dist_mtx>(
             matrix_format, exec, sparse_comm, local_sparsity_,
             non_local_sparsity_));
@@ -273,16 +285,6 @@ public:
         word matrix_format, const Repartitioner &repartitioner,
                 std::shared_ptr<const HostMatrixWrapper> host_A)
     {
-
-        auto exec = exec_handler.get_ref_exec();
-        auto device_exec = exec_handler.get_device_exec();
-        auto ranks_per_gpu = repartitioner.get_ranks_per_gpu();
-        label rank{repartitioner.get_rank(exec_handler)};
-        label owner_rank = repartitioner.get_owner_rank(exec_handler);
-        bool owner = repartitioner.is_owner(exec_handler);
-        label nrows = host_A->get_local_nrows();
-        label local_matrix_nnz = host_A->get_local_matrix_nnz();
-
         update_impl(exec_handler, matrix_format, repartitioner, host_A, dist_mtx_,
                     local_sparsity_, non_local_sparsity_, src_comm_pattern_,
                     local_interfaces_);

From c8fbcf04a7f122c09f880cba40ef3a3b4b56e950 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 7 Jun 2024 09:41:29 +0200
Subject: [PATCH 75/90] add debug message

---
 MatrixWrapper/LDUMatrix/HostMatrix.C | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/MatrixWrapper/LDUMatrix/HostMatrix.C b/MatrixWrapper/LDUMatrix/HostMatrix.C
index f646eb395..d59d8f629 100644
--- a/MatrixWrapper/LDUMatrix/HostMatrix.C
+++ b/MatrixWrapper/LDUMatrix/HostMatrix.C
@@ -395,6 +395,8 @@ HostMatrixWrapper::collect_cells_on_non_local_interface(
 std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
     std::shared_ptr<const gko::Executor> exec) const
 {
+    LOG_1(verbose_, "start init non local sparsity pattern")
+
     auto dim = gko::dim<2>{nrows_, non_local_matrix_nnz_};
 
     SparsityPatternVector sparsity;
@@ -432,6 +434,7 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
         prev_rank = rank;
         element_ctr++;
     }
+    LOG_1(verbose_, "done init non local sparsity pattern")
 
     return std::make_shared<SparsityPattern>(exec, dim, sparsity);
 }
@@ -439,7 +442,7 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_non_local_sparsity(
 std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_local_sparsity(
     std::shared_ptr<const gko::Executor> exec) const
 {
-    LOG_1(verbose_, "start init host sparsity pattern")
+    LOG_1(verbose_, "start init local sparsity pattern")
 
     auto sparsity{std::make_shared<SparsityPattern>(
         exec->get_master(), local_matrix_w_interfaces_nnz_)};
@@ -525,7 +528,7 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_local_sparsity(
         }
     }
 
-    LOG_1(verbose_, "done init host sparsity pattern")
+    LOG_1(verbose_, "done init local sparsity pattern")
     return sparsity;
 }
 

From 2d9597f65bc3b8861efe25c27fdce8d323dbfcc4 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 7 Jun 2024 09:48:21 +0200
Subject: [PATCH 76/90] add debug message

---
 Repartitioner/Repartitioner.H | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index 6c8c8b2d8..f115c2913 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -217,6 +217,7 @@ public:
         std::shared_ptr<SparsityPattern> src_local_pattern,
         std::shared_ptr<SparsityPattern> src_non_local_pattern) const
     {
+    LOG_1(verbose_, "start repartition sparsity pattern")
         // 1. obtain send recv sizes vector
         // here we can reuse code from repartition_comm_pattern
         //
@@ -347,6 +348,7 @@ public:
             gathered_non_local.cols[i] = i;
         }
 
+    LOG_1(verbose_, "done repartition sparsity pattern")
         if (is_owner(exec_handler)) {
             auto new_local_spars_pattern = std::make_shared<SparsityPattern>(
                 exec, merged_local_dim, merged_local);

From d465b2037861bf6ed74b4ae0d15b014ea1444042 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 7 Jun 2024 09:52:31 +0200
Subject: [PATCH 77/90] add debug message

---
 Repartitioner/Repartitioner.H | 5 ++++-
 lduLduBase/lduLduBase.H       | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index f115c2913..e55230f90 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -61,16 +61,19 @@ private:
 
     const label ranks_per_gpu_;
 
+    const label verbose_;
+
     std::shared_ptr<const partition> orig_partition_;
 
 
 public:
-    Repartitioner(label size, label ranks_per_gpu,
+    Repartitioner(label size, label ranks_per_gpu, label verbose_,
                   const ExecutorHandler &exec_handler)
         : size_(size),
           repart_size_(Repartitioner::compute_repart_size(size, ranks_per_gpu,
                                                           exec_handler)),
           ranks_per_gpu_(ranks_per_gpu),
+          verbose_(verbose),
           orig_partition_(gko::share(
               gko::experimental::distributed::build_partition_from_local_size<
                   label, label>(exec_handler.get_device_exec(),
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index bd47fdea9..74725d9b5 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -204,6 +204,7 @@ public:
         Repartitioner repartitioner{
             psi.size(),
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
+            verbose_,
             exec_handler_};
 
         // create a distributed matrix from the host matrix

From 7fd372bce13e2fb7f0a5a086cf3c89da552d1da8 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 7 Jun 2024 09:54:17 +0200
Subject: [PATCH 78/90] add debug message

---
 Repartitioner/Repartitioner.H | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index e55230f90..a4b3ab7ce 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -67,7 +67,7 @@ private:
 
 
 public:
-    Repartitioner(label size, label ranks_per_gpu, label verbose_,
+    Repartitioner(label size, label ranks_per_gpu, label verbose,
                   const ExecutorHandler &exec_handler)
         : size_(size),
           repart_size_(Repartitioner::compute_repart_size(size, ranks_per_gpu,

From 31a9f7c3faa1d25f336c7ceac98542f0e694cb8a Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 7 Jun 2024 09:58:09 +0200
Subject: [PATCH 79/90] add debug message

---
 Repartitioner/Repartitioner.H | 51 ++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index a4b3ab7ce..40d3dd69d 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -87,27 +87,28 @@ public:
     };
 
     /* returns the owner rank for a given rank */
-    label get_owner_rank(
- const ExecutorHandler& exec_handler
-) const { return get_owner_rank(get_rank(exec_handler)); };
+    label get_owner_rank(const ExecutorHandler &exec_handler) const
+    {
+        return get_owner_rank(get_rank(exec_handler));
+    };
 
     /* returns if current rank is an owner  */
-    bool is_owner(
- const ExecutorHandler& exec_handler
-) const { return get_rank(exec_handler) == get_owner_rank(get_rank(exec_handler)); };
+    bool is_owner(const ExecutorHandler &exec_handler) const
+    {
+        return get_rank(exec_handler) == get_owner_rank(get_rank(exec_handler));
+    };
 
     /* @brief check if the given rank gets local after repartitioning
      *
      * */
-    bool reparts_to_local(
- const ExecutorHandler& exec_handler,
-        label rank) const
+    bool reparts_to_local(const ExecutorHandler &exec_handler, label rank) const
     {
-        return get_owner_rank(exec_handler) == compute_owner_rank(rank, ranks_per_gpu_);
+        return get_owner_rank(exec_handler) ==
+               compute_owner_rank(rank, ranks_per_gpu_);
     };
 
     /* shortcut to current rank */
-    label get_rank(const ExecutorHandler& exec_handler) const
+    label get_rank(const ExecutorHandler &exec_handler) const
     {
         return exec_handler.get_communicator().get()->rank();
     };
@@ -153,11 +154,11 @@ public:
      *  the interface was originally from this rank
      */
     std::pair<SparsityPatternVector, std::vector<bool>>
-    build_non_local_interfaces(
-        const ExecutorHandler& exec_handler,
-        SparsityPatternVector &loc,
+    build_non_local_interfaces(const ExecutorHandler &exec_handler,
+                               SparsityPatternVector &loc,
                                const SparsityPatternVector &non_loc) const
     {
+        LOG_1(verbose_, "start build non local interfaces")
         std::vector<label> rows, cols, ldu_mapping, ranks, begins, ends;
         std::vector<bool> is_local;
         label merged_ranks_size = non_loc.ranks.size();
@@ -199,6 +200,7 @@ public:
                 ranks.push_back(get_owner_rank(non_loc.ranks[i]));
             }
         }
+        LOG_1(verbose_, "done build non local interfaces")
         return std::make_pair(
             SparsityPatternVector{rows, cols, ldu_mapping, begins, ends, ranks},
             is_local);
@@ -216,11 +218,11 @@ public:
                std::shared_ptr<SparsityPattern>,
                std::vector<std::pair<bool, label>>>
     repartition_sparsity(
-        const ExecutorHandler& exec_handler,
+        const ExecutorHandler &exec_handler,
         std::shared_ptr<SparsityPattern> src_local_pattern,
         std::shared_ptr<SparsityPattern> src_non_local_pattern) const
     {
-    LOG_1(verbose_, "start repartition sparsity pattern")
+        LOG_1(verbose_, "start repartition sparsity pattern")
         // 1. obtain send recv sizes vector
         // here we can reuse code from repartition_comm_pattern
         //
@@ -266,7 +268,8 @@ public:
                 local_comm_pattern, merged_local.mapping, rank, ranks_per_gpu);
         }
 
-        label rows = (is_owner(exec_handler)) ? merged_local.rows.back() + 1 : 0;
+        label rows =
+            (is_owner(exec_handler)) ? merged_local.rows.back() + 1 : 0;
         gko::dim<2> merged_local_dim{rows, rows};
 
         auto non_local_comm_pattern = compute_send_recv_counts(
@@ -319,8 +322,8 @@ public:
                 rank, ranks_per_gpu);
         }
 
-        auto [gathered_non_local, is_local] =
-            build_non_local_interfaces(exec_handler, merged_local, merged_non_local);
+        auto [gathered_non_local, is_local] = build_non_local_interfaces(
+            exec_handler, merged_local, merged_non_local);
 
         // build vector with locality information
         std::vector<std::pair<bool, label>> locality;
@@ -344,14 +347,14 @@ public:
         // pattern where a particular face is in the send idxs
         // since we have already the row id of the other side
         // it should be doable. Or alternatively we know that we
-        // keep an interface together thus we can just count the idx up to the size.
-        // But we have to make sure that the interfaces are in the same order
-        // on both communication sides.
+        // keep an interface together thus we can just count the idx up to the
+        // size. But we have to make sure that the interfaces are in the same
+        // order on both communication sides.
         for (int i = 0; i < gathered_non_local.cols.size(); i++) {
             gathered_non_local.cols[i] = i;
         }
 
-    LOG_1(verbose_, "done repartition sparsity pattern")
+        LOG_1(verbose_, "done repartition sparsity pattern")
         if (is_owner(exec_handler)) {
             auto new_local_spars_pattern = std::make_shared<SparsityPattern>(
                 exec, merged_local_dim, merged_local);
@@ -406,7 +409,7 @@ public:
     }
 
     std::shared_ptr<const CommunicationPattern> repartition_comm_pattern(
-        const ExecutorHandler& exec_handler,
+        const ExecutorHandler &exec_handler,
         std::shared_ptr<const CommunicationPattern> src_comm_pattern,
         std::shared_ptr<
             const gko::experimental::distributed::Partition<label, label>>

From 43e802c3b249d1ac65fd863a9f9fe88975a6bb27 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sat, 8 Jun 2024 07:41:45 +0200
Subject: [PATCH 80/90] fix gathered_target_id issue

---
 Repartitioner/Repartitioner.H | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index 40d3dd69d..cfbdea84c 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -424,6 +424,9 @@ public:
         auto exec = src_comm_pattern->target_ids.get_executor();
         auto comm = src_comm_pattern->get_comm();
 
+        label rank = comm.rank();
+        bool owner = is_owner(exec_handler);
+
         // Step 1. Check if communication partner is non-local after
         // repartitioning. If it is non-local we keep it. Otherwise local
         // communication can be discarded. Here non-local means:
@@ -460,8 +463,7 @@ public:
 
         // next the send_ixs need to be updated we send them piecewise since
         // the send_idxs are a vector of gko::arrays
-        label rank = get_rank(exec_handler);
-        if (is_owner(exec_handler)) {
+        if (owner) {
             auto recv_counts = std::get<1>(comm_pattern);
             label owner_recv_counts = recv_counts[rank];
             // retrieved from i-th neighbor
@@ -474,6 +476,12 @@ public:
                         gathered_target_sizes[j + owner_recv_counts];
                     std::vector<label> recv_buffer(target_size);
 
+                    // std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                    //           << " recv no  " << j
+                    //           << " recv from  " << rank + i
+                    //           << " recv_count " << recv_count
+                    //           << " target_size " << target_size
+                    //           << " \n";
                     comm.recv(exec, recv_buffer.data(), target_size, rank + i,
                               rank);
 
@@ -499,21 +507,28 @@ public:
             auto send_count = std::get<0>(comm_pattern)[owner];
             for (int i = 0; i < send_count; i++) {
                 auto send_buffer = send_idxs[i].first;
+                // std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                //           << " send to owner " << owner 
+                //           << " count " << i << " of "  <<  send_count
+                //           << " send_buffer.size " << send_buffer.get_size()
+                //           << " \n";
                 comm.send(exec, send_buffer.get_const_data(),
                           send_buffer.get_size(), owner, owner);
             }
         }
 
         // clear communcation neighbors on non owning rank
-        if (!is_owner(exec_handler)) {
+        if (!owner) {
             // TODO NOTE should it be gathered_target_ids and is it needed?
             target_ids.clear();
             target_sizes.clear();
             send_idxs.clear();
         }
 
+
         // early return if no communication partners are left
-        if (target_sizes.size() == 0) {
+        if (gathered_target_ids.size() == 0) {
+
             return std::make_shared<CommunicationPattern>(
                 exec_handler,
                 gko::array<comm_size_type>{exec, target_ids.begin(),
@@ -546,6 +561,7 @@ public:
             send_idxs[0].first.get_data(),
             send_idxs[0].first.get_data() + send_idxs[0].first.get_size()));
 
+
         for (int i = 1; i < target_ids.size(); i++) {
             // communicates with same target rank
             // thus we have only have to adapt the number

From 5149568cb0d00691f6097ff26b13fe9bf081109d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sat, 8 Jun 2024 07:43:00 +0200
Subject: [PATCH 81/90] cleanup

---
 MatrixWrapper/Distributed/Distributed.H | 50 ++++++++++++-------------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 5fc90ac21..9e4fbc0d3 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -142,10 +142,9 @@ public:
     RepartDistMatrix &operator=(const RepartDistMatrix &other)
     {
         if (&other != this) {
-            std::cout << __FILE__ << __LINE__ << "WARN issued a copy of RepartDistMatrix\n";
-        // FatalErrorInFunction << "Copying the RepartDistMatrix is disallowed "
-        //                         "for performance reasons"
-        //                      << abort(FatalError);
+            // FatalErrorInFunction << "Copying the RepartDistMatrix is disallowed "
+            //                         "for performance reasons"
+            //                      << abort(FatalError);
             gko::experimental::EnableDistributedLinOp<
                 RepartDistMatrix>::operator=(std::move(other));
             this->dist_mtx_ = other.dist_mtx_;
@@ -197,13 +196,15 @@ public:
 
         // create original communicator pattern
         auto ranks_per_gpu = repartitioner.get_ranks_per_gpu();
+        label rank{comm.rank()};
+
         auto src_comm_pattern =
             host_A.get()->create_communication_pattern(exec_handler);
         // create partiton here and pass to constructor
+        //
         auto dst_comm_pattern = repartitioner.repartition_comm_pattern(
             exec_handler, src_comm_pattern, orig_partition);
 
-        label rank{src_comm_pattern->get_comm().rank()};
         label owner_rank = repartitioner.get_owner_rank(exec_handler);
         bool owner = repartitioner.is_owner(exec_handler);
 
@@ -214,37 +215,17 @@ public:
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
-        // std::cout << __FILE__ << " rank " << rank
-        //           << " build_localized_partition "
-        //           << " dim " << local_sparsity_->dim[0] << " send idxs size "
-        //           << dst_comm_pattern.send_idxs.size() << " target ids "
-        //           << dst_comm_pattern.target_ids << " target sizes "
-        //           << dst_comm_pattern.target_sizes << "\n";
-
         auto localized_partition = local_part_type::build_from_blocked_recv(
             exec, local_sparsity_->dim[0], dst_comm_pattern->send_idxs,
             dst_comm_pattern->target_ids, dst_comm_pattern->target_sizes);
 
-        // std::cout << __FILE__ << " rank " << rank << " local sparsity size "
-        //           << local_sparsity_->size_ << " local sparsity dim ["
-        //           << local_sparsity_->dim[0] << "x" << local_sparsity_->dim[1]
-        //           << "] non_local sparsity size " << non_local_sparsity_->size_
-        //           << " non local sparsity dim [" << non_local_sparsity_->dim[0]
-        //           << "x" << non_local_sparsity_->dim[1] << "] target_ids "
-        //           << dst_comm_pattern->target_ids << " target_sizes "
-        //           << dst_comm_pattern->target_sizes << " target_send_idxs.size "
-        //           << dst_comm_pattern->send_idxs.size()
-        //           << " non_local_sparsity.size " << non_local_sparsity_->size_
-        //           << " get_recv_indices "
-        //           << localized_partition->get_recv_indices().get_num_elems()
-        //           << " \n";
 
         auto sparse_comm =
             sparse_communicator::create(comm, localized_partition);
 
         auto device_exec = exec_handler.get_device_exec();
         auto dist_A = gko::share(generate_dist_mtx_with_inner_type<dist_mtx>(
-            matrix_format, exec, sparse_comm, local_sparsity_,
+            matrix_format, device_exec, sparse_comm, local_sparsity_,
             non_local_sparsity_));
 
         auto local_coeffs = gko::array<scalar>(exec, local_sparsity_->size_);
@@ -257,6 +238,23 @@ public:
             non_local_coeffs.fill(0.0);
         }
 
+
+         // std::cout << __FILE__ << " rank " << rank << "\n\tlocal sparsity size "
+         //           << local_sparsity_->size_ << " dim ["
+         //           << local_sparsity_->dim[0] << "x" << local_sparsity_->dim[1]
+         //           << "]\n\tnon_local sparsity size " << non_local_sparsity_->size_
+         //           << " dim [" << non_local_sparsity_->dim[0]
+         //           << "x" << non_local_sparsity_->dim[1] << "]\n\tcomm pattern:\n"
+         //           << "\ttarget_ids: "
+         //           << dst_comm_pattern->target_ids << "\n\ttarget_sizes: "
+         //           << dst_comm_pattern->target_sizes << "\n\ttarget_send_idxs.size "
+         //           << dst_comm_pattern->send_idxs.size()
+         //           << "\n\tget_recv_indices "
+         //           << localized_partition->get_recv_indices().get_num_elems()
+         //           << "\n\tget_send_indices "
+         //           << localized_partition->get_send_indices().get_num_elems()
+         //           << "\n";
+
         // FIXME make sure that we work on the device executor
         dist_A->read_distributed(
             device_matrix_data(exec, local_sparsity_->dim,

From 05a5d8ca4630b220dbeb4056fbffeb3ad5bb174d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <go@hpsim.de>
Date: Tue, 11 Jun 2024 08:53:17 +0000
Subject: [PATCH 82/90] WIP fix intel

---
 .../CommunicationPattern.C                    |   9 ++
 MatrixWrapper/Distributed/Distributed.H       | 132 ++++++++++++++----
 .../SparsityPattern/SparsityPattern.H         |  10 ++
 Preconditioner/Preconditioner.H               |   2 +-
 Repartitioner/Repartitioner.H                 |  35 ++++-
 5 files changed, 157 insertions(+), 31 deletions(-)

diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index 4fce7e2a5..3fc94c585 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -103,6 +103,7 @@ compute_send_recv_counts(const ExecutorHandler &exec_handler,
 
     label tot_recv_elements{0};
     label comm_elements_buffer{0};
+    std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
     if (rank == owner_rank) {
         // send and recv to it self
         recv_offsets[owner_rank] = padding_before;
@@ -111,6 +112,7 @@ compute_send_recv_counts(const ExecutorHandler &exec_handler,
         // the start of the next rank data
         tot_recv_elements = padding_before + size + padding_after;
 
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         for (int i = 1; i < ranks_per_gpu; i++) {
             // receive the recv counts
             comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
@@ -166,6 +168,13 @@ void communicate_values (
     // send_buffer should be on the host
     // recv_buffer should be on the device
     // auto rank = comm.rank();
+    std::cout
+	    << __FILE__ << ":" << __LINE__
+	    << " send_counts " <<   send_counts
+	    << " recv_counts " << recv_counts
+	    << " send_offsets " << send_offsets
+	    << " recv_offsets " << recv_offsets
+	    << "\n";
 
     comm.all_to_all_v(exec, send_buffer, send_counts.data(),
                       send_offsets.data(), recv_buffer, recv_counts.data(),
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 9e4fbc0d3..534a06030 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -215,10 +215,30 @@ public:
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " build_localized_partition \n";
+                 //<< " dim " << local_sparsity_->dim[0] << " send idxs size "
+                 //<< dst_comm_pattern.send_idxs.size() << " target ids "
+                 //<< dst_comm_pattern.target_ids << " target sizes "
+                 //<< dst_comm_pattern.target_sizes << "\n";
+
         auto localized_partition = local_part_type::build_from_blocked_recv(
             exec, local_sparsity_->dim[0], dst_comm_pattern->send_idxs,
             dst_comm_pattern->target_ids, dst_comm_pattern->target_sizes);
 
+        std::cout << __FILE__ << " rank " << rank << " local sparsity size "
+                  << local_sparsity_->size_ << " local sparsity dim ["
+                  << local_sparsity_->dim[0] << "x" << local_sparsity_->dim[1]
+                  << "] non_local sparsity size " << non_local_sparsity_->size_
+                  << " non local sparsity dim [" << non_local_sparsity_->dim[0]
+                  << "x" << non_local_sparsity_->dim[1] << "] target_ids "
+                  << dst_comm_pattern->target_ids << " target_sizes "
+                  << dst_comm_pattern->target_sizes << " target_send_idxs.size "
+                  << dst_comm_pattern->send_idxs.size()
+                  << " non_local_sparsity.size " << non_local_sparsity_->size_
+                  << " get_recv_indices "
+                  << localized_partition->get_recv_indices().get_num_elems()
+                  << " \n";
 
         auto sparse_comm =
             sparse_communicator::create(comm, localized_partition);
@@ -264,11 +284,15 @@ public:
                                non_local_sparsity_->row_idxs,
                                non_local_sparsity_->col_idxs, non_local_coeffs),
             sparse_comm);
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " dnoe read distributed  \n";
 
 
         update_impl(exec_handler, matrix_format, repartitioner, host_A, dist_A,
                     local_sparsity_, non_local_sparsity_, src_comm_pattern,
                     local_interfaces);
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " dnoe update impl  \n";
 
         auto ret = std::make_shared<RepartDistMatrix>(
             exec, comm, repartitioner.get_repart_dim(), dist_A->get_size(),
@@ -305,6 +329,8 @@ public:
         auto exec = exec_handler.get_ref_exec();
         auto device_exec = exec_handler.get_device_exec();
         auto ranks_per_gpu = repartitioner.get_ranks_per_gpu();
+	bool requires_host_buffer = exec_handler.get_gko_force_host_buffer();
+
         label rank{repartitioner.get_rank(exec_handler)};
         label owner_rank = repartitioner.get_owner_rank(exec_handler);
         bool owner = repartitioner.is_owner(exec_handler);
@@ -314,29 +340,43 @@ public:
         auto diag_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, nrows, local_matrix_nnz,
             local_matrix_nnz - nrows, 0);
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " diag comm pattern  \n";
 
 
         label upper_nnz = host_A->get_upper_nnz();
         auto upper_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, upper_nnz, local_matrix_nnz, 0,
             local_matrix_nnz - upper_nnz);
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " upper comm pattern  \n";
         auto lower_comm_pattern =
             compute_send_recv_counts(exec_handler, ranks_per_gpu, upper_nnz,
                                      local_matrix_nnz, upper_nnz, nrows);
 
         scalar *local_ptr;
+        scalar *local_ptr_2;
+	label nnz=0;
 
         // update main values
+	std::vector<scalar> loc_buffer;
         if (owner) {
             using Coo = gko::matrix::Coo<scalar, label>;
             auto local_mtx = dist_A->get_local_matrix();
 
+
             std::shared_ptr<const Coo> local =
                 gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
                                  dist_A->get_local_matrix())
                                  ->get_combination()
                                  ->get_operators()[0]);
-            local_ptr = const_cast<scalar *>(local->get_const_values());
+	    if (requires_host_buffer) {
+		    loc_buffer.resize(local->get_num_stored_elements());
+		    local_ptr = loc_buffer.data();
+		    local_ptr_2 = const_cast<scalar *>(local->get_const_values());
+	    } else {
+		    local_ptr = const_cast<scalar *>(local->get_const_values());
+	    }
         }
         communicate_values(exec_handler, diag_comm_pattern, host_A->get_diag(),
                            local_ptr);
@@ -352,6 +392,18 @@ public:
             communicate_values(exec_handler, lower_comm_pattern,
                                host_A->get_lower(), local_ptr);
         }
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " done comm local mtx \n";
+
+	if (requires_host_buffer) {	
+		auto host_buffer_view =
+            gko::array<scalar>::view(exec, nnz, local_ptr);
+		auto target_buffer_view =
+            gko::array<scalar>::view(device_exec, nnz, local_ptr_2);
+		target_buffer_view = host_buffer_view;
+	}
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " done copy to device  \n";
 
         // copy interface values
         auto comm = *exec_handler.get_communicator().get();
@@ -364,6 +416,7 @@ public:
             label tag = 0;
             label comm_rank, comm_size;
             scalar *recv_buffer_ptr;
+	    std::vector<scalar> host_recv_buffer;
             label remain_host_interfaces = host_A->get_interface_size();
             for (auto [is_local, comm_rank] : local_interfaces) {
                 label &ctr = (is_local) ? loc_ctr : nloc_ctr;
@@ -383,9 +436,18 @@ public:
                     comm_size =
                         non_local_sparsity->interface_spans[ctr].length();
                 }
-                recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
+
+		if (requires_host_buffer) {
+			host_recv_buffer.resize(comm_size);
+			recv_buffer_ptr = host_recv_buffer.data();
+		} else {
+			recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
+		}
 
                 if (comm_rank != rank) {
+		   std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+				  << " comm_rank " << comm_rank << " rank " << rank << "  \n";
+
                     comm.recv(exec, recv_buffer_ptr, comm_size, comm_rank, tag);
                 } else {
                     // if data is already on this rank
@@ -427,33 +489,47 @@ public:
             }
         }
 
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " reorder \n";
         // reorder updated values
-        if (owner) {
-            // NOTE local sparsity size includes the interfaces
-            using Coo = gko::matrix::Coo<scalar, label>;
-            using dim_type = gko::dim<2>::dimension_type;
-            std::shared_ptr<const Coo> local =
-                gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
-                                 dist_A->get_local_matrix())
-                                 ->get_combination()
-                                 ->get_operators()[0]);
-            auto local_elements = local->get_num_stored_elements();
-            local_ptr = const_cast<scalar *>(local->get_const_values());
-            // TODO make sure this doesn't copy
-            // create a non owning dense matrix of local_values
-
-            auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
-                exec, gko::dim<2>{static_cast<dim_type>(local_elements), 1},
-                gko::array<scalar>::view(exec, local_elements, local_ptr), 1));
-
-            auto mapping_view = gko::array<label>::view(
-                exec, local_elements, local_sparsity->ldu_mapping.get_data());
-
-
-            // TODO this needs to copy ldu_mapping to the device
-            auto dense_vec = row_collection->clone();
-            dense_vec->row_gather(&mapping_view, row_collection.get());
-        }
+      if (owner) {
+          // NOTE local sparsity size includes the interfaces
+          using Coo = gko::matrix::Coo<scalar, label>;
+          using dim_type = gko::dim<2>::dimension_type;
+          std::shared_ptr<const Coo> local =
+              gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
+                               dist_A->get_local_matrix())
+                               ->get_combination()
+                               ->get_operators()[0]);
+          auto local_elements = local->get_num_stored_elements();
+          local_ptr = const_cast<scalar *>(local->get_const_values());
+          // TODO make sure this doesn't copy
+          // create a non owning dense matrix of local_values
+
+          auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
+              device_exec, gko::dim<2>{static_cast<dim_type>(local_elements), 1},
+              gko::array<scalar>::view(device_exec, local_elements, local_ptr), 1));
+      	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << " local_elements " << local_elements
+      		  << " reorder \n";
+
+          auto mapping_view = gko::array<label>::view(
+              exec, local_elements, local_sparsity->ldu_mapping.get_data());
+      	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+      		  << " reorder \n";
+
+
+          // TODO this needs to copy ldu_mapping to the device
+           auto dense_vec = row_collection->clone();
+        //auto dense_vec = gko::share(gko::matrix::Dense<scalar>::create(exec, row_collection->get_size()));
+
+      	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+      		  << " reorder \n";
+          dense_vec->row_gather(&mapping_view, row_collection.get());
+      	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+      		  << " reorder \n";
+      }
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << "done reorder \n";
     };
 
     RepartDistMatrix(
diff --git a/MatrixWrapper/SparsityPattern/SparsityPattern.H b/MatrixWrapper/SparsityPattern/SparsityPattern.H
index cc4a71b0e..499346d17 100644
--- a/MatrixWrapper/SparsityPattern/SparsityPattern.H
+++ b/MatrixWrapper/SparsityPattern/SparsityPattern.H
@@ -47,6 +47,16 @@ struct SparsityPattern {
           rank(std::vector<label>{})
     {}
 
+    SparsityPattern(std::shared_ptr<const SparsityPattern> other)
+        : size_(other->size_),
+          row_idxs(other->row_idxs),
+          col_idxs(other->col_idxs),
+          ldu_mapping(other->ldu_mapping),
+          dim(other->dim),
+          interface_spans(other->interface_spans),
+          rank(other->rank)
+    {}
+
     SparsityPattern(std::shared_ptr<const gko::Executor> exec, label size)
         : size_(size),
           row_idxs{exec, static_cast<gko::size_type>(size_)},
diff --git a/Preconditioner/Preconditioner.H b/Preconditioner/Preconditioner.H
index ca75213a1..635363068 100644
--- a/Preconditioner/Preconditioner.H
+++ b/Preconditioner/Preconditioner.H
@@ -262,7 +262,7 @@ public:
             auto smoother_gen = gko::share(
                 ir::build()
                     .with_solver(inner_solver_gen)
-                    .with_relaxation_factor(0.9)
+                    //.with_relaxation_factor(0.9)
                     .with_criteria(
                         gko::stop::Iteration::build().with_max_iters(2u).on(
                             device_exec))
diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index cfbdea84c..08530d4f0 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -214,7 +214,8 @@ public:
     ** signals whether this is a new local interface (no communication), and the
     ** second entry (label) tracks the original rank of the interface
     */
-    std::tuple<std::shared_ptr<SparsityPattern>,
+    std::tuple<
+	    std::shared_ptr<SparsityPattern>,
                std::shared_ptr<SparsityPattern>,
                std::vector<std::pair<bool, label>>>
     repartition_sparsity(
@@ -238,12 +239,36 @@ public:
         label rank = get_rank(exec_handler);
         label owner_rank = get_owner_rank(exec_handler);
         label ranks_per_gpu = ranks_per_gpu_;
+	// TODO dont copy
+        if (ranks_per_gpu == 1) {
+               std::vector<std::pair<bool, label>> ret;
+	       for (auto comm_rank:src_non_local_pattern->rank) {
+		       ret.emplace_back(false, rank);
+	       }
+
+
+         	return std::make_tuple<
+	       std::shared_ptr<SparsityPattern>,
+               std::shared_ptr<SparsityPattern>,
+               std::vector<std::pair<bool, label>>>(
+         		std::make_shared<SparsityPattern>(src_local_pattern),
+	        	std::make_shared<SparsityPattern>(src_non_local_pattern),
+	               	std::move(ret)
+        	);
+        }
 
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         auto local_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, src_local_pattern->size_);
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank  << " owner rank" << owner_rank<< "\n";
+
 
-        label offset = orig_partition_->get_range_bounds()[rank] -
+        label offset = 0;
+        if (ranks_per_gpu != 1){
+	offset = 	orig_partition_->get_range_bounds()[rank] -
                        orig_partition_->get_range_bounds()[owner_rank];
+	}
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         auto gather_closure = [&](auto &comm_pattern, auto &data,
                                   label offset) {
@@ -251,6 +276,7 @@ public:
                                    data.get_data(), offset);
         };
 
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         SparsityPatternVector merged_local{
             gather_closure(local_comm_pattern, src_local_pattern->row_idxs,
                            offset),
@@ -267,6 +293,7 @@ public:
             make_ldu_mapping_consecutive(
                 local_comm_pattern, merged_local.mapping, rank, ranks_per_gpu);
         }
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         label rows =
             (is_owner(exec_handler)) ? merged_local.rows.back() + 1 : 0;
@@ -284,6 +311,7 @@ public:
             spans_begin.push_back(elem.begin);
             spans_end.push_back(elem.end);
         }
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         // the non local cols are in local idx of other side
         // thus we need the new offset of the other side
@@ -300,6 +328,7 @@ public:
             std::transform(data, data + size, data,
                            [&](label idx) { return idx + local_offset; });
         }
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         SparsityPatternVector merged_non_local{
             gather_closure(non_local_comm_pattern,
@@ -328,6 +357,7 @@ public:
         // build vector with locality information
         std::vector<std::pair<bool, label>> locality;
         label ctr{0};
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         if (is_owner(exec_handler)) {
             auto recv_counts = std::get<1>(span_comm_pattern);
@@ -354,6 +384,7 @@ public:
             gathered_non_local.cols[i] = i;
         }
 
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         LOG_1(verbose_, "done repartition sparsity pattern")
         if (is_owner(exec_handler)) {
             auto new_local_spars_pattern = std::make_shared<SparsityPattern>(

From 630b0c4125fbc2430d1cc33eb5ebfa88b592d5e7 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 11 Jun 2024 12:19:07 +0200
Subject: [PATCH 83/90] rebase to gko.1.8.0

---
 CMakeLists.txt                          |   3 +-
 DevicePersistent/Base/Base.C            |   2 +-
 DevicePersistent/Partition/Partition.H  |  15 ++--
 MatrixWrapper/Distributed/Distributed.H | 103 +++++++++++-------------
 4 files changed, 59 insertions(+), 64 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 026fc00c9..9e2d19875 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -74,7 +74,8 @@ if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO))
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "sparse-communicator-cmake-modernization-combination-interface-ogl-rebase"
+    "ogl_rebase_1.8.0"
+    # "sparse-communicator-cmake-modernization-combination-interface-ogl-rebase"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)
diff --git a/DevicePersistent/Base/Base.C b/DevicePersistent/Base/Base.C
index 27fdb3361..0e01375e7 100644
--- a/DevicePersistent/Base/Base.C
+++ b/DevicePersistent/Base/Base.C
@@ -28,7 +28,7 @@ defineTemplateTypeNameWithName(DevicePersistentBase<gko::LinOp>,
 
 // typedef needed  to avoid confusion with the comma separated template
 // arguments as macro arguments
-typedef gko::experimental::distributed::localized_partition<label> Partition;
+typedef gko::experimental::distributed::Partition<label, label> Partition;
 defineTemplateTypeNameWithName(DevicePersistentBase<Partition>,
                                "PersistentPartition");
 
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 49f6c87a3..03ea9bf06 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -16,10 +16,11 @@
 namespace Foam {
 
 
+using part_type =
+    gko::experimental::distributed::Partition<label, label>;
+
 struct PartitionInitFunctor {
     using comm_size_type = int;
-    using part_type =
-        gko::experimental::distributed::localized_partition<label>;
 
     const ExecutorHandler &exec_;
 
@@ -57,13 +58,14 @@ struct PartitionInitFunctor {
     std::shared_ptr<part_type> init() const
     {
         auto exec = exec_.get_ref_exec();
+        auto comm = *exec_.get_communicator().get();
 
         word msg{"initialising partition of size " +
                  std::to_string(local_size_)};
         LOG_1(verbose_, msg)
 
-        auto host_partition = part_type::build_from_blocked_recv(
-            exec, local_size_, send_idxs_, target_ids_, target_sizes_);
+        auto host_partition = gko::share(gko::experimental::distributed::build_partition_from_local_size<label,label>(
+            exec, comm, local_size_));
 
         return host_partition;
     }
@@ -78,8 +80,7 @@ struct PartitionInitFunctor {
  * reference or omp.
  * */
 class PersistentPartition
-    : public PersistentBase<
-          gko::experimental::distributed::localized_partition<label>,
+    : public PersistentBase<part_type,
           PartitionInitFunctor> {
     const int ranks_per_gpu_;
 
@@ -90,8 +91,6 @@ class PersistentPartition
     const ExecutorHandler &exec_;
 
 public:
-    using part_type =
-        gko::experimental::distributed::localized_partition<label>;
 
     /* PersistentPartition constructor using existing memory
      *
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 9e4fbc0d3..ac207e22c 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -38,8 +38,7 @@ SourceFiles
 template <typename DistMtxType>
 std::unique_ptr<DistMtxType> generate_dist_mtx_with_inner_type(
     word matrix_format, std::shared_ptr<const gko::Executor> exec,
-    std::shared_ptr<const gko::experimental::distributed::sparse_communicator>
-        comm,
+    gko::experimental::mpi::communicator comm,
     std::shared_ptr<const SparsityPattern> local_sparsity,
     std::shared_ptr<const SparsityPattern> non_local_sparsity)
 {
@@ -110,14 +109,10 @@ class RepartDistMatrix
                                                GlobalIndexType>;
 
     using part_type = gko::experimental::distributed::Partition<label, label>;
-    using local_part_type =
-        gko::experimental::distributed::localized_partition<label>;
 
     using vec = gko::matrix::Dense<ValueType>;
     using device_matrix_data = gko::device_matrix_data<scalar, label>;
     using communicator = gko::experimental::mpi::communicator;
-    using sparse_communicator =
-        gko::experimental::distributed::sparse_communicator;
 
 public:
     using gko::experimental::EnableDistributedLinOp<RepartDistMatrix<
@@ -142,7 +137,8 @@ public:
     RepartDistMatrix &operator=(const RepartDistMatrix &other)
     {
         if (&other != this) {
-            // FatalErrorInFunction << "Copying the RepartDistMatrix is disallowed "
+            // FatalErrorInFunction << "Copying the RepartDistMatrix is
+            // disallowed "
             //                         "for performance reasons"
             //                      << abort(FatalError);
             gko::experimental::EnableDistributedLinOp<
@@ -150,8 +146,8 @@ public:
             this->dist_mtx_ = other.dist_mtx_;
             this->local_sparsity_ = other.local_sparsity_;
             this->non_local_sparsity_ = other.non_local_sparsity_;
-            this->src_comm_pattern_ =other.src_comm_pattern_;
-            this->local_interfaces_ =other.local_interfaces_;
+            this->src_comm_pattern_ = other.src_comm_pattern_;
+            this->local_interfaces_ = other.local_interfaces_;
         }
         return *this;
     }
@@ -164,22 +160,21 @@ public:
     RepartDistMatrix &operator=(RepartDistMatrix &&other)
     {
         if (&other != this) {
-            FatalErrorInFunction << "Not implemented"
-                                << abort(FatalError);
+            FatalErrorInFunction << "Not implemented" << abort(FatalError);
             gko::experimental::EnableDistributedLinOp<
                 RepartDistMatrix>::operator=(std::move(other));
             this->dist_mtx_ = std::move(other.dist_mtx_);
-            this->local_sparsity_ =std::move(other.local_sparsity_);
-            this->non_local_sparsity_ =std::move(other.non_local_sparsity_);
-            this->src_comm_pattern_ =std::move(other.src_comm_pattern_);
-            this->local_interfaces_ =std::move(other.local_interfaces_);
+            this->local_sparsity_ = std::move(other.local_sparsity_);
+            this->non_local_sparsity_ = std::move(other.non_local_sparsity_);
+            this->src_comm_pattern_ = std::move(other.src_comm_pattern_);
+            this->local_interfaces_ = std::move(other.local_interfaces_);
         }
         return *this;
     }
 
     static std::shared_ptr<RepartDistMatrix> create(
-        const ExecutorHandler &exec_handler,
-        word matrix_format, const Repartitioner &repartitioner,
+        const ExecutorHandler &exec_handler, word matrix_format,
+        const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A)
     {
         auto exec = exec_handler.get_ref_exec();
@@ -215,17 +210,9 @@ public:
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
-        auto localized_partition = local_part_type::build_from_blocked_recv(
-            exec, local_sparsity_->dim[0], dst_comm_pattern->send_idxs,
-            dst_comm_pattern->target_ids, dst_comm_pattern->target_sizes);
-
-
-        auto sparse_comm =
-            sparse_communicator::create(comm, localized_partition);
-
         auto device_exec = exec_handler.get_device_exec();
         auto dist_A = gko::share(generate_dist_mtx_with_inner_type<dist_mtx>(
-            matrix_format, device_exec, sparse_comm, local_sparsity_,
+            matrix_format, device_exec, comm, local_sparsity_,
             non_local_sparsity_));
 
         auto local_coeffs = gko::array<scalar>(exec, local_sparsity_->size_);
@@ -238,33 +225,43 @@ public:
             non_local_coeffs.fill(0.0);
         }
 
-
-         // std::cout << __FILE__ << " rank " << rank << "\n\tlocal sparsity size "
-         //           << local_sparsity_->size_ << " dim ["
-         //           << local_sparsity_->dim[0] << "x" << local_sparsity_->dim[1]
-         //           << "]\n\tnon_local sparsity size " << non_local_sparsity_->size_
-         //           << " dim [" << non_local_sparsity_->dim[0]
-         //           << "x" << non_local_sparsity_->dim[1] << "]\n\tcomm pattern:\n"
-         //           << "\ttarget_ids: "
-         //           << dst_comm_pattern->target_ids << "\n\ttarget_sizes: "
-         //           << dst_comm_pattern->target_sizes << "\n\ttarget_send_idxs.size "
-         //           << dst_comm_pattern->send_idxs.size()
-         //           << "\n\tget_recv_indices "
-         //           << localized_partition->get_recv_indices().get_num_elems()
-         //           << "\n\tget_send_indices "
-         //           << localized_partition->get_send_indices().get_num_elems()
-         //           << "\n";
+        // std::cout << __FILE__ << " rank " << rank << "\n\tlocal sparsity size
+        // "
+        //           << local_sparsity_->size_ << " dim ["
+        //           << local_sparsity_->dim[0] << "x" <<
+        //           local_sparsity_->dim[1]
+        //           << "]\n\tnon_local sparsity size " <<
+        //           non_local_sparsity_->size_
+        //           << " dim [" << non_local_sparsity_->dim[0]
+        //           << "x" << non_local_sparsity_->dim[1] << "]\n\tcomm
+        //           pattern:\n"
+        //           << "\ttarget_ids: "
+        //           << dst_comm_pattern->target_ids << "\n\ttarget_sizes: "
+        //           << dst_comm_pattern->target_sizes <<
+        //           "\n\ttarget_send_idxs.size "
+        //           << dst_comm_pattern->send_idxs.size()
+        //           << "\n\tget_recv_indices "
+        //           << localized_partition->get_recv_indices().get_num_elems()
+        //           << "\n\tget_send_indices "
+        //           << localized_partition->get_send_indices().get_num_elems()
+        //           << "\n";
 
         // FIXME make sure that we work on the device executor
         dist_A->read_distributed(
             device_matrix_data(exec, local_sparsity_->dim,
                                local_sparsity_->row_idxs,
                                local_sparsity_->col_idxs, local_coeffs),
-            device_matrix_data(exec, non_local_sparsity_->dim,
-                               non_local_sparsity_->row_idxs,
-                               non_local_sparsity_->col_idxs, non_local_coeffs),
-            sparse_comm);
+            device_matrix_data(
+                exec, non_local_sparsity_->dim, non_local_sparsity_->row_idxs,
+                non_local_sparsity_->col_idxs, non_local_coeffs));
+
 
+        // auto dist_A = DistMtxType::create(
+        //        exec, comm,
+        //        CombinationMatrix<scalar, label, mtx_type>::create(
+        //            exec, local_dim, local_interfaces),
+        //        CombinationMatrix<scalar, label, mtx_type>::create(
+        //            exec, non_local_dim, non_local_interfaces));
 
         update_impl(exec_handler, matrix_format, repartitioner, host_A, dist_A,
                     local_sparsity_, non_local_sparsity_, src_comm_pattern,
@@ -278,20 +275,18 @@ public:
         return ret;
     };
 
-    void update(
-        const ExecutorHandler &exec_handler,
-        word matrix_format, const Repartitioner &repartitioner,
+    void update(const ExecutorHandler &exec_handler, word matrix_format,
+                const Repartitioner &repartitioner,
                 std::shared_ptr<const HostMatrixWrapper> host_A)
     {
-        update_impl(exec_handler, matrix_format, repartitioner, host_A, dist_mtx_,
-                    local_sparsity_, non_local_sparsity_, src_comm_pattern_,
-                    local_interfaces_);
+        update_impl(exec_handler, matrix_format, repartitioner, host_A,
+                    dist_mtx_, local_sparsity_, non_local_sparsity_,
+                    src_comm_pattern_, local_interfaces_);
     };
 
     // TODO matrix_format can be data member
     static void update_impl(
-        const ExecutorHandler &exec_handler,
-        word matrix_format,
+        const ExecutorHandler &exec_handler, word matrix_format,
         const Repartitioner &repartitioner,
         std::shared_ptr<const HostMatrixWrapper> host_A,
         std::shared_ptr<

From e582bf1439435b256d602553ea4194777d8d97a6 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <go@hpsim.de>
Date: Tue, 11 Jun 2024 08:53:17 +0000
Subject: [PATCH 84/90] WIP fix intel

---
 CMakeLists.txt                                |   2 +-
 DevicePersistent/Vector/Vector.H              |  43 +++--
 MatrixWrapper/Combination/Combination.H       |   3 -
 .../CommunicationPattern.C                    |  11 +-
 MatrixWrapper/Distributed/Distributed.H       | 152 +++++++++++++-----
 .../SparsityPattern/SparsityPattern.H         |  10 ++
 Preconditioner/Preconditioner.H               |   2 +-
 Repartitioner/Repartitioner.H                 |  39 ++++-
 StoppingCriterion/StoppingCriterion.C         |   3 +
 9 files changed, 204 insertions(+), 61 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 026fc00c9..ba63f7f16 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -74,7 +74,7 @@ if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO))
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "sparse-communicator-cmake-modernization-combination-interface-ogl-rebase"
+    "ogl_rebase_1.8.0"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 5258353e4..a9397fa98 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -22,23 +22,24 @@ namespace Foam {
 template <class T>
 struct VectorInitFunctor {
     using vec = gko::matrix::Dense<scalar>;
-    using dist_vec = gko::experimental::distributed::Vector<scalar>;
-
-    const word name_;
 
-    const Repartitioner &repartitioner_;
+    using dist_vec = gko::experimental::distributed::Vector<scalar>;
 
     const ExecutorHandler &exec_;
 
-    const label verbose_;
+    const word name_;
 
-    const bool on_device_;
+    const Repartitioner &repartitioner_;
 
     // Memory from which array will be initialised
     const T *other_;
 
     const label size_;
 
+    const label verbose_;
+
+    const bool on_device_;
+
     VectorInitFunctor(const ExecutorHandler &exec, const word name,
                       const Repartitioner &repartitioner, const T *other,
                       const label size, const label verbose,
@@ -82,20 +83,22 @@ struct VectorInitFunctor {
         auto host_view =
             gko::array<T>::const_view(exec_.get_ref_exec(), local_size, other_);
 
+
         // FIXME this shouldn't be needed here also not during an
         // update
         auto comm_pattern = compute_send_recv_counts(
             exec_, repartitioner_.get_ranks_per_gpu(), local_size);
 
-        // this should happen on device
         auto local_coeffs = gko::array<scalar>(exec, repart_size);
 
         communicate_values(exec_, comm_pattern, host_view.get_const_data(),
                            local_coeffs.get_data());
 
         auto ret = gko::share(dist_vec::create(
-            exec, *comm.get(),
+            exec_.get_device_exec(), *comm.get(),
             vec::create(exec, gko::dim<2>{repart_size, 1}, local_coeffs, 1)));
+
+
         return ret;
     }
 };
@@ -175,9 +178,27 @@ public:
         //     << " comm_pattern recv_offs " << std::get<3>(comm_pattern)
         //     << "\n";
 
-        communicate_values(exec_, comm_pattern,
-                           get_vector()->get_local_values(),
-                           const_cast<T *>(memory_));
+        std::vector<scalar> host_send_buffer;
+	scalar  *send_ptr;
+
+	if (true){
+		label repart_size = repartitioner_.get_repart_size();
+		host_send_buffer.resize(repart_size);
+		send_ptr = host_send_buffer.data();
+		auto host_exec = exec_.get_ref_exec(); 
+		auto device_exec = exec_.get_device_exec(); 
+		auto host_buffer_view =
+		    gko::array<scalar>::view(host_exec, repart_size, send_ptr);
+		auto target_buffer_view =
+		    gko::array<scalar>::view(device_exec, repart_size, get_vector()->get_local_values()
+				    );
+		target_buffer_view = host_buffer_view;
+	} else {
+		send_ptr = get_vector()->get_local_values();
+	}
+
+	communicate_values(exec_, comm_pattern, send_ptr,
+				   const_cast<T *>(memory_));
     }
 
     /** Writes the content of the distributed vector to disk
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index e2c8b8c58..4638f6c20 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -33,9 +33,6 @@ public:
 
     void read(const device_matrix_data &data) override
     {
-        // TODO FIXME if no matrix is given because of repartitioning
-        // which might remove non_local matrix
-        // a zero sized dummy matrix needs to be created
         auto exec = this->get_executor();
         auto data_exec = data.get_executor();
 
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index 4fce7e2a5..265baa08f 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -103,6 +103,7 @@ compute_send_recv_counts(const ExecutorHandler &exec_handler,
 
     label tot_recv_elements{0};
     label comm_elements_buffer{0};
+    std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
     if (rank == owner_rank) {
         // send and recv to it self
         recv_offsets[owner_rank] = padding_before;
@@ -111,6 +112,7 @@ compute_send_recv_counts(const ExecutorHandler &exec_handler,
         // the start of the next rank data
         tot_recv_elements = padding_before + size + padding_after;
 
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         for (int i = 1; i < ranks_per_gpu; i++) {
             // receive the recv counts
             comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);
@@ -165,7 +167,14 @@ void communicate_values (
     //
     // send_buffer should be on the host
     // recv_buffer should be on the device
-    // auto rank = comm.rank();
+    // oensrtauto rank = comm.rank();
+// std:::cout
+//     << __FILE__ << ":" << __LINE__
+//     << " send_counts " <<   send_counts
+//     << " recv_counts " << recv_counts
+//     << " send_offsets " << send_offsets
+//     << " recv_offsets " << recv_offsets
+//     << "\n";
 
     comm.all_to_all_v(exec, send_buffer, send_counts.data(),
                       send_offsets.data(), recv_buffer, recv_counts.data(),
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 9e4fbc0d3..3a518f23c 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -146,7 +146,7 @@ public:
             //                         "for performance reasons"
             //                      << abort(FatalError);
             gko::experimental::EnableDistributedLinOp<
-                RepartDistMatrix>::operator=(std::move(other));
+                RepartDistMatrix>::operator=(other);
             this->dist_mtx_ = other.dist_mtx_;
             this->local_sparsity_ = other.local_sparsity_;
             this->non_local_sparsity_ = other.non_local_sparsity_;
@@ -215,10 +215,30 @@ public:
         local_sparsity_ = repart_loc_sparsity;
         non_local_sparsity_ = repart_non_loc_sparsity;
 
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " build_localized_partition \n";
+                 //<< " dim " << local_sparsity_->dim[0] << " send idxs size "
+                 //<< dst_comm_pattern.send_idxs.size() << " target ids "
+                 //<< dst_comm_pattern.target_ids << " target sizes "
+                 //<< dst_comm_pattern.target_sizes << "\n";
+
         auto localized_partition = local_part_type::build_from_blocked_recv(
             exec, local_sparsity_->dim[0], dst_comm_pattern->send_idxs,
             dst_comm_pattern->target_ids, dst_comm_pattern->target_sizes);
 
+        std::cout << __FILE__ << " rank " << rank << " local sparsity size "
+                  << local_sparsity_->size_ << " local sparsity dim ["
+                  << local_sparsity_->dim[0] << "x" << local_sparsity_->dim[1]
+                  << "] non_local sparsity size " << non_local_sparsity_->size_
+                  << " non local sparsity dim [" << non_local_sparsity_->dim[0]
+                  << "x" << non_local_sparsity_->dim[1] << "] target_ids "
+                  << dst_comm_pattern->target_ids << " target_sizes "
+                  << dst_comm_pattern->target_sizes << " target_send_idxs.size "
+                  << dst_comm_pattern->send_idxs.size()
+                  << " non_local_sparsity.size " << non_local_sparsity_->size_
+                  << " get_recv_indices "
+                  << localized_partition->get_recv_indices().get_num_elems()
+                  << " \n";
 
         auto sparse_comm =
             sparse_communicator::create(comm, localized_partition);
@@ -264,11 +284,11 @@ public:
                                non_local_sparsity_->row_idxs,
                                non_local_sparsity_->col_idxs, non_local_coeffs),
             sparse_comm);
-
-
         update_impl(exec_handler, matrix_format, repartitioner, host_A, dist_A,
                     local_sparsity_, non_local_sparsity_, src_comm_pattern,
                     local_interfaces);
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " dnoe update impl  \n";
 
         auto ret = std::make_shared<RepartDistMatrix>(
             exec, comm, repartitioner.get_repart_dim(), dist_A->get_size(),
@@ -305,6 +325,8 @@ public:
         auto exec = exec_handler.get_ref_exec();
         auto device_exec = exec_handler.get_device_exec();
         auto ranks_per_gpu = repartitioner.get_ranks_per_gpu();
+	bool requires_host_buffer = exec_handler.get_gko_force_host_buffer();
+
         label rank{repartitioner.get_rank(exec_handler)};
         label owner_rank = repartitioner.get_owner_rank(exec_handler);
         bool owner = repartitioner.is_owner(exec_handler);
@@ -314,8 +336,6 @@ public:
         auto diag_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, nrows, local_matrix_nnz,
             local_matrix_nnz - nrows, 0);
-
-
         label upper_nnz = host_A->get_upper_nnz();
         auto upper_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, upper_nnz, local_matrix_nnz, 0,
@@ -325,18 +345,29 @@ public:
                                      local_matrix_nnz, upper_nnz, nrows);
 
         scalar *local_ptr;
+        scalar *local_ptr_2;
+	label nnz=0;
 
         // update main values
+	std::vector<scalar> loc_buffer;
         if (owner) {
             using Coo = gko::matrix::Coo<scalar, label>;
             auto local_mtx = dist_A->get_local_matrix();
 
+
             std::shared_ptr<const Coo> local =
                 gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
                                  dist_A->get_local_matrix())
                                  ->get_combination()
                                  ->get_operators()[0]);
-            local_ptr = const_cast<scalar *>(local->get_const_values());
+	    nnz = local->get_num_stored_elements();
+	    if (requires_host_buffer) {
+		    loc_buffer.resize(nnz);
+		    local_ptr = loc_buffer.data();
+		    local_ptr_2 = const_cast<scalar *>(local->get_const_values());
+	    } else {
+		    local_ptr = const_cast<scalar *>(local->get_const_values());
+	    }
         }
         communicate_values(exec_handler, diag_comm_pattern, host_A->get_diag(),
                            local_ptr);
@@ -352,7 +383,26 @@ public:
             communicate_values(exec_handler, lower_comm_pattern,
                                host_A->get_lower(), local_ptr);
         }
-
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " done comm local mtx \n";
+
+	if (requires_host_buffer) {	
+		auto host_buffer_view =
+            gko::array<scalar>::view(exec, nnz, local_ptr);
+		auto target_buffer_view =
+            gko::array<scalar>::view(device_exec, nnz, local_ptr_2);
+		target_buffer_view = host_buffer_view;
+	}
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << " done copy to device  \n";
+
+	if (requires_host_buffer) {	
+	auto host_buffer_view =
+            gko::array<scalar>::view(exec, nnz, local_ptr);
+	auto target_buffer_view =
+            gko::array<scalar>::view(device_exec, nnz, local_ptr_2);
+		target_buffer_view = host_buffer_view;
+	}
         // copy interface values
         auto comm = *exec_handler.get_communicator().get();
         if (owner) {
@@ -364,6 +414,8 @@ public:
             label tag = 0;
             label comm_rank, comm_size;
             scalar *recv_buffer_ptr;
+            scalar *recv_buffer_ptr_2;
+	    std::vector<scalar> host_recv_buffer;
             label remain_host_interfaces = host_A->get_interface_size();
             for (auto [is_local, comm_rank] : local_interfaces) {
                 label &ctr = (is_local) ? loc_ctr : nloc_ctr;
@@ -383,10 +435,25 @@ public:
                     comm_size =
                         non_local_sparsity->interface_spans[ctr].length();
                 }
-                recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
+
+		if (requires_host_buffer) {
+			host_recv_buffer.resize(comm_size);
+			recv_buffer_ptr = host_recv_buffer.data();
+			recv_buffer_ptr_2 = const_cast<scalar *>(mtx->get_const_values());
+		} else {
+			recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
+		}
 
                 if (comm_rank != rank) {
-                    comm.recv(exec, recv_buffer_ptr, comm_size, comm_rank, tag);
+                    comm.recv(device_exec, recv_buffer_ptr, comm_size, comm_rank, tag);
+		if (requires_host_buffer) {
+				auto host_buffer_view =
+				    gko::array<scalar>::view(exec, comm_size, recv_buffer_ptr);
+				auto target_buffer_view =
+				    gko::array<scalar>::view(device_exec, comm_size, recv_buffer_ptr_2);
+				target_buffer_view = host_buffer_view;
+			}
+
                 } else {
                     // if data is already on this rank
                     auto data_view = gko::array<scalar>::const_view(
@@ -394,8 +461,9 @@ public:
                         host_A->get_interface_data(host_interface_ctr));
 
                     // TODO FIXME this needs target executor
+		    recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
                     auto target_view = gko::array<scalar>::view(
-                        exec, comm_size, recv_buffer_ptr);
+                        device_exec, comm_size, recv_buffer_ptr);
 
                     target_view = data_view;
 
@@ -409,7 +477,7 @@ public:
                 auto neg_one = gko::initialize<vec>({-1.0}, exec);
                 auto interface_dense = vec::create(
                     exec, gko::dim<2>{comm_size, 1},
-                    gko::array<scalar>::view(exec, comm_size, recv_buffer_ptr),
+                    gko::array<scalar>::view(device_exec, comm_size, recv_buffer_ptr),
                     1);
 
                 interface_dense->scale(neg_one);
@@ -423,37 +491,43 @@ public:
                 label comm_size =
                     src_comm_pattern->target_sizes.get_const_data()[i];
                 const scalar *send_buffer_ptr = host_A->get_interface_data(i);
-                comm.send(exec, send_buffer_ptr, comm_size, owner_rank, tag);
+                comm.send(device_exec, send_buffer_ptr, comm_size, owner_rank, tag);
             }
         }
-
         // reorder updated values
-        if (owner) {
-            // NOTE local sparsity size includes the interfaces
-            using Coo = gko::matrix::Coo<scalar, label>;
-            using dim_type = gko::dim<2>::dimension_type;
-            std::shared_ptr<const Coo> local =
-                gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
-                                 dist_A->get_local_matrix())
-                                 ->get_combination()
-                                 ->get_operators()[0]);
-            auto local_elements = local->get_num_stored_elements();
-            local_ptr = const_cast<scalar *>(local->get_const_values());
-            // TODO make sure this doesn't copy
-            // create a non owning dense matrix of local_values
-
-            auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
-                exec, gko::dim<2>{static_cast<dim_type>(local_elements), 1},
-                gko::array<scalar>::view(exec, local_elements, local_ptr), 1));
-
-            auto mapping_view = gko::array<label>::view(
-                exec, local_elements, local_sparsity->ldu_mapping.get_data());
-
-
-            // TODO this needs to copy ldu_mapping to the device
-            auto dense_vec = row_collection->clone();
-            dense_vec->row_gather(&mapping_view, row_collection.get());
-        }
+      if (owner) {
+          // NOTE local sparsity size includes the interfaces
+          using Coo = gko::matrix::Coo<scalar, label>;
+          using dim_type = gko::dim<2>::dimension_type;
+          std::shared_ptr<const Coo> local =
+              gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
+                               dist_A->get_local_matrix())
+                               ->get_combination()
+                               ->get_operators()[0]);
+          auto local_elements = local->get_num_stored_elements();
+          local_ptr = const_cast<scalar *>(local->get_const_values());
+          // TODO make sure this doesn't copy
+          // create a non owning dense matrix of local_values
+
+          auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
+              device_exec, gko::dim<2>{static_cast<dim_type>(local_elements), 1},
+              gko::array<scalar>::view(device_exec, local_elements, local_ptr), 1));
+          auto mapping_view = gko::array<label>::view(
+              exec, local_elements, local_sparsity->ldu_mapping.get_data());
+
+
+          // TODO this needs to copy ldu_mapping to the device
+           auto dense_vec = row_collection->clone();
+        //auto dense_vec = gko::share(gko::matrix::Dense<scalar>::create(exec, row_collection->get_size()));
+
+      	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+      		  << " reorder \n";
+          dense_vec->row_gather(&mapping_view, row_collection.get());
+      	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+      		  << " reorder \n";
+      }
+        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
+                  << "done reorder \n";
     };
 
     RepartDistMatrix(
diff --git a/MatrixWrapper/SparsityPattern/SparsityPattern.H b/MatrixWrapper/SparsityPattern/SparsityPattern.H
index cc4a71b0e..499346d17 100644
--- a/MatrixWrapper/SparsityPattern/SparsityPattern.H
+++ b/MatrixWrapper/SparsityPattern/SparsityPattern.H
@@ -47,6 +47,16 @@ struct SparsityPattern {
           rank(std::vector<label>{})
     {}
 
+    SparsityPattern(std::shared_ptr<const SparsityPattern> other)
+        : size_(other->size_),
+          row_idxs(other->row_idxs),
+          col_idxs(other->col_idxs),
+          ldu_mapping(other->ldu_mapping),
+          dim(other->dim),
+          interface_spans(other->interface_spans),
+          rank(other->rank)
+    {}
+
     SparsityPattern(std::shared_ptr<const gko::Executor> exec, label size)
         : size_(size),
           row_idxs{exec, static_cast<gko::size_type>(size_)},
diff --git a/Preconditioner/Preconditioner.H b/Preconditioner/Preconditioner.H
index ca75213a1..635363068 100644
--- a/Preconditioner/Preconditioner.H
+++ b/Preconditioner/Preconditioner.H
@@ -262,7 +262,7 @@ public:
             auto smoother_gen = gko::share(
                 ir::build()
                     .with_solver(inner_solver_gen)
-                    .with_relaxation_factor(0.9)
+                    //.with_relaxation_factor(0.9)
                     .with_criteria(
                         gko::stop::Iteration::build().with_max_iters(2u).on(
                             device_exec))
diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index cfbdea84c..d21ee5984 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -76,9 +76,10 @@ public:
           verbose_(verbose),
           orig_partition_(gko::share(
               gko::experimental::distributed::build_partition_from_local_size<
-                  label, label>(exec_handler.get_device_exec(),
+                  label, label>(exec_handler.get_ref_exec(),
                                 *exec_handler.get_communicator().get(),
-                                size))) {};
+                                size))) {
+	  };
 
     /* returns the owner rank for a given rank */
     label get_owner_rank(label rank) const
@@ -214,7 +215,8 @@ public:
     ** signals whether this is a new local interface (no communication), and the
     ** second entry (label) tracks the original rank of the interface
     */
-    std::tuple<std::shared_ptr<SparsityPattern>,
+    std::tuple<
+	    std::shared_ptr<SparsityPattern>,
                std::shared_ptr<SparsityPattern>,
                std::vector<std::pair<bool, label>>>
     repartition_sparsity(
@@ -233,17 +235,38 @@ public:
         // 4. dim needs to be updated
         // this is not implemented yet, but we can't fail here for
         // debugging reasons
-        auto exec = exec_handler.get_device_exec();
+        auto exec = exec_handler.get_ref_exec();
         auto comm = *exec_handler.get_communicator().get();
         label rank = get_rank(exec_handler);
         label owner_rank = get_owner_rank(exec_handler);
         label ranks_per_gpu = ranks_per_gpu_;
+	// TODO dont copy
+        if (ranks_per_gpu == 1) {
+               std::vector<std::pair<bool, label>> ret;
+	       for (auto comm_rank:src_non_local_pattern->rank) {
+		       ret.emplace_back(false, rank);
+	       }
+
+
+         	return std::make_tuple<
+	       std::shared_ptr<SparsityPattern>,
+               std::shared_ptr<SparsityPattern>,
+               std::vector<std::pair<bool, label>>>(
+         		std::make_shared<SparsityPattern>(src_local_pattern),
+	        	std::make_shared<SparsityPattern>(src_non_local_pattern),
+	               	std::move(ret)
+        	);
+        }
 
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         auto local_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, src_local_pattern->size_);
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank  << " owner rank" << owner_rank<< "\n";
+
 
-        label offset = orig_partition_->get_range_bounds()[rank] -
+auto	offset =	orig_partition_->get_range_bounds()[rank] -
                        orig_partition_->get_range_bounds()[owner_rank];
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         auto gather_closure = [&](auto &comm_pattern, auto &data,
                                   label offset) {
@@ -251,6 +274,7 @@ public:
                                    data.get_data(), offset);
         };
 
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         SparsityPatternVector merged_local{
             gather_closure(local_comm_pattern, src_local_pattern->row_idxs,
                            offset),
@@ -267,6 +291,7 @@ public:
             make_ldu_mapping_consecutive(
                 local_comm_pattern, merged_local.mapping, rank, ranks_per_gpu);
         }
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         label rows =
             (is_owner(exec_handler)) ? merged_local.rows.back() + 1 : 0;
@@ -284,6 +309,7 @@ public:
             spans_begin.push_back(elem.begin);
             spans_end.push_back(elem.end);
         }
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         // the non local cols are in local idx of other side
         // thus we need the new offset of the other side
@@ -300,6 +326,7 @@ public:
             std::transform(data, data + size, data,
                            [&](label idx) { return idx + local_offset; });
         }
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         SparsityPatternVector merged_non_local{
             gather_closure(non_local_comm_pattern,
@@ -328,6 +355,7 @@ public:
         // build vector with locality information
         std::vector<std::pair<bool, label>> locality;
         label ctr{0};
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         if (is_owner(exec_handler)) {
             auto recv_counts = std::get<1>(span_comm_pattern);
@@ -354,6 +382,7 @@ public:
             gathered_non_local.cols[i] = i;
         }
 
+	std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         LOG_1(verbose_, "done repartition sparsity pattern")
         if (is_owner(exec_handler)) {
             auto new_local_spars_pattern = std::make_shared<SparsityPattern>(
diff --git a/StoppingCriterion/StoppingCriterion.C b/StoppingCriterion/StoppingCriterion.C
index f9f2bdd24..3962ef323 100644
--- a/StoppingCriterion/StoppingCriterion.C
+++ b/StoppingCriterion/StoppingCriterion.C
@@ -94,6 +94,9 @@ bool StoppingCriterion::OpenFOAMDistStoppingCriterion::check_impl(
     const auto exec = this->get_executor();
 
     auto dense_r = gko::as<dist_vec>(updater.residual_);
+    auto dense_r_local = dense_r->get_local_vector();
+
+
     auto norm1 = vec::create(exec, gko::dim<2>{1});
     dense_r->compute_norm1(norm1.get());
     auto norm1_host = vec::create(exec->get_master(), gko::dim<2>{1});

From 02517283c0fe1687b2ba60f64668786553990962 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 12 Jun 2024 08:33:55 +0200
Subject: [PATCH 85/90] clean up add copy ctr

---
 MatrixWrapper/Combination/Combination.H | 15 +++++++++++++--
 MatrixWrapper/Distributed/Distributed.H |  9 ---------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index e2c8b8c58..ac29da50f 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -76,8 +76,19 @@ public:
      */
     CombinationMatrix &operator=(const CombinationMatrix &other)
     {
-        FatalErrorInFunction << "Copying CombinationMatrix is not allowed"
-                             << abort(FatalError);
+        // FatalErrorInFunction << "Copying CombinationMatrix is not allowed"
+        //                      << abort(FatalError);
+        std::cout << __FILE__ << ":" << __LINE__ << "FIXME why is copy neeeded?\n";
+        if (&other != this) {
+            gko::EnableLinOp<CombinationMatrix>::operator=(other);
+            for (auto range: other.interface_ranges_){
+                interface_ranges_.emplace_back(range.begin, range.end);
+            }
+            comb_ = other.comb_;
+            // restore other invariant
+            // other.row_ptrs_.resize_and_reset(1);
+        }
+        return *this;
     }
 
     /**
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 67b090adc..44594c2e0 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -236,8 +236,6 @@ public:
                   << dst_comm_pattern->target_sizes << " target_send_idxs.size "
                   << dst_comm_pattern->send_idxs.size()
                   << " non_local_sparsity.size " << non_local_sparsity_->size_
-                  << " get_recv_indices "
-                  << localized_partition->get_recv_indices().get_num_elems()
                   << " \n";
 
         // FIXME make sure that we work on the device executor
@@ -260,8 +258,6 @@ public:
         update_impl(exec_handler, matrix_format, repartitioner, host_A, dist_A,
                     local_sparsity_, non_local_sparsity_, src_comm_pattern,
                     local_interfaces);
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-                  << " dnoe update impl  \n";
 
         auto ret = std::make_shared<RepartDistMatrix>(
             exec, comm, repartitioner.get_repart_dim(), dist_A->get_size(),
@@ -307,16 +303,11 @@ public:
         auto diag_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, nrows, local_matrix_nnz,
             local_matrix_nnz - nrows, 0);
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-                  << " diag comm pattern  \n";
-
 
         label upper_nnz = host_A->get_upper_nnz();
         auto upper_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, upper_nnz, local_matrix_nnz, 0,
             local_matrix_nnz - upper_nnz);
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-                  << " upper comm pattern  \n";
         auto lower_comm_pattern =
             compute_send_recv_counts(exec_handler, ranks_per_gpu, upper_nnz,
                                      local_matrix_nnz, upper_nnz, nrows);

From 6ec930a8c70bf1d4b1cbf76fa053dc2fc7ce4cbb Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 12 Jun 2024 09:42:35 +0200
Subject: [PATCH 86/90] fixup

---
 MatrixWrapper/Distributed/Distributed.H | 157 +++++++++++++-----------
 1 file changed, 84 insertions(+), 73 deletions(-)

diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index 939a023b6..f250ea410 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -315,10 +315,10 @@ public:
 
         scalar *local_ptr;
         scalar *local_ptr_2;
-    label nnz=0;
+        label nnz = 0;
 
         // update main values
-    std::vector<scalar> loc_buffer;
+        std::vector<scalar> loc_buffer;
         if (owner) {
             using Coo = gko::matrix::Coo<scalar, label>;
             auto local_mtx = dist_A->get_local_matrix();
@@ -329,14 +329,14 @@ public:
                                  dist_A->get_local_matrix())
                                  ->get_combination()
                                  ->get_operators()[0]);
-        nnz = local->get_num_stored_elements();
-        if (requires_host_buffer) {
-            loc_buffer.resize(nnz);
-            local_ptr = loc_buffer.data();
-            local_ptr_2 = const_cast<scalar *>(local->get_const_values());
-        } else {
-            local_ptr = const_cast<scalar *>(local->get_const_values());
-        }
+            nnz = local->get_num_stored_elements();
+            if (requires_host_buffer) {
+                loc_buffer.resize(nnz);
+                local_ptr = loc_buffer.data();
+                local_ptr_2 = const_cast<scalar *>(local->get_const_values());
+            } else {
+                local_ptr = const_cast<scalar *>(local->get_const_values());
+            }
         }
         communicate_values(exec_handler, diag_comm_pattern, host_A->get_diag(),
                            local_ptr);
@@ -353,23 +353,23 @@ public:
                                host_A->get_lower(), local_ptr);
         }
 
-    if (requires_host_buffer) {    
-        auto host_buffer_view =
-            gko::array<scalar>::view(exec, nnz, local_ptr);
-        auto target_buffer_view =
-            gko::array<scalar>::view(device_exec, nnz, local_ptr_2);
-        target_buffer_view = host_buffer_view;
-    }
+        if (requires_host_buffer) {
+            auto host_buffer_view =
+                gko::array<scalar>::view(exec, nnz, local_ptr);
+            auto target_buffer_view =
+                gko::array<scalar>::view(device_exec, nnz, local_ptr_2);
+            target_buffer_view = host_buffer_view;
+        }
         std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
                   << " done copy to device  \n";
 
-    if (requires_host_buffer) {    
-    auto host_buffer_view =
-            gko::array<scalar>::view(exec, nnz, local_ptr);
-    auto target_buffer_view =
-            gko::array<scalar>::view(device_exec, nnz, local_ptr_2);
-        target_buffer_view = host_buffer_view;
-    }
+        if (requires_host_buffer) {
+            auto host_buffer_view =
+                gko::array<scalar>::view(exec, nnz, local_ptr);
+            auto target_buffer_view =
+                gko::array<scalar>::view(device_exec, nnz, local_ptr_2);
+            target_buffer_view = host_buffer_view;
+        }
         // copy interface values
         auto comm = *exec_handler.get_communicator().get();
         if (owner) {
@@ -382,7 +382,7 @@ public:
             label comm_rank, comm_size;
             scalar *recv_buffer_ptr;
             scalar *recv_buffer_ptr_2;
-        std::vector<scalar> host_recv_buffer;
+            std::vector<scalar> host_recv_buffer;
             label remain_host_interfaces = host_A->get_interface_size();
             for (auto [is_local, comm_rank] : local_interfaces) {
                 label &ctr = (is_local) ? loc_ctr : nloc_ctr;
@@ -403,23 +403,26 @@ public:
                         non_local_sparsity->interface_spans[ctr].length();
                 }
 
-        if (requires_host_buffer) {
-            host_recv_buffer.resize(comm_size);
-            recv_buffer_ptr = host_recv_buffer.data();
-            recv_buffer_ptr_2 = const_cast<scalar *>(mtx->get_const_values());
-        } else {
-            recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
-        }
+                if (requires_host_buffer) {
+                    host_recv_buffer.resize(comm_size);
+                    recv_buffer_ptr = host_recv_buffer.data();
+                    recv_buffer_ptr_2 =
+                        const_cast<scalar *>(mtx->get_const_values());
+                } else {
+                    recv_buffer_ptr =
+                        const_cast<scalar *>(mtx->get_const_values());
+                }
 
                 if (comm_rank != rank) {
-                    comm.recv(device_exec, recv_buffer_ptr, comm_size, comm_rank, tag);
-        if (requires_host_buffer) {
-                auto host_buffer_view =
-                    gko::array<scalar>::view(exec, comm_size, recv_buffer_ptr);
-                auto target_buffer_view =
-                    gko::array<scalar>::view(device_exec, comm_size, recv_buffer_ptr_2);
-                target_buffer_view = host_buffer_view;
-            }
+                    comm.recv(device_exec, recv_buffer_ptr, comm_size,
+                              comm_rank, tag);
+                    if (requires_host_buffer) {
+                        auto host_buffer_view = gko::array<scalar>::view(
+                            exec, comm_size, recv_buffer_ptr);
+                        auto target_buffer_view = gko::array<scalar>::view(
+                            device_exec, comm_size, recv_buffer_ptr_2);
+                        target_buffer_view = host_buffer_view;
+                    }
 
                 } else {
                     // if data is already on this rank
@@ -428,7 +431,8 @@ public:
                         host_A->get_interface_data(host_interface_ctr));
 
                     // TODO FIXME this needs target executor
-            recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
+                    recv_buffer_ptr =
+                        const_cast<scalar *>(mtx->get_const_values());
                     auto target_view = gko::array<scalar>::view(
                         device_exec, comm_size, recv_buffer_ptr);
 
@@ -442,10 +446,11 @@ public:
                 // interface values need to be multiplied by -1
                 using vec = gko::matrix::Dense<ValueType>;
                 auto neg_one = gko::initialize<vec>({-1.0}, exec);
-                auto interface_dense = vec::create(
-                    exec, gko::dim<2>{comm_size, 1},
-                    gko::array<scalar>::view(device_exec, comm_size, recv_buffer_ptr),
-                    1);
+                auto interface_dense =
+                    vec::create(exec, gko::dim<2>{comm_size, 1},
+                                gko::array<scalar>::view(device_exec, comm_size,
+                                                         recv_buffer_ptr),
+                                1);
 
                 interface_dense->scale(neg_one);
             }
@@ -458,37 +463,43 @@ public:
                 label comm_size =
                     src_comm_pattern->target_sizes.get_const_data()[i];
                 const scalar *send_buffer_ptr = host_A->get_interface_data(i);
-                comm.send(device_exec, send_buffer_ptr, comm_size, owner_rank, tag);
+                comm.send(device_exec, send_buffer_ptr, comm_size, owner_rank,
+                          tag);
             }
         }
         // reorder updated values
-      if (owner) {
-          // NOTE local sparsity size includes the interfaces
-          using Coo = gko::matrix::Coo<scalar, label>;
-          using dim_type = gko::dim<2>::dimension_type;
-          std::shared_ptr<const Coo> local =
-              gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
-                               dist_A->get_local_matrix())
-                               ->get_combination()
-                               ->get_operators()[0]);
-          auto local_elements = local->get_num_stored_elements();
-          local_ptr = const_cast<scalar *>(local->get_const_values());
-          // TODO make sure this doesn't copy
-          // create a non owning dense matrix of local_values
-
-          auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
-              device_exec, gko::dim<2>{static_cast<dim_type>(local_elements), 1},
-              gko::array<scalar>::view(device_exec, local_elements, local_ptr), 1));
-          auto mapping_view = gko::array<label>::view(
-              exec, local_elements, local_sparsity->ldu_mapping.get_data());
-
-
-          // TODO this needs to copy ldu_mapping to the device
-           auto dense_vec = row_collection->clone();
-        //auto dense_vec = gko::share(gko::matrix::Dense<scalar>::create(exec, row_collection->get_size()));
-
-          dense_vec->row_gather(&mapping_view, row_collection.get());
-      }
+        if (owner) {
+            // NOTE local sparsity size includes the interfaces
+            using Coo = gko::matrix::Coo<scalar, label>;
+            using dim_type = gko::dim<2>::dimension_type;
+            std::shared_ptr<const Coo> local =
+                gko::as<Coo>(gko::as<CombinationMatrix<scalar, label, Coo>>(
+                                 dist_A->get_local_matrix())
+                                 ->get_combination()
+                                 ->get_operators()[0]);
+            auto local_elements = local->get_num_stored_elements();
+            local_ptr = const_cast<scalar *>(local->get_const_values());
+            // TODO make sure this doesn't copy
+            // create a non owning dense matrix of local_values
+
+            auto row_collection = gko::share(gko::matrix::Dense<scalar>::create(
+                device_exec,
+                gko::dim<2>{static_cast<dim_type>(local_elements), 1},
+                gko::array<scalar>::view(device_exec, local_elements,
+                                         local_ptr),
+                1));
+            auto mapping_view = gko::array<label>::view(
+                exec, local_elements, local_sparsity->ldu_mapping.get_data());
+
+
+            // TODO this needs to copy ldu_mapping to the device
+            auto dense_vec = row_collection->clone();
+            // auto dense_vec =
+            // gko::share(gko::matrix::Dense<scalar>::create(exec,
+            // row_collection->get_size()));
+
+            dense_vec->row_gather(&mapping_view, row_collection.get());
+        }
     };
 
     RepartDistMatrix(

From 3f6252a5eb59a40fa6ae29c1cc76051d21312e39 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 12 Jun 2024 09:50:34 +0200
Subject: [PATCH 87/90] fixup!

---
 MatrixWrapper/CommunicationPattern/CommunicationPattern.C | 2 --
 1 file changed, 2 deletions(-)

diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index 94b6dc926..dd755ba01 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -101,7 +101,6 @@ CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
 
     label tot_recv_elements{0};
     label comm_elements_buffer{0};
-    std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
     if (rank == owner_rank) {
         // send and recv to it self
         recv_offsets[owner_rank] = padding_before;
@@ -110,7 +109,6 @@ CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
         // the start of the next rank data
         tot_recv_elements = padding_before + size + padding_after;
 
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         for (int i = 1; i < ranks_per_gpu; i++) {
             // receive the recv counts
             comm.recv(exec, &comm_elements_buffer, 1, rank + i, rank);

From 43ca454c7c1eba914cbfb77c5446228462d44104 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 18 Jun 2024 11:37:34 +0200
Subject: [PATCH 88/90] fixup! communication of ranks-per-gpu == 1

---
 DevicePersistent/Vector/Vector.H              | 45 +++++++-------
 MatrixWrapper/Combination/Combination.H       |  3 +-
 .../CommunicationPattern.C                    | 10 +++-
 .../CommunicationPattern.H                    | 58 +++++++++++++++++--
 MatrixWrapper/Distributed/Distributed.H       | 24 +++++---
 Repartitioner/Repartitioner.H                 | 45 +++++++-------
 6 files changed, 122 insertions(+), 63 deletions(-)

diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index a9397fa98..c6a648467 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -98,7 +98,6 @@ struct VectorInitFunctor {
             exec_.get_device_exec(), *comm.get(),
             vec::create(exec, gko::dim<2>{repart_size, 1}, local_coeffs, 1)));
 
-
         return ret;
     }
 };
@@ -126,10 +125,9 @@ class PersistentVector
     // updated even if was found in the object registry
     const bool update_;
 
-    const Repartitioner & repartitioner_;
+    const Repartitioner &repartitioner_;
 
 public:
-
     /* PersistentVector constructor using existing memory
      *
      * @param memory ptr to memory on host from which the gko array is
@@ -171,7 +169,7 @@ public:
         auto comm_pattern = compute_scatter_counts(
             exec_, repartitioner_.get_ranks_per_gpu(), size_);
 
-        // std::cout << __FILE__ << ":" << __LINE__ << " rank " << repartitioner_.get_rank()
+        // std::cout << __FILE__ << ":" << __LINE__
         //     << " comm_pattern send " << std::get<0>(comm_pattern)
         //     << " comm_pattern recv " << std::get<1>(comm_pattern)
         //     << " comm_pattern send_offs " << std::get<2>(comm_pattern)
@@ -179,26 +177,25 @@ public:
         //     << "\n";
 
         std::vector<scalar> host_send_buffer;
-	scalar  *send_ptr;
-
-	if (true){
-		label repart_size = repartitioner_.get_repart_size();
-		host_send_buffer.resize(repart_size);
-		send_ptr = host_send_buffer.data();
-		auto host_exec = exec_.get_ref_exec(); 
-		auto device_exec = exec_.get_device_exec(); 
-		auto host_buffer_view =
-		    gko::array<scalar>::view(host_exec, repart_size, send_ptr);
-		auto target_buffer_view =
-		    gko::array<scalar>::view(device_exec, repart_size, get_vector()->get_local_values()
-				    );
-		target_buffer_view = host_buffer_view;
-	} else {
-		send_ptr = get_vector()->get_local_values();
-	}
-
-	communicate_values(exec_, comm_pattern, send_ptr,
-				   const_cast<T *>(memory_));
+        scalar *send_ptr;
+
+        if (true) {
+            label repart_size = repartitioner_.get_repart_size();
+            host_send_buffer.resize(repart_size);
+            send_ptr = host_send_buffer.data();
+            auto host_exec = exec_.get_ref_exec();
+            auto device_exec = exec_.get_device_exec();
+            auto host_buffer_view =
+                gko::array<scalar>::view(host_exec, repart_size, send_ptr);
+            auto target_buffer_view = gko::array<scalar>::view(
+                device_exec, repart_size, get_vector()->get_local_values());
+            host_buffer_view = target_buffer_view;
+        } else {
+            send_ptr = get_vector()->get_local_values();
+        }
+
+        communicate_values(exec_, comm_pattern, send_ptr,
+                           const_cast<T *>(memory_));
     }
 
     /** Writes the content of the distributed vector to disk
diff --git a/MatrixWrapper/Combination/Combination.H b/MatrixWrapper/Combination/Combination.H
index 8a2d85e01..87c8ba447 100644
--- a/MatrixWrapper/Combination/Combination.H
+++ b/MatrixWrapper/Combination/Combination.H
@@ -75,10 +75,9 @@ public:
     {
         // FatalErrorInFunction << "Copying CombinationMatrix is not allowed"
         //                      << abort(FatalError);
-        std::cout << __FILE__ << ":" << __LINE__ << "FIXME why is copy neeeded?\n";
         if (&other != this) {
             gko::EnableLinOp<CombinationMatrix>::operator=(other);
-            for (auto range: other.interface_ranges_){
+            for (auto range : other.interface_ranges_) {
                 interface_ranges_.emplace_back(range.begin, range.end);
             }
             comb_ = other.comb_;
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
index dd755ba01..156dafc8c 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.C
@@ -96,8 +96,9 @@ CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
     label owner_rank = compute_owner_rank(rank, ranks_per_gpu);
     std::vector<label> send_counts(total_ranks, 0);
     std::vector<label> recv_counts(total_ranks, 0);
-    std::vector<label> send_offsets(total_ranks, 0);
-    std::vector<label> recv_offsets(total_ranks, 0);
+    // last entry of offsets vector for total sum
+    std::vector<label> send_offsets(total_ranks + 1, 0);
+    std::vector<label> recv_offsets(total_ranks + 1, 0);
 
     label tot_recv_elements{0};
     label comm_elements_buffer{0};
@@ -138,6 +139,11 @@ CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
         comm.send(exec, &padding_after, 1, owner_rank, owner_rank);
     }
 
+    send_offsets[total_ranks] =
+        std::accumulate(send_counts.begin(), send_counts.end(), 0);
+    recv_offsets[total_ranks] =
+        std::accumulate(recv_counts.begin(), recv_counts.end(), 0);
+
     return std::make_tuple(send_counts, recv_counts, send_offsets,
                            recv_offsets);
 }
diff --git a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
index 10a3c38ad..66adc8571 100644
--- a/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
+++ b/MatrixWrapper/CommunicationPattern/CommunicationPattern.H
@@ -37,12 +37,10 @@ SourceFiles
 using CommCounts = std::tuple<std::vector<label>, std::vector<label>,
                               std::vector<label>, std::vector<label>>;
 
-CommCounts
-invert_comm_pattern(CommCounts in_pattern);
+CommCounts invert_comm_pattern(CommCounts in_pattern);
 
-CommCounts
-compute_scatter_counts(const ExecutorHandler &exec_handler,
-                         label ranks_per_gpu, label size);
+CommCounts compute_scatter_counts(const ExecutorHandler &exec_handler,
+                                  label ranks_per_gpu, label size);
 
 // TODO rename function to compute_gather_to_owner_comm
 /* @brief  This function computes the send and recv counts vectors for
@@ -72,7 +70,7 @@ CommCounts compute_send_recv_counts(const ExecutorHandler &exec_handler,
  * @parameter size how many elements to send to owner
  * */
 CommCounts compute_scatter_from_owner_comm(const ExecutorHandler &exec_handler,
-                                    label ranks_per_gpu, label size);
+                                           label ranks_per_gpu, label size);
 
 /* Given a rank id and ranks_per_gpus this function computes the corresponding
  * owner rank  */
@@ -126,6 +124,54 @@ struct CommunicationPattern {
     {
         return *exec_handler.get_communicator().get();
     }
+
+
+    gko::array<label> total_rank_send_idx() const
+    {
+        std::vector<label> tmp;
+
+        for (auto &[arr, id] : send_idxs) {
+            label arr_size = arr.get_size();
+            tmp.insert(tmp.end(), arr.get_const_data(),
+                       arr.get_const_data() + arr_size);
+        }
+
+        return gko::array<label>(exec_handler.get_ref_exec(), tmp.begin(),
+                                 tmp.end());
+    }
+
+
+    CommCounts send_recv_pattern() const
+    {
+        auto comm = *exec_handler.get_communicator().get();
+        label total_ranks{comm.size()};
+
+        std::vector<label> send_counts(comm.size());
+        std::vector<label> send_offsets(comm.size() + 1);
+        std::vector<label> recv_counts(comm.size());
+        std::vector<label> recv_offsets(comm.size() + 1);
+
+        label comm_ranks = target_ids.get_size();
+        label tot_comm_size = 0;
+        for (label i = 0; i < comm_ranks; i++) {
+            auto comm_rank = target_ids.get_const_data()[i];
+            auto comm_size = target_sizes.get_const_data()[i];
+            tot_comm_size += comm_size;
+            send_counts[comm_rank] = comm_size;
+            recv_counts[comm_rank] = comm_size;
+        }
+
+        recv_offsets[comm.size()] = tot_comm_size;
+        std::partial_sum(recv_counts.begin(), recv_counts.end(),
+                         recv_offsets.begin() + 1);
+        recv_offsets[0] = 0;
+
+        std::partial_sum(send_counts.begin(), send_counts.end(),
+                         send_offsets.begin() + 1);
+        send_offsets[0] = 0;
+
+        return CommCounts(send_offsets, send_counts, recv_offsets, recv_counts);
+    }
 };
 
 std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e);
diff --git a/MatrixWrapper/Distributed/Distributed.H b/MatrixWrapper/Distributed/Distributed.H
index f250ea410..cfba9606b 100644
--- a/MatrixWrapper/Distributed/Distributed.H
+++ b/MatrixWrapper/Distributed/Distributed.H
@@ -239,13 +239,22 @@ public:
                   << " \n";
 
         // FIXME make sure that we work on the device executor
+        auto comm_pattern = compute_send_recv_counts(
+            exec_handler, 1, non_local_sparsity_->size_);
+
+        auto send_idxs = dst_comm_pattern->total_rank_send_idx();
+        auto comm_counts = dst_comm_pattern->send_recv_pattern();
+
+
         dist_A->read_distributed(
             device_matrix_data(exec, local_sparsity_->dim,
                                local_sparsity_->row_idxs,
                                local_sparsity_->col_idxs, local_coeffs),
-            device_matrix_data(
-                exec, non_local_sparsity_->dim, non_local_sparsity_->row_idxs,
-                non_local_sparsity_->col_idxs, non_local_coeffs));
+            device_matrix_data(exec, non_local_sparsity_->dim,
+                               non_local_sparsity_->row_idxs,
+                               non_local_sparsity_->col_idxs, non_local_coeffs),
+            std::get<0>(comm_counts), std::get<1>(comm_counts),
+            std::get<2>(comm_counts), std::get<3>(comm_counts), send_idxs);
 
 
         // auto dist_A = DistMtxType::create(
@@ -258,8 +267,6 @@ public:
         update_impl(exec_handler, matrix_format, repartitioner, host_A, dist_A,
                     local_sparsity_, non_local_sparsity_, src_comm_pattern,
                     local_interfaces);
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-                  << " dnoe update impl  \n";
 
         auto ret = std::make_shared<RepartDistMatrix>(
             exec, comm, repartitioner.get_repart_dim(), dist_A->get_size(),
@@ -360,8 +367,6 @@ public:
                 gko::array<scalar>::view(device_exec, nnz, local_ptr_2);
             target_buffer_view = host_buffer_view;
         }
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank
-                  << " done copy to device  \n";
 
         if (requires_host_buffer) {
             auto host_buffer_view =
@@ -445,9 +450,10 @@ public:
                 ctr++;
                 // interface values need to be multiplied by -1
                 using vec = gko::matrix::Dense<ValueType>;
-                auto neg_one = gko::initialize<vec>({-1.0}, exec);
+                recv_buffer_ptr = const_cast<scalar *>(mtx->get_const_values());
+                auto neg_one = gko::initialize<vec>({-1.0}, device_exec);
                 auto interface_dense =
-                    vec::create(exec, gko::dim<2>{comm_size, 1},
+                    vec::create(device_exec, gko::dim<2>{comm_size, 1},
                                 gko::array<scalar>::view(device_exec, comm_size,
                                                          recv_buffer_ptr),
                                 1);
diff --git a/Repartitioner/Repartitioner.H b/Repartitioner/Repartitioner.H
index 7783c00cd..a78b6fbb8 100644
--- a/Repartitioner/Repartitioner.H
+++ b/Repartitioner/Repartitioner.H
@@ -239,20 +239,18 @@ public:
         label owner_rank = get_owner_rank(exec_handler);
         label ranks_per_gpu = ranks_per_gpu_;
         // TODO dont copy
-        if (ranks_per_gpu == 1) {
-            std::vector<std::pair<bool, label>> ret;
-            for (auto comm_rank : src_non_local_pattern->rank) {
-                ret.emplace_back(false, rank);
-            }
-
-
-            return std::make_tuple<std::shared_ptr<SparsityPattern>,
-                                   std::shared_ptr<SparsityPattern>,
-                                   std::vector<std::pair<bool, label>>>(
-                std::make_shared<SparsityPattern>(src_local_pattern),
-                std::make_shared<SparsityPattern>(src_non_local_pattern),
-                std::move(ret));
-        }
+        // if (ranks_per_gpu == 1) {
+        //     std::vector<std::pair<bool, label>> ret;
+        //     for (auto comm_rank : src_non_local_pattern->rank) {
+        //         ret.emplace_back(false, rank);
+        //     }
+        //     return std::make_tuple<std::shared_ptr<SparsityPattern>,
+        //                            std::shared_ptr<SparsityPattern>,
+        //                            std::vector<std::pair<bool, label>>>(
+        //         std::make_shared<SparsityPattern>(src_local_pattern),
+        //         std::make_shared<SparsityPattern>(src_non_local_pattern),
+        //         std::move(ret));
+        // }
 
         auto local_comm_pattern = compute_send_recv_counts(
             exec_handler, ranks_per_gpu, src_local_pattern->size_);
@@ -344,7 +342,6 @@ public:
         // build vector with locality information
         std::vector<std::pair<bool, label>> locality;
         label ctr{0};
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
 
         if (is_owner(exec_handler)) {
             auto recv_counts = std::get<1>(span_comm_pattern);
@@ -371,7 +368,6 @@ public:
             gathered_non_local.cols[i] = i;
         }
 
-        std::cout << __FILE__ << ":" << __LINE__ << " rank " << rank << "\n";
         LOG_1(verbose_, "done repartition sparsity pattern")
         if (is_owner(exec_handler)) {
             auto new_local_spars_pattern = std::make_shared<SparsityPattern>(
@@ -396,8 +392,15 @@ public:
             auto new_non_local_spars_pattern =
                 std::make_shared<SparsityPattern>(exec);
 
-            new_local_spars_pattern->dim = merged_local_dim;
-            new_non_local_spars_pattern->dim = merged_local_dim;
+            // FIXME
+            // std::cout << __FILE__ << ":" << __LINE__ << " rank " <<
+            // comm.rank() <<
+            //     " merged_local dim " <<
+            new_local_spars_pattern->dim =
+                gko::dim<2>{0, 0};  // merged_local_dim;
+            new_non_local_spars_pattern->dim =
+                gko::dim<2>{0, 0};  // merged_local_dim;
+
 
             return std::make_tuple<std::shared_ptr<SparsityPattern>,
                                    std::shared_ptr<SparsityPattern>,
@@ -434,13 +437,15 @@ public:
             partition) const
     {
         using comm_size_type = label;
+        auto exec = src_comm_pattern->target_ids.get_executor();
+        auto comm = src_comm_pattern->get_comm();
 
         if (ranks_per_gpu_ == 1) {
+            std::cout << __FILE__ << ":" << __LINE__ << " rank " << comm.rank()
+                      << *src_comm_pattern.get() << "\n";
             return src_comm_pattern;
         }
 
-        auto exec = src_comm_pattern->target_ids.get_executor();
-        auto comm = src_comm_pattern->get_comm();
 
         label rank = comm.rank();
         bool owner = is_owner(exec_handler);

From c45397f80f5a2a97eb50eabdf8e2abdd34f7e522 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 24 Jun 2024 11:06:57 +0200
Subject: [PATCH 89/90] WIP repartition to single device for num_ranks == 0

---
 DevicePersistent/ExecutorHandler/ExecutorHandler.H | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DevicePersistent/ExecutorHandler/ExecutorHandler.H b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
index 8b323eba1..71804d100 100644
--- a/DevicePersistent/ExecutorHandler/ExecutorHandler.H
+++ b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
@@ -31,7 +31,7 @@ struct ExecutorInitFunctor {
                     ? std::make_shared<gko::experimental::mpi::communicator>(
                           MPI_COMM_WORLD, force_host_buffer)
                     : NULL),
-          device_id_(comm_->rank() / gpus_per_rank),
+          device_id_(if (gpus_per_rank != 0) ? comm_->rank() / gpus_per_rank : comm_->get_num_ranks()),
           executor_name_(executor_name),
           field_name_(field_name),
           verbose_(verbose)

From aafb47a841f796c2d492a90da80e2d6c6143c1ef Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 24 Jun 2024 11:28:06 +0200
Subject: [PATCH 90/90] fixup

---
 DevicePersistent/ExecutorHandler/ExecutorHandler.H | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DevicePersistent/ExecutorHandler/ExecutorHandler.H b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
index 71804d100..51983cf72 100644
--- a/DevicePersistent/ExecutorHandler/ExecutorHandler.H
+++ b/DevicePersistent/ExecutorHandler/ExecutorHandler.H
@@ -31,7 +31,7 @@ struct ExecutorInitFunctor {
                     ? std::make_shared<gko::experimental::mpi::communicator>(
                           MPI_COMM_WORLD, force_host_buffer)
                     : NULL),
-          device_id_(if (gpus_per_rank != 0) ? comm_->rank() / gpus_per_rank : comm_->get_num_ranks()),
+          device_id_((gpus_per_rank != 0) ? comm_->rank() / gpus_per_rank : comm_->size()),
           executor_name_(executor_name),
           field_name_(field_name),
           verbose_(verbose)