Compile MFEM externally

This commit puts MFEM on the same footing as the other dependencies.
Instead of being unconditionally built, it is now build when building
external dependencies. This allowed me to add MFEM as a dependency
managed spack.

In doing this, I found a new bug in spack:
spack/spack#51505

So, I changed our approach to testing the spack package. Instead of
adding a local repo, I overwrite palace in the `builtin` repo.

Author: Sbozzolo

.github/workflows/docs.yml

@@ -36,13 +36,19 @@ jobs:
           buildcache: true
           color: true
 
+        # https://github.com/spack/spack/issues/51505
+      - name: Overwrite builtin Palace package
+        run: |
+          cp -r spack_repo/local/packages/palace "$(spack location --repo builtin)/packages"
+          cp extern/patch/mfem/* "$(spack location --repo builtin)/packages/palace"
+
       - name: Setup Environment
         run: |
           # Spack.yaml with most / all settings configured
           cat << EOF > spack.yaml
           spack:
             specs:
-              - palace # we install this without a cache each time
+              - palace@develop # we install this without a cache each time
             view: false
             config:
               install_tree:
@@ -56,8 +62,6 @@ jobs:
             packages:
               petsc:
                 require: ~hdf5
-            repos:
-            - spack_repo/local
             mirrors:
               spack:
                 binary: true
@@ -98,7 +102,7 @@ jobs:
       - name: Concretize Spack
         run: |
           # Using `spack develop` in order to have an in-source build
-          spack -e . develop --path=$(pwd) local.palace@git."${{ github.head_ref || github.ref_name }}"=develop
+          spack -e . develop --path=$(pwd) palace@git."${{ github.head_ref || github.ref_name }}"=develop
           spack -e . concretize -f
 
         # Relies on cache-hit(s)
@@ -111,7 +115,7 @@ jobs:
         run: |
           spack -e . install --only-concrete --show-log-on-error --only package --keep-stage --no-cache
         # If you want to use the branch-local source instead (should we always do this?)
-        # spack -e . develop --path=$(pwd) local.palace@git."${{ github.head_ref || github.ref_name }}"=develop
+        # spack -e . develop --path=$(pwd) palace@git."${{ github.head_ref || github.ref_name }}"=develop
 
       - name: Build and deploy
         env:

.github/workflows/spack.yml

@@ -51,8 +51,9 @@ jobs:
         compiler: [gcc] #, clang, intel]
         gpu: [none] #, cuda, rocm]
         blas: [openblas] # ,intel-oneapi-mkl]
-        spack_version: [develop] # ,v0.23.1]
         mpi: [openmpi] #, mpich, mvapich]
+        eigensolver: [arpack, slepc]
+        solver: [mumps, strumpack, superlu-dist]
 
     runs-on: palace_ubuntu-latest_16-core
     steps:
@@ -89,12 +90,7 @@ jobs:
             GPU_VARIANT=""
           fi
 
-          # Develop requires ^compiler, not %compiler
-          if [[ "${{ matrix.spack_version }}" == 'v0.23.1' ]]; then
-            PALACE_SPEC="local.palace@develop+strumpack+arpack+mumps+tests %${{ matrix.compiler }} ${GPU_VARIANT} ^${{ matrix.mpi }} ^${{ matrix.blas }}"
-          else
-            PALACE_SPEC="local.palace@develop+strumpack+arpack+mumps+tests${GPU_VARIANT} ^${{ matrix.mpi }} ^${{ matrix.blas }} ^${{ matrix.compiler }}"
-          fi
+          PALACE_SPEC="palace@develop+${{ matrix.solver }}+${{ matrix.eigensolver }} +tests %${{ matrix.compiler }} ${GPU_VARIANT} ^${{ matrix.mpi }} ^${{ matrix.blas }}"
 
           # Spack.yaml with most / all settings configured
           cat << EOF > spack.yaml
@@ -116,8 +112,6 @@ jobs:
                 require: ~hdf5
               rocblas:
                 require: ~tensile
-            repos:
-            - spack_repo/local
             mirrors:
               spack:
                 binary: true
@@ -131,6 +125,13 @@ jobs:
                   secret_variable: GITHUB_TOKEN
           EOF
 
+        # https://github.com/spack/spack/issues/51505
+      - name: Overwrite builtin Palace package
+        if: needs.filter.outputs.test == 'true'
+        run: |
+          cp -r spack_repo/local/packages/palace "$(spack location --repo builtin)/packages"
+          cp extern/patch/mfem/* "$(spack location --repo builtin)/packages/palace"
+
       - name: Configure External Packages
         if: needs.filter.outputs.test == 'true'
         run: |
@@ -167,7 +168,7 @@ jobs:
         # Unfortunately it then becomes difficult to know when to re-concretize.
         run: |
           # Using `spack develop` in order to have an in-source build
-          spack -e . develop --path=$(pwd) local.palace@git."${{ github.head_ref || github.ref_name }}"=develop
+          spack -e . develop --path=$(pwd) palace@git."${{ github.head_ref || github.ref_name }}"=develop
           spack -e . concretize -f
 
       - name: Build Dependencies
@@ -188,6 +189,14 @@ jobs:
           palace-unit-tests --skip-benchmarks
           mpirun -np 2 $(which palace-unit-tests) --skip-benchmarks
 
+      - uses: julia-actions/cache@v2
+        if: needs.filter.outputs.test == 'true'
+
+      - uses: julia-actions/setup-julia@v2
+        if: needs.filter.outputs.test == 'true'
+        with:
+          version: '1'
+
       - name: Run Integration Tests
         if: needs.filter.outputs.test == 'true' && matrix.gpu == 'none' # Skip tests for GPU builds
         env:

CHANGELOG.md

@@ -68,9 +68,23 @@ The format of this changelog is based on
     [PR 510](https://github.com/awslabs/palace/pull/510) and [PR 526](https://github.com/awslabs/palace/pull/526).
   - Update EM constants to CODATA Recommended Values of the Fundamental Physical Constants 2022
     [PR 525](https://github.com/awslabs/palace/pull/525).
+  - Fix compatibility with CUDA 12 and GCC 14+. [PR 535](https://github.com/awslabs/palace/pull/535).
   - Scaled Rs/Ls/Cs of impedance boundary conditions affected by mesh cracking, fixing bug where
     `"CrackInternalBoundaryElements"` would lead to incorrect results. [PR 544](https://github.com/awslabs/palace/pull/544).
 
+#### Build system
+
+  - When building on a machine with a GPU, `CMAKE_CUDA_ARCHITECTURES` has now a
+    default value that is automatically determined to be the consistent with the
+    GPU. [PR 530](https://github.com/awslabs/palace/pull/530).
+  - The spack Palace build now supports a new variant `+tests` to compile and
+    install the unit tests. [PR
+    549](https://github.com/awslabs/palace/pull/549).
+  - MFEM is no longer unconditionally compiled alongside of Palace and can now
+    be built separately. This is used in the Palace spack build, which now
+    explicitely depends on MFEM. [PR
+    550](https://github.com/awslabs/palace/pull/550).
+
 ## [0.14.0] - 2025-08-20
 
   - Added `--version` command line flag for displaying Palace version information.

CMakeLists.txt

@@ -212,10 +212,6 @@ if(PALACE_BUILD_EXTERNAL_DEPS)
   add_subdirectory("extern")
 endif()
 
-# Add MFEM (always built as part of Palace)
-message(STATUS "====================== Configuring MFEM dependency =====================")
-include(ExternalMFEM)
-
 # Add the main Palace project
 message(STATUS "========================== Configuring Palace ==========================")
 include(ExternalPalace)

cmake/ExternalPalace.cmake

@@ -6,9 +6,8 @@
 #
 
 # Force build order
-set(PALACE_DEPENDENCIES mfem)
 if(PALACE_BUILD_EXTERNAL_DEPS)
-  list(APPEND PALACE_DEPENDENCIES libCEED json fmt eigen scn)
+  list(APPEND PALACE_DEPENDENCIES mfem libCEED json fmt eigen scn)
   if(PALACE_WITH_SLEPC)
     list(APPEND PALACE_DEPENDENCIES slepc)
   endif()
@@ -24,12 +23,16 @@ list(APPEND PALACE_OPTIONS
   "-DPALACE_WITH_OPENMP=${PALACE_WITH_OPENMP}"
   "-DPALACE_WITH_SLEPC=${PALACE_WITH_SLEPC}"
   "-DPALACE_WITH_ARPACK=${PALACE_WITH_ARPACK}"
+  "-DPALACE_WITH_SUNDIALS=${PALACE_WITH_SUNDIALS}"
+  "-DPALACE_WITH_GSLIB=${PALACE_WITH_GSLIB}"
   "-DANALYZE_SOURCES_CLANG_TIDY=${ANALYZE_SOURCES_CLANG_TIDY}"
   "-DANALYZE_SOURCES_CPPCHECK=${ANALYZE_SOURCES_CPPCHECK}"
-  "-DMFEM_DATA_PATH=${CMAKE_BINARY_DIR}/extern/mfem/data" # Path to meshes for testing
   "-DPALACE_BUILD_EXTERNAL_DEPS=${PALACE_BUILD_EXTERNAL_DEPS}" # For Catch2
   "-DPALACE_BUILD_WITH_COVERAGE=${PALACE_BUILD_WITH_COVERAGE}"
 )
+if(NOT "${MFEM_DIR}" STREQUAL "")
+  list(APPEND PALACE_OPTIONS "-DMFEM_DIR=${MFEM_DIR}")
+endif()
 if(PALACE_WITH_ARPACK)
   list(APPEND PALACE_OPTIONS
     "-DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER}"

extern/CMakeLists.txt

@@ -93,3 +93,7 @@ endif()
 message(STATUS "==================== Configuring libCEED dependency ====================")
 include(ExternalLibCEED)
 
+# Add MFEM
+message(STATUS "====================== Configuring MFEM dependency =====================")
+include(ExternalMFEM)
+

palace/CMakeLists.txt

@@ -112,6 +112,15 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 # Find MPI
 find_package(MPI REQUIRED)
 
+
+if(PALACE_WITH_SUNDIALS AND NOT PALACE_BUILD_EXTERNAL_DEPS)
+  # Spack's sundials requires MPI with C bindings, so we alias
+  # MPI_C to MPI_CXX (if it does not already exist).
+  if(NOT TARGET MPI::MPI_C AND TARGET MPI::MPI_CXX)
+    add_library(MPI::MPI_C ALIAS MPI::MPI_CXX)
+  endif()
+endif()
+
 # Find OpenMP
 if(PALACE_WITH_OPENMP)
   find_package(OpenMP REQUIRED)
@@ -141,15 +150,20 @@ if(NOT "${MFEM_DIR}" STREQUAL "")
   set(MFEM_ROOT ${MFEM_DIR})
 endif()
 find_package(Threads REQUIRED)
-find_package(MFEM REQUIRED CONFIG)
-message(STATUS "Found MFEM: ${MFEM_VERSION} in ${MFEM_DIR}")
-if(NOT MFEM_USE_MPI)
-  message(FATAL_ERROR "Build requires MFEM with MPI support")
+
+# MFEM in Spack is currently using autotools, and this find_package will fail
+# (hence the QUIET), but this is not a problem.
+find_package(MFEM QUIET CONFIG)
+if(MFEM_FOUND)
+  message(STATUS "Found MFEM: ${MFEM_VERSION} in ${MFEM_DIR}")
+  if(NOT MFEM_USE_MPI)
+    message(FATAL_ERROR "Build requires MFEM with MPI support")
+  endif()
+endif()
+
+if(NOT "${HYPRE_DIR}" STREQUAL "")
+  set(HYPRE_ROOT ${HYPRE_DIR})
 endif()
-# if(MFEM_CXX_FLAGS)
-#   # Pull compiler flags from MFEM for OpenMP and optimizations
-#   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MFEM_CXX_FLAGS}")
-# endif()
 
 # Find libCEED
 include(PkgConfigHelpers)
@@ -327,6 +341,29 @@ target_link_libraries(${LIB_TARGET_NAME}
   PUBLIC mfem ${LIBCEED_TARGET} nlohmann_json::nlohmann_json fmt::fmt scn::scn
          Eigen3::Eigen LAPACK::LAPACK MPI::MPI_CXX
 )
+
+if(NOT PALACE_BUILD_EXTERNAL_DEPS)
+  # When using external deps, we need to manually link them to Palace (when
+  # building them, they get linked through mfem).
+
+  # TODO: Switch to find_package when moving to hypre3
+  if(TARGET HYPRE::HYPRE)
+    target_link_libraries(${LIB_TARGET_NAME} PUBLIC HYPRE::HYPRE)
+  else()
+    find_library(HYPRE_LIBRARY NAMES HYPRE)
+    if(HYPRE_LIBRARY)
+      target_link_libraries(${LIB_TARGET_NAME} PUBLIC ${HYPRE_LIBRARY})
+    endif()
+  endif()
+
+  # Explicitly link SUNDIALS if MFEM doesn't export it transitively
+  if(PALACE_WITH_SUNDIALS)
+    find_package(SUNDIALS REQUIRED)
+    message(STATUS "Found SUNDIALS: ${SUNDIALS_VERSION} in ${SUNDIALS_DIR}")
+    target_link_libraries(${LIB_TARGET_NAME} PUBLIC SUNDIALS::arkode SUNDIALS::cvode)
+  endif()
+endif()
+
 target_link_libraries(
   ${LIB_TARGET_NAME}
   PRIVATE $<BUILD_INTERFACE:$<$<CONFIG:Debug,RelWithDebInfo>:common_warnings>>

spack_repo/local/packages/palace/package.py

@@ -144,6 +144,33 @@ class Palace(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("hypre~cuda", when="~cuda")
     depends_on("hypre~rocm", when="~rocm")
 
+    with when("@develop"):
+        depends_on(
+            "mfem+mpi+metis cxxstd=17 commit=0c4c006ef86dc2b2cf415e5bc4ed9118c9768652",
+            patches=[
+                "patch_mesh_vis_dev.diff",
+                "patch_par_tet_mesh_fix_dev.diff",
+                "patch_gmsh_parser_performance.diff",
+                "patch_race_condition_fix.diff",
+            ],
+        )
+        depends_on("mfem+shared", when="+shared")
+        depends_on("mfem~shared", when="~shared")
+        depends_on("mfem+openmp", when="+openmp")
+        depends_on("mfem~openmp", when="~openmp")
+        depends_on("mfem+superlu-dist", when="+superlu-dist")
+        depends_on("mfem~superlu-dist", when="~superlu-dist")
+        depends_on("mfem+strumpack", when="+strumpack")
+        depends_on("mfem~strumpack", when="~strumpack")
+        depends_on("mfem+mumps", when="+mumps")
+        depends_on("mfem~mumps", when="~mumps")
+        depends_on("mfem+sundials", when="+sundials")
+        depends_on("mfem~sundials", when="~sundials")
+        depends_on("mfem+gslib", when="+gslib")
+        depends_on("mfem~gslib", when="~gslib")
+
+        depends_on("mfem+exceptions", when="+tests")
+
     with when("+libxsmm"):
         # NOTE: @=main != @main since libxsmm has a version main-2023-22
         depends_on("libxsmm@=main blas=0")
@@ -190,6 +217,7 @@ class Palace(CMakePackage, CudaPackage, ROCmPackage):
         for arch in CudaPackage.cuda_arch_values:
             cuda_variant = f"+cuda cuda_arch={arch}"
             depends_on(f"hypre{cuda_variant}", when=f"{cuda_variant}")
+            depends_on(f"mfem{cuda_variant}", when=f"{cuda_variant}")
             depends_on(f"magma{cuda_variant}", when=f"{cuda_variant}")
             depends_on(f"libceed{cuda_variant}", when=f"{cuda_variant} @0.14:")
             depends_on(f"sundials{cuda_variant}", when=f"+sundials{cuda_variant} @0.14:")
@@ -200,6 +228,7 @@ class Palace(CMakePackage, CudaPackage, ROCmPackage):
         for arch in ROCmPackage.amdgpu_targets:
             rocm_variant = f"+rocm amdgpu_target={arch}"
             depends_on(f"hypre{rocm_variant}", when=f"{rocm_variant}")
+            depends_on(f"mfem{rocm_variant}", when=f"{rocm_variant}")
             depends_on(f"magma{rocm_variant}", when=f"{rocm_variant}")
             depends_on(f"libceed{rocm_variant}", when=f"{rocm_variant} @0.14:")
             depends_on(f"sundials{rocm_variant}", when=f"+sundials{rocm_variant} @0.14:")
@@ -225,9 +254,12 @@ def cmake_args(self):
             self.define_from_variant("PALACE_WITH_SUNDIALS", "sundials"),
             self.define_from_variant("PALACE_WITH_SUPERLU", "superlu-dist"),
             self.define("PALACE_BUILD_EXTERNAL_DEPS", False),
-            self.define_from_variant("PALACE_MFEM_USE_EXCEPTIONS", "tests"),
         ]
 
+
+        with when("@:0.14"):
+            self.define_from_variant("PALACE_MFEM_USE_EXCEPTIONS", "tests")
+
         # We guarantee that there are arch specs with conflicts above
         if self.spec.satisfies("+cuda"):
             args.append(

test/unit/CMakeLists.txt

@@ -86,17 +86,13 @@ set_property(
 
 # Add unit test mesh file path definition
 set_property(
-  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test-libceed.cpp
+  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test-geodata.cpp
+         ${CMAKE_CURRENT_SOURCE_DIR}/test-libceed.cpp
          ${CMAKE_CURRENT_SOURCE_DIR}/test-materialoperator.cpp
          ${CMAKE_CURRENT_SOURCE_DIR}/test-strattonchu.cpp
+         ${CMAKE_CURRENT_SOURCE_DIR}/test-rap.cpp
   APPEND PROPERTY COMPILE_DEFINITIONS "PALACE_TEST_MESH_DIR=\"${CMAKE_INSTALL_PREFIX}/share/palace/test/mesh\""
 )
-set_property(
-  SOURCE
-  ${CMAKE_CURRENT_SOURCE_DIR}/test-geodata.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/test-rap.cpp
-  APPEND PROPERTY COMPILE_DEFINITIONS "MFEM_DATA_PATH=\"${CMAKE_INSTALL_PREFIX}/share/palace/test/mfem-data\""
-)
 
 target_compile_definitions(
   unit-tests
@@ -128,14 +124,6 @@ install(
   DESTINATION share/palace/test
 )
 
-# Install MFEM data files if available
-if(MFEM_DATA_PATH)
-  install(
-    DIRECTORY ${MFEM_DATA_PATH}/
-    DESTINATION share/palace/test/mfem-data
-  )
-endif()
-
 install(
   DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/mesh/
   DESTINATION share/palace/test/mesh