Workaround OpenMP problem

Author: Sbozzolo

.github/workflows/build-and-test-linux.yml

@@ -361,14 +361,14 @@ jobs:
           $MPICC mpi_test.c -o mpi_test
           mpirun -np 2 ./mpi_test
 
-      - uses: julia-actions/cache@v2
-        if: needs.filter.outputs.test == 'true'
-
       - uses: julia-actions/setup-julia@v2
         if: needs.filter.outputs.test == 'true'
         with:
           version: '1'
 
+      - uses: julia-actions/cache@v2
+        if: needs.filter.outputs.test == 'true'
+
       - name: Build Palace
         if: needs.filter.outputs.test == 'true'
         env:

.github/workflows/build-and-test-macos.yml

@@ -135,6 +135,9 @@ jobs:
         with:
           version: '1'
 
+      - uses: julia-actions/cache@v2
+        if: needs.filter.outputs.test == 'true'
+
       - name: Configure Open MPI
         if: needs.filter.outputs.test == 'true' && matrix.mpi == 'openmpi'
         run: |

.github/workflows/docs.yml

@@ -117,14 +117,14 @@ jobs:
         # If you want to use the branch-local source instead (should we always do this?)
         # spack -e . develop --path=$(pwd) palace@git."${{ github.head_ref || github.ref_name }}"=develop
 
-      - uses: julia-actions/cache@v2
-        if: needs.filter.outputs.test == 'true'
-
       - uses: julia-actions/setup-julia@v2
         if: needs.filter.outputs.test == 'true'
         with:
           version: '1'
 
+      - uses: julia-actions/cache@v2
+        if: needs.filter.outputs.test == 'true'
+
       - name: Build and deploy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/spack.yml

@@ -84,17 +84,16 @@ jobs:
         #   solver: +strumpack
         #   cuda: ~cuda
 
-        # test-libceed gives wrong results
-        # - arch: arm
-        #   compiler: gcc
-        #   mpi: mpich
-        #   math-libs: armpl-gcc
-        #   shared: +shared
-        #   int: +int64
-        #   openmp: +openmp
-        #   eigensolver: +arpack
-        #   solver: +strumpack
-        #   cuda: ~cuda
+        - arch: arm
+          compiler: gcc
+          mpi: mpich
+          math-libs: armpl-gcc
+          shared: +shared
+          int: +int64
+          openmp: +openmp
+          eigensolver: +arpack
+          solver: +strumpack
+          cuda: ~cuda
 
         # Fails building libxsmm with ifx error None not found
         # - arch: x86
@@ -155,17 +154,16 @@ jobs:
         #   solver: +mumps
         #   cuda: ~cuda
 
-        # test-libceed gives wrong results
-        # - arch: arm
-        #   compiler: llvm
-        #   mpi: mpich
-        #   math-libs: openblas
-        #   shared: +shared
-        #   int: ~int64
-        #   openmp: +openmp
-        #   eigensolver: +arpack
-        #   solver: +superlu-dist
-        #   cuda: ~cuda
+        - arch: arm
+          compiler: llvm
+          mpi: mpich
+          math-libs: openblas
+          shared: +shared
+          int: ~int64
+          openmp: +openmp
+          eigensolver: +arpack
+          solver: +superlu-dist
+          cuda: ~cuda
 
     runs-on: ${{ matrix.arch == 'x86' && 'palace_ubuntu-latest_16-core' || 'ubuntu-24.04-arm' }}
     steps:
@@ -228,7 +226,7 @@ jobs:
               mpi:
                 require: ${{ matrix.mpi }}
               blas:
-                require: ${{ matrix.math-libs }}
+                require: ${{ matrix.math-libs }} threads=openmp
               c:
                 require: [${{ matrix.compiler }}]
               cxx:
@@ -259,9 +257,18 @@ jobs:
       - name: Overwrite builtin Palace package
         if: needs.filter.outputs.test == 'true'
         run: |
+          # Bug with OpenMP and libceed
+          # PR https://github.com/spack/spack-packages/pull/2361
+          cd "$(spack location --repo builtin)"
+          git remote add sbozzolo https://github.com/Sbozzolo/spack-packages.git
+          git fetch sbozzolo libceed_openmp
+          git reset --hard sbozzolo/libceed_openmp
+          cd -
+
           cp -r spack_repo/local/packages/palace "$(spack location --repo builtin)/packages"
           cp extern/patch/mfem/* "$(spack location --repo builtin)/packages/palace"
 
+
       - name: Remove Android NDK # it confuses the Intel compiler
         if: needs.filter.outputs.test == 'true' && matrix.compiler == 'intel-oneapi-compilers'
         run: sudo rm -rf /usr/local/lib/android/
@@ -350,14 +357,14 @@ jobs:
           palace-unit-tests --skip-benchmarks
           mpirun -np 2 $(which palace-unit-tests) --skip-benchmarks
 
-      - uses: julia-actions/cache@v2
-        if: needs.filter.outputs.test == 'true'
-
       - uses: julia-actions/setup-julia@v2
         if: needs.filter.outputs.test == 'true'
         with:
           version: '1'
 
+      - uses: julia-actions/cache@v2
+        if: needs.filter.outputs.test == 'true'
+
       - name: Run Integration Tests
         if: needs.filter.outputs.test == 'true'
         run: |

spack_repo/local/packages/palace/package.py

@@ -150,6 +150,9 @@ class Palace(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("hypre~cuda", when="~cuda")
     depends_on("hypre~rocm", when="~rocm")
 
+    depends_on("libceed+openmp", when="+openmp")
+    depends_on("libceed~openmp", when="~openmp")
+
     with when("@0.15:"):
         # +lapack means: use external lapack
         depends_on(
@@ -164,6 +167,7 @@ class Palace(CMakePackage, CudaPackage, ROCmPackage):
         depends_on("mfem+shared", when="+shared")
         depends_on("mfem~shared", when="~shared")
         depends_on("mfem+openmp", when="+openmp")
+        depends_on("mfem+threadsafe", when="+openmp")
         depends_on("mfem~openmp", when="~openmp")
         depends_on("mfem+superlu-dist", when="+superlu-dist")
         depends_on("mfem~superlu-dist", when="~superlu-dist")