Skip to content

Commit 953f9e7

Browse files
committed
Workaround OpenMP problem
1 parent 4fa8eaf commit 953f9e7

File tree

7 files changed

+1689
-32
lines changed

7 files changed

+1689
-32
lines changed

.github/workflows/build-and-test-linux.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -361,14 +361,14 @@ jobs:
361361
$MPICC mpi_test.c -o mpi_test
362362
mpirun -np 2 ./mpi_test
363363
364-
- uses: julia-actions/cache@v2
365-
if: needs.filter.outputs.test == 'true'
366-
367364
- uses: julia-actions/setup-julia@v2
368365
if: needs.filter.outputs.test == 'true'
369366
with:
370367
version: '1'
371368

369+
- uses: julia-actions/cache@v2
370+
if: needs.filter.outputs.test == 'true'
371+
372372
- name: Build Palace
373373
if: needs.filter.outputs.test == 'true'
374374
env:

.github/workflows/build-and-test-macos.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,9 @@ jobs:
135135
with:
136136
version: '1'
137137

138+
- uses: julia-actions/cache@v2
139+
if: needs.filter.outputs.test == 'true'
140+
138141
- name: Configure Open MPI
139142
if: needs.filter.outputs.test == 'true' && matrix.mpi == 'openmpi'
140143
run: |

.github/workflows/docs.yml

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ jobs:
3939
# https://github.com/spack/spack/issues/51505
4040
- name: Overwrite builtin Palace package
4141
run: |
42+
# Bug with OpenMP and libceed
43+
# TODO: Remove when https://github.com/spack/spack-packages/pull/2361 is merged
44+
cp -r spack_repo/local/packages/libceed/package.py "$(spack location --repo builtin)/packages/libceed/"
45+
4246
cp -r spack_repo/local/packages/palace "$(spack location --repo builtin)/packages"
4347
cp extern/patch/mfem/* "$(spack location --repo builtin)/packages/palace"
4448
@@ -117,14 +121,14 @@ jobs:
117121
# If you want to use the branch-local source instead (should we always do this?)
118122
# spack -e . develop --path=$(pwd) palace@git."${{ github.head_ref || github.ref_name }}"=develop
119123

120-
- uses: julia-actions/cache@v2
121-
if: needs.filter.outputs.test == 'true'
122-
123124
- uses: julia-actions/setup-julia@v2
124125
if: needs.filter.outputs.test == 'true'
125126
with:
126127
version: '1'
127128

129+
- uses: julia-actions/cache@v2
130+
if: needs.filter.outputs.test == 'true'
131+
128132
- name: Build and deploy
129133
env:
130134
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/spack.yml

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -84,17 +84,16 @@ jobs:
8484
# solver: +strumpack
8585
# cuda: ~cuda
8686

87-
# test-libceed gives wrong results
88-
# - arch: arm
89-
# compiler: gcc
90-
# mpi: mpich
91-
# math-libs: armpl-gcc
92-
# shared: +shared
93-
# int: +int64
94-
# openmp: +openmp
95-
# eigensolver: +arpack
96-
# solver: +strumpack
97-
# cuda: ~cuda
87+
- arch: arm
88+
compiler: gcc
89+
mpi: mpich
90+
math-libs: armpl-gcc
91+
shared: +shared
92+
int: +int64
93+
openmp: +openmp
94+
eigensolver: +arpack
95+
solver: +strumpack
96+
cuda: ~cuda
9897

9998
# Fails building libxsmm with ifx error None not found
10099
# - arch: x86
@@ -155,17 +154,16 @@ jobs:
155154
# solver: +mumps
156155
# cuda: ~cuda
157156

158-
# test-libceed gives wrong results
159-
# - arch: arm
160-
# compiler: llvm
161-
# mpi: mpich
162-
# math-libs: openblas
163-
# shared: +shared
164-
# int: ~int64
165-
# openmp: +openmp
166-
# eigensolver: +arpack
167-
# solver: +superlu-dist
168-
# cuda: ~cuda
157+
- arch: arm
158+
compiler: llvm
159+
mpi: mpich
160+
math-libs: openblas
161+
shared: +shared
162+
int: ~int64
163+
openmp: +openmp
164+
eigensolver: +arpack
165+
solver: +superlu-dist
166+
cuda: ~cuda
169167

170168
runs-on: ${{ matrix.arch == 'x86' && 'palace_ubuntu-latest_16-core' || 'ubuntu-24.04-arm' }}
171169
steps:
@@ -228,7 +226,7 @@ jobs:
228226
mpi:
229227
require: ${{ matrix.mpi }}
230228
blas:
231-
require: ${{ matrix.math-libs }}
229+
require: ${{ matrix.math-libs }} threads=openmp
232230
c:
233231
require: [${{ matrix.compiler }}]
234232
cxx:
@@ -259,9 +257,14 @@ jobs:
259257
- name: Overwrite builtin Palace package
260258
if: needs.filter.outputs.test == 'true'
261259
run: |
260+
# Bug with OpenMP and libceed
261+
# TODO: Remove when https://github.com/spack/spack-packages/pull/2361 is merged
262+
cp -r spack_repo/local/packages/libceed/package.py "$(spack location --repo builtin)/packages/libceed/"
263+
262264
cp -r spack_repo/local/packages/palace "$(spack location --repo builtin)/packages"
263265
cp extern/patch/mfem/* "$(spack location --repo builtin)/packages/palace"
264266
267+
265268
- name: Remove Android NDK # it confuses the Intel compiler
266269
if: needs.filter.outputs.test == 'true' && matrix.compiler == 'intel-oneapi-compilers'
267270
run: sudo rm -rf /usr/local/lib/android/
@@ -350,14 +353,14 @@ jobs:
350353
palace-unit-tests --skip-benchmarks
351354
mpirun -np 2 $(which palace-unit-tests) --skip-benchmarks
352355
353-
- uses: julia-actions/cache@v2
354-
if: needs.filter.outputs.test == 'true'
355-
356356
- uses: julia-actions/setup-julia@v2
357357
if: needs.filter.outputs.test == 'true'
358358
with:
359359
version: '1'
360360

361+
- uses: julia-actions/cache@v2
362+
if: needs.filter.outputs.test == 'true'
363+
361364
- name: Run Integration Tests
362365
if: needs.filter.outputs.test == 'true'
363366
run: |
Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
# Copyright Spack Project Developers. See COPYRIGHT file for details.
2+
#
3+
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
4+
5+
from spack_repo.builtin.build_systems.cuda import CudaPackage
6+
from spack_repo.builtin.build_systems.makefile import MakefilePackage
7+
from spack_repo.builtin.build_systems.rocm import ROCmPackage
8+
9+
from spack.package import *
10+
11+
12+
class Libceed(MakefilePackage, CudaPackage, ROCmPackage):
13+
"""The CEED API Library: Code for Efficient Extensible Discretizations."""
14+
15+
homepage = "https://github.com/CEED/libCEED"
16+
git = "https://github.com/CEED/libCEED.git"
17+
18+
maintainers("jedbrown", "v-dobrev", "tzanio", "jeremylt")
19+
20+
license("BSD-2-Clause")
21+
22+
version("develop", branch="main")
23+
version("0.12.0", tag="v0.12.0", commit="4018a20a98d451fac24765d3ddb936861647ce8d")
24+
version("0.11.0", tag="v0.11.0", commit="8ec64e9ae9d5df169dba8c8ee61d8ec8907b8f80")
25+
version("0.10.1", tag="v0.10.1", commit="74532b27052d94e943eb8bc76257fbd710103614")
26+
version("0.9", tag="v0.9.0", commit="d66340f5aae79e564186ab7514a1cd08b3a1b06b")
27+
version("0.8", tag="v0.8", commit="e8f234590eddcce2220edb1d6e979af7a3c35f82")
28+
version("0.7", tag="v0.7", commit="0bc92be5158efcbeb80d3d59240233bf5b2f748c")
29+
version(
30+
"0.6", commit="c7f533e01e2f3f6720fbf37aac2af2ffed225f60"
31+
) # tag v0.6 + small portability fixes
32+
version("0.5", tag="v0.5", commit="12804ff7ea2ac608ae5494437379e4f626cf5cb7")
33+
version("0.4", tag="v0.4", commit="40b9dad77dea06a1608fa8b93a0d8b9c993ee43d")
34+
version("0.2", tag="v0.2", commit="113004cb41757b819325a4b3a8a7dfcea5156531")
35+
version("0.1", tag="v0.1", commit="74e0540e2478136394f75869675056eb6aba67cc")
36+
37+
variant("occa", default=False, description="Enable OCCA backends")
38+
variant("debug", default=False, description="Enable debug build")
39+
variant("libxsmm", default=False, description="Enable LIBXSMM backend", when="@0.3:")
40+
variant("magma", default=False, description="Enable MAGMA backend", when="@0.6:")
41+
42+
variant("openmp", default=False, description="Enable OpenMP support")
43+
44+
conflicts("+rocm", when="@:0.7")
45+
46+
depends_on("c", type="build") # generated
47+
depends_on("cxx", type="build") # generated
48+
depends_on("fortran", type="build") # generated
49+
50+
with when("+rocm"):
51+
depends_on("[email protected]:", when="@0.8:")
52+
depends_on("[email protected]:", when="@0.8:")
53+
54+
conflicts("+occa", when="@0.9:")
55+
56+
with when("+occa"):
57+
depends_on("[email protected]", when="@0.7:")
58+
depends_on("[email protected]:", when="@0.4")
59+
depends_on("[email protected],develop", when="@:0.2")
60+
depends_on("occa+cuda", when="+cuda")
61+
depends_on("occa~cuda", when="~cuda")
62+
63+
depends_on("libxsmm", when="+libxsmm")
64+
depends_on("blas", when="+libxsmm", type="link")
65+
66+
depends_on("magma", when="+magma")
67+
68+
patch("libceed-v0.8-hip.patch", when="@0.8+rocm")
69+
patch("pkgconfig-version-0.4.diff", when="@0.4")
70+
71+
# occa: do not occaFree kernels
72+
# Repeated creation and freeing of kernels appears to expose a caching
73+
# bug in Occa.
74+
patch("occaFree-0.2.diff", when="@0.2")
75+
76+
@property
77+
def common_make_opts(self):
78+
spec = self.spec
79+
compiler = self.compiler
80+
# Note: The occa package exports OCCA_DIR in the environment
81+
82+
# Use verbose building output
83+
makeopts = ["V=1"]
84+
85+
if spec.satisfies("@:0.2"):
86+
makeopts += ["NDEBUG=%s" % ("" if spec.satisfies("+debug") else "1")]
87+
88+
elif spec.satisfies("@0.4:"):
89+
# Determine options based on the compiler:
90+
if spec.satisfies("+debug"):
91+
opt = "-g"
92+
elif compiler.name == "gcc":
93+
opt = "-O3 -g -ffp-contract=fast"
94+
if compiler.version >= ver(4.9):
95+
opt += " -fopenmp-simd"
96+
if self.spec.target.family in ["x86_64", "aarch64"]:
97+
opt += " -march=native"
98+
elif compiler.name == "apple-clang":
99+
opt = "-O3 -g -march=native -ffp-contract=fast"
100+
if compiler.version >= ver(10):
101+
opt += " -fopenmp-simd"
102+
elif compiler.name == "clang":
103+
opt = "-O3 -g -march=native -ffp-contract=fast"
104+
if compiler.version >= ver(6):
105+
opt += " -fopenmp-simd"
106+
elif compiler.name in ["xl", "xl_r"]:
107+
opt = "-O -g -qsimd=auto"
108+
elif compiler.name == "intel":
109+
opt = "-O3 -g"
110+
makeopts += ["CC_VENDOR=icc"]
111+
else:
112+
opt = "-O -g"
113+
# Note: spack will inject additional target-specific flags through
114+
# the compiler wrapper.
115+
makeopts += ["OPT=%s" % opt]
116+
117+
if spec.satisfies("@0.7") and compiler.name in ["xl", "xl_r"]:
118+
makeopts += ["CXXFLAGS.XL=-qpic -std=c++11 -MMD"]
119+
120+
if spec.satisfies("@:0.7") and "avx" in self.spec.target:
121+
makeopts.append("AVX=1")
122+
123+
if spec.satisfies("+cuda"):
124+
makeopts += ["CUDA_DIR=%s" % spec["cuda"].prefix]
125+
makeopts += ["CUDA_ARCH=sm_%s" % spec.variants["cuda_arch"].value]
126+
if spec.satisfies("@:0.4"):
127+
nvccflags = [
128+
'-ccbin %s -Xcompiler "%s" -Xcompiler %s'
129+
% (compiler.cxx, opt, compiler.cc_pic_flag)
130+
]
131+
nvccflags = " ".join(nvccflags)
132+
makeopts += ["NVCCFLAGS=%s" % nvccflags]
133+
else:
134+
# Disable CUDA auto-detection:
135+
makeopts += ["CUDA_DIR=/disable-cuda"]
136+
137+
if spec.satisfies("+rocm"):
138+
makeopts += ["HIP_DIR=%s" % spec["hip"].prefix]
139+
amdgpu_target = ",".join(spec.variants["amdgpu_target"].value)
140+
makeopts += ["HIP_ARCH=%s" % amdgpu_target]
141+
if spec.satisfies("@0.8"):
142+
makeopts += ["HIPBLAS_DIR=%s" % spec["hipblas"].prefix]
143+
144+
if spec.satisfies("+libxsmm"):
145+
makeopts += ["XSMM_DIR=%s" % spec["libxsmm"].prefix]
146+
makeopts += ["BLAS_LIB=%s" % spec["blas"].libs]
147+
148+
if spec.satisfies("+magma"):
149+
makeopts += ["MAGMA_DIR=%s" % spec["magma"].prefix]
150+
151+
if spec.satisfies("+openmp"):
152+
makeopts += ["OPENMP=1"]
153+
154+
return makeopts
155+
156+
def edit(self, spec, prefix):
157+
make("info", *self.common_make_opts)
158+
159+
@property
160+
def build_targets(self):
161+
return self.common_make_opts
162+
163+
@property
164+
def install_targets(self):
165+
return ["install", "prefix={0}".format(self.prefix)] + self.common_make_opts
166+
167+
def check(self):
168+
make("prove", *self.common_make_opts, parallel=False)
169+
170+
@when("@0.1")
171+
def install(self, spec, prefix):
172+
mkdirp(prefix.include)
173+
install("ceed.h", prefix.include)
174+
mkdirp(prefix.lib)
175+
install("libceed.%s" % dso_suffix, prefix.lib)
176+
filter_file(r"^prefix=.*$", "prefix=%s" % prefix, "ceed.pc")
177+
filter_file(r"^includedir=\$\{prefix\}$", "includedir=${prefix}/include", "ceed.pc")
178+
filter_file(r"^libdir=\$\{prefix\}$", "libdir=${prefix}/lib", "ceed.pc")
179+
filter_file(r"Version:.*$", "Version: 0.1", "ceed.pc")
180+
mkdirp(prefix.lib.pkgconfig)
181+
install("ceed.pc", prefix.lib.pkgconfig)

0 commit comments

Comments
 (0)