Skip to content

Commit 5284fcf

Browse files
committed
merge jobordner/main <- enzo-project/main
2 parents 5457219 + f999ad2 commit 5284fcf

File tree

433 files changed

+23557
-5636
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

433 files changed

+23557
-5636
lines changed

.circleci/config.yml

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ commands:
6767
- run:
6868
name: "Install grackle."
6969
command: |
70-
git clone -b master https://github.com/grackle-project/grackle $HOME/grackle
70+
git clone -b main https://github.com/grackle-project/grackle $HOME/grackle
7171
cd $HOME/grackle
7272
./configure
7373
cd src/clib
@@ -134,19 +134,11 @@ commands:
134134
# remove build directory in case we've already compiled before
135135
rm -rf build
136136
137-
# this is used for tests involving Grackle
138-
if [[ $USE_GRACKLE == 1 ]]; then
139-
GRACKLE_INPUT_DATA_DIR="$HOME/grackle/grackle_data_files/input/"
140-
else
141-
GRACKLE_INPUT_DATA_DIR=""
142-
fi
143-
144137
if [ ! -f << parameters.skipfile >> ]; then
145138
cmake -DEnzo-E_CONFIG=linux_gcc \
146139
-GNinja \
147140
-DUSE_DOUBLE_PREC=<< parameters.usedouble >> \
148141
-DUSE_GRACKLE=<< parameters.usegrackle >> \
149-
-DGRACKLE_INPUT_DATA_DIR="$GRACKLE_INPUT_DATA_DIR" \
150142
-Bbuild \
151143
-DPARALLEL_LAUNCHER_NPROC_ARG="++local;+p" \
152144
-DPython3_FIND_VIRTUALENV=ONLY
@@ -206,6 +198,9 @@ commands:
206198
usedouble:
207199
type: boolean
208200
default: false
201+
usegrackle:
202+
type: boolean
203+
default: false
209204
generate:
210205
type: boolean
211206
default: false
@@ -216,6 +211,14 @@ commands:
216211
source $BASH_ENV
217212
source $HOME/venv/bin/activate
218213
214+
# do a little setup based on the value of usegrackle:
215+
# 1. convert the parameter to an env var storing 0 or 1
216+
# 2. if that variable holds true, export GRACKLE_INPUT_DATA_DIR
217+
USE_GRACKLE=$(( 0 <<# parameters.usegrackle >> + 1 <</ parameters.usegrackle >> ))
218+
if [[ $USE_GRACKLE == 1 ]]; then
219+
export GRACKLE_INPUT_DATA_DIR="$HOME/grackle/grackle_data_files/input/"
220+
fi
221+
219222
export USE_DOUBLE=<< parameters.usedouble >>
220223
export GENERATE_TEST_RESULTS=<< parameters.generate >>
221224
pytest test/answer_tests
@@ -315,12 +318,13 @@ jobs:
315318

316319
- compile-enzoe:
317320
usedouble: << parameters.usedouble >>
318-
tag: gold-standard-001
321+
tag: gold-standard-004
319322
skipfile: notafile
320323
usegrackle: << parameters.usegrackle >>
321324

322325
- run-answer-tests:
323326
usedouble: << parameters.usedouble >>
327+
usegrackle: << parameters.usegrackle >>
324328
generate: true
325329

326330
- compile-enzoe:
@@ -331,6 +335,7 @@ jobs:
331335

332336
- run-answer-tests:
333337
usedouble: << parameters.usedouble >>
338+
usegrackle: << parameters.usegrackle >>
334339
generate: false
335340

336341
docs-build:

.readthedocs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,14 @@
44

55
version: 2
66

7+
build:
8+
os: ubuntu-22.04
9+
tools:
10+
# the python version currently just matches the version used for building
11+
# the docs with CircleCi
12+
python: "3.10"
13+
14+
715
python:
816
install:
917
- requirements: doc/source/requirements.txt

CMakeLists.txt

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ endif()
1919
#Define project and languages
2020
project(Enzo-E VERSION 1.0.0 LANGUAGES C CXX Fortran)
2121

22+
# We need C++11
23+
set(CMAKE_CXX_STANDARD 11)
24+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
25+
2226
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
2327

2428
# Don't allow in-source builds
@@ -50,15 +54,13 @@ find_package(Charm REQUIRED)
5054
# Link executables with the charmc wrapper
5155
STRING(REGEX REPLACE "<CMAKE_CXX_COMPILER>" "${CHARM_COMPILER}"
5256
CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE}")
53-
# for book-keeping purposes create a library representing the external
54-
# Charm-library. We are going to make this an interface library (a header-only)
55-
# library since the charmc wrapper makes sure we link to the correct library
56-
add_library(CHARM_EXTERNAL INTERFACE)
57-
target_include_directories(CHARM_EXTERNAL INTERFACE ${CHARM_INCLUDE_DIRS})
5857

5958

6059
# Need to process Grackle here as both Cello and Enzo-E depend on it
6160
option(USE_GRACKLE "Use Grackle Chemistry" ON)
61+
# don't bother advertising the following option (but users can
62+
# overwrite it if they really want to - e.g. to reduce binary size)
63+
option(GRACKLE_USE_STATIC_LIBS "sets Grackle's lib-type if USE_GRACKLE=ON" ON)
6264
if (USE_GRACKLE)
6365
find_package(Grackle)
6466
if (Grackle_FOUND)
@@ -292,10 +294,13 @@ if (use_papi)
292294
endif()
293295
endif()
294296

295-
option(USE_SIMD "Use OpenMP SIMD directives. Also enables value-unsafe floating-point optimizations which may already be enable on some compilers (like icc)." OFF)
296-
include("cmake/EnableOMPSIMD.cmake")
297-
if (USE_SIMD)
298-
enableOMPSIMD()
297+
option(OPTIMIZE_FP "Enables value-unsafe floating-point optimizations (this may already be enabled on some compilers like icc)." OFF)
298+
option(USE_SIMD "Use OpenMP SIMD directives. This may already be enabled on some compilers (like icc)." OFF)
299+
include("cmake/EnableFPOptimizations.cmake")
300+
if (OPTIMIZE_FP)
301+
enableFPOptimizations(USE_SIMD)
302+
elseif (USE_SIMD)
303+
message(FATAL_ERROR "Can only use `USE_SIMD=ON` when `OPTIMIZE_FP=ON`.")
299304
endif()
300305

301306
# Include machine file second time (if used initially) to set additional options that may depend

cmake/EnableFPOptimizations.cmake

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
# See LICENSE_ENZO file for license and copyright information
2+
3+
#.rst:
4+
# EnableFPOptimizations
5+
# ---------------------
6+
#
7+
# Defines a function that enables value-unsafe floating point optimization
8+
# flags for the C and C++ compilers.
9+
#
10+
# This can also add the openmp-simd flags to the C and C++ compilers (and any
11+
# other flags necessary for simd optimizations).
12+
13+
if(__enableFPOptimizations)
14+
return()
15+
endif()
16+
set(__enableFPOptimizations YES)
17+
18+
# Function 'enableFPOptimizations' is used to add C and C++ flags to the
19+
# various build types that compile the program with value unsafe floating point
20+
# optimizations.
21+
#
22+
# ARGUMENTS
23+
# ---------
24+
# USE_SIMD
25+
# When this a passed a TRUE value, this function also adds C and C++ compiler
26+
# flags to enable OpenMP's SIMD directives
27+
# - For clang, gnu and intel compilers this does NOT enable other OpenMP
28+
# directives and should not link the openmp runtime library
29+
#
30+
# NOTES
31+
# -----
32+
# This function assumes that the global CONFIG_ARCH_FLAGS variable has been
33+
# defined by the machine configuration file (this flag should provide the
34+
# compiler about the architecture of the CPU where the code will be executed,
35+
# which lets the compiler choose optimal instructions). If this variable has
36+
# not be defined, the function will produce an error and provide the user with
37+
# a sensible default that they may want to use.
38+
#
39+
# This function may have less impact on the performance of programs compiled by
40+
# the Intel compiler compared to code compiled by other compilers. This is
41+
# because the Intel compilers enable some of these options by default (this
42+
# strongly to the Intel Compiler's reputation for producing faster code)
43+
function(enableFPOptimizations USE_SIMD)
44+
45+
# ToDo: check assumption that CMAKE_C_COMPILER_ID and CMAKE_CXX_COMPILER_ID
46+
# are equal to each other...
47+
48+
# First, determine the values for the following variables based on the
49+
# compiler type:
50+
# - DFLT_HOSTARCHFLAG:
51+
# * this stores the flag telling the compiler that it can assume that the
52+
# CPU architecture of the machine currently performing the compilation
53+
# is identical to that of the machine where the compiled program is run
54+
# * this variable is only used to describe a potential default value that
55+
# the user could use in the error message raised when the
56+
# CONFIG_ARCH_FLAGS global variable was not defined in the machine
57+
# configuration file.
58+
# - SPEED_FLAGS:
59+
# * Specifies flags that prioritize code speed over code size.
60+
# * Only the "RELEASE" and "RELWITHDEBINFO" build types will make use of
61+
# these flags
62+
# - FP_FLAGS:
63+
# * Specifies flags enabling value-unsafe floating point optimizations.
64+
# - OMPSIMD_FLAGS:
65+
# * Specifies flags enabling OpenMP's SIMD directives
66+
67+
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
68+
# theoretically includes Clang and AppleClang
69+
70+
set(DFLT_HOSTARCHFLAG "-march=native")
71+
set(SPEED_FLAGS "-O3 -funroll-loops")
72+
set(FP_FLAGS "-fopenmp-simd")
73+
set(OMPSIMD_FLAGS "-fopenmp-simd")
74+
75+
message(FATAL_ERROR "enableOFPOptimizations is untested for Clang.")
76+
77+
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
78+
79+
# in case we ever want to support a version with OpenMP-SIMD, that does not
80+
# enable value-unsafe optimizations of floating-point operations, try the
81+
# following flags:
82+
# - "-fopenmp-simd -funroll-loops -fno-trapping-math -fno-signed-zeros"
83+
# - if we also include -fno-math-errno, it will cause 1D MHD shock tube
84+
# problems to have slightly different L1 error norms for the VL+CT
85+
# solver, depending on the position in the transverse direction
86+
# (see MHD_shock_tube_test)
87+
88+
set(DFLT_HOSTARCHFLAG "-march=native")
89+
set(SPEED_FLAGS "-O3 -funroll-loops")
90+
set(FP_FLAGS "-ffast-math")
91+
set(OMPSIMD_FLAGS "-fopenmp-simd")
92+
93+
elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel")
94+
# this matches both the legacy icc and icpc compiler...
95+
# For CMake Ver >= 20, CMAKE_CXX_COMPILER_ID is "IntelLLVM" for the latter
96+
# (but this will continue to match both)
97+
98+
# technically -qopenmp-simd is already enabled by default when compiling
99+
# with -O2 or higher
100+
set(DFLT_HOSTARCHFLAG "-xHost")
101+
set(SPEED_FLAGS "-O3")
102+
set(FP_FLAGS "")
103+
set(OMPSIMD_FLAGS "-qopenmp-simd")
104+
105+
else()
106+
message(FATAL_ERROR
107+
"OpenMP-SIMD handling is not yet implemented for the "
108+
"${CMAKE_CXX_COMPILER_ID} compiler."
109+
)
110+
endif()
111+
112+
# Second, check if the CONFIG_ARCH_FLAGS variable is enabled
113+
if (NOT DEFINED CONFIG_ARCH_FLAGS)
114+
message(FATAL_ERROR
115+
"The CONFIG_ARCH_FLAGS variable is not defined.\n"
116+
"This variable is strongly-recommended while compiling with OpenMP-SIMD, in order to inform the compiler which vector instructions are available/prefered.\n"
117+
"This variable should be defined in the machine config file.\n"
118+
"A reasonable default value for this variable, when using the ${CMAKE_CXX_COMPILER_ID} compiler, might be \"${HOSTARCHFLAG}\" (this tells the compiler to target the CPU architecture of the machine that is performing the compilation)."
119+
)
120+
endif()
121+
122+
# Finally, update the flags
123+
# - we NEED to update CMAKE_<LANG>_FLAGS_<CONFIG>
124+
# - if we update CMAKE_<LANG>_FLAGS the optimization level flags in
125+
# CMAKE_<LANG>_FLAGS_<CONFIG> get precedence. This is specifically an
126+
# issue for RelWithDebInfo builds
127+
# - we could theoretically use add_compile_options(), but that would also
128+
# pass these flags to the FORTRAN compiler
129+
foreach(BTYPE IN ITEMS "DEBUG" "RELEASE" "MINSIZEREL" "RELWITHDEBINFO")
130+
foreach(LANG IN ITEMS "C" "CXX")
131+
132+
# make a copy of global variable to be modified
133+
set(localCopy ${CMAKE_${LANG}_FLAGS_${BTYPE}})
134+
135+
# append the new flags to the local copy
136+
string(APPEND localCopy " ${FP_FLAGS} ${CONFIG_ARCH_FLAGS}")
137+
138+
if(USE_SIMD)
139+
string(APPEND localCopy " ${OMPSIMD_FLAGS}")
140+
endif()
141+
142+
if ((BTYPE STREQUAL "RELEASE") OR
143+
(BTYPE STREQUAL "RELWITHDEBINFO"))
144+
# append strings that prioritize speed (at expense of size)
145+
string(APPEND localCopy " ${SPEED_FLAGS}")
146+
endif()
147+
148+
# overwrite the global variable
149+
set(CMAKE_${LANG}_FLAGS_${BTYPE} ${localCopy} PARENT_SCOPE)
150+
151+
endforeach(LANG)
152+
endforeach(BTYPE)
153+
endfunction(enableFPOptimizations)
154+
155+
# REQUIRE TARGET_ARCH_FLAGS for gcc compiler...

0 commit comments

Comments
 (0)