Skip to content

Commit 6c96c75

Browse files
ax3lax3l
authored
Doc: Major Cori Update (#2978)
* Doc: Major Cori Update Update Cori instructions after the major upgrade yesterday. Before using the new modules, remove the old, pre-build software: ```bash rm -rf $HOME/sw/knl $HOME/sw/haswell $HOME/sw/gpu $HOME/sw/cori_gpu ``` * Cori: Work-Around ADIOS 2.7.1 ZeroMQ Disable due to ornladios/ADIOS2#2709 * Fix haswell/knl mismatch
1 parent c58445b commit 6c96c75

File tree

4 files changed

+38
-38
lines changed

4 files changed

+38
-38
lines changed

‎Docs/source/install/hpc/cori.rst‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,25 +45,25 @@ And install ADIOS2, BLAS++ and LAPACK++:
4545
# c-blosc (I/O compression)
4646
git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc
4747
rm -rf src/c-blosc-knl-build
48-
cmake -S src/c-blosc -B src/c-blosc-knl-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/c-blosc-1.12.1-knl-install
48+
cmake -S src/c-blosc -B src/c-blosc-knl-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/c-blosc-1.12.1-install
4949
cmake --build src/c-blosc-knl-build --target install --parallel 16
5050
5151
# ADIOS2
5252
git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2
5353
rm -rf src/adios2-knl-build
54-
cmake -S src/adios2 -B src/adios2-knl-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/adios2-2.7.1-knl-install
54+
cmake -S src/adios2 -B src/adios2-knl-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/adios2-2.7.1-install
5555
cmake --build src/adios2-knl-build --target install --parallel 16
5656
5757
# BLAS++ (for PSATD+RZ)
5858
git clone https://bitbucket.org/icl/blaspp.git src/blaspp
5959
rm -rf src/blaspp-knl-build
60-
cmake -S src/blaspp -B src/blaspp-knl-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/blaspp-master-knl-install
60+
cmake -S src/blaspp -B src/blaspp-knl-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/blaspp-master-install
6161
cmake --build src/blaspp-knl-build --target install --parallel 16
6262
6363
# LAPACK++ (for PSATD+RZ)
6464
git clone https://bitbucket.org/icl/lapackpp.git src/lapackpp
6565
rm -rf src/lapackpp-knl-build
66-
CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-knl-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lapackpp-master-knl-install
66+
CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-knl-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/lapackpp-master-install
6767
cmake --build src/lapackpp-knl-build --target install --parallel 16
6868
6969
For PICMI and Python workflows, also install a virtual environment:
@@ -74,8 +74,8 @@ For PICMI and Python workflows, also install a virtual environment:
7474
python3 -m pip install --user --upgrade pip
7575
python3 -m pip install --user virtualenv
7676
77-
python3 -m venv $HOME/sw/venvs/knl_warpx
78-
source $HOME/sw/venvs/knl_warpx/bin/activate
77+
python3 -m venv $HOME/sw/knl/venvs/knl_warpx
78+
source $HOME/sw/knl/venvs/knl_warpx/bin/activate
7979
8080
python3 -m pip install --upgrade pip
8181
python3 -m pip install --upgrade wheel
@@ -108,13 +108,13 @@ And install ADIOS2, BLAS++ and LAPACK++:
108108
# c-blosc (I/O compression)
109109
git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc
110110
rm -rf src/c-blosc-haswell-build
111-
cmake -S src/c-blosc -B src/c-blosc-haswell-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/c-blosc-1.12.1-haswell-install
111+
cmake -S src/c-blosc -B src/c-blosc-haswell-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/haswell/c-blosc-1.12.1-install
112112
cmake --build src/c-blosc-haswell-build --target install --parallel 16
113113
114114
# ADIOS2
115115
git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2
116116
rm -rf src/adios2-haswell-build
117-
cmake -S src/adios2 -B src/adios2-haswell-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/adios2-2.7.1-haswell-install
117+
cmake -S src/adios2 -B src/adios2-haswell-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/haswell/adios2-2.7.1-install
118118
cmake --build src/adios2-haswell-build --target install --parallel 16
119119
120120
# BLAS++ (for PSATD+RZ)
@@ -126,7 +126,7 @@ And install ADIOS2, BLAS++ and LAPACK++:
126126
# LAPACK++ (for PSATD+RZ)
127127
git clone https://bitbucket.org/icl/lapackpp.git src/lapackpp
128128
rm -rf src/lapackpp-haswell-build
129-
CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-haswell-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lapackpp-master-haswell-install
129+
CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-haswell-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/haswell/lapackpp-master-install
130130
cmake --build src/lapackpp-haswell-build --target install --parallel 16
131131
132132
For PICMI and Python workflows, also install a virtual environment:
@@ -137,8 +137,8 @@ For PICMI and Python workflows, also install a virtual environment:
137137
python3 -m pip install --user --upgrade pip
138138
python3 -m pip install --user virtualenv
139139
140-
python3 -m venv $HOME/sw/venvs/haswell_warpx
141-
source $HOME/sw/venvs/haswell_warpx/bin/activate
140+
python3 -m venv $HOME/sw/haswell/venvs/haswell_warpx
141+
source $HOME/sw/haswell/venvs/haswell_warpx/bin/activate
142142
143143
python3 -m pip install --upgrade pip
144144
python3 -m pip install --upgrade wheel
@@ -173,12 +173,12 @@ And install ADIOS2:
173173
# c-blosc (I/O compression)
174174
git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc
175175
rm -rf src/c-blosc-gpu-build
176-
cmake -S src/c-blosc -B src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/c-blosc-1.12.1-gpu-install
176+
cmake -S src/c-blosc -B src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/cori_gpu/c-blosc-1.12.1-install
177177
cmake --build src/c-blosc-gpu-build --target install --parallel 16
178178
179179
git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2
180180
rm -rf src/adios2-gpu-build
181-
cmake -S src/adios2 -B src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/adios2-2.7.1-gpu-install
181+
cmake -S src/adios2 -B src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/cori_gpu/adios2-2.7.1-install
182182
cmake --build src/adios2-gpu-build --target install --parallel 16
183183
184184
For PICMI and Python workflows, also install a virtual environment:
@@ -189,8 +189,8 @@ For PICMI and Python workflows, also install a virtual environment:
189189
python3 -m pip install --user --upgrade pip
190190
python3 -m pip install --user virtualenv
191191
192-
python3 -m venv $HOME/sw/venvs/gpu_warpx
193-
source $HOME/sw/venvs/gpu_warpx/bin/activate
192+
python3 -m venv $HOME/sw/cori_gpu/venvs/gpu_warpx
193+
source $HOME/sw/cori_gpu/venvs/gpu_warpx/bin/activate
194194
195195
python3 -m pip install --upgrade pip
196196
python3 -m pip install --upgrade wheel

‎Tools/machines/cori-nersc/gpu_warpx.profile.example‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@ module purge
44
module load modules
55
module load cgpu
66
module load esslurm
7-
module load gcc/8.3.0 cuda/11.4.0 cmake/3.21.3
7+
module load gcc/8.3.0 cuda/11.4.0 cmake/3.22.1
88
module load openmpi
99

10-
export CMAKE_PREFIX_PATH=$HOME/sw/c-blosc-1.12.1-gpu-install:$CMAKE_PREFIX_PATH
11-
export CMAKE_PREFIX_PATH=$HOME/sw/adios2-2.7.1-gpu-install:$CMAKE_PREFIX_PATH
10+
export CMAKE_PREFIX_PATH=$HOME/sw/cori_gpu/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH
11+
export CMAKE_PREFIX_PATH=$HOME/sw/cori_gpu/adios2-2.7.1-install:$CMAKE_PREFIX_PATH
1212

13-
if [ -d "$HOME/sw/venvs/gpu_warpx" ]
13+
if [ -d "$HOME/sw/cori_gpu/venvs/cori_gpu_warpx" ]
1414
then
15-
source $HOME/sw/venvs/gpu_warpx/bin/activate
15+
source $HOME/sw/cori_gpu/venvs/cori_gpu_warpx/bin/activate
1616
fi
1717

1818
# compiler environment hints

‎Tools/machines/cori-nersc/haswell_warpx.profile.example‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
module swap PrgEnv-intel PrgEnv-gnu
2-
module load cmake/3.21.3
2+
module load cmake/3.22.1
33
module switch cray-libsci cray-libsci/20.09.1
44
module load cray-hdf5-parallel/1.10.5.2
5-
module load cray-fftw/3.3.8.4
6-
module load cray-python/3.7.3.2
5+
module load cray-fftw/3.3.8.10
6+
module load cray-python/3.9.7.1
77

88
export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH
9-
export CMAKE_PREFIX_PATH=$HOME/sw/c-blosc-1.12.1-haswell-install:$CMAKE_PREFIX_PATH
10-
export CMAKE_PREFIX_PATH=$HOME/sw/adios2-2.7.1-haswell-install:$CMAKE_PREFIX_PATH
11-
export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-master-haswell-install:$CMAKE_PREFIX_PATH
12-
export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-master-haswell-install:$CMAKE_PREFIX_PATH
9+
export CMAKE_PREFIX_PATH=$HOME/sw/haswell/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH
10+
export CMAKE_PREFIX_PATH=$HOME/sw/haswell/adios2-2.7.1-install:$CMAKE_PREFIX_PATH
11+
export CMAKE_PREFIX_PATH=$HOME/sw/haswell/blaspp-master-install:$CMAKE_PREFIX_PATH
12+
export CMAKE_PREFIX_PATH=$HOME/sw/haswell/lapackpp-master-install:$CMAKE_PREFIX_PATH
1313

14-
if [ -d "$HOME/sw/venvs/haswell_warpx" ]
14+
if [ -d "$HOME/sw/haswell/venvs/haswell_warpx" ]
1515
then
16-
source $HOME/sw/venvs/haswell_warpx/bin/activate
16+
source $HOME/sw/haswell/venvs/haswell_warpx/bin/activate
1717
fi

‎Tools/machines/cori-nersc/knl_warpx.profile.example‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
module swap craype-haswell craype-mic-knl
22
module swap PrgEnv-intel PrgEnv-gnu
3-
module load cmake/3.21.3
3+
module load cmake/3.22.1
44
module switch cray-libsci cray-libsci/20.09.1
55
module load cray-hdf5-parallel/1.10.5.2
6-
module load cray-fftw/3.3.8.4
7-
module load cray-python/3.7.3.2
6+
module load cray-fftw/3.3.8.10
7+
module load cray-python/3.9.7.1
88

99
export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH
10-
export CMAKE_PREFIX_PATH=$HOME/sw/c-blosc-1.12.1-knl-install:$CMAKE_PREFIX_PATH
11-
export CMAKE_PREFIX_PATH=$HOME/sw/adios2-2.7.1-knl-install:$CMAKE_PREFIX_PATH
12-
export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-master-knl-install:$CMAKE_PREFIX_PATH
13-
export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-master-knl-install:$CMAKE_PREFIX_PATH
10+
export CMAKE_PREFIX_PATH=$HOME/sw/knl/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH
11+
export CMAKE_PREFIX_PATH=$HOME/sw/knl/adios2-2.7.1-install:$CMAKE_PREFIX_PATH
12+
export CMAKE_PREFIX_PATH=$HOME/sw/knl/blaspp-master-install:$CMAKE_PREFIX_PATH
13+
export CMAKE_PREFIX_PATH=$HOME/sw/knl/lapackpp-master-install:$CMAKE_PREFIX_PATH
1414

15-
if [ -d "$HOME/sw/venvs/knl_warpx" ]
15+
if [ -d "$HOME/sw/knl/venvs/knl_warpx" ]
1616
then
17-
source $HOME/sw/venvs/knl_warpx/bin/activate
17+
source $HOME/sw/knl/venvs/knl_warpx/bin/activate
1818
fi
1919

2020
export CXXFLAGS="-march=knl"

0 commit comments

Comments
 (0)