Skip to content

Commit 1bd5667

Browse files
fluidnumerics-joefluidnumerics-joe
committed
Add check for gpu aware mpi support in cmake stage
1 parent e536846 commit 1bd5667

File tree

5 files changed

+165
-55
lines changed

5 files changed

+165
-55
lines changed

‎CMakeLists.txt

+55-44
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,15 @@ project(self VERSION 1.0.0
5454
LANGUAGES Fortran C CXX)
5555

5656
# Fortran compiler requirements
57-
INCLUDE(FortranCInterface)
57+
include(FortranCInterface)
5858
FortranCInterface_VERIFY()
59-
IF(NOT FortranCInterface_VERIFIED_C)
60-
MESSAGE(FATAL_ERROR "Fortran compiler must support C Interface")
61-
ENDIF(NOT FortranCInterface_VERIFIED_C)
59+
if(NOT FortranCInterface_VERIFIED_C)
60+
message(FATAL_ERROR "Fortran compiler must support C Interface")
61+
endif(NOT FortranCInterface_VERIFIED_C)
6262

63-
IF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
63+
if(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
6464
MESSAGE(FATAL_ERROR "Fortran compiler does not support F90")
65-
ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
65+
endif(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
6666

6767
# Default Fortran compiler flags
6868
# Fortran compiler flags
@@ -130,9 +130,8 @@ if( SELF_ENABLE_MULTITHREADING )
130130

131131
endif()
132132

133-
134133
# MPI
135-
find_package(MPI COMPONENTS Fortran REQUIRED)
134+
find_package(MPI COMPONENTS Fortran C REQUIRED)
136135

137136
# HDF5 : See https://cmake.org/cmake/help/latest/module/FindHDF5.html
138137
find_package(HDF5 REQUIRED Fortran)
@@ -162,7 +161,7 @@ if(SELF_ENABLE_DOUBLE_PRECISION)
162161
endif()
163162

164163
if(SELF_ENABLE_GPU)
165-
message("Enabling GPU Support")
164+
message("-- Enabling GPU Support")
166165
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DENABLE_GPU" )
167166
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DENABLE_GPU" )
168167
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DENABLE_GPU")
@@ -175,56 +174,68 @@ if(SELF_ENABLE_GPU)
175174
set( CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_PROFILE} -DENABLE_GPU")
176175
set( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DENABLE_GPU" )
177176

177+
# Check MPI for GPU awareness
178+
# Add SELF's cmake module directory to the search path
179+
set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake")
180+
181+
# Include the custom module
182+
include(CheckMPIFeatures)
183+
CheckMPIFeatures()
184+
178185
# HIP (Optional)
179186
# IF HIP is found, we prefer HIP. IF HIP is not found, then we fall back to look for CUDA
180187
# only. If neither HIP nor CUDA are found, Cmake will error out.
181188
find_package(hip)
182189
if(hip_FOUND)
183-
find_package(hipblas REQUIRED)
184-
message("HIP found. Enabling HIP language.")
185-
enable_language(HIP)
186-
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" )
187-
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" )
188-
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP")
189-
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP")
190-
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" )
191-
192-
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_HIP" )
193-
set( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DHAVE_HIP" )
194-
set( CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS_COVERAGE} -DHAVE_HIP")
195-
set( CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_PROFILE} -DHAVE_HIP")
196-
set( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DHAVE_HIP" )
197-
198-
set( BACKEND_LIBRARIES hip::device roc::hipblas)
190+
if(MPI_HAS_QUERY_HIP_SUPPORT)
191+
find_package(hipblas REQUIRED)
192+
#message("-- HIP found. Enabling HIP language.")
193+
#enable_language(HIP)
194+
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" )
195+
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" )
196+
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP")
197+
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP")
198+
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" )
199+
200+
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_HIP" )
201+
set( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DHAVE_HIP" )
202+
set( CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS_COVERAGE} -DHAVE_HIP")
203+
set( CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_PROFILE} -DHAVE_HIP")
204+
set( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DHAVE_HIP" )
205+
206+
set( BACKEND_LIBRARIES hip::device roc::hipblas)
207+
else()
208+
message( FATAL_ERROR "MPI installation is not GPU-aware" )
209+
endif()
199210
else()
200211
# CUDA (Optional)
201212
find_package(cuda)
202213
if(cuda_FOUND)
203-
message("CUDA found. Enabling CUDA language.")
204-
enable_language(CUDA)
205-
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" )
206-
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" )
207-
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA")
208-
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA")
209-
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" )
210-
211-
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_CUDA" )
212-
set( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DHAVE_CUDA" )
213-
set( CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS_COVERAGE} -DHAVE_CUDA")
214-
set( CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_PROFILE} -DHAVE_CUDA")
215-
set( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DHAVE_CUDA" )
214+
if(MPI_HAS_QUERY_CUDA_SUPPORT)
215+
#message("-- CUDA found. Enabling CUDA language.")
216+
#enable_language(CUDA)
217+
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" )
218+
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" )
219+
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA")
220+
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA")
221+
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" )
222+
223+
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_CUDA" )
224+
set( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DHAVE_CUDA" )
225+
set( CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS_COVERAGE} -DHAVE_CUDA")
226+
set( CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_PROFILE} -DHAVE_CUDA")
227+
set( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DHAVE_CUDA" )
216228

217-
# TO DO - need cuda libraries and hipblas libraries
229+
# TO DO - need cuda libraries and hipblas libraries
230+
else()
231+
message( FATAL_ERROR "MPI installation is not GPU-aware" )
232+
endif()
218233
else()
219234
message( FATAL_ERROR "Enabling GPU support requires either HIP or CUDA." )
220235
endif()
221236
endif()
222237
endif()
223-
# get_cmake_property(_variableNames VARIABLES)
224-
# list (SORT _variableNames)
225-
# foreach (_variableName ${_variableNames})
226-
# message(STATUS "${_variableName}=${${_variableName}}")
227-
# endforeach()
238+
228239
# ----- end dependencies ----- #
229240

230241
# Libraries

‎cmake/CheckMPIFeatures.cmake

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# From https://fossies.org/linux/misc/abinit-10.0.9.tar.gz/abinit-10.0.9/cmake/CheckMPIFeatures.cmake?m=t
2+
# This file is borrowed and slightly modified from
3+
# https://github.com/eschnett/MPIwrapper/blob/main/cmake/CheckMPIFeatures.cmake
4+
#
5+
# function CheckMPIFeatures provides helper to check if MPI implementation
6+
# has the runtime ability to probe GPU-awareness
7+
# - cuda-aware (Nvidia GPU),
8+
# - hip-aware (AMD GPU),
9+
# - ze-aware (INTEL GPU)
10+
#
11+
#
12+
# Apparently Intel MPI (as of version 2021.7.0) doesn't provide header mpi-ext.h, too bad.
13+
#
14+
15+
include(CheckCSourceCompiles)
16+
function(CheckMPIFeatures)
17+
if (NOT DEFINED HAVE_MPI_EXT OR NOT DEFINED MPI_HAS_QUERY_CUDA_SUPPORT)
18+
list(JOIN MPI_COMPILE_FLAGS " " CMAKE_REQUIRED_FLAGS)
19+
20+
set(CMAKE_REQUIRED_LIBRARIES MPI::MPI_C)
21+
# We cannot use check_include_file here as <mpi.h> needs to be
22+
# included before <mpi-ext.h>, and check_include_file doesn't
23+
# support this.
24+
check_c_source_compiles(
25+
"
26+
#include <mpi.h>
27+
#include <mpi-ext.h>
28+
int main() {
29+
return 0;
30+
}
31+
"
32+
HAVE_MPI_EXT)
33+
34+
if(NOT HAVE_MPI_EXT)
35+
set(HAVE_MPI_EXT 0)
36+
else()
37+
set(HAVE_MPI_EXT 1)
38+
endif()
39+
40+
list(APPEND CMAKE_REQUIRED_DEFINITIONS -DHAVE_MPI_EXT=${HAVE_MPI_EXT})
41+
42+
check_c_source_compiles(
43+
"
44+
#include <mpi.h>
45+
#if HAVE_MPI_EXT
46+
#include <mpi-ext.h>
47+
#endif
48+
int main() {
49+
int result = MPIX_Query_cuda_support();
50+
return 0;
51+
}
52+
"
53+
MPI_HAS_QUERY_CUDA_SUPPORT)
54+
55+
if(NOT MPI_HAS_QUERY_CUDA_SUPPORT)
56+
set(MPI_HAS_QUERY_CUDA_SUPPORT 0)
57+
else()
58+
set(MPI_HAS_QUERY_CUDA_SUPPORT 1)
59+
endif()
60+
61+
check_c_source_compiles(
62+
"
63+
#include <mpi.h>
64+
#if HAVE_MPI_EXT
65+
#include <mpi-ext.h>
66+
#endif
67+
int main() {
68+
int result = MPIX_Query_rocm_support();
69+
return 0;
70+
}
71+
"
72+
MPI_HAS_QUERY_HIP_SUPPORT)
73+
74+
if(NOT MPI_HAS_QUERY_HIP_SUPPORT)
75+
set(MPI_HAS_QUERY_HIP_SUPPORT 0)
76+
else()
77+
set(MPI_HAS_QUERY_HIP_SUPPORT 1)
78+
endif()
79+
80+
check_c_source_compiles(
81+
"
82+
#include <mpi.h>
83+
#if HAVE_MPI_EXT
84+
#include <mpi-ext.h>
85+
#endif
86+
int main() {
87+
int result = MPIX_Query_ze_support();
88+
return 0;
89+
}
90+
"
91+
MPI_HAS_QUERY_ZE_SUPPORT)
92+
93+
if(NOT MPI_HAS_QUERY_ZE_SUPPORT)
94+
set(MPI_HAS_QUERY_ZE_SUPPORT 0)
95+
else()
96+
set(MPI_HAS_QUERY_ZE_SUPPORT 1)
97+
endif()
98+
99+
list(REMOVE_ITEM CMAKE_REQUIRED_DEFINITIONS -DHAVE_MPI_EXT)
100+
endif()
101+
endfunction()
102+
103+
CheckMPIFeatures()

‎src/CMakeLists.txt

+5-5
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@ file(GLOB SELF_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/*.f*")
2929
if(SELF_ENABLE_GPU)
3030
file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.f*")
3131
file(GLOB SELF_BACKEND_CPPSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.cpp*")
32-
if(hip_FOUND)
33-
set_source_files_properties(${SELF_CPPSRC} PROPERTIES LANGUAGE HIP)
34-
elseif(cuda_FOUND)
35-
set_source_files_properties(${SELF_CPPSRC} PROPERTIES LANGUAGE CUDA)
36-
endif()
32+
# if(hip_FOUND)
33+
# set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE HIP)
34+
# elseif(cuda_FOUND)
35+
# set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE CUDA)
36+
# endif()
3737
else()
3838
file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/cpu/*.f*")
3939
endif()

‎src/gpu/SELF_GPU_Macros.h

+1-5
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,6 @@
1919

2020
#include <hip/hip_runtime.h>
2121

22-
#ifdef __HIPCC__
23-
#define __CUDACC__
24-
#endif
25-
2622
__attribute__((unused))
2723
static void check(const hipError_t err, const char *const file, const int line)
2824
{
@@ -32,7 +28,7 @@ static void check(const hipError_t err, const char *const file, const int line)
3228
exit(err);
3329
}
3430

35-
#elif HAVE_CUDA
31+
#else
3632

3733
#include <cuda_runtime.h>
3834

‎test/scalarboundaryinterp_2d_constant.f90

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ integer function scalarboundaryinterp_2d_constant() result(r)
7272

7373
! Calculate diff from exact
7474
f%boundary = abs(f%boundary-1.0_prec)
75-
75+
print*, "absmax(error), tolerance : ",maxval(f%boundary),tolerance
7676
if(maxval(f%boundary) <= tolerance) then
7777
r = 0
7878
else

0 commit comments

Comments
 (0)