From e43e84de47e5192d1c1b21f78bc6f55bafdfaa5d Mon Sep 17 00:00:00 2001 From: "Daniel J. Magee" Date: Fri, 24 May 2024 13:47:25 -0600 Subject: [PATCH 1/2] Make OpenMP_FOUND depend on whether OpenMP_CXX_FLAGS are found, Add OpenMP_CXX_FLAGS to cmake deps directly. Take compute capability (variable SMS) as environment variable, set default to 80 (Ampere). Use CMAKE_CXX_COMPILER as host compiler for nvcc and add -allow-unsupported-compiler flag so it works with everything. Move printouts to end of CMakeLists.txt to print CUDA compiler config. Update lanl-setup config for new machines. --- src/CMakeLists.txt | 58 ++++++++++++++++++++----------------- src/config/find_tpls.cmake | 18 ++++++------ src/config/lanl-setup.cmake | 28 ++++++++++++++---- 3 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 56ffa6e..b8475a1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -130,27 +130,6 @@ feature_summary( WHAT ALL FATAL_ON_MISSING_REQUIRED_PACKAGES QUIET_ON_EMPTY ) -# Summary of user-selectable build options -message( "\nBuild Summary:\n") -message( STATUS "Machine name : ${SITENAME}") -report_lanl_hpc_features() -if( ${BRANSON_N_GROUPS} GREATER 1 ) - message( STATUS "Energy groups : ${BRANSON_N_GROUPS}") -else() - message( STATUS "Energy groups : Gray (1-group)") -endif() -message( STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") -message( STATUS "Compiler : ${CMAKE_CXX_COMPILER}") -if( CMAKE_CONFIGURATION_TYPES ) - message( STATUS "Compiler Flags (All) : ${CMAKE_CXX_FLAGS}") - message( STATUS "Compiler Flags (Debug) : ${CMAKE_CXX_FLAGS_DEBUG}") - message( STATUS "Compiler Flags (Release): ${CMAKE_CXX_FLAGS_RELEASE}") -else() - string( TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER ) - message( STATUS "Compiler Flags : ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}") -endif() -message("\n") - #------------------------------------------------------------------------------# # Set up libraries for Cray RMA routines @@ -169,7 +148,7 @@ set( branson_deps pugixml) if(OpenMP_FOUND) - set(branson_deps "OpenMP::OpenMP_CXX;${branson_deps}") + set(branson_deps "OpenMP::OpenMP_CXX;${OpenMP_CXX_FLAGS};${branson_deps}") endif() if(METIS_FOUND) @@ -192,12 +171,17 @@ configure_file(config.h.in ${PROJECT_BINARY_DIR}/config.h) if("${CUDA_DBS_STRING}" STREQUAL "CUDA" ) message("Setting CUDA compiler options") - set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "70") # V100 - #set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "80") # A100 + if ( NOT DEFINED ENV{SMS} ) + set( SMS "80" ) + else() + set( SMS $ENV{SMS} ) + endif() + set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "${SMS}") # V100 set_target_properties(BRANSON PROPERTIES CUDA_STANDARD 17) - string(APPEND CMAKE_CUDA_FLAGS " -g --expt-relaxed-constexpr") - string(APPEND CMAKE_CUDA_FLAGS " --expt-extended-lambda" ) + string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CMAKE_CXX_COMPILER} --expt-relaxed-constexpr") + string(APPEND CMAKE_CUDA_FLAGS " --expt-extended-lambda -allow-unsupported-compiler" ) set_source_files_properties("main.cc" PROPERTIES LANGUAGE CUDA) + message( STATUS "SMS: ${SMS}") endif() target_link_libraries( BRANSON PRIVATE ${branson_deps} ) @@ -216,6 +200,28 @@ else() "Building tests disabled, set BUILD_TESTING=TRUE or don't set BUILD_TESTING to enable test builds") endif() +# Summary of user-selectable build options +message( "\nBuild Summary:\n") +message( STATUS "Machine name : ${SITENAME}") +report_lanl_hpc_features() +if( ${BRANSON_N_GROUPS} GREATER 1 ) + message( STATUS "Energy groups : ${BRANSON_N_GROUPS}") +else() + message( STATUS "Energy groups : Gray (1-group)") +endif() +message( STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") +message( STATUS "Compiler : ${CMAKE_CXX_COMPILER}") +if( CMAKE_CONFIGURATION_TYPES ) + message( STATUS "Compiler Flags (All) : ${CMAKE_CXX_FLAGS}") + message( STATUS "Compiler Flags (Debug) : ${CMAKE_CXX_FLAGS_DEBUG}") + message( STATUS "Compiler Flags (CUDA) : ${CMAKE_CUDA_FLAGS}") + message( STATUS "Compiler Flags (Release): ${CMAKE_CXX_FLAGS_RELEASE}") +else() + string( TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER ) + message( STATUS "Compiler Flags : ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}") +endif() +message("\n") + #------------------------------------------------------------------------------# # Targets for installation diff --git a/src/config/find_tpls.cmake b/src/config/find_tpls.cmake index 22c2ea9..aaf264e 100644 --- a/src/config/find_tpls.cmake +++ b/src/config/find_tpls.cmake @@ -56,12 +56,12 @@ macro(setupTPLs) # Find package if desired: if(USE_OPENMP) - find_package(OpenMP QUIET) + find_package(OpenMP) else() set(OpenMP_FOUND FALSE) endif() - if(OpenMP_FOUND) + if(OpenMP_CXX_FLAGS) # [2022-10-27 KT] cmake/3.22 doesn't report OpenMP_C_VERSION for nvc++. Fake it for now. if("${OpenMP_C_VERSION}x" STREQUAL "x" AND CMAKE_CXX_COMPILER_ID MATCHES "NVHPC") set(OpenMP_C_VERSION @@ -71,13 +71,13 @@ macro(setupTPLs) endif() message(STATUS "Looking for OpenMP... ${OpenMP_C_FLAGS} (supporting the ${OpenMP_C_VERSION} " "standard)") - if(OpenMP_C_VERSION VERSION_LESS 3.0) - message(STATUS "OpenMP standard support is too old (< 3.0). Disabling OpenMP build features.") - set(OpenMP_FOUND FALSE) - set(OpenMP_C_FLAGS - "" - CACHE BOOL "OpenMP disabled (too old)." FORCE) - endif() + # if(OpenMP_C_VERSION VERSION_LESS 3.0) + # message(STATUS "OpenMP standard support is too old (< 3.0). Disabling OpenMP build features.") + # set(OpenMP_FOUND FALSE) + # set(OpenMP_C_FLAGS + # "" + # CACHE BOOL "OpenMP disabled (too old)." FORCE) + # endif() set(OpenMP_FOUND ${OpenMP_FOUND} CACHE BOOL "Is OpenMP available?" FORCE) diff --git a/src/config/lanl-setup.cmake b/src/config/lanl-setup.cmake index 3f941c2..d3662ce 100644 --- a/src/config/lanl-setup.cmake +++ b/src/config/lanl-setup.cmake @@ -10,21 +10,39 @@ if( ${SITENAME} MATCHES "sn" OR ${SITENAME} MATCHES "ic" OR ${SITENAME} MATCHES "fi") set( SITENAME "Snow") set(SNOW_NODE TRUE) -elseif( ${SITENAME} MATCHES "ba") +elseif( ${SITENAME} MATCHES "ba-") set( SITENAME "Badger") set(BADGER_NODE TRUE) -elseif( ${SITENAME} MATCHES "tt") +elseif( ${SITENAME} MATCHES "tt-") set( SITENAME "Trinitite") set(TRINITITE_NODE TRUE) -elseif( ${SITENAME} MATCHES "tr") +elseif( ${SITENAME} MATCHES "tr-") + set( SITENAME "Trinity") + set(TRINITY_NODE TRUE) +elseif( ${SITENAME} MATCHES "ch-") + set( SITENAME "Chicoma") + set(CHICOMA_NODE TRUE) +elseif( ${SITENAME} MATCHES "ve-") + set( SITENAME "Venado") + set(VENADO_NODE TRUE) +elseif( ${SITENAME} MATCHES "ro-") + set( SITENAME "Rocinante") + set(ROCINANTE_NODE TRUE) +elseif( ${SITENAME} MATCHES "xr-") + set( SITENAME "Crossroads") + set(CROSSROADS_NODE TRUE) +elseif( ${SITENAME} MATCHES "ve-") + set( SITENAME "Venado") + set(VENADO_NODE TRUE) +elseif( ${SITENAME} MATCHES "tr-") set( SITENAME "Trinity") set(TRINITY_NODE TRUE) elseif( ${SITENAME} MATCHES "ccscs[0-9]+" ) # do nothing (keep the fullname) set(TRINITY_NODE TRUE) elseif( ${SITENAME} MATCHES "nid[0-9]+" ) - set( SITENAME "Trinitite") - set(TRINITITE_NODE TRUE) + set( SITENAME "CRAY_BACKEND") + set(CRAY_BACKEND_NODE TRUE) endif() #------------------------------------------------------------------------------# From 6e3b13bd55cc41cb7cab5763ba0339ad677c5681 Mon Sep 17 00:00:00 2001 From: "Daniel J. Magee" Date: Thu, 30 May 2024 11:46:39 -0600 Subject: [PATCH 2/2] Cray compiler id is 'Clang' change CMakeLists.txt accordingly. Fence GPU status messages with ENABLE_VERBOSE_GPU_TRANSPORT option. Add message to header if CUDA is enabled in binary. --- src/CMakeLists.txt | 9 ++++++--- src/gpu_setup.h | 5 ++++- src/input.h | 6 ++++++ src/transport_photon.h | 5 ++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b8475a1..02ecf0a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -106,9 +106,12 @@ elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Intel" ) # branson but it makes it the hardware counter data match Jayenne more closely (Jayenne uses # this flag) set( CMAKE_CXX_FLAGS_RELEASE "-g -O3 -fp-speculation=fast -fp-model=precise -qno-opt-dynamic-align") -elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Cray" ) - set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -DR123_USE_GNU_UINT128=0") - set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DR123_USE_GNU_UINT128=0") +elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Clang" ) + set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fopenmp -DR123_USE_GNU_UINT128=0") + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fopenmp -DR123_USE_GNU_UINT128=0") +# elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Cray" ) +# set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fopenmp -g -DR123_USE_GNU_UINT128=0") +# set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -g -DR123_USE_GNU_UINT128=0") endif() diff --git a/src/gpu_setup.h b/src/gpu_setup.h index 7153e7c..176ee7b 100644 --- a/src/gpu_setup.h +++ b/src/gpu_setup.h @@ -27,8 +27,9 @@ class GPU_Setup { if(m_use_gpu_transporter) { // MPI rank to GPU mapping set_device_ID(rank, n_ranks); - +#ifdef ENABLE_VERBOSE_GPU_TRANSPORT std::cout<<"Allocating and transferring "< (sigma_s / ((1.0 - f) * sigma_a + sigma_s))) + if (rng.generate_random_number() > (sigma_s / total_sigma_s )) phtn.set_group(sample_emission_group(rng, *cell)); phtn.set_descriptor(Constants::SCATTER); } @@ -253,8 +253,11 @@ void gpu_transport_photons(const uint32_t rank_cell_offset, cudaDeviceSynchronize(); +#ifdef ENABLE_VERBOSE_GPU_TRANSPORT std::cout << "Launching with " << n_blocks << " blocks and "; std::cout << n_batch_photons << " photons" << std::endl; +#endif + gpu_no_accel_transport<<>>( rank_cell_offset, device_photons_ptr, device_cells_ptr, device_cell_tallies_ptr, n_batch_photons);