lanl · dmageeLANL · May 24, 2024 · May 30, 2024
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -106,9 +106,12 @@ elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Intel" )
   # branson but it makes it the hardware counter data match Jayenne more closely (Jayenne uses
   # this flag)
   set( CMAKE_CXX_FLAGS_RELEASE "-g -O3 -fp-speculation=fast -fp-model=precise -qno-opt-dynamic-align")
-elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Cray" )
-  set( CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -g -DR123_USE_GNU_UINT128=0")
-  set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DR123_USE_GNU_UINT128=0")
+elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Clang" )
+  set( CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -g -fopenmp -DR123_USE_GNU_UINT128=0")
+  set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fopenmp -DR123_USE_GNU_UINT128=0")
+# elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Cray" )
+#   set( CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -fopenmp -g -DR123_USE_GNU_UINT128=0")
+#   set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -g -DR123_USE_GNU_UINT128=0")
 endif()
 
 
@@ -130,27 +133,6 @@ feature_summary( WHAT ALL
   FATAL_ON_MISSING_REQUIRED_PACKAGES
   QUIET_ON_EMPTY )
 
-# Summary of user-selectable build options
-message( "\nBuild Summary:\n")
-message( STATUS "Machine name    : ${SITENAME}")
-report_lanl_hpc_features()
-if( ${BRANSON_N_GROUPS} GREATER 1 )
-  message( STATUS "Energy groups   : ${BRANSON_N_GROUPS}")
-else()
-  message( STATUS "Energy groups   : Gray (1-group)")
-endif()
-message( STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
-message( STATUS "Compiler        : ${CMAKE_CXX_COMPILER}")
-if( CMAKE_CONFIGURATION_TYPES )
-  message( STATUS "Compiler Flags (All)    : ${CMAKE_CXX_FLAGS}")
-  message( STATUS "Compiler Flags (Debug)  : ${CMAKE_CXX_FLAGS_DEBUG}")
-  message( STATUS "Compiler Flags (Release): ${CMAKE_CXX_FLAGS_RELEASE}")
-else()
-  string( TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER )
-  message( STATUS "Compiler Flags  : ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}")
-endif()
-message("\n")
-
 #------------------------------------------------------------------------------#
 # Set up libraries for Cray RMA routines
 
@@ -169,7 +151,7 @@ set( branson_deps
   pugixml)
 
 if(OpenMP_FOUND)
-  set(branson_deps "OpenMP::OpenMP_CXX;${branson_deps}")
+  set(branson_deps "OpenMP::OpenMP_CXX;${OpenMP_CXX_FLAGS};${branson_deps}")
 endif()
 
 if(METIS_FOUND)
@@ -192,12 +174,17 @@ configure_file(config.h.in ${PROJECT_BINARY_DIR}/config.h)
 
 if("${CUDA_DBS_STRING}" STREQUAL "CUDA" )
   message("Setting CUDA compiler options")
-  set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "70") # V100
-  #set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "80") # A100
+  if ( NOT DEFINED ENV{SMS} )
+    set( SMS "80" )
+  else()
+    set( SMS $ENV{SMS} )
+  endif()
+  set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "${SMS}") # V100
   set_target_properties(BRANSON PROPERTIES CUDA_STANDARD 17)
-  string(APPEND CMAKE_CUDA_FLAGS " -g --expt-relaxed-constexpr")
-  string(APPEND CMAKE_CUDA_FLAGS " --expt-extended-lambda"  )
+  string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CMAKE_CXX_COMPILER} --expt-relaxed-constexpr")
+  string(APPEND CMAKE_CUDA_FLAGS " --expt-extended-lambda -allow-unsupported-compiler"  )
   set_source_files_properties("main.cc" PROPERTIES LANGUAGE CUDA)
+  message( STATUS "SMS: ${SMS}")
 endif()
 
 target_link_libraries( BRANSON PRIVATE ${branson_deps} )
@@ -216,6 +203,28 @@ else()
     "Building tests disabled, set BUILD_TESTING=TRUE or don't set BUILD_TESTING to enable test builds")
 endif()
 
+# Summary of user-selectable build options
+message( "\nBuild Summary:\n")
+message( STATUS "Machine name    : ${SITENAME}")
+report_lanl_hpc_features()
+if( ${BRANSON_N_GROUPS} GREATER 1 )
+  message( STATUS "Energy groups   : ${BRANSON_N_GROUPS}")
+else()
+  message( STATUS "Energy groups   : Gray (1-group)")
+endif()
+message( STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
+message( STATUS "Compiler        : ${CMAKE_CXX_COMPILER}")
+if( CMAKE_CONFIGURATION_TYPES )
+  message( STATUS "Compiler Flags (All)    : ${CMAKE_CXX_FLAGS}")
+  message( STATUS "Compiler Flags (Debug)  : ${CMAKE_CXX_FLAGS_DEBUG}")
+  message( STATUS "Compiler Flags (CUDA)   : ${CMAKE_CUDA_FLAGS}")
+  message( STATUS "Compiler Flags (Release): ${CMAKE_CXX_FLAGS_RELEASE}")
+else()
+  string( TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER )
+  message( STATUS "Compiler Flags  : ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}")
+endif()
+message("\n")
+
 
 #------------------------------------------------------------------------------#
 # Targets for installation

diff --git a/src/config/find_tpls.cmake b/src/config/find_tpls.cmake
@@ -56,12 +56,12 @@ macro(setupTPLs)
 
   # Find package if desired:
   if(USE_OPENMP)
-    find_package(OpenMP QUIET)
+    find_package(OpenMP)
   else()
     set(OpenMP_FOUND FALSE)
   endif()
 
-  if(OpenMP_FOUND)
+  if(OpenMP_CXX_FLAGS)
     # [2022-10-27 KT] cmake/3.22 doesn't report OpenMP_C_VERSION for nvc++. Fake it for now.
     if("${OpenMP_C_VERSION}x" STREQUAL "x" AND CMAKE_CXX_COMPILER_ID MATCHES "NVHPC")
       set(OpenMP_C_VERSION
@@ -71,13 +71,13 @@ macro(setupTPLs)
     endif()
     message(STATUS "Looking for OpenMP... ${OpenMP_C_FLAGS} (supporting the ${OpenMP_C_VERSION} "
                    "standard)")
-    if(OpenMP_C_VERSION VERSION_LESS 3.0)
-      message(STATUS "OpenMP standard support is too old (< 3.0). Disabling OpenMP build features.")
-      set(OpenMP_FOUND FALSE)
-      set(OpenMP_C_FLAGS
-          ""
-          CACHE BOOL "OpenMP disabled (too old)." FORCE)
-    endif()
+    # if(OpenMP_C_VERSION VERSION_LESS 3.0)
+    #   message(STATUS "OpenMP standard support is too old (< 3.0). Disabling OpenMP build features.")
+    #   set(OpenMP_FOUND FALSE)
+    #   set(OpenMP_C_FLAGS
+    #       ""
+    #       CACHE BOOL "OpenMP disabled (too old)." FORCE)
+    # endif()
     set(OpenMP_FOUND
         ${OpenMP_FOUND}
         CACHE BOOL "Is OpenMP available?" FORCE)

diff --git a/src/config/lanl-setup.cmake b/src/config/lanl-setup.cmake
@@ -10,21 +10,39 @@
 if( ${SITENAME} MATCHES "sn" OR ${SITENAME} MATCHES "ic" OR ${SITENAME} MATCHES "fi")
    set( SITENAME "Snow")
    set(SNOW_NODE TRUE)
-elseif( ${SITENAME} MATCHES "ba")
+elseif( ${SITENAME} MATCHES "ba-")
   set( SITENAME "Badger")
   set(BADGER_NODE TRUE)
-elseif( ${SITENAME} MATCHES "tt")
+elseif( ${SITENAME} MATCHES "tt-")
   set( SITENAME "Trinitite")
   set(TRINITITE_NODE TRUE)
-elseif( ${SITENAME} MATCHES "tr")
+elseif( ${SITENAME} MATCHES "tr-")
+  set( SITENAME "Trinity")
+  set(TRINITY_NODE TRUE)
+elseif( ${SITENAME} MATCHES "ch-")
+  set( SITENAME "Chicoma")
+  set(CHICOMA_NODE TRUE)
+elseif( ${SITENAME} MATCHES "ve-")
+  set( SITENAME "Venado")
+  set(VENADO_NODE TRUE)
+elseif( ${SITENAME} MATCHES "ro-")
+  set( SITENAME "Rocinante")
+  set(ROCINANTE_NODE TRUE)
+elseif( ${SITENAME} MATCHES "xr-")
+  set( SITENAME "Crossroads")
+  set(CROSSROADS_NODE TRUE)
+elseif( ${SITENAME} MATCHES "ve-")
+  set( SITENAME "Venado")
+  set(VENADO_NODE TRUE)
+elseif( ${SITENAME} MATCHES "tr-")
   set( SITENAME "Trinity")
   set(TRINITY_NODE TRUE)
 elseif( ${SITENAME} MATCHES "ccscs[0-9]+" )
   # do nothing (keep the fullname)
   set(TRINITY_NODE TRUE)
 elseif( ${SITENAME} MATCHES "nid[0-9]+" )
-  set( SITENAME "Trinitite")
-  set(TRINITITE_NODE TRUE)
+  set( SITENAME "CRAY_BACKEND")
+  set(CRAY_BACKEND_NODE TRUE)
 endif()
 
 #------------------------------------------------------------------------------#

diff --git a/src/gpu_setup.h b/src/gpu_setup.h
@@ -27,8 +27,9 @@ class GPU_Setup {
     if(m_use_gpu_transporter) {
       // MPI rank to GPU mapping
       set_device_ID(rank, n_ranks);
-
+#ifdef ENABLE_VERBOSE_GPU_TRANSPORT
       std::cout<<"Allocating and transferring "<<cpu_cells.size()<<" cell(s) to the GPU"<<std::endl;
+#endif
       // allocate and copy cells
       cudaError_t err = cudaMalloc((void **)&device_cells_ptr, sizeof(Cell) * cpu_cells.size());
       Insist(!err, "CUDA error in allocating cells data");
@@ -78,10 +79,12 @@ void set_device_ID(const int rank, const int n_ranks) {
   cudaSetDevice(my_device);
   int my_bus_id = 0;
   cudaDeviceGetAttribute(&my_bus_id, cudaDevAttrPciBusId, my_device);
+#ifdef ENABLE_VERBOSE_GPU_TRANSPORT
   std::cout << "N devices: " << n_devices << std::endl;
   std::cout << "rank: " << rank << ", device: " << my_device << " bus id: ";
   std::cout << my_bus_id << std::endl;
 #endif
+#endif
 }
 
 

diff --git a/src/input.h b/src/input.h
@@ -722,6 +722,12 @@ class Input {
     std::cout<<" no affect"<<std::endl;
 #endif
 
+#ifdef USE_GPU
+    std::cout<<"Branson built with CUDA"<<std::endl;
+#else
+    std::cout<<"Branson built without CUDA"<<std::endl;
+#endif
+
     cout << "Mesh decomposition: ";
     if (decomp_mode == METIS && dd_mode != REPLICATED)
       cout << "METIS" << endl;

diff --git a/src/transport_photon.h b/src/transport_photon.h
@@ -114,7 +114,7 @@ void transport_photon(const uint32_t rank_cell_offset,
       // EVENT TYPE: SCATTER
       if (dist_to_event == dist_to_scatter) {
         phtn.set_angle(get_uniform_angle(rng));
-        if (rng.generate_random_number() > (sigma_s / ((1.0 - f) * sigma_a + sigma_s)))
+        if (rng.generate_random_number() > (sigma_s / total_sigma_s ))
           phtn.set_group(sample_emission_group(rng, *cell));
         phtn.set_descriptor(Constants::SCATTER);
       }
@@ -253,8 +253,11 @@ void gpu_transport_photons(const uint32_t rank_cell_offset,
 
   cudaDeviceSynchronize();
 
+#ifdef ENABLE_VERBOSE_GPU_TRANSPORT
   std::cout << "Launching with " << n_blocks << " blocks and ";
   std::cout << n_batch_photons << " photons" << std::endl;
+#endif
+
   gpu_no_accel_transport<<<n_blocks, Constants::n_threads_per_block>>>(
       rank_cell_offset, device_photons_ptr, device_cells_ptr, device_cell_tallies_ptr, n_batch_photons);