Pr from fork/137

.gitignore

@@ -1,5 +1,7 @@
 *.pyc
 *.o
 *.exe
+*.swp
+*.swo
 build*/
 install*/

.gitmodules

@@ -1,6 +1,7 @@
 [submodule "blt"]
 	path = blt
-	url = https://github.com/LLNL/blt.git
+   url = https://github.com/LLNL/blt.git
+   ignore = dirty
 [submodule "tpl/RAJA"]
 	path = tpl/RAJA
 	url = https://github.com/LLNL/RAJA.git

CMakeLists.txt

@@ -120,6 +120,28 @@ if ((ENABLE_HIP) AND (NOT ENABLE_KOKKOS))
   list(APPEND RAJA_PERFSUITE_DEPENDS blt::hip_runtime)
 endif()
 
+#
+# Are we using Caliper
+#
+set(ENABLE_CALIPER off CACHE BOOL "")
+set(RAJAPERF_USE_CALIPER off)
+if (ENABLE_CALIPER)
+   find_package(caliper REQUIRED)
+   list(APPEND RAJA_PERFSUITE_DEPENDS caliper)
+   set(RAJAPERF_USE_CALIPER on)
+   add_definitions(-DRAJAPERF_USE_CALIPER)
+   message(STATUS "Using Caliper")
+   find_package(adiak REQUIRED)
+   # use ${adiak_LIBRARIES} since version could have adiak vs adiak::adiak export
+   list(APPEND RAJA_PERFSUITE_DEPENDS ${adiak_LIBRARIES})
+   message(STATUS "Using Adiak")
+   if (ENABLE_CUDA)
+      # Adiak will propagate -pthread from spectrum mpi from a spack install of Caliper with +mpi; and needs to be handled even if RAJAPerf is non MPI program
+      # We should delegate to BLT to handle unguarded -pthread from any dependencies, but currently BLT doesn't
+      set_target_properties(adiak PROPERTIES INTERFACE_COMPILE_OPTIONS "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:-pthread>;$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-pthread>")
+   endif ()
+endif ()
+
 set(RAJAPERF_BUILD_SYSTYPE $ENV{SYS_TYPE})
 set(RAJAPERF_BUILD_HOST $ENV{HOSTNAME})
 
@@ -135,7 +157,7 @@ if (ENABLE_CUDA)
   set(RAJAPERF_COMPILER "${CUDA_NVCC_EXECUTABLE}")
   list(APPEND RAJAPERF_COMPILER ${CMAKE_CXX_COMPILER})
   set(RAJAPERF_COMPILER_OPTIONS "${CUDA_NVCC_FLAGS}")
-elseif (ENABLE_HIP)
+  elseif (ENABLE_HIP)
   set(RAJAPERF_COMPILER "${HIP_HIPCC_EXECUTABLE}")
   list(APPEND RAJAPERF_COMPILER ${CMAKE_CXX_COMPILER})
   set(RAJAPERF_COMPILER_OPTIONS "${HIP_HIPCC_FLAGS}")
@@ -147,7 +169,7 @@ else()
 endif()
 
 configure_file(${CMAKE_SOURCE_DIR}/src/rajaperf_config.hpp.in
-  ${CMAKE_CURRENT_BINARY_DIR}/include/rajaperf_config.hpp)
+               ${CMAKE_CURRENT_BINARY_DIR}/include/rajaperf_config.hpp)
 
 include_directories($<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/include>)
 

scripts/lc-builds/blueos_nvcc10_caliper25_gcc8.3.1.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# Copyright (c) 2017-20, Lawrence Livermore National Security, LLC
+# and RAJA Performance Suite project contributors.
+# See the RAJAPerf/COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+#################################################################################
+
+BUILD_SUFFIX=lc_blueos-nvcc10-caliper-gcc8.3.1
+RAJA_HOSTCONFIG=../tpl/RAJA/host-configs/lc-builds/blueos/nvcc_gcc_X.cmake
+
+rm -rf build_${BUILD_SUFFIX} >/dev/null
+mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX}
+
+module load cmake/3.14.5
+module load caliper-2.5.0-gcc-8.3.1-cu3vy3k
+
+CALIPER_PREFIX=/usr/WS2/holger/spack/opt/spack/linux-rhel7-power9le/gcc-8.3.1/caliper-2.5.0-cu3vy3kjwjerpdm6xis2kauhz4s6wto2/
+
+ADIAK_PREFIX=/usr/WS2/holger/spack/opt/spack/linux-rhel7-power9le/gcc-8.3.1/adiak-0.2.1-hsv444o7ofb6s2znkvvnh6hcmr774g73/
+
+cmake \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CXX_COMPILER=/usr/tce/packages/gcc/gcc-8.3.1/bin/g++ \
+  -C ${RAJA_HOSTCONFIG} \
+  -DCMAKE_PREFIX_PATH="${CALIPER_PREFIX}/share/cmake/caliper;${ADIAK_PREFIX}/lib/cmake/adiak" \
+  -DENABLE_OPENMP=On \
+  -DENABLE_CUDA=On \
+  -DCMAKE_CUDA_FLAGS="-Xcompiler -mno-float128" \
+  -DCUDA_TOOLKIT_ROOT_DIR=/usr/tce/packages/cuda/cuda-10.2.89 \
+  -DCMAKE_CUDA_COMPILER=/usr/tce/packages/cuda/cuda-10.1.243/bin/nvcc \
+  -DCUDA_ARCH=sm_70 \
+  -DENABLE_CALIPER=On \
+  -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \
+  -DCMAKE_EXPORT_COMPILE_COMMANDS=On \
+  -DCMAKE_VERBOSE_MAKEFILE=On \
+  "$@" \
+  ..

src/CMakeLists.txt

@@ -8,6 +8,13 @@
 
 include_directories(.)
 
+if(RAJAPERF_USE_CALIPER)
+   message(STATUS "Caliper includes : ${caliper_INCLUDE_DIR}")
+   message(STATUS "Adiak includes : ${adiak_INCLUDE_DIRS}")
+   include_directories(${caliper_INCLUDE_DIR})
+   include_directories(${adiak_INCLUDE_DIRS})
+endif ()
+
 add_subdirectory(common)
 add_subdirectory(basic)
 add_subdirectory(basic-kokkos)
@@ -240,6 +247,10 @@ else()
 blt_add_executable(
   NAME raja-perf.exe
   SOURCES RAJAPerfSuiteDriver.cpp
+  INCLUDES ${PROJECT_BINARY_DIR}/include
   DEPENDS_ON ${RAJA_PERFSUITE_EXECUTABLE_DEPENDS}
   )
 endif()
+
+#blt_print_target_properties(TARGET raja-perf.exe)
+

src/basic/PI_ATOMIC-OMP.cpp

@@ -31,7 +31,6 @@ void PI_ATOMIC::runOpenMPVariant(VariantID vid, size_t RAJAPERF_UNUSED_ARG(tune_
   switch ( vid ) {
 
     case Base_OpenMP : {
-
       startTimer();
       for (RepIndex_type irep = 0; irep < run_reps; ++irep) {
 
@@ -46,7 +45,6 @@ void PI_ATOMIC::runOpenMPVariant(VariantID vid, size_t RAJAPERF_UNUSED_ARG(tune_
 
       }
       stopTimer();
-
       break;
     }
 
@@ -70,7 +68,6 @@ void PI_ATOMIC::runOpenMPVariant(VariantID vid, size_t RAJAPERF_UNUSED_ARG(tune_
 
       }
       stopTimer();
-
       break;
     }
 

src/common/CMakeLists.txt

@@ -14,5 +14,6 @@ blt_add_library(
           OutputUtils.cpp 
           RAJAPerfSuite.cpp 
           RunParams.cpp
+  INCLUDES ${PROJECT_BINARY_DIR}/include/
   DEPENDS_ON ${RAJA_PERFSUITE_DEPENDS}
   )

src/common/Executor.cpp

@@ -28,6 +28,8 @@
 #include <string>
 #include <unordered_map>
 
+#include <regex>
+
 #include <iostream>
 #include <iomanip>
 #include <sstream>
@@ -37,11 +39,22 @@
 
 #include <unistd.h>
 
-
 namespace rajaperf {
 
 using namespace std;
 
+#ifdef RAJAPERF_USE_CALIPER
+vector<string> split(const string str, const string regex_str)
+{
+  regex regexz(regex_str);
+  vector<string> list( regex_token_iterator(str.begin(), str.end(),
+                                            regexz, -1),
+                       sregex_token_iterator() );
+  return list;
+}
+#endif
+
+
 namespace {
 
 #ifdef RAJA_PERFSUITE_ENABLE_MPI
@@ -98,13 +111,76 @@ void Allreduce(const Checksum_type* send, Checksum_type* recv, int count,
 
 #endif
 
-}
+} // close unnamed namespace 
+
 
 Executor::Executor(int argc, char** argv)
   : run_params(argc, argv),
     reference_vid(NumVariants),
     reference_tune_idx(KernelBase::getUnknownTuningIdx())
 {
+#ifdef RAJAPERF_USE_CALIPER
+  struct configuration cc;
+  adiak::init(NULL);
+  adiak::user();
+  adiak::launchdate();
+  adiak::libraries();
+  adiak::cmdline();
+  adiak::clustername();
+  adiak::value("perfsuite_version", cc.perfsuite_version);
+  adiak::value("raja_version", cc.raja_version);
+  adiak::value("cmake_build_type", cc.cmake_build_type);
+
+  adiak::value("compiler_path", cc.compiler);
+  cout << "Compiler path: " << cc.compiler << "\n";
+  auto tokens = split(cc.compiler, "/");
+  string compiler_exec = tokens.back();
+  adiak::value("compiler_version", cc.compiler_version);
+  string compiler = compiler_exec + "-" + cc.compiler_version;
+  cout << "Compiler: " << compiler << "\n";
+  adiak::value("compiler", compiler.c_str());
+  auto tsize = tokens.size();
+  if (tsize >= 3) {
+    // pickup path version <compiler-version-hash|date>/bin/exec
+    string path_version = tokens[tsize-3];
+    cout << "Compiler path version: " << path_version << "\n";
+    auto s = split(path_version,"-");
+    if (s.size() >= 2) {
+      string path_version_short = s[0] + "-" + s[1];
+      cout << "Compiler path version short: " << path_version_short << "\n";
+      adiak::value("Compiler_path_version",path_version_short.c_str());
+    } 
+  }
+
+
+  adiak::value("compiler_flags", cc.compiler_flags);
+  if (!strcmp(cc.cmake_build_type, "Release"))
+    adiak::value("compiler_flags_release", cc.compiler_flags_release);
+  else if (!strcmp(cc.cmake_build_type, "RelWithDebInfo"))
+    adiak::value("compiler_flags_relwithdebinfo", cc.compiler_flags_relwithdebinfo);
+  else if (!strcmp(cc.cmake_build_type, "Debug"))
+    adiak::value("compiler_flags_debug", cc.compiler_flags_debug);
+
+  if (strlen(cc.cuda_compiler_version) > 0) {
+    adiak::value("cuda_compiler_version", cc.cuda_compiler_version);
+    adiak::value("cuda_flags", cc.cuda_flags);
+    adiak::value("cuda_flags_release", cc.cuda_flags_release);
+  }
+
+  if (strlen(cc.systype_build) > 0)
+    adiak::value("systype_build", cc.systype_build);
+  if (strlen(cc.machine_build) > 0)
+    adiak::value("machine_build", cc.machine_build);
+
+  adiak::value("ProblemSize",1.0);
+  adiak::value("SizeMeaning",run_params.SizeMeaningToStr(run_params.getSizeMeaning()).c_str());
+  if (run_params.getSizeMeaning() == RunParams::SizeMeaning::Factor) {
+    adiak::value("ProblemSize",run_params.getSizeFactor());
+  } else if (run_params.getSizeMeaning() == RunParams::SizeMeaning::Direct) {
+    adiak::value("ProblemSize",run_params.getSize());
+  }
+
+#endif
 }
 
 
@@ -113,6 +189,9 @@ Executor::~Executor()
   for (size_t ik = 0; ik < kernels.size(); ++ik) {
     delete kernels[ik];
   }
+#ifdef RAJAPERF_USE_CALIPER 
+  adiak::fini();
+#endif
 }
 
 
@@ -611,6 +690,9 @@ void Executor::setupSuite()
       for (VIDset::iterator vid = run_var.begin();
            vid != run_var.end(); ++vid) {
         variant_ids.push_back( *vid );
+#ifdef RAJAPERF_USE_CALIPER
+        KernelBase::setCaliperMgrVariant(*vid);
+#endif
       }
 
       //
@@ -861,12 +943,21 @@ void Executor::runSuite()
 
   for (size_t ik = 0; ik < warmup_kernels.size(); ++ik) {
     KernelBase* warmup_kernel = warmup_kernels[ik];
+#ifdef RAJAPERF_USE_CALIPER
+    warmup_kernel->caliperOff();
+#endif
     runKernel(warmup_kernel, true);
+#ifdef RAJAPERF_USE_CALIPER
+    warmup_kernel->caliperOn();
+#endif
     delete warmup_kernel;
     warmup_kernels[ik] = nullptr;
   }
 
 
+//
+// Now, run kernels for real....
+//
   getCout() << "\n\nRunning specified kernels and variants...\n";
 
   const int npasses = run_params.getNumPasses();
@@ -882,6 +973,11 @@ void Executor::runSuite()
 
   } // loop over passes through suite
 
+#ifdef RAJAPERF_USE_CALIPER
+  // Flush Caliper data
+  KernelBase::setCaliperMgrFlush();
+#endif
+
 }
 
 template < typename Kernel >

src/common/Executor.hpp

@@ -12,6 +12,10 @@
 #include "common/RAJAPerfSuite.hpp"
 #include "common/RunParams.hpp"
 
+#ifdef RAJAPERF_USE_CALIPER
+#include "rajaperf_config.hpp"
+#endif
+
 #include <iosfwd>
 #include <streambuf>
 #include <memory>

src/common/KernelBase.cpp

@@ -191,6 +191,12 @@ void KernelBase::runKernel(VariantID vid, size_t tune_idx)
     return;
   }
 
+#ifdef RAJAPERF_USE_CALIPER
+  if(doCaliperTiming) {
+    KernelBase::setCaliperMgrStart(vid);
+  }
+#endif
+
   switch ( vid ) {
 
     case Base_Seq :
@@ -261,6 +267,11 @@ void KernelBase::runKernel(VariantID vid, size_t tune_idx)
     }
 
   }
+#ifdef RAJAPERF_USE_CALIPER
+  if(doCaliperTiming) {
+    setCaliperMgrStop(vid); 
+  }
+#endif
 }
 
 void KernelBase::print(std::ostream& os) const
@@ -330,5 +341,10 @@ void KernelBase::print(std::ostream& os) const
   }
   os << std::endl;
 }
-
+#ifdef RAJAPERF_USE_CALIPER
+// initialize a KernelBase static 
+std::map<rajaperf::VariantID, cali::ConfigManager> KernelBase::mgr;
+#endif
 }  // closing brace for rajaperf namespace
+
+