-
Notifications
You must be signed in to change notification settings - Fork 51
Feature/kokkos #188
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: develop
Are you sure you want to change the base?
Feature/kokkos #188
Changes from 142 commits
792224f
9456d76
4e4c2dd
19afb97
d658a4d
f7631d1
0f74831
c70a7ca
b52d974
37c2897
ae5007a
7f707f9
88f98f9
8b12325
8b828f5
529746d
cfda905
b227e70
b3ef7f5
104ac3d
c9cf2a7
fddf904
7a37806
7aa5d8d
6858ba3
3f8acf9
8bac825
c1193dc
9d0501c
5ed43c0
ca36ef0
b171109
3381d88
71901b6
f252173
27d63c9
eec425e
c1f72a5
9835eff
8bdcaf7
c69ce1a
e1b705a
91f7910
9b4cfdc
7753a5d
233072e
a62d462
2d35bc9
25d3ac4
cb4bd10
6cb52f2
c34a43d
8fca574
8dc5c01
944e58a
a3d4061
4f738c8
ce4079e
fc4bb07
1ca2a3c
86c1283
5c32d68
497741c
5ef80af
ce5738a
0be15e4
5af2f06
0261b3b
a4f2ef0
e306920
210bd2c
28c76e5
e76979e
97f9e5d
4dbf1d0
a3196d6
ae6286d
37659d6
6f16423
05d6ab7
8c35199
9d8292e
18253c3
4f730cd
7f17ba4
b406206
af54587
009388c
7be49db
f238c1c
c9ab748
a0873de
1514d53
2ce3e28
ba8678e
80dea71
4093557
12af5ca
b9a1381
8d193ed
bee1341
d90c1fa
115a116
a26044d
2a65ef7
4a24d9a
0611696
fa7f247
a8a7553
c13b61c
298cd4c
3b0ef87
92f8f4d
efbfb4f
cf7e93a
da4f686
59ecd95
8f7e27b
8d4723c
4c4f103
170490d
4766ccb
3e331dc
6af5a71
a9dbc63
1fd7f0f
8e56aa4
ff48c3a
621e382
f19da0a
29c3460
038c34e
f9d829d
ec0421b
520357d
c064d50
29dfbfd
393504b
201d7ad
b7de26a
3dc1751
18cca87
487bce9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,11 +7,11 @@ | |
| ############################################################################### | ||
|
|
||
| project(RAJAPerfSuite CXX) | ||
|
|
||
| cmake_minimum_required(VERSION 3.9) | ||
|
|
||
| option(ENABLE_RAJA_SEQUENTIAL "Run sequential variants of RAJA kernels. Disable | ||
| this, and all other variants, to run _only_ raw C loops." On) | ||
| option(ENABLE_KOKKOS "Include Kokkos implementations of the kernels in the RAJA Perfsuite" Off) | ||
|
|
||
| # | ||
| # Initialize the BLT build system | ||
|
|
@@ -25,14 +25,14 @@ set(ENABLE_TESTS Off CACHE BOOL "Enable BLT and RAJA tests") | |
| set(ENABLE_EXAMPLES Off CACHE BOOL "Enable RAJA examples") | ||
| set(RAJA_ENABLE_EXERCISES Off CACHE BOOL "Enable RAJA exercises") | ||
|
|
||
| set(CMAKE_CXX_STANDARD 11) | ||
| set(BLT_CXX_STANDARD 11) | ||
|
|
||
| if(ENABLE_KOKKOS) | ||
| set(CMAKE_CXX_STANDARD 14) | ||
| set(BLT_CXX_STANDARD 14) | ||
| endif() | ||
| include(blt/SetupBLT.cmake) | ||
|
|
||
| # | ||
| # Define RAJA settings... | ||
| # | ||
|
|
||
| set(ENABLE_TESTS Off CACHE BOOL "") | ||
| set(ENABLE_EXAMPLES Off CACHE BOOL "") | ||
|
|
@@ -53,11 +53,8 @@ add_subdirectory(tpl/RAJA) | |
| get_property(RAJA_INCLUDE_DIRS DIRECTORY tpl/RAJA PROPERTY INCLUDE_DIRECTORIES) | ||
| include_directories(${RAJA_INCLUDE_DIRS}) | ||
|
|
||
|
|
||
| # | ||
| # Setup variables to pass to Perf suite | ||
| # | ||
|
|
||
| # | ||
| # These (hopefully temporary) macro constants are needed to work-around | ||
| # performance issues in the xl compiler. | ||
|
|
@@ -80,18 +77,25 @@ if (ENABLE_OPENMP) | |
| endif() | ||
| if (ENABLE_CUDA) | ||
| list(APPEND RAJA_PERFSUITE_DEPENDS cuda) | ||
| endif() | ||
| if (ENABLE_HIP) | ||
| endif() | ||
|
|
||
| # Kokkos requires hipcc as the CMAKE_CXX_COMPILER for HIP AMD/VEGA GPU | ||
| # platforms, whereas RAJAPerf Suite uses blt/CMake FindHIP to set HIP compiler | ||
| # Separate RAJAPerf Suite and Kokkos handling of HIP compilers | ||
|
|
||
| if ((ENABLE_HIP) AND (NOT ENABLE_KOKKOS)) | ||
| #if (ENABLE_HIP) | ||
| list(APPEND RAJA_PERFSUITE_DEPENDS hip) | ||
| endif() | ||
|
|
||
| set(RAJAPERF_BUILD_SYSTYPE $ENV{SYS_TYPE}) | ||
| set(RAJAPERF_BUILD_HOST $ENV{HOSTNAME}) | ||
|
|
||
| if (ENABLE_CUDA) | ||
| set(CMAKE_CUDA_STANDARD 11) | ||
| if (ENABLE_CUDA AND ENABLE_KOKKOS) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The ENABLE_CUDA here is redundant (since we are already inside an |
||
| set(CMAKE_CUDA_STANDARD 14) | ||
| endif() | ||
| set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -restrict -arch ${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr") | ||
|
|
||
| set(RAJAPERF_COMPILER "${CUDA_NVCC_EXECUTABLE}") | ||
| list(APPEND RAJAPERF_COMPILER ${CMAKE_CXX_COMPILER}) | ||
| set(RAJAPERF_COMPILER_OPTIONS "${CUDA_NVCC_FLAGS}") | ||
|
|
@@ -106,14 +110,64 @@ else() | |
| list(APPEND RAJAPERF_COMPILER_OPTIONS ${CMAKE_CXX_FLAGS}) | ||
| endif() | ||
|
|
||
| configure_file(${CMAKE_SOURCE_DIR}/src/rajaperf_config.hpp.in | ||
| configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/rajaperf_config.hpp.in | ||
| ${CMAKE_CURRENT_BINARY_DIR}/bin/rajaperf_config.hpp) | ||
|
|
||
| # Make sure RAJA flag propagate (we need to do some house cleaning to | ||
| # Make sure RAJA flag propagate (we need to do some tidying to | ||
| # remove project-specific CMake variables that are no longer needed) | ||
| set (CUDA_NVCC_FLAGS ${RAJA_NVCC_FLAGS}) | ||
|
|
||
| # | ||
| # The statement below is required for Kokkos compilation. | ||
| if(ENABLE_KOKKOS) | ||
| include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/tpl/RAJA/include/) | ||
| endif() | ||
|
|
||
|
|
||
| # ENABLE KOKKOS IS A RAJA PERFSUITE OPTION | ||
| if(ENABLE_KOKKOS) | ||
| add_definitions(-DRUN_KOKKOS) | ||
| if(ENABLE_HIP) | ||
| set(Kokkos_ENABLE_HIP ON CACHE BOOL "Kokkos builds for AMD HIP set the | ||
| Kokkos_ENABLE_HIP variable to ON") | ||
| #set(Kokkos_ARCH_VEGA900 ON CACHE BOOL "Docstring") #TODO: better | ||
| #set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE /ascldap/users/ajpowel/RAJAPerf/amd_build/compiler_unscrewer) | ||
| endif() | ||
| if(ENABLE_TARGET_OPENMP) | ||
| set(Kokkos_ENABLE_OPENMPTARGET ON CACHE BOOL "Docstring") | ||
| set(Kokkos_ARCH_VOLTA70 ON CACHE BOOL "Docstring") #TODO: better | ||
| set(CMAKE_CXX_STANDARD 17) | ||
| set(BLT_CXX_STANDARD 17) | ||
| set(RAJA_ENABLE_TARGET_OPENMP ON CACHE BOOL "Docstring") | ||
| if(NOT CMAKE_BUILD_TYPE MATCHES Debug) | ||
| if(NOT EXPERIMENTAL_BUILD) | ||
| message(FATAL_ERROR "Kokkos builds with OpenMPTarget require a Debug build to succeed at the moment. Rebuild with CMAKE_BUILD_TYPE=Debug. If you're a compiler developer, rebuild with -DEXPERIMENTAL_BUILD=ON") | ||
| endif() | ||
| endif() | ||
|
|
||
| #add_definitions(-DRAJA_ENABLE_TARGET_OPENMP) | ||
| endif() | ||
|
|
||
| # ENABLE_CUDA IS A RAJA PERFSUITE OPTION | ||
| if(ENABLE_CUDA) | ||
| set(Kokkos_ENABLE_CUDA ON CACHE BOOL "Docstring") | ||
| set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "Docstring") | ||
| set(Kokkos_ARCH_VOLTA70 ON CACHE BOOL "Docstring") #TODO: better | ||
| enable_language(CUDA) | ||
| endif() | ||
| if(ENABLE_OPENMP) | ||
| #set(Kokkos_ENABLE_OPENMP CACHE BOOL ON) | ||
| set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "Docstring") | ||
| endif() | ||
|
|
||
| add_subdirectory(tpl/kokkos) | ||
| if(ENABLE_CUDA) | ||
ajpowelsnl marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| endif() | ||
| get_property(KOKKOS_INCLUDE_DIRS DIRECTORY tpl/kokkos PROPERTY INCLUDE_DIRECTORIES) | ||
| include_directories(${KOKKOS_INCLUDE_DIRS}) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this required? I think that having |
||
|
|
||
| list(APPEND RAJA_PERFSUITE_DEPENDS kokkos) | ||
| endif() | ||
|
|
||
| # Each directory in the perf suite has its own CMakeLists.txt file. | ||
| # | ||
| # DZP, AJP, DB, DA fixes | ||
| add_subdirectory(src) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| { | ||
| "plots" : { | ||
| "files" : { | ||
| "fileName": "RAJAPerf*", | ||
| "type" : "xml", | ||
| "ignoreOldFiles" : true, | ||
| "recurseDirectories" : true | ||
| }, | ||
| "categories": [ | ||
| "kokkos_lambda", | ||
| "lambda_cuda", | ||
| "base_cuda", | ||
| "base_seq", | ||
| "lambda_seq", | ||
| "raja_cuda", | ||
| "raja_seq" | ||
| ], | ||
| "plot" : [ | ||
| { | ||
| "autoname" : { | ||
| "useProperty" : "y/path", | ||
| "formatByRemovingPrefix" : "\\/RAJAPerf\\d{4}\\-\\d{2}\\-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.xml\\/" | ||
| }, | ||
| "category" : "kokkos_lambda", | ||
| "template" : "kokkos_template", | ||
| "dataLines" : [ | ||
| { | ||
| "name" : "Data Line", | ||
| "template" : "Line_Template", | ||
| "x" : { | ||
| "getPath": "*", | ||
| "getElement" : "performance-report", | ||
| "getKey" : "date", | ||
| "unit" : "timestamp" | ||
| }, | ||
| "y" : { | ||
| "getElement" : "performance-report|timing", | ||
| "getPath": "*/kokkos_perf_suite/*", | ||
| "getPathAttribute": "name", | ||
| "getKey" : "kokkos_lambda", | ||
| "unit" : "seconds", | ||
| "strategy" : { | ||
| "getFirstMatchOnly" : "false", | ||
| "recurseChildGraphs" : "true" | ||
| } | ||
| }, | ||
| "color" : "202,77,77" | ||
| } | ||
| ] | ||
| },{ | ||
| "inherit" : "kokkos_template", | ||
| "category" : "base_cuda", | ||
| "dataLines" : [ | ||
| { | ||
| "inherit" : "Line_Template", | ||
| "y" : { | ||
| "getKey" : "base_cuda" | ||
| } | ||
| } | ||
| ] | ||
| }, { | ||
| "inherit" : "kokkos_template", | ||
| "category" : "base_seq", | ||
| "dataLines" : [ | ||
| { | ||
| "inherit" : "Line_Template", | ||
| "y" : { | ||
| "getKey" : "base_seq" | ||
| } | ||
| } | ||
| ] | ||
| }, { | ||
|
|
||
| "inherit" : "kokkos_template", | ||
| "category" : "lambda_seq", | ||
| "dataLines" : [ | ||
| { | ||
| "inherit" : "Line_Template", | ||
| "y" : { | ||
| "getKey" : "lambda_seq" | ||
| } | ||
| } | ||
| ] | ||
|
|
||
| },{ | ||
| "inherit" : "kokkos_template", | ||
| "category" : "lambda_cuda", | ||
| "dataLines" : [ | ||
| { | ||
| "inherit" : "Line_Template", | ||
| "y" : { | ||
| "getKey" : "lambda_cuda" | ||
| } | ||
| } | ||
| ] | ||
| },{ | ||
| "inherit" : "kokkos_template", | ||
| "category" : "raja_cuda", | ||
| "dataLines" : [ | ||
| { | ||
| "inherit" : "Line_Template", | ||
| "y" : { | ||
| "getKey" : "raja_cuda" | ||
| } | ||
| } | ||
| ] | ||
| }, { | ||
| "inherit" : "kokkos_template", | ||
| "category" : "raja_seq", | ||
| "dataLines" : [ | ||
| { | ||
| "inherit" : "Line_Template", | ||
| "y" : { | ||
| "getKey" : "raja_seq" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| }, | ||
| "graphDisplay": { | ||
| "dbLocation" : "root", | ||
| "page" : 1, | ||
| "displayCategory" : "kokkos_lambda", | ||
| "displayRange" : 30, | ||
| "graphWidth" : 450, | ||
| "graphHeight" : 450, | ||
| "graphsPerRow" : 3, | ||
| "graphsPerPage" : 15, | ||
| "displayedDecimalPlaces" : 3 | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| ############################################################################### | ||
| # Copyright (c) 2017-20, Lawrence Livermore National Security, LLC | ||
| # and RAJA Performance Suite project contributors. | ||
| # See the RAJAPerf/COPYRIGHT file for details. | ||
| # | ||
| # SPDX-License-Identifier: (BSD-3-Clause) | ||
| ################################################################################# | ||
|
|
||
| BUILD_SUFFIX=snl_rhel7-hipcc-4.0.0 | ||
|
|
||
| rm -rf build_${BUILD_SUFFIX} 2>/dev/null | ||
| mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} | ||
|
|
||
| ################################## | ||
| #Caraway Build (AMD) | ||
| ################################# | ||
| module purge | ||
|
|
||
| module load cmake/3.19.3 | ||
|
|
||
| module load git/2.9.4 | ||
|
|
||
| ################################## | ||
| # FOR COMPUTE NODE (caraway04 GPU): | ||
|
|
||
| module load rocm/4.0.0 | ||
|
|
||
| module load python/3.7.3 | ||
|
|
||
| cmake \ | ||
| -DCMAKE_BUILD_TYPE=Release \ | ||
| -DENABLE_KOKKOS=ON \ | ||
| -DENABLE_HIP=ON \ | ||
| -DKokkos_ARCH_VEGA900=ON \ | ||
| -DCMAKE_CXX_FLAGS="--gcc-toolchain=/home/projects/x86-64/gcc/8.2.0/" \ | ||
| -DHIP_HIPCC_FLAGS="--gcc-toolchain=/home/projects/x86-64/gcc/8.2.0/ -std=c++17" \ | ||
| -DCMAKE_CXX_STANDARD=17 \ | ||
| -DCMAKE_CXX_COMPILER=hipcc .. \ | ||
|
|
||
| make -j24;make | ||
|
|
||
| cd bin/ | ||
| ./raja-perf.exe | ||
|
|
||
|
|
||
|
|
Uh oh!
There was an error while loading. Please reload this page.