Merged in syclMacroTesting (pull request #674)

SyclMacroTesting

Approved-by: Phani Motamarri
Approved-by: Sambit Das

Author: phanimotamarri

CMakeLists.txt

@@ -5,6 +5,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.17)
 set(CMAKE_CXX_STANDARD 17
     CACHE STRING "Minimum required C++ standard (default is 17)")
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 # Set the name of the project and target:
 SET(TARGET "dftfe_exe")
@@ -30,6 +31,7 @@ IF (WITH_GPU)
       enable_language(CUDA)
     ELSEIF ("${GPU_LANG}" STREQUAL "hip")
       enable_language(HIP)
+    ELSEIF ("${GPU_LANG}" STREQUAL "sycl")
     ELSE ()
       MESSAGE(FATAL_ERROR "\n"
     "*** Could not find valid GPU_LANG=cuda/hip ***\n\n"
@@ -274,6 +276,26 @@ SET(DEVICE_SRC
   ./utils/FEBasisOperationsKernelsInternalDevice.cc
   ./src/atom/AtomicCenteredNonLocalOperatorKernelsDevice.cc
   )
+ELSEIF ("${GPU_LANG}" STREQUAL "sycl")
+SET(DEVICE_SRC
+  ./utils/MemoryTransferKernelsDevice.cc
+  ./utils/DeviceKernelsGeneric.cc
+  ./utils/DeviceDirectCCLWrapper.cc
+  ./src/dft/densityCalculatorDeviceKernels.cc
+  ./src/dft/densityFirstOrderResponseCalculatorDeviceKernels.cc
+  ./src/dft/solveVselfInBinsDeviceKernels.cc
+  ./src/dft/kineticEnergyDensityCalculatorDeviceKernels.cc
+  ./src/linAlg/linearAlgebraOperationsDeviceKernels.cc
+  ./src/force/forceWfcContractionsDeviceKernels.cc
+  ./utils/constraintMatrixInfoDeviceKernels.cc
+  ./utils/DeviceAPICalls.sycl.cc
+  ./utils/BLASWrapperDevice.cc
+  ./utils/MPICommunicatorP2PKernelsDevice.cc
+  ./src/solvers/linearSolverCGDeviceKernels.cc
+  ./utils/matrixFreeDeviceKernels.cc
+  ./utils/FEBasisOperationsKernelsInternalDevice.cc
+  ./src/atom/AtomicCenteredNonLocalOperatorKernelsDevice.cc
+  )
 
 ENDIF()
 
@@ -288,6 +310,8 @@ IF (WITH_GPU)
        ELSEIF ("${GPU_VENDOR}" STREQUAL "nvidia")
           ADD_DEFINITIONS(-D__HIP_PLATFORM_NVIDIA__)
        ENDIF()
+    ELSEIF ("${GPU_LANG}" STREQUAL "sycl")
+      set_source_files_properties(${DEVICE_SRC} PROPERTIES COMPILE_FLAGS ${CMAKE_SYCL_FLAGS})
     ENDIF()
 ENDIF()
 IF (WITH_GPU)
@@ -353,6 +377,9 @@ ENDIF()
 
 IF (USE_64BIT_INT)
   ADD_DEFINITIONS(-DDFTFE_WITH_64BIT_INT)
+  IF ("${GPU_VENDOR}" STREQUAL "intel")
+    add_compile_options("-fno-sycl-id-queries-fit-in-int")
+  ENDIF()
 ENDIF()
 
 #
@@ -379,6 +406,9 @@ IF (WITH_GPU)
     ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_NVIDIA)
   ELSEIF ("${GPU_VENDOR}" STREQUAL "amd")
     ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_AMD)
+  ELSEIF ("${GPU_VENDOR}" STREQUAL "intel")
+    ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_INTEL)
+    ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_MKL)
   ENDIF()
 
   IF ("${GPU_LANG}" STREQUAL "cuda")
@@ -393,6 +423,34 @@ IF (WITH_GPU)
     ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_LANG_HIP)
     find_package(hipblas)
     target_link_libraries(${TARGETLIB} PUBLIC roc::hipblas)
+  ELSEIF ("${GPU_LANG}" STREQUAL "sycl")
+    MESSAGE(STATUS "DFTFE with GPU LANG SYCL")
+    ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_LANG_SYCL)
+    find_program(MPI_BASE_CXX_COMPILER "icpx")
+    get_filename_component(_REALPATH_SYCL_COMPILER "${MPI_BASE_CXX_COMPILER}" REALPATH)
+    get_filename_component(SYCL_BIN_DIR "${_REALPATH_SYCL_COMPILER}" DIRECTORY)
+    get_filename_component(SYCL_PACKAGE_DIR "${SYCL_BIN_DIR}" DIRECTORY CACHE)
+    find_file(SYCL_INCLUDE_DIR
+    NAMES
+    include
+    HINTS
+    ${SYCL_PACKAGE_DIR}
+    NO_DEFAULT_PATH)
+    target_include_directories(${TARGETLIB} PUBLIC ${SYCL_INCLUDE_DIR})
+    target_include_directories(${TARGETLIB} PUBLIC ${SYCL_INCLUDE_DIR}/sycl)
+    set(MKL_INTERFACE lp64)
+    set(ENABLE_SCALAPACK ON)
+    set(MKL_MPI mpich)
+    find_PACKAGE(IntelSYCL REQUIRED)
+    find_package(MKL REQUIRED)
+    target_link_libraries(${TARGETLIB} PUBLIC MKL::MKL)
+    get_target_property(MKL_SYCL_INCLUDE_DIRS MKL::MKL_SYCL INTERFACE_INCLUDE_DIRECTORIES)
+    target_include_directories(${TARGETLIB} PUBLIC ${MKL_SYCL_INCLUDE_DIRS})
+    get_target_property(mkl_sycl_includes MKL::MKL_SYCL INTERFACE_INCLUDE_DIRECTORIES)
+    target_include_directories(${TARGETLIB} PUBLIC ${mkl_sycl_includes})
+    get_target_property(mkl_sycl_libs MKL::MKL_SYCL INTERFACE_LINK_LIBRARIES)
+    target_link_libraries(${TARGETLIB} PUBLIC ${mkl_sycl_libs})
+    set_target_properties(${TARGETLIB} PROPERTIES LINK_FLAGS ${CMAKE_SYCL_FLAGS})
   ENDIF()
   IF (WITH_GPU_AWARE_MPI)
     ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_AWARE_MPI)

include/AtomicCenteredNonLocalOperatorKernelsDevice.h

@@ -23,7 +23,7 @@
 #include <DeviceAPICalls.h>
 #include <DeviceDataTypeOverloads.h>
 #include <DeviceTypeConfig.h>
-#include <DeviceKernelLauncherConstants.h>
+#include <DeviceKernelLauncherHelpers.h>
 #include <MemoryStorage.h>
 namespace dftfe
 {