Fix logic for adding -allow-unsupported-compiler to nvcc (#7337)

The previous logic would always add -allow-unsupported-compiler and -D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH, even when it was not needed.
The revised logic only adds the flags if MSVC>=1940 and CUDA<=12.4. The new code also uses CMAKE_CUDA_FLAGS for the define, which makes CMake's try_compile work reliably if a CMake version >=3.29.4 is used.
---------
Co-authored-by: Patrik Huber <[email protected]>
Co-authored-by: Sameer Sheorey <[email protected]>

Author: patrikhuber

CHANGELOG.md

@@ -63,6 +63,7 @@
 -   Exposed `get_plotly_fig` and modified `draw_plotly` to return the `Figure` it creates. (PR #7258)
 -   Fix build with librealsense v2.44.0 and upcoming VS 2022 17.13 (PR #7074)
 -   Fix `deprecated-declarations` warnings when compiling code with C++20 standard (PR #7303)
+-   Fix logic for adding -allow-unsupported-compiler to nvcc (PR #7337)
 -   Fix linker error "library limit of 65535 objects exceeded" with Ninja generator on MSVC (PR #7335)
 
 ## 0.13

CMakeLists.txt

@@ -395,14 +395,22 @@ cmake_language(EVAL CODE "cmake_language(DEFER CALL open3d_patch_findthreads_mod
 
 # Build CUDA module by default if CUDA is available
 if(BUILD_CUDA_MODULE)
-    # Suppress nvcc unsupported compiler error for MSVC 2022 with CUDA 11.7 to 12.4
-    # https://forums.developer.nvidia.com/t/problems-with-latest-vs2022-update/294150/12
-    if (MSVC AND MSVC_VERSION VERSION_LESS_EQUAL "1949")
-        # Set this before any CUDA checks
-        add_compile_definitions(_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH)
-        set(CMAKE_CUDA_FLAGS "--allow-unsupported-compiler" CACHE STRING "Additional flags for nvcc" FORCE)
-        message(WARNING "Using --allow-unsupported-compiler flag for nvcc with MSVC 2022. "
-        "Set $Env:NVCC_PREPEND_FLAGS='--allow-unsupported-compiler' if nvcc still fails.")
+    if(MSVC)
+        # Handle/suppress nvcc unsupported compiler error for MSVC>=1940 with CUDA 11.7 to 12.4:
+        # (https://forums.developer.nvidia.com/t/problems-with-latest-vs2022-update/294150/12)
+        # Find CUDAToolkit first to get CUDAToolkit_VERSION:
+        find_package(CUDAToolkit REQUIRED)
+        if (CUDAToolkit_VERSION VERSION_LESS "12.5" AND MSVC_VERSION GREATER_EQUAL 1940)
+            # Set required nvcc flags before enable_language(CUDA).
+            # Note: CMake >=3.29.4 might be needed for CMAKE_CUDA_FLAGS to be passed correctly to
+            # CMake's try_compile environment.
+            set(CMAKE_CUDA_FLAGS
+                  "${CMAKE_CUDA_FLAGS} -allow-unsupported-compiler -D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH"
+                  CACHE STRING "Flags for NVCC" FORCE)
+            endif()
+            message(WARNING "Using --allow-unsupported-compiler flag and -D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH for nvcc<=12.4 with MSVC>=1940. "
+            "Set $Env:NVCC_PREPEND_FLAGS='--allow-unsupported-compiler' if nvcc still fails.")
+        endif()
     endif()
     if (CMAKE_CUDA_ARCHITECTURES)
         message(STATUS "Building with user-provided CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}")