From 4316d4dcfd00bd261f7902de68013522bdb9933b Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 26 Sep 2023 12:16:14 -0400 Subject: FindCUDAToolkit: Search all of `nvcc` implicit includes and library dirs Improves the handling of CUDA layouts where we have multiple include and library directories listed in the output of `nvcc -v`. This updates both when the CUDA language is enabled or not. Fixes: #24915 --- .gitlab-ci.yml | 10 ++ .gitlab/ci/configure_cuda11.8_splayed_nvidia.cmake | 3 + .gitlab/ci/env_cuda11.8_splayed_nvidia.sh | 36 +++++++ .gitlab/os-linux.yml | 6 ++ Modules/FindCUDAToolkit.cmake | 114 +++++++++++++++++---- Tests/Cuda/Toolkit/CMakeLists.txt | 8 +- Tests/CudaOnly/Toolkit/CMakeLists.txt | 16 ++- 7 files changed, 170 insertions(+), 23 deletions(-) create mode 100644 .gitlab/ci/configure_cuda11.8_splayed_nvidia.cmake create mode 100644 .gitlab/ci/env_cuda11.8_splayed_nvidia.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c13ba72..3c1c9cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -350,6 +350,16 @@ t:cuda11.8-minimal-ninja: variables: CMAKE_CI_NO_MR: "true" +t:cuda11.8-minimal-splayed-ninja: + extends: + - .cuda11.8_splayed_nvidia + - .cmake_test_linux_release + - .linux_x86_64_tags_cuda + - .run_dependent + - .needs_centos7_x86_64 + variables: + CMAKE_CI_NO_MR: "true" + t:hip5.5-nvidia: extends: - .hip5.5_nvidia diff --git a/.gitlab/ci/configure_cuda11.8_splayed_nvidia.cmake b/.gitlab/ci/configure_cuda11.8_splayed_nvidia.cmake new file mode 100644 index 0000000..519699b --- /dev/null +++ b/.gitlab/ci/configure_cuda11.8_splayed_nvidia.cmake @@ -0,0 +1,3 @@ +set(CMake_TEST_CUDA "NVIDIA" CACHE STRING "") + +include("${CMAKE_CURRENT_LIST_DIR}/configure_external_test.cmake") diff --git a/.gitlab/ci/env_cuda11.8_splayed_nvidia.sh b/.gitlab/ci/env_cuda11.8_splayed_nvidia.sh new file mode 100644 index 0000000..38e788d --- /dev/null +++ b/.gitlab/ci/env_cuda11.8_splayed_nvidia.sh @@ -0,0 +1,36 @@ +# +# Splay the libraries and includes to emulate conda where +# things are split between the host and build prefix +# +# /usr/local/cuda/include/crt/ -> /tmp/cuda/include/crt +# /usr/local/cuda/lib64/stubs/ -> /tmp/cuda/stubs/ +# /usr/local/cuda/lib64/libcudart* -> /tmp/cuda/libs/ +# +# Also reduce to minimal subset of libraries by removing +# static libraries to emulate a minimal cuda install +mkdir -p /tmp/cuda/libs +mkdir -p /tmp/cuda/stubs +mkdir -p /tmp/cuda/include + +mv /usr/local/cuda/lib64/libcuda* /tmp/cuda/libs +mv /usr/local/cuda/lib64/stubs/ /tmp/cuda/stubs/ +mv /usr/local/cuda/include/crt/ /tmp/cuda/include/ + +# patch the nvcc.profile to handle the splayed layout +# which allows verification +mv /usr/local/cuda/bin/nvcc.profile /usr/local/cuda/bin/nvcc.profile.orig +echo " +TOP = \$(_HERE_)/.. + +NVVMIR_LIBRARY_DIR = \$(TOP)/\$(_NVVM_BRANCH_)/libdevice + +LD_LIBRARY_PATH += \$(TOP)/lib: +PATH += \$(TOP)/\$(_NVVM_BRANCH_)/bin:\$(_HERE_): + +INCLUDES += \"-I\$(TOP)/\$(_TARGET_DIR_)/include\" \$(_SPACE_) \"-I/tmp/cuda/include\" \$(_SPACE_) + +LIBRARIES =+ \$(_SPACE_) \"-L\$(TOP)/\$(_TARGET_DIR_)/lib\$(_TARGET_SIZE_)\" \"-L/tmp/cuda/stubs/\" \"-L/tmp/cuda/libs\" + +CUDAFE_FLAGS += +PTXAS_FLAGS += +" > /usr/local/cuda/bin/nvcc.profile diff --git a/.gitlab/os-linux.yml b/.gitlab/os-linux.yml index e70dc07..8894057 100644 --- a/.gitlab/os-linux.yml +++ b/.gitlab/os-linux.yml @@ -362,6 +362,12 @@ CMAKE_CONFIGURATION: cuda11.8_minimal_nvidia CTEST_NO_WARNINGS_ALLOWED: 1 +.cuda11.8_splayed_nvidia: + extends: .cuda11.8_minimal + variables: + CMAKE_CONFIGURATION: cuda11.8_splayed_nvidia + CTEST_NO_WARNINGS_ALLOWED: 1 + ### HIP builds .hip5.5: diff --git a/Modules/FindCUDAToolkit.cmake b/Modules/FindCUDAToolkit.cmake index 12aca8d..bd9e0ec 100644 --- a/Modules/FindCUDAToolkit.cmake +++ b/Modules/FindCUDAToolkit.cmake @@ -514,7 +514,7 @@ Result variables executable ``nvcc``. ``CUDAToolkit_INCLUDE_DIRS`` - The path to the CUDA Toolkit ``include`` folder containing the header files + List of paths to all the CUDA Toolkit folders containing header files required to compile a project linking against CUDA. ``CUDAToolkit_LIBRARY_DIR`` @@ -579,13 +579,28 @@ Result variables # ############################################################################### +function(_CUDAToolkit_build_include_dirs result_variable default_paths_variable) + set(content "${${default_paths_variable}}") + set(${result_variable} "${content}" PARENT_SCOPE) +endfunction() + +function(_CUDAToolkit_build_library_dirs result_variable default_paths_variable) + set(content "${${default_paths_variable}}") + set(${result_variable} "${content}" PARENT_SCOPE) +endfunction() + # The toolkit is located during compiler detection for CUDA and stored in CMakeCUDACompiler.cmake as -# CMAKE_CUDA_COMPILER_TOOLKIT_ROOT and CMAKE_CUDA_COMPILER_LIBRARY_ROOT. +# - CMAKE_CUDA_COMPILER_TOOLKIT_ROOT +# - CMAKE_CUDA_COMPILER_LIBRARY_ROOT +# - CMAKE_CUDA_COMPILER_LIBRARY_DIRECTORIES_FROM_IMPLICIT_LIBRARIES +# - CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES # We compute the rest based on those here to avoid re-searching and to avoid finding a possibly # different installation. if(CMAKE_CUDA_COMPILER_TOOLKIT_ROOT) set(CUDAToolkit_ROOT_DIR "${CMAKE_CUDA_COMPILER_TOOLKIT_ROOT}") set(CUDAToolkit_LIBRARY_ROOT "${CMAKE_CUDA_COMPILER_LIBRARY_ROOT}") + _CUDAToolkit_build_library_dirs(CUDAToolkit_IMPLICIT_LIBRARY_DIRECTORIES CMAKE_CUDA_HOST_IMPLICIT_LINK_DIRECTORIES) + _CUDAToolkit_build_include_dirs(CUDAToolkit_INCLUDE_DIRECTORIES CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES) set(CUDAToolkit_BIN_DIR "${CUDAToolkit_ROOT_DIR}/bin") set(CUDAToolkit_NVCC_EXECUTABLE "${CUDAToolkit_BIN_DIR}/nvcc${CMAKE_EXECUTABLE_SUFFIX}") set(CUDAToolkit_VERSION "${CMAKE_CUDA_COMPILER_TOOLKIT_VERSION}") @@ -622,11 +637,45 @@ else() # NVIDIA HPC SDK, and distro's splayed layouts execute_process(COMMAND ${CUDAToolkit_NVCC_EXECUTABLE} "-v" "__cmake_determine_cuda" OUTPUT_VARIABLE _CUDA_NVCC_OUT ERROR_VARIABLE _CUDA_NVCC_OUT) + message(CONFIGURE_LOG + "Executed nvcc to extract CUDAToolkit information:\n${_CUDA_NVCC_OUT}\n\n") if(_CUDA_NVCC_OUT MATCHES "\\#\\$ TOP=([^\r\n]*)") get_filename_component(CUDAToolkit_BIN_DIR "${CMAKE_MATCH_1}/bin" ABSOLUTE) + message(CONFIGURE_LOG + "Parsed CUDAToolkit nvcc location:\n${CUDAToolkit_BIN_DIR}\n\n") else() get_filename_component(CUDAToolkit_BIN_DIR "${CUDAToolkit_NVCC_EXECUTABLE}" DIRECTORY) endif() + if(_CUDA_NVCC_OUT MATCHES "\\#\\$ INCLUDES=([^\r\n]*)") + separate_arguments(_nvcc_output NATIVE_COMMAND "${CMAKE_MATCH_1}") + foreach(line IN LISTS _nvcc_output) + string(REGEX REPLACE "^-I" "" line "${line}") + get_filename_component(line "${line}" ABSOLUTE) + list(APPEND _cmake_CUDAToolkit_include_directories "${line}") + endforeach() + message(CONFIGURE_LOG + "Parsed CUDAToolkit nvcc implicit include information:\n${_cmake_CUDAToolkit_include_directories}\n\n") + + set(_cmake_CUDAToolkit_include_directories "${_cmake_CUDAToolkit_include_directories}" CACHE INTERNAL "CUDAToolkit internal list of include directories") + endif() + if(_CUDA_NVCC_OUT MATCHES "\\#\\$ LIBRARIES=([^\r\n]*)") + include(${CMAKE_ROOT}/Modules/CMakeParseImplicitLinkInfo.cmake) + set(_nvcc_link_line "cuda-fake-ld ${CMAKE_MATCH_1}") + CMAKE_PARSE_IMPLICIT_LINK_INFO("${_nvcc_link_line}" + _cmake_CUDAToolkit_implicit_link_libs + _cmake_CUDAToolkit_implicit_link_directories + _cmake_CUDAToolkit_implicit_frameworks + _nvcc_log + "${CMAKE_CUDA_IMPLICIT_OBJECT_REGEX}" + LANGUAGE CUDA) + message(CONFIGURE_LOG + "Parsed CUDAToolkit nvcc implicit link information:\n${_nvcc_log}\n${_cmake_CUDAToolkit_implicit_link_directories}\n\n") + unset(_nvcc_link_line) + unset(_cmake_CUDAToolkit_implicit_link_libs) + unset(_cmake_CUDAToolkit_implicit_frameworks) + + set(_cmake_CUDAToolkit_implicit_link_directories "${_cmake_CUDAToolkit_implicit_link_directories}" CACHE INTERNAL "CUDAToolkit internal list of implicit link directories") + endif() unset(_CUDA_NVCC_OUT) set(CUDAToolkit_BIN_DIR "${CUDAToolkit_BIN_DIR}" CACHE PATH "" FORCE) @@ -642,6 +691,15 @@ else() endif() endif() + if(DEFINED _cmake_CUDAToolkit_include_directories) + _CUDAToolkit_build_include_dirs(_cmake_CUDAToolkit_contents _cmake_CUDAToolkit_include_directories) + set(CUDAToolkit_INCLUDE_DIRECTORIES "${_cmake_CUDAToolkit_contents}" PARENT_SCOPE) + endif() + if(DEFINED _cmake_CUDAToolkit_implicit_link_directories) + _CUDAToolkit_build_library_dirs(_cmake_CUDAToolkit_contents _cmake_CUDAToolkit_implicit_link_directories) + set(CUDAToolkit_IMPLICIT_LIBRARY_DIRECTORIES "${_cmake_CUDAToolkit_contents}" PARENT_SCOPE) + endif() + if(CUDAToolkit_BIN_DIR) get_filename_component(CUDAToolkit_ROOT_DIR ${CUDAToolkit_BIN_DIR} DIRECTORY ABSOLUTE) set(CUDAToolkit_ROOT_DIR "${CUDAToolkit_ROOT_DIR}" PARENT_SCOPE) @@ -885,18 +943,27 @@ if(NOT CUDAToolkit_TARGET_DIR) set(_CUDAToolkit_Pop_Prefix True) endif() -# CUDAToolkit_TARGET_DIR always points to the directory containing the include directory. -# On a scattered installation /usr, on a non-scattered something like /usr/local/cuda or /usr/local/cuda-10.2/targets/aarch64-linux. -if(EXISTS "${CUDAToolkit_TARGET_DIR}/include/cuda_runtime.h") - set(CUDAToolkit_INCLUDE_DIR "${CUDAToolkit_TARGET_DIR}/include") -elseif(NOT CUDAToolkit_FIND_QUIETLY) - message(STATUS "Unable to find cuda_runtime.h in \"${CUDAToolkit_TARGET_DIR}/include\" for CUDAToolkit_INCLUDE_DIR.") + +# We don't need to verify the cuda_runtime header when we are using `nvcc` include paths +# as the compiler being enabled means the header was found +if(NOT CUDAToolkit_INCLUDE_DIRECTORIES) + # Otherwise use CUDAToolkit_TARGET_DIR to guess where the `cuda_runtime.h` is located + # On a scattered installation /usr, on a non-scattered something like /usr/local/cuda or /usr/local/cuda-10.2/targets/aarch64-linux. + if(EXISTS "${CUDAToolkit_TARGET_DIR}/include/cuda_runtime.h") + set(CUDAToolkit_INCLUDE_DIRECTORIES "${CUDAToolkit_TARGET_DIR}/include") + else() + message(STATUS "Unable to find cuda_runtime.h in \"${CUDAToolkit_TARGET_DIR}/include\" for CUDAToolkit_INCLUDE_DIRECTORIES.") + endif() endif() # The NVHPC layout moves math library headers and libraries to a sibling directory and it could be nested under # the version of the CUDA toolchain # Create a separate variable so this directory can be selectively added to math targets. -if(NOT EXISTS "${CUDAToolkit_INCLUDE_DIR}/cublas_v2.h") +find_path(CUDAToolkit_CUBLAS_INCLUDE_DIR cublas_v2.h PATHS + "${CUDAToolkit_INCLUDE_DIRECTORIES}" + NO_DEFAULT_PATH) + +if(NOT CUDAToolkit_CUBLAS_INCLUDE_DIR) file(REAL_PATH "${CUDAToolkit_TARGET_DIR}" CUDAToolkit_MATH_INCLUDE_DIR) cmake_path(APPEND CUDAToolkit_MATH_INCLUDE_DIR "../../math_libs/") if(EXISTS "${CUDAToolkit_MATH_INCLUDE_DIR}/${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}/") @@ -905,22 +972,26 @@ if(NOT EXISTS "${CUDAToolkit_INCLUDE_DIR}/cublas_v2.h") cmake_path(APPEND CUDAToolkit_MATH_INCLUDE_DIR "include") cmake_path(NORMAL_PATH CUDAToolkit_MATH_INCLUDE_DIR) - if(NOT EXISTS "${CUDAToolkit_MATH_INCLUDE_DIR}/cublas_v2.h") - if(NOT CUDAToolkit_FIND_QUIETLY) - message(STATUS "Unable to find cublas_v2.h in either \"${CUDAToolkit_INCLUDE_DIR}\" or \"${CUDAToolkit_MATH_INCLUDE_DIR}\"") - endif() - unset(CUDAToolkit_MATH_INCLUDE_DIR) + find_path(CUDAToolkit_CUBLAS_INCLUDE_DIR cublas_v2.h PATHS + "${CUDAToolkit_INCLUDE_DIRECTORIES}" + ) + if(CUDAToolkit_CUBLAS_INCLUDE_DIR) + list(APPEND CUDAToolkit_INCLUDE_DIRECTORIES "${CUDAToolkit_CUBLAS_INCLUDE_DIR}") endif() endif() +unset(CUDAToolkit_CUBLAS_INCLUDE_DIR CACHE) +unset(CUDAToolkit_CUBLAS_INCLUDE_DIR) # Find the CUDA Runtime Library libcudart find_library(CUDA_CUDART NAMES cudart + PATHS ${CUDAToolkit_IMPLICIT_LIBRARY_DIRECTORIES} PATH_SUFFIXES lib64 lib/x64 ) find_library(CUDA_CUDART NAMES cudart - PATH_SUFFIXES lib64/stubs lib/x64/stubs + PATHS ${CUDAToolkit_IMPLICIT_LIBRARY_DIRECTORIES} + PATH_SUFFIXES lib64/stubs lib/x64/stubs lib/stubs stubs ) if(NOT CUDA_CUDART AND NOT CUDAToolkit_FIND_QUIETLY) @@ -937,7 +1008,7 @@ endif() include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) find_package_handle_standard_args(CUDAToolkit REQUIRED_VARS - CUDAToolkit_INCLUDE_DIR + CUDAToolkit_INCLUDE_DIRECTORIES CUDA_CUDART CUDAToolkit_BIN_DIR VERSION_VAR @@ -946,7 +1017,6 @@ find_package_handle_standard_args(CUDAToolkit unset(CUDAToolkit_ROOT_DIR) mark_as_advanced(CUDA_CUDART - CUDAToolkit_INCLUDE_DIR CUDAToolkit_NVCC_EXECUTABLE CUDAToolkit_SENTINEL_FILE ) @@ -954,7 +1024,7 @@ mark_as_advanced(CUDA_CUDART #----------------------------------------------------------------------------- # Construct result variables if(CUDAToolkit_FOUND) - set(CUDAToolkit_INCLUDE_DIRS ${CUDAToolkit_INCLUDE_DIR}) + set(CUDAToolkit_INCLUDE_DIRS "${CUDAToolkit_INCLUDE_DIRECTORIES}") get_filename_component(CUDAToolkit_LIBRARY_DIR ${CUDA_CUDART} DIRECTORY ABSOLUTE) # Build search paths without any symlinks @@ -976,6 +1046,10 @@ if(CUDAToolkit_FOUND) endblock() endif() + if(DEFINED CUDAToolkit_IMPLICIT_LIBRARY_DIRECTORIES) + list(APPEND CUDAToolkit_LIBRARY_SEARCH_DIRS "${CUDAToolkit_IMPLICIT_LIBRARY_DIRECTORIES}") + endif() + # If no `CUDAToolkit_LIBRARY_ROOT` exists set it based on CUDAToolkit_LIBRARY_DIR if(NOT DEFINED CUDAToolkit_LIBRARY_ROOT) foreach(CUDAToolkit_search_loc IN LISTS CUDAToolkit_LIBRARY_DIR CUDAToolkit_BIN_DIR) @@ -989,7 +1063,8 @@ if(CUDAToolkit_FOUND) unset(CUDAToolkit_possible_lib_root) endif() endif() - +unset(CUDAToolkit_IMPLICIT_LIBRARY_DIRECTORIES) +unset(CUDAToolkit_INCLUDE_DIRECTORIES) #----------------------------------------------------------------------------- # Construct import targets @@ -1257,6 +1332,7 @@ if(CUDAToolkit_FOUND) _CUDAToolkit_find_and_add_import_lib(OpenCL) endif() +unset(CUDAToolkit_LIBRARY_SEARCH_DIRS) if(_CUDAToolkit_Pop_ROOT_PATH) list(REMOVE_AT CMAKE_FIND_ROOT_PATH 0) unset(_CUDAToolkit_Pop_ROOT_PATH) diff --git a/Tests/Cuda/Toolkit/CMakeLists.txt b/Tests/Cuda/Toolkit/CMakeLists.txt index 8432b71..c2989f0 100644 --- a/Tests/Cuda/Toolkit/CMakeLists.txt +++ b/Tests/Cuda/Toolkit/CMakeLists.txt @@ -26,9 +26,11 @@ set(should_exist CUDAToolkit_LIBRARY_ROOT ) foreach (cuda_loc_var IN LISTS should_exist) - if(NOT EXISTS "${${cuda_loc_var}}") - message(FATAL_ERROR "${cuda_loc_var} variable is expected to be set to valid path") - endif() + foreach (entry IN LISTS ${cuda_loc_var}) + if(NOT EXISTS "${entry}") + message(FATAL_ERROR "${cuda_loc_var} variable is expected to be set to valid path") + endif() + endforeach() endforeach() diff --git a/Tests/CudaOnly/Toolkit/CMakeLists.txt b/Tests/CudaOnly/Toolkit/CMakeLists.txt index e0801a3..506fc9f 100644 --- a/Tests/CudaOnly/Toolkit/CMakeLists.txt +++ b/Tests/CudaOnly/Toolkit/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.15) -project(CudaOnlyToolkit CUDA) +project(CudaOnlyToolkit LANGUAGES CUDA) find_package(CUDAToolkit REQUIRED) if(NOT DEFINED CUDAToolkit_VERSION) @@ -13,8 +13,22 @@ message(STATUS "CUDAToolkit_VERSION_PATCH: ${CUDAToolkit_VERSION_PATCH}") message(STATUS "CUDAToolkit_BIN_DIR: ${CUDAToolkit_BIN_DIR}") message(STATUS "CUDAToolkit_INCLUDE_DIRS: ${CUDAToolkit_INCLUDE_DIRS}") message(STATUS "CUDAToolkit_LIBRARY_DIR: ${CUDAToolkit_LIBRARY_DIR}") +message(STATUS "CUDAToolkit_LIBRARY_ROOT: ${CUDAToolkit_LIBRARY_ROOT}") message(STATUS "CUDAToolkit_NVCC_EXECUTABLE ${CUDAToolkit_NVCC_EXECUTABLE}") +set(should_exist + CUDAToolkit_BIN_DIR + CUDAToolkit_INCLUDE_DIRS + CUDAToolkit_LIBRARY_DIR + CUDAToolkit_LIBRARY_ROOT + ) +foreach (cuda_loc_var IN LISTS should_exist) + foreach (entry IN LISTS ${cuda_loc_var}) + if(NOT EXISTS "${entry}") + message(FATAL_ERROR "${cuda_loc_var}[${entry}] variable is expected to be set to valid path") + endif() + endforeach() +endforeach() set(cuda_libs cudart cuda_driver cublas cufft cufftw curand cusolver cusparse) if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 10.1) -- cgit v0.12