Merge topic 'hip-nvidia'

6546aa2a2a ci: Add HIP job using CUDA on NVIDIA GPUs
18158bf81c HIP: Add support for NVIDIA GPUs
127b6fa06b HIP: Add CMAKE_HIP_PLATFORM variable to specify GPU platform
90e23f40ee Tests/HIP/WithDefs: Clean up test case
9ebdf3281f Tests/HIP/ArchitectureOff: Cover HIP_ARCHITECTURES initialization
cfec29196e ci: Add CUDA 11.8 to HIP 5.5 image
26470eb987 ci: Put HIP GPU platform in CMake_TEST_HIP

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !8817

8 files changed, 46 insertions, 6 deletions

diff --git a/.gitlab/ci/configure_debian12_hip_radeon.cmake b/.gitlab/ci/configure_debian12_hip_radeon.cmake
index 58036b0..c7d7004 100644
--- a/.gitlab/ci/configure_debian12_hip_radeon.cmake
+++ b/.gitlab/ci/configure_debian12_hip_radeon.cmake
@@ -1,3 +1,3 @@
-set(CMake_TEST_HIP "ON" CACHE BOOL "")
+set(CMake_TEST_HIP "amd" CACHE BOOL "")
 
 include("${CMAKE_CURRENT_LIST_DIR}/configure_external_test.cmake")
diff --git a/.gitlab/ci/configure_fedora38_hip_radeon.cmake b/.gitlab/ci/configure_fedora38_hip_radeon.cmake
index 58036b0..c7d7004 100644
--- a/.gitlab/ci/configure_fedora38_hip_radeon.cmake
+++ b/.gitlab/ci/configure_fedora38_hip_radeon.cmake
@@ -1,3 +1,3 @@
-set(CMake_TEST_HIP "ON" CACHE BOOL "")
+set(CMake_TEST_HIP "amd" CACHE BOOL "")
 
 include("${CMAKE_CURRENT_LIST_DIR}/configure_external_test.cmake")
diff --git a/.gitlab/ci/configure_hip5.5_nvidia.cmake b/.gitlab/ci/configure_hip5.5_nvidia.cmake
new file mode 100644
index 0000000..4b3511a
--- /dev/null
+++ b/.gitlab/ci/configure_hip5.5_nvidia.cmake
@@ -0,0 +1,3 @@
+set(CMake_TEST_HIP "nvidia" CACHE BOOL "")
+
+include("${CMAKE_CURRENT_LIST_DIR}/configure_external_test.cmake")
diff --git a/.gitlab/ci/configure_hip5.5_radeon.cmake b/.gitlab/ci/configure_hip5.5_radeon.cmake
index 58036b0..c7d7004 100644
--- a/.gitlab/ci/configure_hip5.5_radeon.cmake
+++ b/.gitlab/ci/configure_hip5.5_radeon.cmake
@@ -1,3 +1,3 @@
-set(CMake_TEST_HIP "ON" CACHE BOOL "")
+set(CMake_TEST_HIP "amd" CACHE BOOL "")
 
 include("${CMAKE_CURRENT_LIST_DIR}/configure_external_test.cmake")
diff --git a/.gitlab/ci/docker/hip5.5/Dockerfile b/.gitlab/ci/docker/hip5.5/Dockerfile
index 2deb3c6..3a4aa53 100644
--- a/.gitlab/ci/docker/hip5.5/Dockerfile
+++ b/.gitlab/ci/docker/hip5.5/Dockerfile
@@ -2,7 +2,13 @@
 
 ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:5.5
 
-FROM ${BASE_IMAGE} AS apt-cache
+FROM ${BASE_IMAGE} AS cuda-keyring
+ADD https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb /root/
+RUN --mount=type=tmpfs,target=/var/log \
+    dpkg -i /root/cuda-keyring_1.1-1_all.deb \
+ && rm /root/cuda-keyring_1.1-1_all.deb
+
+FROM cuda-keyring AS apt-cache
 # Populate APT cache w/ the fresh metadata and prefetch packages.
 # Use an empty `docker-clean` file to "hide" the image-provided
 # file to disallow removing packages after `apt-get` operations.
@@ -12,9 +18,12 @@ RUN --mount=type=tmpfs,target=/var/log \
     apt-get update \
  && apt-get --download-only -y install $(grep -h '^[^#]\+$' /root/*.lst)
 
-FROM ${BASE_IMAGE}
+FROM cuda-keyring
 MAINTAINER Brad King <brad.king@kitware.com>
 
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+ENV NVIDIA_REQUIRE_CUDA=cuda>=11.8
+ENV NVIDIA_VISIBLE_DEVICES=all
 ENV PATH="/opt/rocm/bin:$PATH"
 
 RUN --mount=type=bind,source=install_deps.sh,target=/root/install_deps.sh \
diff --git a/.gitlab/ci/docker/hip5.5/deps_packages.lst b/.gitlab/ci/docker/hip5.5/deps_packages.lst
index 9847925..3276055 100644
--- a/.gitlab/ci/docker/hip5.5/deps_packages.lst
+++ b/.gitlab/ci/docker/hip5.5/deps_packages.lst
@@ -2,3 +2,20 @@
 g++
 curl
 git
+
+# NVIDIA CUDA Compiler
+cuda-keyring
+cuda-nvcc-11-8
+cuda-profiler-api-11-8
+
+# NVIDIA CUDA Toolkit
+# These are not needed for HIP, but having them in
+# the environment allows us to run CUDA tests too.
+cuda-nvrtc-dev-11-8
+cuda-nvtx-11-8
+libcublas-dev-11-8
+libcufft-dev-11-8
+libcurand-dev-11-8
+libcusolver-dev-11-8
+libcusparse-dev-11-8
+libnpp-dev-11-8
diff --git a/.gitlab/ci/env_hip5.5_nvidia.sh b/.gitlab/ci/env_hip5.5_nvidia.sh
new file mode 100644
index 0000000..67d1ef2
--- /dev/null
+++ b/.gitlab/ci/env_hip5.5_nvidia.sh
@@ -0,0 +1,4 @@
+export HIP_PLATFORM=nvidia
+export CUDA_PATH=/usr/local/cuda-11.8
+export PATH=/usr/local/cuda-11.8/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/cuda-11.8/lib64
diff --git a/.gitlab/os-linux.yml b/.gitlab/os-linux.yml
index 8398108..d42340c 100644
--- a/.gitlab/os-linux.yml
+++ b/.gitlab/os-linux.yml
@@ -345,7 +345,7 @@
 ### HIP builds
 
 .hip5.5:
-    image: "kitware/cmake:ci-hip5.5-x86_64-2023-06-01"
+    image: "kitware/cmake:ci-hip5.5-x86_64-2023-09-18"
 
     variables:
         GIT_CLONE_PATH: "$CI_BUILDS_DIR/cmake ci"
@@ -373,6 +373,13 @@
         CMAKE_CONFIGURATION: fedora38_hip_radeon
         CTEST_LABELS: "HIP"
 
+.hip5.5_nvidia:
+    extends: .hip5.5
+
+    variables:
+        CMAKE_CONFIGURATION: hip5.5_nvidia
+        CTEST_LABELS: "HIP"
+
 ### C++ modules
 
 .gcc_cxx_modules_x86_64: