Merge topic 'test-vs-generator'

9812e87ee4 gitlab-ci: add a test of the VS generator based off of the ninja build
e63fe4a588 ci: add a script to run the tests with an external CMake
9893e159cc ci: support setting the generator platform and toolset
e0b9e8fba8 ci: test the configuration that was built
f00ab86f1f ci: use BOOL for boolean settings
deb5194a32 tests: avoid a warning for ctresalloc
f5e6738051 gitlab-ci: get the VS2019 location from the runner
79f3790c1a gitlab-ci: get sccache location from the runner on Windows
...

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !4794

8 files changed, 126 insertions, 6 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bd8a0d3..a8b31da 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
 .manual_rules_settings: &manual_rules_settings
     - if: '$CI_PROJECT_PATH == "cmake/cmake"'
-      when: always
+      when: delayed
+      start_in: 5 minutes
     - if: '$CI_MERGE_REQUEST_ID'
       when: manual
     - when: never
@@ -113,9 +114,17 @@
 
     variables:
         CMAKE_CONFIGURATION: windows_vs2019_x64_ninja
-        VCVARSALL: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Professional\\VC\\Auxiliary\\Build\\vcvarsall.bat"
+        VCVARSALL: "${VS160COMNTOOLS}\\..\\..\\VC\\Auxiliary\\Build\\vcvarsall.bat"
         VCVARSPLATFORM: x64
 
+.windows_vs2019_x64: &windows_vs2019_x64
+    extends: .windows
+
+    variables:
+        CMAKE_CONFIGURATION: windows_vs2019_x64
+        CMAKE_GENERATOR: "Visual Studio 16 2019"
+        CMAKE_GENERATOR_PLATFORM: "x64"
+
 .linux_builder_tags: &linux_builder_tags
     tags:
         - build
@@ -176,7 +185,7 @@
 
     script:
         - *before_script_windows
-        - Invoke-Expression -Command .gitlab/ci/sccache.ps1
+        - Set-Item -Force -Path "env:PATH" -Value "$env:PATH;$env:SCCACHE_PATH"
         - Invoke-Expression -Command .gitlab/ci/vcvarsall.ps1
         - sccache --start-server
         - sccache --show-stats
@@ -277,9 +286,18 @@
 
     interruptible: true
 
+.cmake_test_windows_external: &cmake_test_windows_external
+    stage: test-ext
+
+    script:
+        - build/install/bin/ctest --output-on-failure -V -S .gitlab/ci/ctest_test_external.cmake
+
+    interruptible: true
+
 stages:
     - build
     - test
+    - test-ext
 
 build:debian10-iwyu:
     <<:
@@ -391,8 +409,20 @@ test:windows-vs2019-x64-ninja:
         - *windows_vs2019_x64_ninja
         - *cmake_test_windows
         - *windows_builder_tags
+        - *cmake_test_artifacts
     rules: *rules_settings
     dependencies:
         - build:windows-vs2019-x64-ninja
     needs:
         - build:windows-vs2019-x64-ninja
+
+test:windows-vs2019-x64:
+    <<:
+        - *windows_vs2019_x64
+        - *cmake_test_windows_external
+        - *windows_builder_tags
+    rules: *rules_settings
+    dependencies:
+        - test:windows-vs2019-x64-ninja
+    needs:
+        - test:windows-vs2019-x64-ninja
diff --git a/.gitlab/ci/configure_common.cmake b/.gitlab/ci/configure_common.cmake
index 4585224..df49b2d 100644
--- a/.gitlab/ci/configure_common.cmake
+++ b/.gitlab/ci/configure_common.cmake
@@ -1,7 +1,7 @@
-set(CTEST_USE_LAUNCHERS "ON" CACHE STRING "")
+set(CTEST_USE_LAUNCHERS "ON" CACHE BOOL "")
 
 # We run the install right after the build. Avoid rerunning it when installing.
-set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY "ON" CACHE STRING "")
+set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY "ON" CACHE BOOL "")
 # Install CMake under the build tree.
 set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install" CACHE PATH "")
 set(CMake_TEST_INSTALL "OFF" CACHE BOOL "")
diff --git a/.gitlab/ci/configure_external_test.cmake b/.gitlab/ci/configure_external_test.cmake
new file mode 100644
index 0000000..71397d1
--- /dev/null
+++ b/.gitlab/ci/configure_external_test.cmake
@@ -0,0 +1,3 @@
+set(CMake_TEST_HOST_CMAKE "ON" CACHE BOOL "")
+
+include("${CMAKE_CURRENT_LIST_DIR}/configure_common.cmake")
diff --git a/.gitlab/ci/configure_windows_vs2019_x64.cmake b/.gitlab/ci/configure_windows_vs2019_x64.cmake
new file mode 100644
index 0000000..f6ece57
--- /dev/null
+++ b/.gitlab/ci/configure_windows_vs2019_x64.cmake
@@ -0,0 +1,3 @@
+set(CMake_TEST_WIX_NO_VERIFY "ON" CACHE BOOL "")
+
+include("${CMAKE_CURRENT_LIST_DIR}/configure_external_test.cmake")
diff --git a/.gitlab/ci/ctest_exclusions.cmake b/.gitlab/ci/ctest_exclusions.cmake
index d8980f6..eb9b724 100644
--- a/.gitlab/ci/ctest_exclusions.cmake
+++ b/.gitlab/ci/ctest_exclusions.cmake
@@ -1,6 +1,15 @@
 set(test_exclusions
 )
 
+if (CTEST_CMAKE_GENERATOR MATCHES "Visual Studio")
+  list(APPEND test_exclusions
+    # This test takes around 5 minutes with Visual Studio.
+    # https://gitlab.kitware.com/cmake/cmake/-/issues/20733
+    "^ExternalProjectUpdate$"
+    # This test is a dependency of the above and is only required for it.
+    "^ExternalProjectUpdateSetup$")
+endif ()
+
 string(REPLACE ";" "|" test_exclusions "${test_exclusions}")
 if (test_exclusions)
   set(test_exclusions "(${test_exclusions})")
diff --git a/.gitlab/ci/ctest_test_external.cmake b/.gitlab/ci/ctest_test_external.cmake
new file mode 100644
index 0000000..1e61d52
--- /dev/null
+++ b/.gitlab/ci/ctest_test_external.cmake
@@ -0,0 +1,66 @@
+cmake_minimum_required(VERSION 3.8)
+
+include("${CMAKE_CURRENT_LIST_DIR}/gitlab_ci.cmake")
+
+set(cmake_args
+  -C "${CMAKE_CURRENT_LIST_DIR}/configure_$ENV{CMAKE_CONFIGURATION}.cmake")
+
+# Create an entry in CDash.
+ctest_start(Experimental TRACK "${ctest_track}")
+
+# Gather update information.
+find_package(Git)
+set(CTEST_UPDATE_VERSION_ONLY ON)
+set(CTEST_UPDATE_COMMAND "${GIT_EXECUTABLE}")
+ctest_update()
+
+# Configure the project.
+ctest_configure(
+  OPTIONS "${cmake_args}"
+  RETURN_VALUE configure_result)
+
+# Read the files from the build directory.
+ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}")
+
+# We can now submit because we've configured. This is a cmb-superbuild-ism.
+ctest_submit(PARTS Update)
+ctest_submit(PARTS Configure)
+
+if (configure_result)
+  message(FATAL_ERROR
+    "Failed to configure")
+endif ()
+
+include(ProcessorCount)
+ProcessorCount(nproc)
+
+if (CTEST_CMAKE_GENERATOR STREQUAL "Unix Makefiles")
+  set(CTEST_BUILD_FLAGS "-j${nproc}")
+endif ()
+
+ctest_build(
+  NUMBER_WARNINGS num_warnings
+  RETURN_VALUE build_result)
+ctest_submit(PARTS Build)
+
+if (build_result)
+  message(FATAL_ERROR
+    "Failed to build")
+endif ()
+
+if ("$ENV{CTEST_NO_WARNINGS_ALLOWED}" AND num_warnings GREATER 0)
+  message(FATAL_ERROR
+    "Found ${num_warnings} warnings (treating as fatal).")
+endif ()
+
+include("${CMAKE_CURRENT_LIST_DIR}/ctest_exclusions.cmake")
+ctest_test(
+  PARALLEL_LEVEL "${nproc}"
+  RETURN_VALUE test_result
+  EXCLUDE "${test_exclusions}")
+ctest_submit(PARTS Test)
+
+if (test_result)
+  message(FATAL_ERROR
+    "Failed to test")
+endif ()
diff --git a/.gitlab/ci/gitlab_ci.cmake b/.gitlab/ci/gitlab_ci.cmake
index b49f51d..e9c9f80 100644
--- a/.gitlab/ci/gitlab_ci.cmake
+++ b/.gitlab/ci/gitlab_ci.cmake
@@ -27,6 +27,7 @@ endif ()
 if (NOT CTEST_BUILD_CONFIGURATION)
   set(CTEST_BUILD_CONFIGURATION "Release")
 endif ()
+set(CTEST_CONFIGURATION_TYPE "${CTEST_BUILD_CONFIGURATION}")
 
 # Default to using Ninja.
 if (NOT "$ENV{CMAKE_GENERATOR}" STREQUAL "")
@@ -36,6 +37,14 @@ if (NOT CTEST_CMAKE_GENERATOR)
   set(CTEST_CMAKE_GENERATOR "Ninja")
 endif ()
 
+# Set the toolset and platform if requested.
+if (NOT "$ENV{CMAKE_GENERATOR_PLATFORM}" STREQUAL "")
+  set(CTEST_CMAKE_GENERATOR_PLATFORM "$ENV{CMAKE_GENERATOR_PLATFORM}")
+endif ()
+if (NOT "$ENV{CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
+  set(CTEST_CMAKE_GENERATOR_TOOLSET "$ENV{CMAKE_GENERATOR_TOOLSET}")
+endif ()
+
 # Determine the track to submit to.
 set(ctest_track "Experimental")
 if (NOT "$ENV{CI_MERGE_REQUEST_ID}" STREQUAL "")
diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index 983f7e4..868eb24 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -416,7 +416,7 @@ endif()
 if(TARGET ctresalloc)
   add_RunCMake_test(CTestResourceAllocation -DCTRESALLOC_COMMAND=$<TARGET_FILE:ctresalloc>)
 else()
-  message(WARNING "Could not find or build ctresalloc")
+  message(STATUS "Could not find ctresalloc")
 endif()
 
 find_package(Qt4 QUIET)