Merge topic 'ctest-disable-tests'

b070947d Add 'DISABLED' test property
42c68aad CTest: Improve 'Completion Status' reported to CDash for 'Not Run' tests

Acked-by: Kitware Robot <kwrobot@kitware.com>
Reviewed-by: Brad King <brad.king@kitware.com>
Merge-request: !571

18 files changed, 201 insertions, 0 deletions

diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index 217f0d5..e2dc128 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -202,6 +202,7 @@ endif()
 add_RunCMake_test(ctest_start)
 add_RunCMake_test(ctest_submit)
 add_RunCMake_test(ctest_test)
+add_RunCMake_test(ctest_disabled_test)
 add_RunCMake_test(ctest_upload)
 add_RunCMake_test(ctest_fixtures)
 add_RunCMake_test(file)
diff --git a/Tests/RunCMake/ctest_disabled_test/CMakeLists.txt.in b/Tests/RunCMake/ctest_disabled_test/CMakeLists.txt.in
new file mode 100644
index 0000000..d34fcac
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/CMakeLists.txt.in
@@ -0,0 +1,6 @@
+cmake_minimum_required(VERSION 3.7)
+project(@CASE_NAME@ NONE)
+include(CTest)
+
+add_test(NAME SuccessfulTest COMMAND "${CMAKE_COMMAND}" --version)
+@CASE_CMAKELISTS_SUFFIX_CODE@
diff --git a/Tests/RunCMake/ctest_disabled_test/CTestConfig.cmake.in b/Tests/RunCMake/ctest_disabled_test/CTestConfig.cmake.in
new file mode 100644
index 0000000..c0d7e42
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/CTestConfig.cmake.in
@@ -0,0 +1 @@
+set(CTEST_PROJECT_NAME "@CASE_NAME@")
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableAllTests-result.txt b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stderr.txt b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stderr.txt
new file mode 100644
index 0000000..eafba1c
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stderr.txt
@@ -0,0 +1 @@
+No tests were found!!!
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stdout.txt
new file mode 100644
index 0000000..6c824f6
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stdout.txt
@@ -0,0 +1,2 @@
+    Start 1: SuccessfulTest
+1/1 Test #1: SuccessfulTest ...................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt
new file mode 100644
index 0000000..ee0dc51
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt
@@ -0,0 +1,11 @@
+    Start 1: SuccessfulTest
+1/2 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: CleanupTest
+2/2 Test #2: CleanupTest ......................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- CleanupTest
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt
new file mode 100644
index 0000000..e2c9f92
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt
@@ -0,0 +1,9 @@
+50% tests passed, 1 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*3 \- DisabledFailingTest
++
+The following tests FAILED:
+.*2 \- FailingTest \(Failed\)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-result.txt b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stderr.txt b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stderr.txt
new file mode 100644
index 0000000..83c332b
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stderr.txt
@@ -0,0 +1 @@
+Unable to find executable: invalidCommand
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt
new file mode 100644
index 0000000..d8bf966
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt
@@ -0,0 +1,17 @@
+    Start 1: SuccessfulTest
+1/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: DisabledTest
+2/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
+    Start 3: NotRunTest
+.*
+3/3 Test #3: NotRunTest .......................\*\*\*\Not Run +[0-9.]+ sec
++
+50% tests passed, 1 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- DisabledTest
++
+The following tests FAILED:
+.*3 - NotRunTest \(Not Run\)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt
new file mode 100644
index 0000000..886efb8
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt
@@ -0,0 +1,13 @@
+    Start 1: SuccessfulTest
+1/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: DisabledTest
+2/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
+    Start 3: SuccessfulCleanupTest
+3/3 Test #3: SuccessfulCleanupTest ............   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- DisabledTest
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt
new file mode 100644
index 0000000..dc27950
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt
@@ -0,0 +1,13 @@
+    Start 2: DisabledTest
+1/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
+    Start 1: SuccessfulTest
+2/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 3: SuccessfulCleanupTest
+3/3 Test #3: SuccessfulCleanupTest ............   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- DisabledTest
diff --git a/Tests/RunCMake/ctest_disabled_test/DisabledTest-result.txt b/Tests/RunCMake/ctest_disabled_test/DisabledTest-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisabledTest-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisabledTest-stderr.txt b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stderr.txt
new file mode 100644
index 0000000..83c332b
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stderr.txt
@@ -0,0 +1 @@
+Unable to find executable: invalidCommand
diff --git a/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt
new file mode 100644
index 0000000..d8bf966
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt
@@ -0,0 +1,17 @@
+    Start 1: SuccessfulTest
+1/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: DisabledTest
+2/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
+    Start 3: NotRunTest
+.*
+3/3 Test #3: NotRunTest .......................\*\*\*\Not Run +[0-9.]+ sec
++
+50% tests passed, 1 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- DisabledTest
++
+The following tests FAILED:
+.*3 - NotRunTest \(Not Run\)
diff --git a/Tests/RunCMake/ctest_disabled_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_disabled_test/RunCMakeTest.cmake
new file mode 100644
index 0000000..12541c4
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/RunCMakeTest.cmake
@@ -0,0 +1,89 @@
+include(RunCTest)
+
+function(run_DisableNotRunTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME DisabledTest COMMAND notACommand --version)
+add_test(NAME NotRunTest COMMAND invalidCommand --version)
+
+set_tests_properties(SuccessfulTest PROPERTIES DISABLED false)
+set_tests_properties(DisabledTest PROPERTIES DISABLED true)
+  ]])
+  run_ctest(DisableNotRunTest)
+endfunction()
+run_DisableNotRunTest()
+
+function(run_DisableFailingTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+set(someFile "${CMAKE_CURRENT_SOURCE_DIR}/test.cmake")
+add_test(NAME FailingTest
+          COMMAND ${CMAKE_COMMAND} -E compare_files "${someFile}" "${someFile}xxx")
+add_test(NAME DisabledFailingTest
+          COMMAND ${CMAKE_COMMAND} -E compare_files "${someFile}" "${someFile}xxx")
+
+set_tests_properties(FailingTest PROPERTIES DISABLED false)
+set_tests_properties(DisabledFailingTest PROPERTIES DISABLED true)
+  ]])
+  run_ctest(DisableFailingTest)
+endfunction()
+run_DisableFailingTest()
+
+function(run_DisableSetupTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME DisabledTest COMMAND "${CMAKE_COMMAND}" --version)
+add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(DisabledTest PROPERTIES DISABLED true
+                                             FIXTURES_SETUP "Foo")
+set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo")
+set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo")
+  ]])
+run_ctest(DisableSetupTest)
+endfunction()
+run_DisableSetupTest()
+
+function(run_DisableRequiredTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME DisabledTest COMMAND "${CMAKE_COMMAND}" --version)
+add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_SETUP "Foo")
+set_tests_properties(DisabledTest PROPERTIES DISABLED true
+                                             FIXTURES_REQUIRED "Foo")
+set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo")
+  ]])
+run_ctest(DisableRequiredTest)
+endfunction()
+run_DisableRequiredTest()
+
+function(run_DisableCleanupTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME CleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo")
+set_tests_properties(CleanupTest PROPERTIES DISABLED true
+                                            FIXTURES_CLEANUP "Foo")
+  ]])
+run_ctest(DisableCleanupTest)
+endfunction()
+run_DisableCleanupTest()
+
+# Consider a fixture that has a setup test, a cleanup test and a disabled test
+# which requires that fixture. Limit the test list with a regular expression
+# that matches the disabled test but not the setup or cleanup tests, so the
+# initial set of tests to be executed contains just the disabled test. Since
+# the only test requiring the fixture is disabled, CTest should not
+# automatically add in the setup and cleanup tests for the fixture, since no
+# enabled test requires them.
+function(run_DisableAllTests)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME SetupTest COMMAND "${CMAKE_COMMAND}" --version)
+add_test(NAME CleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(SetupTest PROPERTIES FIXTURES_SETUP "Foo")
+set_tests_properties(SuccessfulTest PROPERTIES DISABLED true
+                                               FIXTURES_REQUIRED "Foo")
+set_tests_properties(CleanupTest PROPERTIES FIXTURES_CLEANUP "Foo")
+  ]])
+run_ctest(DisableAllTests -R Successful)
+endfunction()
+run_DisableAllTests()
diff --git a/Tests/RunCMake/ctest_disabled_test/test.cmake.in b/Tests/RunCMake/ctest_disabled_test/test.cmake.in
new file mode 100644
index 0000000..ca23c83
--- /dev/null
+++ b/Tests/RunCMake/ctest_disabled_test/test.cmake.in
@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.7)
+
+set(CTEST_SITE                          "test-site")
+set(CTEST_BUILD_NAME                    "test-build-name")
+set(CTEST_SOURCE_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@")
+set(CTEST_BINARY_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
+set(CTEST_CMAKE_GENERATOR               "@RunCMake_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_PLATFORM      "@RunCMake_GENERATOR_PLATFORM@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET       "@RunCMake_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+
+set(ctest_test_args "@CASE_CTEST_TEST_ARGS@")
+ctest_start(Experimental)
+ctest_configure()
+ctest_build()
+ctest_test(${ctest_test_args})