Merge topic 'ctest_test-ignore-skipped-tests'

851b6c15 cmCTestTestHandler: indicate why a test did not run
25a7f14f Help: add release notes
ab8bbef9 cmCTestTestHandler: count skipped tests as disabled
202a44a4 cmCTestRunTest: do not count skipped tests as failed

Acked-by: Kitware Robot <kwrobot@kitware.com>
Reviewed-by: Craig Scott <craig.scott@crascit.com>
Merge-request: !741

17 files changed, 145 insertions, 12 deletions

diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index 7db5243..cf778b6 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -199,6 +199,7 @@ add_RunCMake_test(ctest_start)
 add_RunCMake_test(ctest_submit)
 add_RunCMake_test(ctest_test)
 add_RunCMake_test(ctest_disabled_test)
+add_RunCMake_test(ctest_skipped_test)
 add_RunCMake_test(ctest_upload)
 add_RunCMake_test(ctest_fixtures)
 add_RunCMake_test(file)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt
index ee0dc51..9449e65 100644
--- a/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt
+++ b/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt
@@ -7,5 +7,5 @@
 +
 Total Test time \(real\) = +[0-9.]+ sec
 +
-The following tests are disabled and did not run:
-.*2 \- CleanupTest
+The following tests did not run:
+.*2 \- CleanupTest \(Disabled\)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt
index e2c9f92..486722e 100644
--- a/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt
+++ b/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt
@@ -2,8 +2,8 @@
 +
 Total Test time \(real\) = +[0-9.]+ sec
 +
-The following tests are disabled and did not run:
-.*3 \- DisabledFailingTest
+The following tests did not run:
+.*3 \- DisabledFailingTest \(Disabled\)
 +
 The following tests FAILED:
 .*2 \- FailingTest \(Failed\)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt
index d8bf966..9078aeb 100644
--- a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt
+++ b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt
@@ -10,8 +10,8 @@
 +
 Total Test time \(real\) = +[0-9.]+ sec
 +
-The following tests are disabled and did not run:
-.*2 \- DisabledTest
+The following tests did not run:
+.*2 \- DisabledTest \(Disabled\)
 +
 The following tests FAILED:
 .*3 - NotRunTest \(Not Run\)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt
index 886efb8..10d385e 100644
--- a/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt
+++ b/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt
@@ -9,5 +9,5 @@
 +
 Total Test time \(real\) = +[0-9.]+ sec
 +
-The following tests are disabled and did not run:
-.*2 \- DisabledTest
+The following tests did not run:
+.*2 \- DisabledTest \(Disabled\)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt
index dc27950..2dfd10d 100644
--- a/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt
+++ b/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt
@@ -9,5 +9,5 @@
 +
 Total Test time \(real\) = +[0-9.]+ sec
 +
-The following tests are disabled and did not run:
-.*2 \- DisabledTest
+The following tests did not run:
+.*2 \- DisabledTest \(Disabled\)
diff --git a/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt
index d8bf966..9078aeb 100644
--- a/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt
+++ b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt
@@ -10,8 +10,8 @@
 +
 Total Test time \(real\) = +[0-9.]+ sec
 +
-The following tests are disabled and did not run:
-.*2 \- DisabledTest
+The following tests did not run:
+.*2 \- DisabledTest \(Disabled\)
 +
 The following tests FAILED:
 .*3 - NotRunTest \(Not Run\)
diff --git a/Tests/RunCMake/ctest_skipped_test/CMakeLists.txt.in b/Tests/RunCMake/ctest_skipped_test/CMakeLists.txt.in
new file mode 100644
index 0000000..cc4b8ed
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/CMakeLists.txt.in
@@ -0,0 +1,12 @@
+cmake_minimum_required(VERSION 3.8)
+project(@CASE_NAME@ C)
+include(CTest)
+
+if (WIN32)
+  set(skip_command "@CMAKE_CURRENT_LIST_DIR@/skip.bat")
+else ()
+  set(skip_command "@CMAKE_CURRENT_LIST_DIR@/skip.sh")
+endif ()
+
+add_test(NAME SuccessfulTest COMMAND "${CMAKE_COMMAND}" --version)
+@CASE_CMAKELISTS_SUFFIX_CODE@
diff --git a/Tests/RunCMake/ctest_skipped_test/CTestConfig.cmake.in b/Tests/RunCMake/ctest_skipped_test/CTestConfig.cmake.in
new file mode 100644
index 0000000..c0d7e42
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/CTestConfig.cmake.in
@@ -0,0 +1 @@
+set(CTEST_PROJECT_NAME "@CASE_NAME@")
diff --git a/Tests/RunCMake/ctest_skipped_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_skipped_test/RunCMakeTest.cmake
new file mode 100644
index 0000000..dcf5cd4
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/RunCMakeTest.cmake
@@ -0,0 +1,51 @@
+include(RunCTest)
+
+function(run_SkipTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME SkipTest COMMAND ${skip_command})
+
+set_tests_properties(SkipTest PROPERTIES SKIP_RETURN_CODE 125)
+  ]])
+  run_ctest(SkipTest)
+endfunction()
+run_SkipTest()
+
+function(run_SkipSetupTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME SkipTest COMMAND ${skip_command})
+add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(SkipTest PROPERTIES SKIP_RETURN_CODE 125
+                                         FIXTURES_SETUP "Foo")
+set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo")
+set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo")
+  ]])
+  run_ctest(SkipSetupTest)
+endfunction()
+run_SkipSetupTest()
+
+function(run_SkipRequiredTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME SkipTest COMMAND ${skip_command})
+add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_SETUP "Foo")
+set_tests_properties(SkipTest PROPERTIES SKIP_RETURN_CODE 125
+                                         FIXTURES_REQUIRED "Foo")
+set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo")
+  ]])
+  run_ctest(SkipRequiredTest)
+endfunction()
+run_SkipRequiredTest()
+
+function(run_SkipCleanupTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME CleanupTest COMMAND ${skip_command})
+
+set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo")
+set_tests_properties(CleanupTest PROPERTIES SKIP_RETURN_CODE 125
+                                            FIXTURES_CLEANUP "Foo")
+  ]])
+  run_ctest(SkipCleanupTest)
+endfunction()
+run_SkipCleanupTest()
diff --git a/Tests/RunCMake/ctest_skipped_test/SkipCleanupTest-stdout.txt b/Tests/RunCMake/ctest_skipped_test/SkipCleanupTest-stdout.txt
new file mode 100644
index 0000000..3b14b7a
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/SkipCleanupTest-stdout.txt
@@ -0,0 +1,11 @@
+    Start 1: SuccessfulTest
+1/2 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: CleanupTest
+2/2 Test #2: CleanupTest ......................\*\*\*\Skipped +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests did not run:
+.*2 \- CleanupTest \(Skipped\)
diff --git a/Tests/RunCMake/ctest_skipped_test/SkipRequiredTest-stdout.txt b/Tests/RunCMake/ctest_skipped_test/SkipRequiredTest-stdout.txt
new file mode 100644
index 0000000..8ecc6e3
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/SkipRequiredTest-stdout.txt
@@ -0,0 +1,13 @@
+    Start 1: SuccessfulTest
+1/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: SkipTest
+2/3 Test #2: SkipTest .........................\*\*\*\Skipped +[0-9.]+ sec
+    Start 3: SuccessfulCleanupTest
+3/3 Test #3: SuccessfulCleanupTest ............   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 3
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests did not run:
+.*2 \- SkipTest \(Skipped\)
diff --git a/Tests/RunCMake/ctest_skipped_test/SkipSetupTest-stdout.txt b/Tests/RunCMake/ctest_skipped_test/SkipSetupTest-stdout.txt
new file mode 100644
index 0000000..fe9bf34
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/SkipSetupTest-stdout.txt
@@ -0,0 +1,13 @@
+    Start 2: SkipTest
+1/3 Test #2: SkipTest .........................\*\*\*\Skipped +[0-9.]+ sec
+    Start 1: SuccessfulTest
+2/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 3: SuccessfulCleanupTest
+3/3 Test #3: SuccessfulCleanupTest ............   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 3
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests did not run:
+.*2 \- SkipTest \(Skipped\)
diff --git a/Tests/RunCMake/ctest_skipped_test/SkipTest-stdout.txt b/Tests/RunCMake/ctest_skipped_test/SkipTest-stdout.txt
new file mode 100644
index 0000000..52e7a0b
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/SkipTest-stdout.txt
@@ -0,0 +1,11 @@
+    Start 1: SuccessfulTest
+1/2 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: SkipTest
+2/2 Test #2: SkipTest .........................\*\*\*\Skipped +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests did not run:
+.*2 \- SkipTest \(Skipped\)
diff --git a/Tests/RunCMake/ctest_skipped_test/skip.bat b/Tests/RunCMake/ctest_skipped_test/skip.bat
new file mode 100755
index 0000000..80e1290
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/skip.bat
@@ -0,0 +1 @@
+EXIT 125
diff --git a/Tests/RunCMake/ctest_skipped_test/skip.sh b/Tests/RunCMake/ctest_skipped_test/skip.sh
new file mode 100755
index 0000000..f9c4603
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/skip.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exit 125
diff --git a/Tests/RunCMake/ctest_skipped_test/test.cmake.in b/Tests/RunCMake/ctest_skipped_test/test.cmake.in
new file mode 100644
index 0000000..ca23c83
--- /dev/null
+++ b/Tests/RunCMake/ctest_skipped_test/test.cmake.in
@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.7)
+
+set(CTEST_SITE                          "test-site")
+set(CTEST_BUILD_NAME                    "test-build-name")
+set(CTEST_SOURCE_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@")
+set(CTEST_BINARY_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
+set(CTEST_CMAKE_GENERATOR               "@RunCMake_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_PLATFORM      "@RunCMake_GENERATOR_PLATFORM@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET       "@RunCMake_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+
+set(ctest_test_args "@CASE_CTEST_TEST_ARGS@")
+ctest_start(Experimental)
+ctest_configure()
+ctest_build()
+ctest_test(${ctest_test_args})