Merge topic 'test-fixtures'

73f47c9e CTest: Add support for test fixtures

16 files changed, 207 insertions, 0 deletions

diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index a1f35d7..778982f 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -193,6 +193,7 @@ add_RunCMake_test(ctest_start)
 add_RunCMake_test(ctest_submit)
 add_RunCMake_test(ctest_test)
 add_RunCMake_test(ctest_upload)
+add_RunCMake_test(ctest_fixtures)
 add_RunCMake_test(file)
 add_RunCMake_test(find_file)
 add_RunCMake_test(find_library)
diff --git a/Tests/RunCMake/ctest_fixtures/CMakeLists.txt.in b/Tests/RunCMake/ctest_fixtures/CMakeLists.txt.in
new file mode 100644
index 0000000..ba1c77a
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/CMakeLists.txt.in
@@ -0,0 +1,81 @@
+cmake_minimum_required (VERSION 3.6.2)
+project(ctest_fixtures LANGUAGES NONE)
+include(CTest)
+
+macro(passTest testName)
+  set(someFile "${CMAKE_CURRENT_SOURCE_DIR}/test.cmake")
+  add_test(NAME ${testName}
+           COMMAND ${CMAKE_COMMAND} -E compare_files "${someFile}" "${someFile}")
+endmacro()
+
+macro(failTest testName)
+  set(someFile "${CMAKE_CURRENT_SOURCE_DIR}/test.cmake")
+  add_test(NAME ${testName}
+           COMMAND ${CMAKE_COMMAND} -E compare_files "${someFile}" "${someFile}xxx")
+endmacro()
+
+# Intersperse actual tests among setup/cleanup tests so that we don't
+# define them in the same order as they need to be executed. Numbers
+# at the end of each line correspond to the test numbers ctest will
+# use for each test.
+passTest(one)           # 1
+passTest(setupBoth)     # 2
+passTest(setupFoo)      # 3
+passTest(setupMeta)     # 4
+passTest(cleanupFoo)    # 5
+passTest(two)           # 6
+passTest(cleanupBar)    # 7
+passTest(three)         # 8
+failTest(setupFails)    # 9
+passTest(wontRun)       # 10
+passTest(cyclicSetup)   # 11
+passTest(cyclicCleanup) # 12
+
+# Define fixture dependencies and ordering
+set_tests_properties(setupFoo   PROPERTIES FIXTURES_SETUP    "Foo")
+set_tests_properties(cleanupFoo PROPERTIES FIXTURES_CLEANUP  "Foo")
+
+set_tests_properties(setupBoth  PROPERTIES FIXTURES_SETUP    "Foo;Bar")
+set_tests_properties(cleanupBar PROPERTIES FIXTURES_CLEANUP  "Bar")
+
+set_tests_properties(setupMeta  PROPERTIES FIXTURES_SETUP    "Meta"
+                                           FIXTURES_REQUIRED "Foo;Bar")
+
+set_tests_properties(setupBoth  PROPERTIES DEPENDS           setupFoo)
+
+set_tests_properties(setupFails PROPERTIES FIXTURES_SETUP    "Fails")
+
+set_tests_properties(one        PROPERTIES FIXTURES_REQUIRED "Other;Foo")
+set_tests_properties(two        PROPERTIES FIXTURES_REQUIRED "Bar")
+set_tests_properties(three      PROPERTIES FIXTURES_REQUIRED "Meta;Bar")
+set_tests_properties(wontRun    PROPERTIES FIXTURES_REQUIRED "Fails")
+
+@CASE_CMAKELISTS_CYCLIC_CODE@
+
+# These are the cases verified by the main cmake build
+#
+#   Regex:         Test case list (in order)
+#    one            3, 2, 1, 5
+#    two            2, 6, 7
+#    three          3, 2, 4, 5, 8, 7
+#    setupFoo       3
+#    wontRun        9, 10
+#    cyclicSetup    -NA- (configure fails)
+#    cyclicCleanup  -NA- (configure fails)
+#
+# In the case of asking for just setupFoo, since there are
+# no tests using the Foo fixture, we do NOT expect cleanupFoo
+# to be executed. It is important not to pull in cleanupFoo
+# if setupFoo is explicitly requested and no other test requires
+# the Foo fixture, otherwise it would not be possible to run
+# just a setup or cleanup test in isolation (likely to be
+# needed during initial creation of such test cases).
+#
+# For the wontRun case, test 9 fails and test 10 should not run.
+# The result of the set of tests should be failure, which is
+# verified by the main cmake build's tests.
+#
+# For the two cyclic test cases invoked by the main cmake build,
+# FIXTURES_... properties are added to the relevant test at the
+# location marked with CASE_CMAKELISTS_CYCLIC_CODE. This creates
+# a self-dependency which causes the configure step to fail.
diff --git a/Tests/RunCMake/ctest_fixtures/CTestConfig.cmake.in b/Tests/RunCMake/ctest_fixtures/CTestConfig.cmake.in
new file mode 100644
index 0000000..9823562
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/CTestConfig.cmake.in
@@ -0,0 +1 @@
+set(CTEST_PROJECT_NAME "CTestTestFixtures.@CASE_NAME@")
diff --git a/Tests/RunCMake/ctest_fixtures/RunCMakeTest.cmake b/Tests/RunCMake/ctest_fixtures/RunCMakeTest.cmake
new file mode 100644
index 0000000..f13289a
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/RunCMakeTest.cmake
@@ -0,0 +1,36 @@
+include(RunCTest)
+
+# Isolate our ctest runs from external environment.
+unset(ENV{CTEST_PARALLEL_LEVEL})
+unset(ENV{CTEST_OUTPUT_ON_FAILURE})
+
+function(run_ctest_test CASE_NAME)
+  set(CASE_CTEST_FIXTURES_ARGS "${ARGN}")
+  run_ctest(${CASE_NAME})
+endfunction()
+
+#------------------------------------------------------------
+# CMake configure will pass
+#------------------------------------------------------------
+run_ctest_test(one      INCLUDE one)
+run_ctest_test(two      INCLUDE two)
+run_ctest_test(three    INCLUDE three)
+run_ctest_test(setupFoo INCLUDE setupFoo)
+run_ctest_test(wontRun  INCLUDE wontRun)
+
+#------------------------------------------------------------
+# CMake configure will fail due to cyclic test dependencies
+#------------------------------------------------------------
+set(CASE_CMAKELISTS_CYCLIC_CODE [[
+    set_tests_properties(cyclicSetup PROPERTIES
+                         FIXTURES_SETUP    "Foo"
+                         FIXTURES_REQUIRED "Foo")
+]])
+run_ctest(cyclicSetup)
+
+set(CASE_CMAKELISTS_CYCLIC_CODE [[
+    set_tests_properties(cyclicCleanup PROPERTIES
+                         FIXTURES_CLEANUP  "Foo"
+                         FIXTURES_REQUIRED "Foo")
+]])
+run_ctest(cyclicCleanup)
diff --git a/Tests/RunCMake/ctest_fixtures/cyclicCleanup-result.txt b/Tests/RunCMake/ctest_fixtures/cyclicCleanup-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/cyclicCleanup-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stderr.txt b/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stderr.txt
new file mode 100644
index 0000000..1a45994
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stderr.txt
@@ -0,0 +1,3 @@
+Error: a cycle exists in the test dependency graph for the test "cyclicCleanup".
+Please fix the cycle and run ctest again.
+No tests were found!!!
diff --git a/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stdout.txt b/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stdout.txt
new file mode 100644
index 0000000..8b7fdf7
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stdout.txt
@@ -0,0 +1 @@
+Test project .*/Tests/RunCMake/ctest_fixtures/cyclicCleanup-build$
diff --git a/Tests/RunCMake/ctest_fixtures/cyclicSetup-result.txt b/Tests/RunCMake/ctest_fixtures/cyclicSetup-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/cyclicSetup-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_fixtures/cyclicSetup-stderr.txt b/Tests/RunCMake/ctest_fixtures/cyclicSetup-stderr.txt
new file mode 100644
index 0000000..2aba6c9
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/cyclicSetup-stderr.txt
@@ -0,0 +1,3 @@
+Error: a cycle exists in the test dependency graph for the test "cyclicSetup".
+Please fix the cycle and run ctest again.
+No tests were found!!!$
diff --git a/Tests/RunCMake/ctest_fixtures/cyclicSetup-stdout.txt b/Tests/RunCMake/ctest_fixtures/cyclicSetup-stdout.txt
new file mode 100644
index 0000000..22c564d
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/cyclicSetup-stdout.txt
@@ -0,0 +1 @@
+Test project .*/Tests/RunCMake/ctest_fixtures/cyclicSetup-build$
diff --git a/Tests/RunCMake/ctest_fixtures/one-stdout.txt b/Tests/RunCMake/ctest_fixtures/one-stdout.txt
new file mode 100644
index 0000000..e2276a4
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/one-stdout.txt
@@ -0,0 +1,13 @@
+Test project .*/Tests/RunCMake/ctest_fixtures/one-build
+    Start 3: setupFoo
+1/4 Test #3: setupFoo +\.+ +Passed +[0-9.]+ sec
+    Start 2: setupBoth
+2/4 Test #2: setupBoth +\.+ +Passed +[0-9.]+ sec
+    Start 1: one
+3/4 Test #1: one +\.+ +Passed +[0-9.]+ sec
+    Start 5: cleanupFoo
+4/4 Test #5: cleanupFoo +\.+ +Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 4
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_fixtures/setupFoo-stdout.txt b/Tests/RunCMake/ctest_fixtures/setupFoo-stdout.txt
new file mode 100644
index 0000000..7e21b34
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/setupFoo-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_fixtures/setupFoo-build
+    Start 3: setupFoo
+1/1 Test #3: setupFoo +\.+ +Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_fixtures/test.cmake.in b/Tests/RunCMake/ctest_fixtures/test.cmake.in
new file mode 100644
index 0000000..43df172
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/test.cmake.in
@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.6.2)
+
+set(CTEST_SITE                          "test-site")
+set(CTEST_BUILD_NAME                    "test-build-name")
+set(CTEST_SOURCE_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@")
+set(CTEST_BINARY_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
+set(CTEST_CMAKE_GENERATOR               "@RunCMake_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_PLATFORM      "@RunCMake_GENERATOR_PLATFORM@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET       "@RunCMake_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+
+set(ctest_fixtures_args "@CASE_CTEST_FIXTURES_ARGS@")
+
+ctest_start(Experimental)
+ctest_configure()
+ctest_test(${ctest_fixtures_args})
diff --git a/Tests/RunCMake/ctest_fixtures/three-stdout.txt b/Tests/RunCMake/ctest_fixtures/three-stdout.txt
new file mode 100644
index 0000000..f192e61
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/three-stdout.txt
@@ -0,0 +1,17 @@
+Test project .*/Tests/RunCMake/ctest_fixtures/three-build
+    Start 3: setupFoo
+1/6 Test #3: setupFoo +\.+ +Passed +[0-9.]+ sec
+    Start 2: setupBoth
+2/6 Test #2: setupBoth +\.+ +Passed +[0-9.]+ sec
+    Start 4: setupMeta
+3/6 Test #4: setupMeta +\.+ +Passed +[0-9.]+ sec
+    Start 5: cleanupFoo
+4/6 Test #5: cleanupFoo +\.+ +Passed +[0-9.]+ sec
+    Start 8: three
+5/6 Test #8: three +\.+ +Passed +[0-9.]+ sec
+    Start 7: cleanupBar
+6/6 Test #7: cleanupBar +\.+ +Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 6
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_fixtures/two-stdout.txt b/Tests/RunCMake/ctest_fixtures/two-stdout.txt
new file mode 100644
index 0000000..2e667a7
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/two-stdout.txt
@@ -0,0 +1,11 @@
+Test project .*/Tests/RunCMake/ctest_fixtures/two-build
+    Start 2: setupBoth
+1/3 Test #2: setupBoth +\.+ +Passed +[0-9.]+ sec
+    Start 6: two
+2/3 Test #6: two +\.+ +Passed +[0-9.]+ sec
+    Start 7: cleanupBar
+3/3 Test #7: cleanupBar +\.+ +Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 3
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_fixtures/wontRun-stdout.txt b/Tests/RunCMake/ctest_fixtures/wontRun-stdout.txt
new file mode 100644
index 0000000..526ca4b
--- /dev/null
+++ b/Tests/RunCMake/ctest_fixtures/wontRun-stdout.txt
@@ -0,0 +1,14 @@
+Test project .*/Tests/RunCMake/ctest_fixtures/wontRun-build
+    Start  9: setupFails
+1/2 Test  #9: setupFails +\.+\*\*\*Failed +[0-9.]+ sec
+    Start 10: wontRun
+Failed test dependencies: setupFails
+2/2 Test #10: wontRun +\.+\*\*\*Not Run +[0-9.]+ sec
++
+0% tests passed, 2 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests FAILED:
+.* +9 - setupFails \(Failed\)
+.* +10 - wontRun \(Not Run\)$