Add CTestLimitDashJ test (#12904)

Add a test that verifies that when ctest -j 4 is called, at most, 4 tests
are executed at any one time.

The test works by running the same script as each of 100 tests. And then setting
up test properties for DEPENDS, RUN_SERIAL, PROCESSORS and COST in order to get
the tests to run in a semi-deterministic ordering, even in parallel. The script
writes a file, sleeps for a bit, and then deletes the file. In the meantime, it
counts files that currently exist, and emits output that triggers a test failure
if the count of files is ever greater than 4.

Prior to the commit that fixed bug #12904, this would result in a failed test
because the output of some of the tests would indicate that more than 4 tests
were running simultaneously.

Now that this issue is resolved, this test will help guarantee that it stays
resolved moving forward.

3 files changed, 101 insertions, 0 deletions

diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt
index f443b5b..b3302f9 100644
--- a/Tests/CMakeLists.txt
+++ b/Tests/CMakeLists.txt
@@ -1959,6 +1959,9 @@ ${CMake_BINARY_DIR}/bin/cmake -DVERSION=master -P ${CMake_SOURCE_DIR}/Utilities/
     --output-log "${CMake_BINARY_DIR}/Tests/CTestTestParallel/testOutput.log"
     )
 
+  add_test_macro(CTestLimitDashJ ${CMAKE_CTEST_COMMAND} -j 4
+    --output-on-failure -C "\${CTestTest_CONFIG}")
+
   add_test(CTestTestPrintLabels ${CMAKE_CTEST_COMMAND} --print-labels)
   set_tests_properties(CTestTestPrintLabels PROPERTIES LABELS "Label1;Label2")
   set_tests_properties(CTestTestPrintLabels PROPERTIES PASS_REGULAR_EXPRESSION
diff --git a/Tests/CTestLimitDashJ/CMakeLists.txt b/Tests/CTestLimitDashJ/CMakeLists.txt
new file mode 100644
index 0000000..0c5950c
--- /dev/null
+++ b/Tests/CTestLimitDashJ/CMakeLists.txt
@@ -0,0 +1,50 @@
+cmake_minimum_required(VERSION 2.8)
+project(CTestLimitDashJ NONE)
+
+# This file demonstrates http://public.kitware.com/Bug/view.php?id=12904
+# when configured with CMake 2.8.10.2 and earlier, and when running
+# "ctest -j 4" in the resulting build tree. This example is hard-coded
+# to assume -j 4 just to reproduce the issue easily. Adjust the
+# FAIL_REGULAR_EXPRESSION and PROCESSORS values to reproduce this problem
+# with a different ctest -j value...
+
+if(EXISTS "${CMAKE_BINARY_DIR}/Testing/Temporary/CTestCostData.txt")
+  message(STATUS "Removing CTestCostData.txt to force ordering by COST PROPERTY value rather than prior run data")
+  file(REMOVE "${CMAKE_BINARY_DIR}/Testing/Temporary/CTestCostData.txt")
+endif()
+
+include(CTest)
+
+configure_file(
+  ${CMAKE_CURRENT_SOURCE_DIR}/CreateSleepDelete.cmake
+  ${CMAKE_CURRENT_BINARY_DIR}/CreateSleepDelete.cmake
+  @ONLY
+  )
+
+foreach(n RANGE 1 100)
+  add_test(NAME t${n}
+    COMMAND ${CMAKE_CTEST_COMMAND}
+      -D basefilename=f${n}
+      -S ${CMAKE_CURRENT_BINARY_DIR}/CreateSleepDelete.cmake
+    )
+  set_property(TEST t${n} PROPERTY FAIL_REGULAR_EXPRESSION "(c='[5-9]'|c='[1-9][0-9]+')")
+endforeach()
+
+set_property(TEST t1 PROPERTY RUN_SERIAL 1)
+set_property(TEST t1 PROPERTY PROCESSORS 4)
+
+set_property(TEST t51 PROPERTY RUN_SERIAL 1)
+set_property(TEST t51 PROPERTY PROCESSORS 4)
+
+foreach(n RANGE 2 50)
+  set_property(TEST t${n} PROPERTY DEPENDS t1)
+endforeach()
+set_property(TEST t1 PROPERTY DEPENDS t51)
+set_property(TEST t51 PROPERTY DEPENDS t100)
+
+foreach(n 50)
+  set_property(TEST t${n} PROPERTY COST 6)
+endforeach()
+foreach(n RANGE 52 99)
+  set_property(TEST t${n} PROPERTY COST 3)
+endforeach()
diff --git a/Tests/CTestLimitDashJ/CreateSleepDelete.cmake b/Tests/CTestLimitDashJ/CreateSleepDelete.cmake
new file mode 100644
index 0000000..b09307f
--- /dev/null
+++ b/Tests/CTestLimitDashJ/CreateSleepDelete.cmake
@@ -0,0 +1,48 @@
+set(CTEST_RUN_CURRENT_SCRIPT 0)
+
+if(NOT DEFINED basefilename)
+  message(FATAL_ERROR "pass -Dbasefilename=f1")
+endif()
+
+if(NOT DEFINED ext)
+  set(ext "jkqvxz")
+endif()
+
+if(NOT DEFINED sleep_interval)
+  set(sleep_interval 1)
+endif()
+
+get_filename_component(self_dir "${CMAKE_CURRENT_LIST_FILE}" PATH)
+set(filename "${self_dir}/${basefilename}.${ext}")
+
+# count files
+file(GLOB f1 *.${ext})
+list(LENGTH f1 c1)
+message("c='${c1}'")
+
+# write a new file
+message("Writing file: filename='${filename}'")
+file(WRITE "${filename}" "${filename}")
+
+# count files again
+file(GLOB f2 *.${ext})
+list(LENGTH f2 c2)
+message("c='${c2}'")
+
+# snooze
+message("Sleeping: sleep_interval='${sleep_interval}'")
+ctest_sleep(${sleep_interval})
+
+# count files again
+file(GLOB f3 *.${ext})
+list(LENGTH f3 c3)
+message("c='${c3}'")
+
+# delete the file we wrote earlier
+message("Removing file: filename='${filename}'")
+file(REMOVE "${filename}")
+
+# count files again
+file(GLOB f4 *.${ext})
+list(LENGTH f4 c4)
+message("c='${c4}'")