Merge topic 'ctest-hardware-allocation'

e9500271a3 Help: Add documentation for CTest hardware allocation
d1f100a415 CTest: Add Json output for PROCESSES property
b741ee820d Tests: Add test for CTest hardware allocation feature
3c8a5aad46 Tests: Write tests for cthwalloc helper tool
2d74e54661 Tests: Write cthwalloc helper tool
e34de0691b CTest: Allocate hardware to tests
aee0964851 CTest: Add bin-packing algorithm
c494b2973a CTest: Add cmCTestHardwareAllocator class
...

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !3858

69 files changed, 952 insertions, 4 deletions

diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index bd068fd..0925c0e 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -335,6 +335,44 @@ add_RunCMake_test(no_install_prefix)
 add_RunCMake_test(configure_file)
 add_RunCMake_test(CTestTimeoutAfterMatch)
 
+# cthwalloc links against CMakeLib and CTestLib, which means it can't be built
+# if CMake_TEST_EXTERNAL_CMAKE is activated (the compiler might be different.)
+# So, it has to be provided in the original build tree.
+if(CMake_TEST_EXTERNAL_CMAKE)
+  set(no_package_root_path)
+  if(NOT CMAKE_VERSION VERSION_LESS 3.12)
+    set(no_package_root_path NO_PACKAGE_ROOT_PATH)
+  endif()
+  find_program(cthwalloc cthwalloc PATHS ${CMake_TEST_EXTERNAL_CMAKE}
+    NO_DEFAULT_PATH
+    ${no_package_root_path}
+    NO_CMAKE_PATH
+    NO_CMAKE_ENVIRONMENT_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH
+    NO_CMAKE_SYSTEM_PATH
+    NO_CMAKE_FIND_ROOT_PATH
+    )
+  if(cthwalloc)
+    add_executable(cthwalloc IMPORTED)
+    set_property(TARGET cthwalloc PROPERTY IMPORTED_LOCATION ${cthwalloc})
+  endif()
+else()
+  add_executable(cthwalloc CTestHardwareAllocation/cthwalloc.cxx)
+  target_link_libraries(cthwalloc CTestLib)
+  target_include_directories(cthwalloc PRIVATE
+    ${CMake_BINARY_DIR}/Source
+    ${CMake_SOURCE_DIR}/Source
+    ${CMake_SOURCE_DIR}/Source/CTest
+    )
+  set_property(TARGET cthwalloc PROPERTY RUNTIME_OUTPUT_DIRECTORY ${CMake_BIN_DIR})
+endif()
+
+if(TARGET cthwalloc)
+  add_RunCMake_test(CTestHardwareAllocation -DCTHWALLOC_COMMAND=$<TARGET_FILE:cthwalloc>)
+else()
+  message(WARNING "Could not find or build cthwalloc")
+endif()
+
 find_package(Qt4 QUIET)
 find_package(Qt5Core QUIET)
 if (QT4_FOUND AND Qt5Core_FOUND AND NOT Qt5Core_VERSION VERSION_LESS 5.1.0)
diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index ca2975a..fd2c97f 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -258,6 +258,7 @@ function(run_ShowOnly)
     add_test(ShowOnly \"${CMAKE_COMMAND}\" -E echo)
     set_tests_properties(ShowOnly PROPERTIES
       WILL_FAIL true
+      PROCESSES \"2,threads:2,gpus:4;gpus:2,threads:4\"
       REQUIRED_FILES RequiredFileDoesNotExist
       _BACKTRACE_TRIPLES \"file1;1;add_test;file0;;\"
       )
diff --git a/Tests/RunCMake/CTestCommandLine/show-only_json-v1_check.py b/Tests/RunCMake/CTestCommandLine/show-only_json-v1_check.py
index 3ad5768..6eb8624 100644
--- a/Tests/RunCMake/CTestCommandLine/show-only_json-v1_check.py
+++ b/Tests/RunCMake/CTestCommandLine/show-only_json-v1_check.py
@@ -80,6 +80,62 @@ def check_willfail_property(p):
     assert p["name"] == "WILL_FAIL"
     assert p["value"] == True
 
+def check_processes_property(p):
+    assert is_dict(p)
+    assert sorted(p.keys()) == ["name", "value"]
+    assert is_string(p["name"])
+    assert is_list(p["value"])
+    assert p["name"] == "PROCESSES"
+    assert len(p["value"]) == 3
+
+    assert is_dict(p["value"][0])
+    assert sorted(p["value"][0].keys()) == ["requirements"]
+    assert is_list(p["value"][0]["requirements"])
+    assert len(p["value"][0]["requirements"]) == 2
+    assert is_dict(p["value"][0]["requirements"][0])
+    assert sorted(p["value"][0]["requirements"][0].keys()) == \
+        [".type", "slots"]
+    assert is_string(p["value"][0]["requirements"][0][".type"])
+    assert p["value"][0]["requirements"][0][".type"] == "threads"
+    assert is_int(p["value"][0]["requirements"][0]["slots"])
+    assert p["value"][0]["requirements"][0]["slots"] == 2
+    assert is_string(p["value"][0]["requirements"][1][".type"])
+    assert p["value"][0]["requirements"][1][".type"] == "gpus"
+    assert is_int(p["value"][0]["requirements"][1]["slots"])
+    assert p["value"][0]["requirements"][1]["slots"] == 4
+
+    assert is_dict(p["value"][1])
+    assert sorted(p["value"][1].keys()) == ["requirements"]
+    assert is_list(p["value"][1]["requirements"])
+    assert len(p["value"][1]["requirements"]) == 2
+    assert is_dict(p["value"][1]["requirements"][0])
+    assert sorted(p["value"][1]["requirements"][0].keys()) == \
+        [".type", "slots"]
+    assert is_string(p["value"][1]["requirements"][0][".type"])
+    assert p["value"][1]["requirements"][0][".type"] == "threads"
+    assert is_int(p["value"][1]["requirements"][0]["slots"])
+    assert p["value"][1]["requirements"][0]["slots"] == 2
+    assert is_string(p["value"][1]["requirements"][1][".type"])
+    assert p["value"][1]["requirements"][1][".type"] == "gpus"
+    assert is_int(p["value"][1]["requirements"][1]["slots"])
+    assert p["value"][1]["requirements"][1]["slots"] == 4
+
+    assert is_dict(p["value"][2])
+    assert sorted(p["value"][2].keys()) == ["requirements"]
+    assert is_list(p["value"][2]["requirements"])
+    assert len(p["value"][2]["requirements"]) == 2
+    assert is_dict(p["value"][2]["requirements"][0])
+    assert sorted(p["value"][2]["requirements"][0].keys()) == \
+        [".type", "slots"]
+    assert is_string(p["value"][2]["requirements"][0][".type"])
+    assert p["value"][2]["requirements"][0][".type"] == "gpus"
+    assert is_int(p["value"][2]["requirements"][0]["slots"])
+    assert p["value"][2]["requirements"][0]["slots"] == 2
+    assert is_string(p["value"][2]["requirements"][1][".type"])
+    assert p["value"][2]["requirements"][1][".type"] == "threads"
+    assert is_int(p["value"][2]["requirements"][1]["slots"])
+    assert p["value"][2]["requirements"][1]["slots"] == 4
+
 def check_workingdir_property(p):
     assert is_dict(p)
     assert sorted(p.keys()) == ["name", "value"]
@@ -90,10 +146,11 @@ def check_workingdir_property(p):
 
 def check_properties(p):
     assert is_list(p)
-    assert len(p) == 3
-    check_reqfiles_property(p[0])
-    check_willfail_property(p[1])
-    check_workingdir_property(p[2])
+    assert len(p) == 4
+    check_processes_property(p[0])
+    check_reqfiles_property(p[1])
+    check_willfail_property(p[2])
+    check_workingdir_property(p[3])
 
 def check_tests(t):
     assert is_list(t)
diff --git a/Tests/RunCMake/CTestHardwareAllocation/CMakeLists.txt.in b/Tests/RunCMake/CTestHardwareAllocation/CMakeLists.txt.in
new file mode 100644
index 0000000..d6cff63
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/CMakeLists.txt.in
@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.15)
+set(CASE_NAME "@CASE_NAME@")
+if(CASE_NAME MATCHES "^(.*)-ctest-s")
+  set(projname "${CMAKE_MATCH_1}")
+  project(${projname} NONE)
+  include(CTest)
+  include("@RunCMake_SOURCE_DIR@/HardwareCommon.cmake")
+  include("@RunCMake_SOURCE_DIR@/${projname}.cmake")
+endif()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/HardwareCommon.cmake b/Tests/RunCMake/CTestHardwareAllocation/HardwareCommon.cmake
new file mode 100644
index 0000000..3893d40
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/HardwareCommon.cmake
@@ -0,0 +1,23 @@
+function(setup_hardware_tests)
+  if(CTEST_HARDWARE_ALLOC_ENABLED)
+    add_test(NAME HardwareSetup COMMAND "${CMAKE_COMMAND}" -E remove -f "${CMAKE_BINARY_DIR}/cthwalloc.log")
+  endif()
+endfunction()
+
+function(add_hardware_test name sleep_time proc)
+  if(CTEST_HARDWARE_ALLOC_ENABLED)
+    add_test(NAME "${name}" COMMAND "${CTHWALLOC_COMMAND}" write "${CMAKE_BINARY_DIR}/cthwalloc.log" "${name}" "${sleep_time}" "${proc}")
+    set_property(TEST "${name}" PROPERTY DEPENDS HardwareSetup)
+  else()
+    add_test(NAME "${name}" COMMAND "${CTHWALLOC_COMMAND}" write "${CMAKE_BINARY_DIR}/cthwalloc.log" "${name}" "${sleep_time}")
+  endif()
+  set_property(TEST "${name}" PROPERTY PROCESSES "${proc}")
+  list(APPEND HARDWARE_TESTS "${name}")
+  set(HARDWARE_TESTS "${HARDWARE_TESTS}" PARENT_SCOPE)
+endfunction()
+
+function(cleanup_hardware_tests)
+  if(CTEST_HARDWARE_ALLOC_ENABLED)
+    file(WRITE "${CMAKE_BINARY_DIR}/hwtests.txt" "${HARDWARE_TESTS}")
+  endif()
+endfunction()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/RunCMakeTest.cmake b/Tests/RunCMake/CTestHardwareAllocation/RunCMakeTest.cmake
new file mode 100644
index 0000000..d666922
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/RunCMakeTest.cmake
@@ -0,0 +1,167 @@
+include(RunCMake)
+include(RunCTest)
+
+###############################################################################
+# Test cthwalloc itself - we want to make sure it's not just rubber-stamping
+# the test results
+###############################################################################
+
+function(cthwalloc_verify_log expected_contents)
+  if(NOT EXISTS "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log")
+    string(APPEND RunCMake_TEST_FAILED "Log file was not written\n")
+    set(RunCMake_TEST_FAILED "${RunCMake_TEST_FAILED}" PARENT_SCOPE)
+    return()
+  endif()
+  file(READ "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log" actual_contents)
+  if(NOT actual_contents STREQUAL expected_contents)
+    string(APPEND RunCMake_TEST_FAILED "Actual log did not match expected log\n")
+    set(RunCMake_TEST_FAILED "${RunCMake_TEST_FAILED}" PARENT_SCOPE)
+  endif()
+endfunction()
+
+function(run_cthwalloc_write_proc name proc)
+  file(REMOVE "${RunCMake_BINARY_DIR}/${name}-build/cthwalloc.log")
+  run_cthwalloc_write_proc_nodel("${name}" "${proc}" "${ARGN}")
+endfunction()
+
+function(run_cthwalloc_write_proc_nodel name proc)
+  string(REPLACE ";" "\\;" proc "${proc}")
+  run_cmake_command(${name} "${CMAKE_COMMAND}" -E env "${ARGN}" "${CTHWALLOC_COMMAND}" write "${RunCMake_BINARY_DIR}/${name}-build/cthwalloc.log" "${name}" 0 "${proc}")
+endfunction()
+
+function(run_cthwalloc_write_noproc name)
+  run_cmake_command(${name} "${CMAKE_COMMAND}" -E env "${ARGN}" "${CTHWALLOC_COMMAND}" write "${RunCMake_BINARY_DIR}/${name}-build/cthwalloc.log" "${name}" 0)
+endfunction()
+
+function(run_cthwalloc_verify name tests)
+  string(REPLACE ";" "\\;" tests "${tests}")
+  run_cmake_command(${name} "${CTHWALLOC_COMMAND}" verify "${RunCMake_SOURCE_DIR}/${name}.log" "${CMAKE_CURRENT_LIST_DIR}/hwspec.json" "${tests}")
+endfunction()
+
+unset(ENV{CTEST_PROCESS_COUNT})
+set(RunCMake_TEST_NO_CLEAN 1)
+file(REMOVE_RECURSE "${RunCMake_BINARY_DIR}/cthwalloc-write-proc-good1-build")
+file(MAKE_DIRECTORY "${RunCMake_BINARY_DIR}/cthwalloc-write-proc-good1-build")
+file(WRITE "${RunCMake_BINARY_DIR}/cthwalloc-write-proc-good1-build/cthwalloc.log"
+[[begin test1
+alloc widgets 0 1
+dealloc widgets 0 1
+end test1
+]])
+run_cthwalloc_write_proc_nodel(cthwalloc-write-proc-good1 "1,widgets:2,transmogrifiers:1;2,widgets:1,widgets:2"
+  CTEST_PROCESS_COUNT=3
+  CTEST_PROCESS_0=widgets,transmogrifiers
+  CTEST_PROCESS_0_WIDGETS=id:0,slots:2
+  CTEST_PROCESS_0_TRANSMOGRIFIERS=id:calvin,slots:1
+  CTEST_PROCESS_1=widgets
+  "CTEST_PROCESS_1_WIDGETS=id:0,slots:1\\;id:2,slots:2"
+  CTEST_PROCESS_2=widgets
+  "CTEST_PROCESS_2_WIDGETS=id:0,slots:1\\;id:2,slots:2"
+  )
+set(RunCMake_TEST_NO_CLEAN 0)
+run_cthwalloc_write_proc(cthwalloc-write-proc-good2 "widgets:8"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets
+  CTEST_PROCESS_0_WIDGETS=id:3,slots:8
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-nocount "widgets:8")
+run_cthwalloc_write_proc(cthwalloc-write-proc-badcount "widgets:8"
+  CTEST_PROCESS_COUNT=2
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-nores "widgets:8"
+  CTEST_PROCESS_COUNT=1
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-badres "widgets:8"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets,transmogrifiers
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-nowidgets "widgets:8"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets1 "widgets:8"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets
+  CTEST_PROCESS_0_WIDGETS=
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets2 "widgets:8"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets
+  "CTEST_PROCESS_0_WIDGETS=id:3,slots:8\\;id:0,slots:1"
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets3 "widgets:8"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets
+  CTEST_PROCESS_0_WIDGETS=id:3,slots:7
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets4 "widgets:8"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets
+  CTEST_PROCESS_0_WIDGETS=invalid
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets5 "widgets:2,widgets:2"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets
+  "CTEST_PROCESS_0_WIDGETS=id:0,slots:2\\;id:0,slots:1"
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets6 "widgets:2"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets
+  "CTEST_PROCESS_0_WIDGETS=id:0,slots:2\\;id:0,slots:1"
+  )
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets7 "widgets:2,widgets:2"
+  CTEST_PROCESS_COUNT=1
+  CTEST_PROCESS_0=widgets
+  CTEST_PROCESS_0_WIDGETS=id:0,slots:2
+  )
+
+run_cthwalloc_write_noproc(cthwalloc-write-noproc-good1)
+run_cthwalloc_write_noproc(cthwalloc-write-noproc-count
+  CTEST_PROCESS_COUNT=1
+  )
+
+run_cthwalloc_verify(cthwalloc-verify-good1 "test1;test2")
+run_cthwalloc_verify(cthwalloc-verify-good2 "")
+run_cthwalloc_verify(cthwalloc-verify-nolog "")
+run_cthwalloc_verify(cthwalloc-verify-nores "")
+run_cthwalloc_verify(cthwalloc-verify-noid "")
+run_cthwalloc_verify(cthwalloc-verify-notenough "")
+run_cthwalloc_verify(cthwalloc-verify-baddealloc "")
+run_cthwalloc_verify(cthwalloc-verify-leak "")
+run_cthwalloc_verify(cthwalloc-verify-badtest1 "")
+run_cthwalloc_verify(cthwalloc-verify-badtest2 "test1")
+run_cthwalloc_verify(cthwalloc-verify-badtest3 "test1")
+run_cthwalloc_verify(cthwalloc-verify-badtest4 "test1")
+run_cthwalloc_verify(cthwalloc-verify-badtest5 "test1")
+run_cthwalloc_verify(cthwalloc-verify-nobegin "test1")
+run_cthwalloc_verify(cthwalloc-verify-noend "test1")
+
+###############################################################################
+# Now test the hardware allocation feature of CTest
+###############################################################################
+
+function(run_ctest_hardware name parallel random)
+  run_ctest("${name}-ctest-s-hw" "-DCTEST_HARDWARE_ALLOC_ENABLED=1" "-DCTHWALLOC_COMMAND=${CTHWALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
+  run_ctest("${name}-ctest-s-nohw" "-DCTEST_HARDWARE_ALLOC_ENABLED=0" "-DCTHWALLOC_COMMAND=${CTHWALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
+endfunction()
+
+function(verify_ctest_hardware)
+  file(READ "${RunCMake_TEST_BINARY_DIR}/hwtests.txt" hwtests)
+  execute_process(COMMAND "${CTHWALLOC_COMMAND}" verify "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log" "${CMAKE_CURRENT_LIST_DIR}/hwspec.json" "${hwtests}"
+    OUTPUT_VARIABLE output ERROR_QUIET RESULT_VARIABLE result)
+  if(result)
+    string(APPEND RunCMake_TEST_FAILED "${output}")
+    set(RunCMake_TEST_FAILED "${RunCMake_TEST_FAILED}" PARENT_SCOPE)
+  endif()
+endfunction()
+
+run_ctest_hardware(lotsoftests 10 1)
+run_ctest_hardware(checkfree1 2 0)
+run_ctest_hardware(checkfree2 1 0)
+run_ctest_hardware(notenough1 1 0)
+run_ctest_hardware(notenough2 1 0)
+run_ctest_hardware(ensure_parallel 2 0)
+
+set(ENV{CTEST_PROCESS_COUNT} 2)
+run_ctest_hardware(process_count 1 0)
+unset(ENV{CTEST_PROCESS_COUNT})
diff --git a/Tests/RunCMake/CTestHardwareAllocation/checkfree1-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/checkfree1-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..94b1fa7
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/checkfree1-ctest-s-hw-check.cmake
@@ -0,0 +1 @@
+verify_ctest_hardware()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/checkfree1.cmake b/Tests/RunCMake/CTestHardwareAllocation/checkfree1.cmake
new file mode 100644
index 0000000..0e997b5
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/checkfree1.cmake
@@ -0,0 +1,7 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 1 "widgets:8")
+add_hardware_test(Test2 1 "fluxcapacitors:50;fluxcapacitors:50,widgets:8")
+add_hardware_test(Test3 1 "fluxcapacitors:121")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/checkfree2-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/checkfree2-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..94b1fa7
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/checkfree2-ctest-s-hw-check.cmake
@@ -0,0 +1 @@
+verify_ctest_hardware()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/checkfree2.cmake b/Tests/RunCMake/CTestHardwareAllocation/checkfree2.cmake
new file mode 100644
index 0000000..3c2b666
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/checkfree2.cmake
@@ -0,0 +1,8 @@
+setup_hardware_tests()
+
+# This test is an attack on the hardware scheduling algorithm. It has been
+# carefully crafted to fool the algorithm into thinking there isn't sufficient
+# hardware for it.
+add_hardware_test(Test1 1 "widgets:2;4,widgets:4")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-baddealloc-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-baddealloc-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-baddealloc-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-baddealloc.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-baddealloc.log
new file mode 100644
index 0000000..abd6bad
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-baddealloc.log
@@ -0,0 +1,2 @@
+alloc widgets 0 1
+dealloc widgets 0 2
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest1-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest1-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest1-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest1.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest1.log
new file mode 100644
index 0000000..605104b
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest1.log
@@ -0,0 +1 @@
+begin test1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest2-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest2-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest2-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest2.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest2.log
new file mode 100644
index 0000000..1ff1b0d
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest2.log
@@ -0,0 +1,2 @@
+begin test1
+begin test1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest3-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest3-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest3-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest3.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest3.log
new file mode 100644
index 0000000..1925e6a
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest3.log
@@ -0,0 +1,3 @@
+begin test1
+end test1
+begin test1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest4-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest4-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest4-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest4.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest4.log
new file mode 100644
index 0000000..3fe7da1
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest4.log
@@ -0,0 +1,3 @@
+begin test1
+end test1
+end test1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest5-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest5-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest5-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest5.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest5.log
new file mode 100644
index 0000000..3a2e7e3
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-badtest5.log
@@ -0,0 +1 @@
+end test1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-good1.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-good1.log
new file mode 100644
index 0000000..2cca0c3
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-good1.log
@@ -0,0 +1,14 @@
+begin test1
+alloc widgets 3 4
+alloc widgets 4 1
+alloc transmogrifiers calvin 2
+alloc fluxcapacitors outatime 121
+begin test2
+alloc widgets 3 4
+dealloc widgets 3 4
+dealloc widgets 4 1
+dealloc transmogrifiers calvin 2
+dealloc fluxcapacitors outatime 121
+end test1
+dealloc widgets 3 4
+end test2
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-good2.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-good2.log
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-good2.log
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-leak-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-leak-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-leak-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-leak.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-leak.log
new file mode 100644
index 0000000..b900d86
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-leak.log
@@ -0,0 +1 @@
+alloc widgets 0 1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nobegin-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nobegin-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nobegin-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nobegin.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nobegin.log
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nobegin.log
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noend-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noend-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noend-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noend.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noend.log
new file mode 100644
index 0000000..605104b
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noend.log
@@ -0,0 +1 @@
+begin test1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noid-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noid-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noid-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noid.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noid.log
new file mode 100644
index 0000000..c718975
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-noid.log
@@ -0,0 +1,2 @@
+alloc fluxcapacitors train 1
+dealloc fluxcapacitors train 1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nolog-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nolog-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nolog-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nores-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nores-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nores-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nores.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nores.log
new file mode 100644
index 0000000..a18202b
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-nores.log
@@ -0,0 +1,2 @@
+alloc gpus 0 1
+dealloc gpus 0 1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-notenough-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-notenough-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-notenough-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-notenough.log b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-notenough.log
new file mode 100644
index 0000000..ac78d5a
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-verify-notenough.log
@@ -0,0 +1,2 @@
+alloc widgets 0 8
+dealloc widgets 0 8
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-noproc-count-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-noproc-count-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-noproc-count-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badcount-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badcount-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badcount-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badres-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badres-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badres-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets1-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets1-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets1-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets2-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets2-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets2-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets3-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets3-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets3-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets4-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets4-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets4-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets5-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets5-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets5-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets6-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets6-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets6-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets7-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets7-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-badwidgets7-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-good1-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-good1-check.cmake
new file mode 100644
index 0000000..949d2d7
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-good1-check.cmake
@@ -0,0 +1,20 @@
+cthwalloc_verify_log(
+[[begin test1
+alloc widgets 0 1
+dealloc widgets 0 1
+end test1
+begin cthwalloc-write-proc-good1
+alloc transmogrifiers calvin 1
+alloc widgets 0 2
+alloc widgets 0 1
+alloc widgets 2 2
+alloc widgets 0 1
+alloc widgets 2 2
+dealloc transmogrifiers calvin 1
+dealloc widgets 0 2
+dealloc widgets 0 1
+dealloc widgets 2 2
+dealloc widgets 0 1
+dealloc widgets 2 2
+end cthwalloc-write-proc-good1
+]])
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-good2-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-good2-check.cmake
new file mode 100644
index 0000000..ca0c6b8
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-good2-check.cmake
@@ -0,0 +1,6 @@
+cthwalloc_verify_log(
+[[begin cthwalloc-write-proc-good2
+alloc widgets 3 8
+dealloc widgets 3 8
+end cthwalloc-write-proc-good2
+]])
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-nocount-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-nocount-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-nocount-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-nores-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-nores-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-nores-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-nowidgets-result.txt b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-nowidgets-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc-write-proc-nowidgets-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc.cxx b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc.cxx
new file mode 100644
index 0000000..eee2c7f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc.cxx
@@ -0,0 +1,396 @@
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdlib>
+#include <iostream>
+#include <map>
+#include <set>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "cmsys/Encoding.hxx"
+#include "cmsys/FStream.hxx"
+
+#include "cmCTestHardwareAllocator.h"
+#include "cmCTestHardwareSpec.h"
+#include "cmCTestMultiProcessHandler.h"
+#include "cmCTestTestHandler.h"
+#include "cmFileLock.h"
+#include "cmFileLockResult.h"
+#include "cmStringAlgorithms.h"
+#include "cmSystemTools.h"
+
+/*
+ * This helper program is used to verify that the CTest hardware allocation
+ * feature is working correctly. It consists of two stages:
+ *
+ * 1) write - This stage receives the PROCESSES property of the test and
+ *    compares it with the values passed in the CTEST_PROCESS_* environment
+ *    variables. If it received all of the resources it expected, then it
+ *    writes this information to a log file, which will be read in the verify
+ *    stage.
+ * 2) verify - This stage compares the log file with the hardware spec file to
+ *    make sure that no resources were over-subscribed, deallocated without
+ *    being allocated, or allocated without being deallocated.
+ */
+
+static int usage(const char* argv0)
+{
+  std::cout << "Usage: " << argv0 << " (write|verify) <args...>" << std::endl;
+  return 1;
+}
+
+static int usageWrite(const char* argv0)
+{
+  std::cout << "Usage: " << argv0
+            << " write <log-file> <test-name> <sleep-time-secs>"
+               " [<processes-property>]"
+            << std::endl;
+  return 1;
+}
+
+static int usageVerify(const char* argv0)
+{
+  std::cout << "Usage: " << argv0
+            << " verify <log-file> <hardware-spec-file> [<test-names>]"
+            << std::endl;
+  return 1;
+}
+
+static int doWrite(int argc, char const* const* argv)
+{
+  if (argc < 5 || argc > 6) {
+    return usageWrite(argv[0]);
+  }
+  std::string logFile = argv[2];
+  std::string testName = argv[3];
+  unsigned int sleepTime = std::atoi(argv[4]);
+  std::vector<std::map<
+    std::string, std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>>
+    hardware;
+  if (argc == 6) {
+    // Parse processes property
+    std::string processesProperty = argv[5];
+    std::vector<
+      std::vector<cmCTestTestHandler::cmCTestTestResourceRequirement>>
+      processes;
+    bool result =
+      cmCTestTestHandler::ParseProcessesProperty(processesProperty, processes);
+    (void)result;
+    assert(result);
+
+    // Verify process count
+    const char* processCountEnv = cmSystemTools::GetEnv("CTEST_PROCESS_COUNT");
+    if (!processCountEnv) {
+      std::cout << "CTEST_PROCESS_COUNT should be defined" << std::endl;
+      return 1;
+    }
+    int processCount = std::atoi(processCountEnv);
+    if (processes.size() != std::size_t(processCount)) {
+      std::cout << "CTEST_PROCESS_COUNT does not match expected processes"
+                << std::endl
+                << "Expected: " << processes.size() << std::endl
+                << "Actual: " << processCount << std::endl;
+      return 1;
+    }
+
+    if (!cmSystemTools::Touch(logFile + ".lock", true)) {
+      std::cout << "Could not create lock file" << std::endl;
+      return 1;
+    }
+    cmFileLock lock;
+    auto lockResult =
+      lock.Lock(logFile + ".lock", static_cast<unsigned long>(-1));
+    if (!lockResult.IsOk()) {
+      std::cout << "Could not lock file" << std::endl;
+      return 1;
+    }
+    std::size_t i = 0;
+    cmsys::ofstream fout(logFile.c_str(), std::ios::app);
+    fout << "begin " << testName << std::endl;
+    for (auto& process : processes) {
+      try {
+        // Build and verify set of expected resources
+        std::set<std::string> expectedResources;
+        for (auto const& it : process) {
+          expectedResources.insert(it.ResourceType);
+        }
+
+        std::string prefix = "CTEST_PROCESS_";
+        prefix += std::to_string(i);
+        const char* actualResourcesCStr = cmSystemTools::GetEnv(prefix);
+        if (!actualResourcesCStr) {
+          std::cout << prefix << " should be defined" << std::endl;
+          return 1;
+        }
+
+        auto actualResourcesVec =
+          cmSystemTools::SplitString(actualResourcesCStr, ',');
+        std::set<std::string> actualResources;
+        for (auto const& r : actualResourcesVec) {
+          if (!r.empty()) {
+            actualResources.insert(r);
+          }
+        }
+
+        if (actualResources != expectedResources) {
+          std::cout << prefix << " did not list expected resources"
+                    << std::endl;
+          return 1;
+        }
+
+        // Verify that we got what we asked for and write it to the log
+        prefix += '_';
+        std::map<std::string,
+                 std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>
+          hwEntry;
+        for (auto const& type : actualResources) {
+          auto it = process.begin();
+
+          std::string varName = prefix;
+          varName += cmSystemTools::UpperCase(type);
+          const char* varVal = cmSystemTools::GetEnv(varName);
+          if (!varVal) {
+            std::cout << varName << " should be defined" << std::endl;
+            return 1;
+          }
+
+          auto received = cmSystemTools::SplitString(varVal, ';');
+          for (auto const& r : received) {
+            while (it->ResourceType != type || it->UnitsNeeded == 0) {
+              ++it;
+              if (it == process.end()) {
+                std::cout << varName << " did not list expected resources"
+                          << std::endl;
+                return 1;
+              }
+            }
+            auto split = cmSystemTools::SplitString(r, ',');
+            if (split.size() != 2) {
+              std::cout << varName << " was ill-formed" << std::endl;
+              return 1;
+            }
+            if (!cmHasLiteralPrefix(split[0], "id:")) {
+              std::cout << varName << " was ill-formed" << std::endl;
+              return 1;
+            }
+            auto id = split[0].substr(3);
+            if (!cmHasLiteralPrefix(split[1], "slots:")) {
+              std::cout << varName << " was ill-formed" << std::endl;
+              return 1;
+            }
+            auto slots = split[1].substr(6);
+            unsigned int amount = std::atoi(slots.c_str());
+            if (amount != static_cast<unsigned int>(it->SlotsNeeded)) {
+              std::cout << varName << " did not list expected resources"
+                        << std::endl;
+              return 1;
+            }
+            --it->UnitsNeeded;
+
+            fout << "alloc " << type << " " << id << " " << amount
+                 << std::endl;
+            hwEntry[type].push_back({ id, amount });
+          }
+
+          bool ended = false;
+          while (it->ResourceType != type || it->UnitsNeeded == 0) {
+            ++it;
+            if (it == process.end()) {
+              ended = true;
+              break;
+            }
+          }
+
+          if (!ended) {
+            std::cout << varName << " did not list expected resources"
+                      << std::endl;
+            return 1;
+          }
+        }
+        hardware.push_back(hwEntry);
+
+        ++i;
+      } catch (...) {
+        std::cout << "Unknown error while processing resources" << std::endl;
+        return 1;
+      }
+    }
+
+    auto unlockResult = lock.Release();
+    if (!unlockResult.IsOk()) {
+      std::cout << "Could not unlock file" << std::endl;
+      return 1;
+    }
+  } else {
+    if (cmSystemTools::GetEnv("CTEST_PROCESS_COUNT")) {
+      std::cout << "CTEST_PROCESS_COUNT should not be defined" << std::endl;
+      return 1;
+    }
+  }
+
+  std::this_thread::sleep_for(std::chrono::seconds(sleepTime));
+
+  if (argc == 6) {
+    if (!cmSystemTools::Touch(logFile + ".lock", true)) {
+      std::cout << "Could not create lock file" << std::endl;
+      return 1;
+    }
+    cmFileLock lock;
+    auto lockResult =
+      lock.Lock(logFile + ".lock", static_cast<unsigned long>(-1));
+    if (!lockResult.IsOk()) {
+      std::cout << "Could not lock file" << std::endl;
+      return 1;
+    }
+    cmsys::ofstream fout(logFile.c_str(), std::ios::app);
+    for (auto const& process : hardware) {
+      for (auto const& it : process) {
+        for (auto const& it2 : it.second) {
+          fout << "dealloc " << it.first << " " << it2.Id << " " << it2.Slots
+               << std::endl;
+        }
+      }
+    }
+
+    fout << "end " << testName << std::endl;
+
+    auto unlockResult = lock.Release();
+    if (!unlockResult.IsOk()) {
+      std::cout << "Could not unlock file" << std::endl;
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+static int doVerify(int argc, char const* const* argv)
+{
+  if (argc < 4 || argc > 5) {
+    return usageVerify(argv[0]);
+  }
+  std::string logFile = argv[2];
+  std::string hwFile = argv[3];
+  std::string testNames;
+  if (argc == 5) {
+    testNames = argv[4];
+  }
+  auto testNameList = cmExpandedList(testNames, false);
+  std::set<std::string> testNameSet(testNameList.begin(), testNameList.end());
+
+  cmCTestHardwareSpec spec;
+  if (!spec.ReadFromJSONFile(hwFile)) {
+    std::cout << "Could not read hardware spec " << hwFile << std::endl;
+    return 1;
+  }
+
+  cmCTestHardwareAllocator allocator;
+  allocator.InitializeFromHardwareSpec(spec);
+
+  cmsys::ifstream fin(logFile.c_str(), std::ios::in);
+  if (!fin) {
+    std::cout << "Could not open log file " << logFile << std::endl;
+    return 1;
+  }
+
+  std::string command;
+  std::string resourceName;
+  std::string resourceId;
+  std::string testName;
+  unsigned int amount;
+  std::set<std::string> inProgressTests;
+  std::set<std::string> completedTests;
+  try {
+    while (fin >> command) {
+      if (command == "begin") {
+        if (!(fin >> testName)) {
+          std::cout << "Could not read begin line" << std::endl;
+          return 1;
+        }
+        if (!testNameSet.count(testName) || inProgressTests.count(testName) ||
+            completedTests.count(testName)) {
+          std::cout << "Could not begin test" << std::endl;
+          return 1;
+        }
+        inProgressTests.insert(testName);
+      } else if (command == "alloc") {
+        if (!(fin >> resourceName) || !(fin >> resourceId) ||
+            !(fin >> amount)) {
+          std::cout << "Could not read alloc line" << std::endl;
+          return 1;
+        }
+        if (!allocator.AllocateResource(resourceName, resourceId, amount)) {
+          std::cout << "Could not allocate resources" << std::endl;
+          return 1;
+        }
+      } else if (command == "dealloc") {
+        if (!(fin >> resourceName) || !(fin >> resourceId) ||
+            !(fin >> amount)) {
+          std::cout << "Could not read dealloc line" << std::endl;
+          return 1;
+        }
+        if (!allocator.DeallocateResource(resourceName, resourceId, amount)) {
+          std::cout << "Could not deallocate resources" << std::endl;
+          return 1;
+        }
+      } else if (command == "end") {
+        if (!(fin >> testName)) {
+          std::cout << "Could not read end line" << std::endl;
+          return 1;
+        }
+        if (!inProgressTests.erase(testName)) {
+          std::cout << "Could not end test" << std::endl;
+          return 1;
+        }
+        if (!completedTests.insert(testName).second) {
+          std::cout << "Could not end test" << std::endl;
+          return 1;
+        }
+      }
+    }
+  } catch (...) {
+    std::cout << "Unknown error while reading log file" << std::endl;
+    return 1;
+  }
+
+  auto const& avail = allocator.GetResources();
+  for (auto const& it : avail) {
+    for (auto const& it2 : it.second) {
+      if (it2.second.Locked != 0) {
+        std::cout << "Resource was not unlocked" << std::endl;
+        return 1;
+      }
+    }
+  }
+
+  if (completedTests != testNameSet) {
+    std::cout << "Tests were not ended" << std::endl;
+    return 1;
+  }
+
+  return 0;
+}
+
+int main(int argc, char const* const* argv)
+{
+  cmsys::Encoding::CommandLineArguments args =
+    cmsys::Encoding::CommandLineArguments::Main(argc, argv);
+  argc = args.argc();
+  argv = args.argv();
+
+  if (argc < 2) {
+    return usage(argv[0]);
+  }
+
+  std::string argv1 = argv[1];
+  if (argv1 == "write") {
+    return doWrite(argc, argv);
+  }
+  if (argv1 == "verify") {
+    return doVerify(argc, argv);
+  }
+  return usage(argv[0]);
+}
diff --git a/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..e5f6828
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel-ctest-s-hw-check.cmake
@@ -0,0 +1,16 @@
+verify_ctest_hardware()
+
+set(expected_contents [[
+begin Test1
+alloc transmogrifiers calvin 2
+begin Test2
+alloc transmogrifiers hobbes 2
+dealloc transmogrifiers calvin 2
+end Test1
+dealloc transmogrifiers hobbes 2
+end Test2
+]])
+file(READ "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log" actual_contents)
+if(NOT actual_contents STREQUAL expected_contents)
+  string(APPEND RunCMake_TEST_FAILED "cthwalloc.log contents did not match expected\n")
+endif()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel.cmake b/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel.cmake
new file mode 100644
index 0000000..1dafb8f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel.cmake
@@ -0,0 +1,11 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 4 "transmogrifiers:2")
+
+# Mitigate possible race conditions to ensure that the events are logged in the
+# exact order we want
+add_test(NAME Test2Sleep COMMAND "${CMAKE_COMMAND}" -E sleep 2)
+add_hardware_test(Test2 4 "transmogrifiers:2")
+set_property(TEST Test2 APPEND PROPERTY DEPENDS Test2Sleep)
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/hwspec.json b/Tests/RunCMake/CTestHardwareAllocation/hwspec.json
new file mode 100644
index 0000000..c67fcca
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/hwspec.json
@@ -0,0 +1,55 @@
+{
+  "local": [
+    {
+      "widgets": [
+        {
+          "id": "0",
+          "slots": 4
+        },
+        {
+          "id": "1",
+          "slots": 2
+        },
+        {
+          "id": "2",
+          "slots": 4
+        },
+        {
+          "id": "3",
+          "slots": 8
+        },
+        {
+          "id": "4",
+          "slots": 1
+        },
+        {
+          "id": "5",
+          "slots": 1
+        },
+        {
+          "id": "6",
+          "slots": 1
+        },
+        {
+          "id": "7"
+        }
+      ],
+      "transmogrifiers": [
+        {
+          "id": "calvin",
+          "slots": 2
+        },
+        {
+          "id": "hobbes",
+          "slots": 2
+        }
+      ],
+      "fluxcapacitors": [
+        {
+          "id": "outatime",
+          "slots": 121
+        }
+      ]
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CTestHardwareAllocation/lotsoftests-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/lotsoftests-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..94b1fa7
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/lotsoftests-ctest-s-hw-check.cmake
@@ -0,0 +1 @@
+verify_ctest_hardware()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/lotsoftests.cmake b/Tests/RunCMake/CTestHardwareAllocation/lotsoftests.cmake
new file mode 100644
index 0000000..c684434
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/lotsoftests.cmake
@@ -0,0 +1,16 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 2 "widgets:8;2,widgets:2")
+add_hardware_test(Test2 5 "fluxcapacitors:40")
+add_hardware_test(Test3 1 "10,widgets:1,fluxcapacitors:2")
+add_hardware_test(Test4 4 "fluxcapacitors:121")
+
+foreach(i RANGE 5 50)
+  add_hardware_test(Test${i} 1 "2,widgets:1")
+endforeach()
+
+foreach(i RANGE 51 100)
+  add_hardware_test(Test${i} 1 "2,transmogrifiers:2")
+endforeach()
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..9c730be
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-check.cmake
@@ -0,0 +1,3 @@
+if(EXISTS "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log")
+  set(RunCMake_TEST_FAILED "cthwalloc.log should not exist")
+endif()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-result.txt b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-stderr.txt b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-stderr.txt
new file mode 100644
index 0000000..d465cd3
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-stderr.txt
@@ -0,0 +1,4 @@
+^Insufficient hardware
+CMake Error at [^
+]*/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough1.cmake b/Tests/RunCMake/CTestHardwareAllocation/notenough1.cmake
new file mode 100644
index 0000000..3e1f620
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough1.cmake
@@ -0,0 +1,5 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 1 "fluxcapacitors:200")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..9c730be
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-check.cmake
@@ -0,0 +1,3 @@
+if(EXISTS "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log")
+  set(RunCMake_TEST_FAILED "cthwalloc.log should not exist")
+endif()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-result.txt b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-stderr.txt b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-stderr.txt
new file mode 100644
index 0000000..912f0fb
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-stderr.txt
@@ -0,0 +1,4 @@
+^Insufficient hardware
+CMake Error at [^
+]*/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough2.cmake b/Tests/RunCMake/CTestHardwareAllocation/notenough2.cmake
new file mode 100644
index 0000000..8205c95
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough2.cmake
@@ -0,0 +1,5 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 1 "terminators:2")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/process_count-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/process_count-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..94b1fa7
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/process_count-ctest-s-hw-check.cmake
@@ -0,0 +1 @@
+verify_ctest_hardware()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/process_count.cmake b/Tests/RunCMake/CTestHardwareAllocation/process_count.cmake
new file mode 100644
index 0000000..c969648
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/process_count.cmake
@@ -0,0 +1,5 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 1 "widgets:1")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/test.cmake.in b/Tests/RunCMake/CTestHardwareAllocation/test.cmake.in
new file mode 100644
index 0000000..5ba3587
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/test.cmake.in
@@ -0,0 +1,23 @@
+set(CTEST_SITE "test-site")
+set(CTEST_BUILD_NAME "test-build-name")
+set(CTEST_SOURCE_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@")
+set(CTEST_BINARY_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
+set(CTEST_CMAKE_GENERATOR "@RunCMake_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_PLATFORM "@RunCMake_GENERATOR_PLATFORM@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}")
+set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC")
+
+ctest_start(Experimental QUIET)
+ctest_configure(OPTIONS
+  "-DCTEST_HARDWARE_ALLOC_ENABLED=${CTEST_HARDWARE_ALLOC_ENABLED};-DCTHWALLOC_COMMAND=${CTHWALLOC_COMMAND}"
+  )
+ctest_build()
+
+if(CTEST_HARDWARE_ALLOC_ENABLED)
+  set(hwspec HARDWARE_SPEC_FILE "@RunCMake_SOURCE_DIR@/hwspec.json")
+endif()
+ctest_test(${hwspec} RETURN_VALUE retval PARALLEL_LEVEL ${CTEST_PARALLEL} SCHEDULE_RANDOM ${CTEST_RANDOM})
+if(retval)
+  message(FATAL_ERROR "Tests did not pass")
+endif()