Tests: Add test for CTest hardware allocation feature

23 files changed, 204 insertions, 26 deletions

diff --git a/Tests/RunCMake/CTestHardwareAllocation/CMakeLists.txt.in b/Tests/RunCMake/CTestHardwareAllocation/CMakeLists.txt.in
new file mode 100644
index 0000000..d6cff63
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/CMakeLists.txt.in
@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.15)
+set(CASE_NAME "@CASE_NAME@")
+if(CASE_NAME MATCHES "^(.*)-ctest-s")
+  set(projname "${CMAKE_MATCH_1}")
+  project(${projname} NONE)
+  include(CTest)
+  include("@RunCMake_SOURCE_DIR@/HardwareCommon.cmake")
+  include("@RunCMake_SOURCE_DIR@/${projname}.cmake")
+endif()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/HardwareCommon.cmake b/Tests/RunCMake/CTestHardwareAllocation/HardwareCommon.cmake
new file mode 100644
index 0000000..3893d40
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/HardwareCommon.cmake
@@ -0,0 +1,23 @@
+function(setup_hardware_tests)
+  if(CTEST_HARDWARE_ALLOC_ENABLED)
+    add_test(NAME HardwareSetup COMMAND "${CMAKE_COMMAND}" -E remove -f "${CMAKE_BINARY_DIR}/cthwalloc.log")
+  endif()
+endfunction()
+
+function(add_hardware_test name sleep_time proc)
+  if(CTEST_HARDWARE_ALLOC_ENABLED)
+    add_test(NAME "${name}" COMMAND "${CTHWALLOC_COMMAND}" write "${CMAKE_BINARY_DIR}/cthwalloc.log" "${name}" "${sleep_time}" "${proc}")
+    set_property(TEST "${name}" PROPERTY DEPENDS HardwareSetup)
+  else()
+    add_test(NAME "${name}" COMMAND "${CTHWALLOC_COMMAND}" write "${CMAKE_BINARY_DIR}/cthwalloc.log" "${name}" "${sleep_time}")
+  endif()
+  set_property(TEST "${name}" PROPERTY PROCESSES "${proc}")
+  list(APPEND HARDWARE_TESTS "${name}")
+  set(HARDWARE_TESTS "${HARDWARE_TESTS}" PARENT_SCOPE)
+endfunction()
+
+function(cleanup_hardware_tests)
+  if(CTEST_HARDWARE_ALLOC_ENABLED)
+    file(WRITE "${CMAKE_BINARY_DIR}/hwtests.txt" "${HARDWARE_TESTS}")
+  endif()
+endfunction()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/RunCMakeTest.cmake b/Tests/RunCMake/CTestHardwareAllocation/RunCMakeTest.cmake
index aa249d4..d666922 100644
--- a/Tests/RunCMake/CTestHardwareAllocation/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestHardwareAllocation/RunCMakeTest.cmake
@@ -1,4 +1,5 @@
 include(RunCMake)
+include(RunCTest)
 
 ###############################################################################
 # Test cthwalloc itself - we want to make sure it's not just rubber-stamping
@@ -47,7 +48,7 @@ alloc widgets 0 1
 dealloc widgets 0 1
 end test1
 ]])
-run_cthwalloc_write_proc_nodel(cthwalloc-write-proc-good1 "1,widgets=2,transmogrifiers=1;2,widgets=1,widgets=2"
+run_cthwalloc_write_proc_nodel(cthwalloc-write-proc-good1 "1,widgets:2,transmogrifiers:1;2,widgets:1,widgets:2"
   CTEST_PROCESS_COUNT=3
   CTEST_PROCESS_0=widgets,transmogrifiers
   CTEST_PROCESS_0_WIDGETS=id:0,slots:2
@@ -58,57 +59,57 @@ run_cthwalloc_write_proc_nodel(cthwalloc-write-proc-good1 "1,widgets=2,transmogr
   "CTEST_PROCESS_2_WIDGETS=id:0,slots:1\\;id:2,slots:2"
   )
 set(RunCMake_TEST_NO_CLEAN 0)
-run_cthwalloc_write_proc(cthwalloc-write-proc-good2 "widgets=8"
+run_cthwalloc_write_proc(cthwalloc-write-proc-good2 "widgets:8"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets
   CTEST_PROCESS_0_WIDGETS=id:3,slots:8
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-nocount "widgets=8")
-run_cthwalloc_write_proc(cthwalloc-write-proc-badcount "widgets=8"
+run_cthwalloc_write_proc(cthwalloc-write-proc-nocount "widgets:8")
+run_cthwalloc_write_proc(cthwalloc-write-proc-badcount "widgets:8"
   CTEST_PROCESS_COUNT=2
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-nores "widgets=8"
+run_cthwalloc_write_proc(cthwalloc-write-proc-nores "widgets:8"
   CTEST_PROCESS_COUNT=1
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-badres "widgets=8"
+run_cthwalloc_write_proc(cthwalloc-write-proc-badres "widgets:8"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets,transmogrifiers
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-nowidgets "widgets=8"
+run_cthwalloc_write_proc(cthwalloc-write-proc-nowidgets "widgets:8"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets1 "widgets=8"
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets1 "widgets:8"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets
   CTEST_PROCESS_0_WIDGETS=
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets2 "widgets=8"
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets2 "widgets:8"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets
   "CTEST_PROCESS_0_WIDGETS=id:3,slots:8\\;id:0,slots:1"
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets3 "widgets=8"
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets3 "widgets:8"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets
   CTEST_PROCESS_0_WIDGETS=id:3,slots:7
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets4 "widgets=8"
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets4 "widgets:8"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets
   CTEST_PROCESS_0_WIDGETS=invalid
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets5 "widgets=2,widgets=2"
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets5 "widgets:2,widgets:2"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets
   "CTEST_PROCESS_0_WIDGETS=id:0,slots:2\\;id:0,slots:1"
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets6 "widgets=2"
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets6 "widgets:2"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets
   "CTEST_PROCESS_0_WIDGETS=id:0,slots:2\\;id:0,slots:1"
   )
-run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets7 "widgets=2,widgets=2"
+run_cthwalloc_write_proc(cthwalloc-write-proc-badwidgets7 "widgets:2,widgets:2"
   CTEST_PROCESS_COUNT=1
   CTEST_PROCESS_0=widgets
   CTEST_PROCESS_0_WIDGETS=id:0,slots:2
@@ -134,3 +135,33 @@ run_cthwalloc_verify(cthwalloc-verify-badtest4 "test1")
 run_cthwalloc_verify(cthwalloc-verify-badtest5 "test1")
 run_cthwalloc_verify(cthwalloc-verify-nobegin "test1")
 run_cthwalloc_verify(cthwalloc-verify-noend "test1")
+
+###############################################################################
+# Now test the hardware allocation feature of CTest
+###############################################################################
+
+function(run_ctest_hardware name parallel random)
+  run_ctest("${name}-ctest-s-hw" "-DCTEST_HARDWARE_ALLOC_ENABLED=1" "-DCTHWALLOC_COMMAND=${CTHWALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
+  run_ctest("${name}-ctest-s-nohw" "-DCTEST_HARDWARE_ALLOC_ENABLED=0" "-DCTHWALLOC_COMMAND=${CTHWALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
+endfunction()
+
+function(verify_ctest_hardware)
+  file(READ "${RunCMake_TEST_BINARY_DIR}/hwtests.txt" hwtests)
+  execute_process(COMMAND "${CTHWALLOC_COMMAND}" verify "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log" "${CMAKE_CURRENT_LIST_DIR}/hwspec.json" "${hwtests}"
+    OUTPUT_VARIABLE output ERROR_QUIET RESULT_VARIABLE result)
+  if(result)
+    string(APPEND RunCMake_TEST_FAILED "${output}")
+    set(RunCMake_TEST_FAILED "${RunCMake_TEST_FAILED}" PARENT_SCOPE)
+  endif()
+endfunction()
+
+run_ctest_hardware(lotsoftests 10 1)
+run_ctest_hardware(checkfree1 2 0)
+run_ctest_hardware(checkfree2 1 0)
+run_ctest_hardware(notenough1 1 0)
+run_ctest_hardware(notenough2 1 0)
+run_ctest_hardware(ensure_parallel 2 0)
+
+set(ENV{CTEST_PROCESS_COUNT} 2)
+run_ctest_hardware(process_count 1 0)
+unset(ENV{CTEST_PROCESS_COUNT})
diff --git a/Tests/RunCMake/CTestHardwareAllocation/checkfree1-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/checkfree1-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..94b1fa7
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/checkfree1-ctest-s-hw-check.cmake
@@ -0,0 +1 @@
+verify_ctest_hardware()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/checkfree1.cmake b/Tests/RunCMake/CTestHardwareAllocation/checkfree1.cmake
new file mode 100644
index 0000000..0e997b5
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/checkfree1.cmake
@@ -0,0 +1,7 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 1 "widgets:8")
+add_hardware_test(Test2 1 "fluxcapacitors:50;fluxcapacitors:50,widgets:8")
+add_hardware_test(Test3 1 "fluxcapacitors:121")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/checkfree2-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/checkfree2-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..94b1fa7
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/checkfree2-ctest-s-hw-check.cmake
@@ -0,0 +1 @@
+verify_ctest_hardware()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/checkfree2.cmake b/Tests/RunCMake/CTestHardwareAllocation/checkfree2.cmake
new file mode 100644
index 0000000..3c2b666
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/checkfree2.cmake
@@ -0,0 +1,8 @@
+setup_hardware_tests()
+
+# This test is an attack on the hardware scheduling algorithm. It has been
+# carefully crafted to fool the algorithm into thinking there isn't sufficient
+# hardware for it.
+add_hardware_test(Test1 1 "widgets:2;4,widgets:4")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..e5f6828
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel-ctest-s-hw-check.cmake
@@ -0,0 +1,16 @@
+verify_ctest_hardware()
+
+set(expected_contents [[
+begin Test1
+alloc transmogrifiers calvin 2
+begin Test2
+alloc transmogrifiers hobbes 2
+dealloc transmogrifiers calvin 2
+end Test1
+dealloc transmogrifiers hobbes 2
+end Test2
+]])
+file(READ "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log" actual_contents)
+if(NOT actual_contents STREQUAL expected_contents)
+  string(APPEND RunCMake_TEST_FAILED "cthwalloc.log contents did not match expected\n")
+endif()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel.cmake b/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel.cmake
new file mode 100644
index 0000000..1dafb8f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/ensure_parallel.cmake
@@ -0,0 +1,11 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 4 "transmogrifiers:2")
+
+# Mitigate possible race conditions to ensure that the events are logged in the
+# exact order we want
+add_test(NAME Test2Sleep COMMAND "${CMAKE_COMMAND}" -E sleep 2)
+add_hardware_test(Test2 4 "transmogrifiers:2")
+set_property(TEST Test2 APPEND PROPERTY DEPENDS Test2Sleep)
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/hwspec.json b/Tests/RunCMake/CTestHardwareAllocation/hwspec.json
index bf8daf6..c67fcca 100644
--- a/Tests/RunCMake/CTestHardwareAllocation/hwspec.json
+++ b/Tests/RunCMake/CTestHardwareAllocation/hwspec.json
@@ -4,51 +4,50 @@
       "widgets": [
         {
           "id": "0",
-          "capacity": 4
+          "slots": 4
         },
         {
           "id": "1",
-          "capacity": 2
+          "slots": 2
         },
         {
           "id": "2",
-          "capacity": 4
+          "slots": 4
         },
         {
           "id": "3",
-          "capacity": 8
+          "slots": 8
         },
         {
           "id": "4",
-          "capacity": 1
+          "slots": 1
         },
         {
           "id": "5",
-          "capacity": 1
+          "slots": 1
         },
         {
           "id": "6",
-          "capacity": 1
+          "slots": 1
         },
         {
-          "id": "7",
-          "capacity": 1
+          "id": "7"
         }
       ],
       "transmogrifiers": [
         {
           "id": "calvin",
-          "capacity": 2
+          "slots": 2
         },
         {
           "id": "hobbes",
-          "capacity": 2
+          "slots": 2
         }
       ],
       "fluxcapacitors": [
         {
           "id": "outatime",
-          "capacity": 121
+          "slots": 121
         }
       ]
     }
diff --git a/Tests/RunCMake/CTestHardwareAllocation/lotsoftests-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/lotsoftests-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..94b1fa7
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/lotsoftests-ctest-s-hw-check.cmake
@@ -0,0 +1 @@
+verify_ctest_hardware()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/lotsoftests.cmake b/Tests/RunCMake/CTestHardwareAllocation/lotsoftests.cmake
new file mode 100644
index 0000000..c684434
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/lotsoftests.cmake
@@ -0,0 +1,16 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 2 "widgets:8;2,widgets:2")
+add_hardware_test(Test2 5 "fluxcapacitors:40")
+add_hardware_test(Test3 1 "10,widgets:1,fluxcapacitors:2")
+add_hardware_test(Test4 4 "fluxcapacitors:121")
+
+foreach(i RANGE 5 50)
+  add_hardware_test(Test${i} 1 "2,widgets:1")
+endforeach()
+
+foreach(i RANGE 51 100)
+  add_hardware_test(Test${i} 1 "2,transmogrifiers:2")
+endforeach()
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..9c730be
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-check.cmake
@@ -0,0 +1,3 @@
+if(EXISTS "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log")
+  set(RunCMake_TEST_FAILED "cthwalloc.log should not exist")
+endif()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-result.txt b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-stderr.txt b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-stderr.txt
new file mode 100644
index 0000000..d465cd3
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw-stderr.txt
@@ -0,0 +1,4 @@
+^Insufficient hardware
+CMake Error at [^
+]*/Tests/RunCMake/CTestHardwareAllocation/notenough1-ctest-s-hw/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough1.cmake b/Tests/RunCMake/CTestHardwareAllocation/notenough1.cmake
new file mode 100644
index 0000000..3e1f620
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough1.cmake
@@ -0,0 +1,5 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 1 "fluxcapacitors:200")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..9c730be
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-check.cmake
@@ -0,0 +1,3 @@
+if(EXISTS "${RunCMake_TEST_BINARY_DIR}/cthwalloc.log")
+  set(RunCMake_TEST_FAILED "cthwalloc.log should not exist")
+endif()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-result.txt b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-stderr.txt b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-stderr.txt
new file mode 100644
index 0000000..912f0fb
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw-stderr.txt
@@ -0,0 +1,4 @@
+^Insufficient hardware
+CMake Error at [^
+]*/Tests/RunCMake/CTestHardwareAllocation/notenough2-ctest-s-hw/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestHardwareAllocation/notenough2.cmake b/Tests/RunCMake/CTestHardwareAllocation/notenough2.cmake
new file mode 100644
index 0000000..8205c95
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/notenough2.cmake
@@ -0,0 +1,5 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 1 "terminators:2")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/process_count-ctest-s-hw-check.cmake b/Tests/RunCMake/CTestHardwareAllocation/process_count-ctest-s-hw-check.cmake
new file mode 100644
index 0000000..94b1fa7
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/process_count-ctest-s-hw-check.cmake
@@ -0,0 +1 @@
+verify_ctest_hardware()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/process_count.cmake b/Tests/RunCMake/CTestHardwareAllocation/process_count.cmake
new file mode 100644
index 0000000..c969648
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/process_count.cmake
@@ -0,0 +1,5 @@
+setup_hardware_tests()
+
+add_hardware_test(Test1 1 "widgets:1")
+
+cleanup_hardware_tests()
diff --git a/Tests/RunCMake/CTestHardwareAllocation/test.cmake.in b/Tests/RunCMake/CTestHardwareAllocation/test.cmake.in
new file mode 100644
index 0000000..5ba3587
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/test.cmake.in
@@ -0,0 +1,23 @@
+set(CTEST_SITE "test-site")
+set(CTEST_BUILD_NAME "test-build-name")
+set(CTEST_SOURCE_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@")
+set(CTEST_BINARY_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
+set(CTEST_CMAKE_GENERATOR "@RunCMake_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_PLATFORM "@RunCMake_GENERATOR_PLATFORM@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}")
+set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC")
+
+ctest_start(Experimental QUIET)
+ctest_configure(OPTIONS
+  "-DCTEST_HARDWARE_ALLOC_ENABLED=${CTEST_HARDWARE_ALLOC_ENABLED};-DCTHWALLOC_COMMAND=${CTHWALLOC_COMMAND}"
+  )
+ctest_build()
+
+if(CTEST_HARDWARE_ALLOC_ENABLED)
+  set(hwspec HARDWARE_SPEC_FILE "@RunCMake_SOURCE_DIR@/hwspec.json")
+endif()
+ctest_test(${hwspec} RETURN_VALUE retval PARALLEL_LEVEL ${CTEST_PARALLEL} SCHEDULE_RANDOM ${CTEST_RANDOM})
+if(retval)
+  message(FATAL_ERROR "Tests did not pass")
+endif()