CTest: Throw error if resource spec is invalid

19 files changed, 72 insertions, 19 deletions

diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index 7b72f30..34f9d43 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -83,7 +83,7 @@ cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
 cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler() = default;
 
 // Set the tests
-void cmCTestMultiProcessHandler::SetTests(TestMap tests,
+bool cmCTestMultiProcessHandler::SetTests(TestMap tests,
                                           PropertiesMap properties)
 {
   this->PendingTests = std::move(tests);
@@ -95,10 +95,11 @@ void cmCTestMultiProcessHandler::SetTests(TestMap tests,
     this->HasInvalidGeneratedResourceSpec =
       !this->CheckGeneratedResourceSpec();
     if (this->HasCycles || this->HasInvalidGeneratedResourceSpec) {
-      return;
+      return false;
     }
     this->CreateTestCostList();
   }
+  return true;
 }
 
 // Set the max number of tests that can be run at the same time.
diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h
index 02589ca..d66c348 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.h
+++ b/Source/CTest/cmCTestMultiProcessHandler.h
@@ -61,7 +61,7 @@ public:
   cmCTestMultiProcessHandler();
   virtual ~cmCTestMultiProcessHandler();
   // Set the tests
-  void SetTests(TestMap tests, PropertiesMap properties);
+  bool SetTests(TestMap tests, PropertiesMap properties);
   // Set the max number of tests that can be run at the same time.
   void SetParallelLevel(size_t);
   void SetTestLoad(unsigned long load);
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index dc16f66..e83bdfc 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -1410,7 +1410,9 @@ bool cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
     properties[p.Index] = &p;
   }
   parallel->SetResourceSpecFile(this->ResourceSpecFile);
-  parallel->SetTests(std::move(tests), std::move(properties));
+  if (!parallel->SetTests(std::move(tests), std::move(properties))) {
+    return false;
+  }
   parallel->SetPassFailVectors(&passed, &failed);
   this->TestResults.clear();
   parallel->SetTestResults(&this->TestResults);
diff --git a/Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake b/Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake
index 42e13fc..3c3e88f 100644
--- a/Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake
@@ -167,6 +167,16 @@ function(verify_ctest_resources)
   endif()
 endfunction()
 
+set(RunCMake_TEST_BINARY_DIR "${RunCMake_BINARY_DIR}/invalid-build")
+set(RunCMake_TEST_NO_CLEAN 1)
+file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "add_test(true \"${CMAKE_COMMAND}\" -E true)\n")
+run_cmake_command(invalid-nofile-ctest ${CMAKE_CTEST_COMMAND} --resource-spec-file "${RunCMake_BINARY_DIR}/noexist.json")
+run_cmake_command(invalid-not-json-ctest ${CMAKE_CTEST_COMMAND} --resource-spec-file "${RunCMake_SOURCE_DIR}/invalid.json")
+unset(RunCMake_TEST_NO_CLEAN)
+unset(RunCMake_TEST_BINARY_DIR)
+
 run_ctest_resource(lotsoftests 10 1 0)
 run_ctest_resource(checkfree1 2 0 1)
 run_ctest_resource(checkfree2 1 0 0)
diff --git a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-ctest-s-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-ctest-s-stderr.txt
index 397ca38..0a927e7 100644
--- a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-ctest-s-stderr.txt
+++ b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-ctest-s-stderr.txt
@@ -1,3 +1,5 @@
 ^Error: a cycle exists in the test dependency graph for the test "GenerateSpecFile"\.
-Please fix the cycle and run ctest again.
-No tests were found!!!$
+Please fix the cycle and run ctest again\.
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures-ctest-s-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures-ctest-s-stderr.txt
index 06ea90f..533b07c 100644
--- a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures-ctest-s-stderr.txt
+++ b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures-ctest-s-stderr.txt
@@ -1,2 +1,4 @@
 ^All tests that have RESOURCE_GROUPS must include the resource spec generator fixture in their FIXTURES_REQUIRED
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec-ctest-s-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec-ctest-s-stderr.txt
index 4e4c01c..d91d6dc 100644
--- a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec-ctest-s-stderr.txt
+++ b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec-ctest-s-stderr.txt
@@ -1,2 +1,4 @@
 ^GENERATED_RESOURCE_SPEC_FILE test property cannot be used in conjunction with ResourceSpecFile option
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators-ctest-s-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators-ctest-s-stderr.txt
index 273cb80..def7a8b 100644
--- a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators-ctest-s-stderr.txt
+++ b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators-ctest-s-stderr.txt
@@ -1,2 +1,4 @@
 ^Only one test may define the GENERATED_RESOURCE_SPEC_FILE property
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures-ctest-s-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures-ctest-s-stderr.txt
index 39ee275..3301318 100644
--- a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures-ctest-s-stderr.txt
+++ b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures-ctest-s-stderr.txt
@@ -1,2 +1,4 @@
 ^Test that defines GENERATED_RESOURCE_SPEC_FILE must have exactly one FIXTURES_SETUP
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture-ctest-s-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture-ctest-s-stderr.txt
index 06ea90f..a3c4d20 100644
--- a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture-ctest-s-stderr.txt
+++ b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture-ctest-s-stderr.txt
@@ -1,2 +1,4 @@
 ^All tests that have RESOURCE_GROUPS must include the resource spec generator fixture in their FIXTURES_REQUIRED
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture-ctest-s-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture-ctest-s-stderr.txt
index 39ee275..92a93bb 100644
--- a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture-ctest-s-stderr.txt
+++ b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture-ctest-s-stderr.txt
@@ -1,2 +1,4 @@
 ^Test that defines GENERATED_RESOURCE_SPEC_FILE must have exactly one FIXTURES_SETUP
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path-ctest-s-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path-ctest-s-stderr.txt
index 2c4dff8..6829ffe 100644
--- a/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path-ctest-s-stderr.txt
+++ b/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path-ctest-s-stderr.txt
@@ -1,2 +1,4 @@
 ^GENERATED_RESOURCE_SPEC_FILE must be an absolute path
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$
diff --git a/Tests/RunCMake/CTestResourceAllocation/invalid-nofile-ctest-result.txt b/Tests/RunCMake/CTestResourceAllocation/invalid-nofile-ctest-result.txt
new file mode 100644
index 0000000..45a4fb7
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/invalid-nofile-ctest-result.txt
@@ -0,0 +1 @@
+8
diff --git a/Tests/RunCMake/CTestResourceAllocation/invalid-nofile-ctest-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/invalid-nofile-ctest-stderr.txt
new file mode 100644
index 0000000..df1135e
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/invalid-nofile-ctest-stderr.txt
@@ -0,0 +1,8 @@
+^Could not read/parse resource spec file [^
+]*/Tests/RunCMake/CTestResourceAllocation/noexist\.json:[ ]
+File not found: [^
+]*/Tests/RunCMake/CTestResourceAllocation/noexist.json
+Errors while running CTest
+Output from these tests are in: [^
+]*/Tests/RunCMake/CTestResourceAllocation/invalid-build/Testing/Temporary/LastTest\.log
+Use "--rerun-failed --output-on-failure" to re-run the failed cases verbosely\.$
diff --git a/Tests/RunCMake/CTestResourceAllocation/invalid-not-json-ctest-result.txt b/Tests/RunCMake/CTestResourceAllocation/invalid-not-json-ctest-result.txt
new file mode 100644
index 0000000..45a4fb7
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/invalid-not-json-ctest-result.txt
@@ -0,0 +1 @@
+8
diff --git a/Tests/RunCMake/CTestResourceAllocation/invalid-not-json-ctest-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/invalid-not-json-ctest-stderr.txt
new file mode 100644
index 0000000..5bd9d97
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/invalid-not-json-ctest-stderr.txt
@@ -0,0 +1,15 @@
+^Could not read/parse resource spec file [^
+]*/Tests/RunCMake/CTestResourceAllocation/invalid\.json:[ ]
+JSON Parse Error: [^
+]*/Tests/RunCMake/CTestResourceAllocation/invalid\.json:
+\* Line 1, Column 1
+  Syntax error: value, object or array expected\.
+(\* Line 1, Column 2
+  Extra non-whitespace after JSON value\.
+|\* Line 1, Column 1
+  A valid JSON document must be either an array or an object value\.
+)
+Errors while running CTest
+Output from these tests are in: [^
+]*/Tests/RunCMake/CTestResourceAllocation/invalid-build/Testing/Temporary/LastTest\.log
+Use "--rerun-failed --output-on-failure" to re-run the failed cases verbosely\.$
diff --git a/Tests/RunCMake/CTestResourceAllocation/invalid.json b/Tests/RunCMake/CTestResourceAllocation/invalid.json
new file mode 100644
index 0000000..4c861f8
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/invalid.json
@@ -0,0 +1 @@
+This is not a valid JSON file!
diff --git a/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stderr.txt b/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stderr.txt
index 1a45994..1973b2a 100644
--- a/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stderr.txt
+++ b/Tests/RunCMake/ctest_fixtures/cyclicCleanup-stderr.txt
@@ -1,3 +1,2 @@
-Error: a cycle exists in the test dependency graph for the test "cyclicCleanup".
-Please fix the cycle and run ctest again.
-No tests were found!!!
+^Error: a cycle exists in the test dependency graph for the test "cyclicCleanup"\.
+Please fix the cycle and run ctest again\.$
diff --git a/Tests/RunCMake/ctest_fixtures/cyclicSetup-stderr.txt b/Tests/RunCMake/ctest_fixtures/cyclicSetup-stderr.txt
index 2aba6c9..aee735a 100644
--- a/Tests/RunCMake/ctest_fixtures/cyclicSetup-stderr.txt
+++ b/Tests/RunCMake/ctest_fixtures/cyclicSetup-stderr.txt
@@ -1,3 +1,2 @@
-Error: a cycle exists in the test dependency graph for the test "cyclicSetup".
-Please fix the cycle and run ctest again.
-No tests were found!!!$
+^Error: a cycle exists in the test dependency graph for the test "cyclicSetup"\.
+Please fix the cycle and run ctest again\.$