CTest: Add support for test fixtures

Add new test properties:

* FIXTURES_SETUP
* FIXTURES_CLEANUP
* FIXTURES_REQUIRED

to specify the roles and dependencies of tests providing/using
test fixtures.

5 files changed, 233 insertions, 2 deletions

diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index d65c915..d8e1312 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -137,6 +137,16 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
   testRun->SetIndex(test);
   testRun->SetTestProperties(this->Properties[test]);
 
+  // Find any failed dependencies for this test. We assume the more common
+  // scenario has no failed tests, so make it the outer loop.
+  for (std::vector<std::string>::const_iterator it = this->Failed->begin();
+       it != this->Failed->end(); ++it) {
+    if (this->Properties[test]->RequireSuccessDepends.find(*it) !=
+        this->Properties[test]->RequireSuccessDepends.end()) {
+      testRun->AddFailedDependency(*it);
+    }
+  }
+
   std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
   cmSystemTools::ChangeDirectory(this->Properties[test]->Directory);
 
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index e7e51d5..fb53a7e 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -177,7 +177,8 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
     passIt;
   bool forceFail = false;
   bool outputTestErrorsToConsole = false;
-  if (!this->TestProperties->RequiredRegularExpressions.empty()) {
+  if (!this->TestProperties->RequiredRegularExpressions.empty() &&
+      this->FailedDependencies.empty()) {
     bool found = false;
     for (passIt = this->TestProperties->RequiredRegularExpressions.begin();
          passIt != this->TestProperties->RequiredRegularExpressions.end();
@@ -201,7 +202,8 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
     }
     reason += "]";
   }
-  if (!this->TestProperties->ErrorRegularExpressions.empty()) {
+  if (!this->TestProperties->ErrorRegularExpressions.empty() &&
+      this->FailedDependencies.empty()) {
     for (passIt = this->TestProperties->ErrorRegularExpressions.begin();
          passIt != this->TestProperties->ErrorRegularExpressions.end();
          ++passIt) {
@@ -437,6 +439,23 @@ bool cmCTestRunTest::StartTest(size_t total)
   this->TestResult.Name = this->TestProperties->Name;
   this->TestResult.Path = this->TestProperties->Directory;
 
+  if (!this->FailedDependencies.empty()) {
+    this->TestProcess = new cmProcess;
+    std::string msg = "Failed test dependencies:";
+    for (std::set<std::string>::const_iterator it =
+           this->FailedDependencies.begin();
+         it != this->FailedDependencies.end(); ++it) {
+      msg += " " + *it;
+    }
+    *this->TestHandler->LogFile << msg << std::endl;
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, msg << std::endl);
+    this->TestResult.Output = msg;
+    this->TestResult.FullCommandLine = "";
+    this->TestResult.CompletionStatus = "Not Run";
+    this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
+    return false;
+  }
+
   if (args.size() >= 2 && args[1] == "NOT_AVAILABLE") {
     this->TestProcess = new cmProcess;
     std::string msg;
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 0ca434d..bae633d 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -49,6 +49,11 @@ public:
 
   int GetIndex() { return this->Index; }
 
+  void AddFailedDependency(const std::string& failedTest)
+  {
+    this->FailedDependencies.insert(failedTest);
+  }
+
   std::string GetProcessOutput() { return this->ProcessOutput; }
 
   bool IsStopTimePassed() { return this->StopTimePassed; }
@@ -106,6 +111,7 @@ private:
   // The test results
   cmCTestTestHandler::cmCTestTestResult TestResult;
   int Index;
+  std::set<std::string> FailedDependencies;
   std::string StartTime;
   std::string ActualCommand;
   std::vector<std::string> Arguments;
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index 479c516..18e24d6 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -33,6 +33,7 @@
 #include <cmsys/RegularExpression.hxx>
 #include <functional>
 #include <iomanip>
+#include <iterator>
 #include <set>
 #include <sstream>
 #include <stdio.h>
@@ -762,6 +763,9 @@ void cmCTestTestHandler::ComputeTestList()
     it->Index = cnt; // save the index into the test list for this test
     finalList.push_back(*it);
   }
+
+  UpdateForFixtures(finalList);
+
   // Save the total number of tests before exclusions
   this->TotalNumberOfTests = this->TestList.size();
   // Set the TestList to the final list of all test
@@ -791,6 +795,8 @@ void cmCTestTestHandler::ComputeTestListForRerunFailed()
     finalList.push_back(*it);
   }
 
+  UpdateForFixtures(finalList);
+
   // Save the total number of tests before exclusions
   this->TotalNumberOfTests = this->TestList.size();
 
@@ -800,6 +806,169 @@ void cmCTestTestHandler::ComputeTestListForRerunFailed()
   this->UpdateMaxTestNameWidth();
 }
 
+void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const
+{
+  cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+                     "Updating test list for fixtures" << std::endl,
+                     this->Quiet);
+
+  // Prepare some maps to help us find setup and cleanup tests for
+  // any given fixture
+  typedef std::set<ListOfTests::const_iterator> TestIteratorSet;
+  typedef std::map<std::string, TestIteratorSet> FixtureDependencies;
+  FixtureDependencies fixtureSetups;
+  FixtureDependencies fixtureDeps;
+
+  for (ListOfTests::const_iterator it = this->TestList.begin();
+       it != this->TestList.end(); ++it) {
+    const cmCTestTestProperties& p = *it;
+
+    const std::set<std::string>& setups = p.FixturesSetup;
+    for (std::set<std::string>::const_iterator depsIt = setups.begin();
+         depsIt != setups.end(); ++depsIt) {
+      fixtureSetups[*depsIt].insert(it);
+      fixtureDeps[*depsIt].insert(it);
+    }
+
+    const std::set<std::string>& cleanups = p.FixturesCleanup;
+    for (std::set<std::string>::const_iterator depsIt = cleanups.begin();
+         depsIt != cleanups.end(); ++depsIt) {
+      fixtureDeps[*depsIt].insert(it);
+    }
+  }
+
+  // Prepare fast lookup of tests already included in our list of tests
+  std::set<std::string> addedTests;
+  for (ListOfTests::const_iterator it = tests.begin(); it != tests.end();
+       ++it) {
+    const cmCTestTestProperties& p = *it;
+    addedTests.insert(p.Name);
+  }
+
+  // This is a lookup of fixture name to a list of indices into the
+  // final tests array for tests which require that fixture. It is
+  // needed at the end to populate dependencies of the cleanup tests
+  // in our final list of tests.
+  std::map<std::string, std::vector<size_t> > fixtureRequirements;
+
+  // Use integer index for iteration because we append to
+  // the tests vector as we go
+  size_t fixtureTestsAdded = 0;
+  std::set<std::string> addedFixtures;
+  for (size_t i = 0; i < tests.size(); ++i) {
+    if (tests[i].FixturesRequired.empty()) {
+      continue;
+    }
+    // Must copy the set of fixtures because we may invalidate
+    // the tests array by appending to it
+    const std::set<std::string> fixtures = tests[i].FixturesRequired;
+    for (std::set<std::string>::const_iterator fixturesIt = fixtures.begin();
+         fixturesIt != fixtures.end(); ++fixturesIt) {
+
+      const std::string& requiredFixtureName = *fixturesIt;
+      if (requiredFixtureName.empty()) {
+        continue;
+      }
+
+      fixtureRequirements[requiredFixtureName].push_back(i);
+
+      // Add dependencies to this test for all of the setup tests
+      // associated with the required fixture. If any of those setup
+      // tests fail, this test should not run. We make the fixture's
+      // cleanup tests depend on this test case later.
+      FixtureDependencies::const_iterator setupIt =
+        fixtureSetups.find(requiredFixtureName);
+      if (setupIt != fixtureSetups.end()) {
+        for (TestIteratorSet::const_iterator sIt = setupIt->second.begin();
+             sIt != setupIt->second.end(); ++sIt) {
+          const std::string& setupTestName = (**sIt).Name;
+          tests[i].RequireSuccessDepends.insert(setupTestName);
+          if (std::find(tests[i].Depends.begin(), tests[i].Depends.end(),
+                        setupTestName) == tests[i].Depends.end()) {
+            tests[i].Depends.push_back(setupTestName);
+          }
+        }
+      }
+
+      // Append any fixture setup/cleanup tests to our test list if they
+      // are not already in it (they could have been in the original
+      // set of tests passed to us at the outset or have already been
+      // added from a previously checked test). A fixture isn't required
+      // to have setup/cleanup tests.
+      if (!addedFixtures.insert(requiredFixtureName).second) {
+        // Already added this fixture
+        continue;
+      }
+      FixtureDependencies::const_iterator fixtureIt =
+        fixtureDeps.find(requiredFixtureName);
+      if (fixtureIt == fixtureDeps.end()) {
+        // No setup or cleanup tests for this fixture
+        continue;
+      }
+
+      const TestIteratorSet& testIters = fixtureIt->second;
+      for (TestIteratorSet::const_iterator depsIt = testIters.begin();
+           depsIt != testIters.end(); ++depsIt) {
+        ListOfTests::const_iterator lotIt = *depsIt;
+        const cmCTestTestProperties& p = *lotIt;
+
+        if (!addedTests.insert(p.Name).second) {
+          // Already have p in our test list
+          continue;
+        }
+
+        // This is a test not yet in our list, so add it and
+        // update its index to reflect where it was in the original
+        // full list of all tests (needed to track individual tests
+        // across ctest runs for re-run failed, etc.)
+        tests.push_back(p);
+        tests.back().Index =
+          1 + static_cast<int>(std::distance(this->TestList.begin(), lotIt));
+        ++fixtureTestsAdded;
+
+        cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Added test "
+                             << p.Name << " required by fixture "
+                             << requiredFixtureName << std::endl,
+                           this->Quiet);
+      }
+    }
+  }
+
+  // Now that we have the final list of tests, we can update all cleanup
+  // tests to depend on those tests which require that fixture
+  for (ListOfTests::iterator tIt = tests.begin(); tIt != tests.end(); ++tIt) {
+    cmCTestTestProperties& p = *tIt;
+    const std::set<std::string>& cleanups = p.FixturesCleanup;
+    for (std::set<std::string>::const_iterator fIt = cleanups.begin();
+         fIt != cleanups.end(); ++fIt) {
+      const std::string& fixture = *fIt;
+      std::map<std::string, std::vector<size_t> >::const_iterator cIt =
+        fixtureRequirements.find(fixture);
+      if (cIt == fixtureRequirements.end()) {
+        // No test cases require the fixture this cleanup test is for.
+        // This cleanup test must have been part of the original test
+        // list passed in (which is not an error)
+        continue;
+      }
+
+      const std::vector<size_t>& indices = cIt->second;
+      for (std::vector<size_t>::const_iterator indexIt = indices.begin();
+           indexIt != indices.end(); ++indexIt) {
+        const std::string& reqTestName = tests[*indexIt].Name;
+        if (std::find(p.Depends.begin(), p.Depends.end(), reqTestName) ==
+            p.Depends.end()) {
+          p.Depends.push_back(reqTestName);
+        }
+      }
+    }
+  }
+
+  cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Added "
+                       << fixtureTestsAdded
+                       << " tests to meet fixture requirements" << std::endl,
+                     this->Quiet);
+}
+
 void cmCTestTestHandler::UpdateMaxTestNameWidth()
 {
   std::string::size_type max = this->CTest->GetMaxTestNameWidth();
@@ -1829,6 +1998,24 @@ bool cmCTestTestHandler::SetTestsProperties(
 
             rtit->LockedResources.insert(lval.begin(), lval.end());
           }
+          if (key == "FIXTURES_SETUP") {
+            std::vector<std::string> lval;
+            cmSystemTools::ExpandListArgument(val, lval);
+
+            rtit->FixturesSetup.insert(lval.begin(), lval.end());
+          }
+          if (key == "FIXTURES_CLEANUP") {
+            std::vector<std::string> lval;
+            cmSystemTools::ExpandListArgument(val, lval);
+
+            rtit->FixturesCleanup.insert(lval.begin(), lval.end());
+          }
+          if (key == "FIXTURES_REQUIRED") {
+            std::vector<std::string> lval;
+            cmSystemTools::ExpandListArgument(val, lval);
+
+            rtit->FixturesRequired.insert(lval.begin(), lval.end());
+          }
           if (key == "TIMEOUT") {
             rtit->Timeout = atof(val.c_str());
             rtit->ExplicitTimeout = true;
diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h
index a5e62dc..beaf8e0 100644
--- a/Source/CTest/cmCTestTestHandler.h
+++ b/Source/CTest/cmCTestTestHandler.h
@@ -139,6 +139,10 @@ public:
     std::vector<std::string> Environment;
     std::vector<std::string> Labels;
     std::set<std::string> LockedResources;
+    std::set<std::string> FixturesSetup;
+    std::set<std::string> FixturesCleanup;
+    std::set<std::string> FixturesRequired;
+    std::set<std::string> RequireSuccessDepends;
   };
 
   struct cmCTestTestResult
@@ -251,6 +255,11 @@ private:
   // based on LastTestFailed.log
   void ComputeTestListForRerunFailed();
 
+  // add required setup/cleanup tests not already in the
+  // list of tests to be run and update dependencies between
+  // tests to account for fixture setup/cleanup
+  void UpdateForFixtures(ListOfTests& tests) const;
+
   void UpdateMaxTestNameWidth();
 
   bool GetValue(const char* tag, std::string& value, std::istream& fin);