ctest: Add a new --repeat-until-fail option

This option tells ctest to run each test N times until the test fails or
the N times have run. This is useful for finding random failing tests.

3 files changed, 87 insertions, 8 deletions

diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index eb33d8e..bd090db 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -121,6 +121,11 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
   this->RunningCount += GetProcessorsUsed(test);
 
   cmCTestRunTest* testRun = new cmCTestRunTest(this->TestHandler);
+  if(this->CTest->GetRepeatUntilFail())
+    {
+    testRun->SetRunUntilFailOn();
+    testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
+    }
   testRun->SetIndex(test);
   testRun->SetTestProperties(this->Properties[test]);
 
@@ -289,7 +294,13 @@ bool cmCTestMultiProcessHandler::CheckOutput()
     cmCTestRunTest* p = *i;
     int test = p->GetIndex();
 
-    if(p->EndTest(this->Completed, this->Total, true))
+    bool testResult = p->EndTest(this->Completed, this->Total, true);
+    if(p->StartAgain())
+      {
+      this->Completed--; // remove the completed test because run again
+      continue;
+      }
+    if(testResult)
       {
       this->Passed->push_back(p->GetTestProperties()->Name);
       }
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index 03131fd..6f72684 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -33,6 +33,9 @@ cmCTestRunTest::cmCTestRunTest(cmCTestTestHandler* handler)
   this->CompressedOutput = "";
   this->CompressionRatio = 2;
   this->StopTimePassed = false;
+  this->NumberOfRunsLeft = 1; // default to 1 run of the test
+  this->RunUntilFail = false; // default to run the test once
+  this->RunAgain = false;   // default to not having to run again
 }
 
 cmCTestRunTest::~cmCTestRunTest()
@@ -357,13 +360,50 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
     this->MemCheckPostProcess();
     this->ComputeWeightedCost();
     }
-  // Always push the current TestResult onto the
+  // If the test does not need to rerun push the current TestResult onto the
   // TestHandler vector
-  this->TestHandler->TestResults.push_back(this->TestResult);
+  if(!this->NeedsToRerun())
+    {
+    this->TestHandler->TestResults.push_back(this->TestResult);
+    }
   delete this->TestProcess;
   return passed;
 }
 
+bool cmCTestRunTest::StartAgain()
+{
+  if(!this->RunAgain)
+    {
+    return false;
+    }
+  this->RunAgain = false; // reset
+  // change to tests directory
+  std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
+  cmSystemTools::ChangeDirectory(this->TestProperties->Directory);
+  this->StartTest(this->TotalNumberOfTests);
+  // change back
+  cmSystemTools::ChangeDirectory(current_dir);
+  return true;
+}
+
+bool cmCTestRunTest::NeedsToRerun()
+{
+  this->NumberOfRunsLeft--;
+  if(this->NumberOfRunsLeft == 0)
+    {
+    return false;
+    }
+  // if number of runs left is not 0, and we are running until
+  // we find a failed test, then return true so the test can be
+  // restarted
+  if(this->RunUntilFail
+     && this->TestResult.Status == cmCTestTestHandler::COMPLETED)
+    {
+    this->RunAgain = true;
+    return true;
+    }
+  return false;
+}
 //----------------------------------------------------------------------
 void cmCTestRunTest::ComputeWeightedCost()
 {
@@ -400,6 +440,7 @@ void cmCTestRunTest::MemCheckPostProcess()
 // Starts the execution of a test.  Returns once it has started
 bool cmCTestRunTest::StartTest(size_t total)
 {
+  this->TotalNumberOfTests = total; // save for rerun case
   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(2*getNumWidth(total) + 8)
     << "Start "
     << std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
@@ -494,10 +535,10 @@ bool cmCTestRunTest::StartTest(size_t total)
 //----------------------------------------------------------------------
 void cmCTestRunTest::ComputeArguments()
 {
+  this->Arguments.clear(); // reset becaue this might be a rerun
   std::vector<std::string>::const_iterator j =
     this->TestProperties->Args.begin();
   ++j; // skip test name
-
   // find the test executable
   if(this->TestHandler->MemCheck)
     {
@@ -682,10 +723,28 @@ bool cmCTestRunTest::ForkProcess(double testTimeOut, bool explicitTimeout,
 
 void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
 {
-  cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
-             << completed << "/");
-  cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
-             << total << " ");
+  // if this is the last or only run of this test
+  // then print out completed / total
+  // Only issue is if a test fails and we are running until fail
+  // then it will never print out the completed / total, same would
+  // got for run until pass.  Trick is when this is called we don't
+  // yet know if we are passing or failing.
+  if(this->NumberOfRunsLeft == 1)
+    {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << completed << "/");
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << total << " ");
+    }
+  // if this is one of several runs of a test just print blank space
+  // to keep things neat
+  else
+    {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << " " << " ");
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << " " << " ");
+    }
 
   if ( this->TestHandler->MemCheck )
     {
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 476f3e1..3b5c831 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -27,6 +27,8 @@ public:
   cmCTestRunTest(cmCTestTestHandler* handler);
   ~cmCTestRunTest();
 
+  void SetNumberOfRuns(int n) {this->NumberOfRunsLeft = n;}
+  void SetRunUntilFailOn() { this->RunUntilFail = true;}
   void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties * prop)
   { this->TestProperties = prop; }
 
@@ -58,7 +60,10 @@ public:
   void ComputeArguments();
 
   void ComputeWeightedCost();
+
+  bool StartAgain();
 private:
+  bool NeedsToRerun();
   void DartProcessing();
   void ExeNotFound(std::string exe);
   // Figures out a final timeout which is min(STOP_TIME, NOW+TIMEOUT)
@@ -92,6 +97,10 @@ private:
   std::string ActualCommand;
   std::vector<std::string> Arguments;
   bool StopTimePassed;
+  bool RunUntilFail;
+  int NumberOfRunsLeft;
+  bool RunAgain;
+  size_t TotalNumberOfTests;
 };
 
 inline int getNumWidth(size_t n)