ctest: Add a new --repeat-until-fail option

This option tells ctest to run each test N times until the test fails or
the N times have run. This is useful for finding random failing tests.

6 files changed, 131 insertions, 15 deletions

diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index eb33d8e..bd090db 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -121,6 +121,11 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
   this->RunningCount += GetProcessorsUsed(test);
 
   cmCTestRunTest* testRun = new cmCTestRunTest(this->TestHandler);
+  if(this->CTest->GetRepeatUntilFail())
+    {
+    testRun->SetRunUntilFailOn();
+    testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
+    }
   testRun->SetIndex(test);
   testRun->SetTestProperties(this->Properties[test]);
 
@@ -289,7 +294,13 @@ bool cmCTestMultiProcessHandler::CheckOutput()
     cmCTestRunTest* p = *i;
     int test = p->GetIndex();
 
-    if(p->EndTest(this->Completed, this->Total, true))
+    bool testResult = p->EndTest(this->Completed, this->Total, true);
+    if(p->StartAgain())
+      {
+      this->Completed--; // remove the completed test because run again
+      continue;
+      }
+    if(testResult)
       {
       this->Passed->push_back(p->GetTestProperties()->Name);
       }
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index 03131fd..6f72684 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -33,6 +33,9 @@ cmCTestRunTest::cmCTestRunTest(cmCTestTestHandler* handler)
   this->CompressedOutput = "";
   this->CompressionRatio = 2;
   this->StopTimePassed = false;
+  this->NumberOfRunsLeft = 1; // default to 1 run of the test
+  this->RunUntilFail = false; // default to run the test once
+  this->RunAgain = false;   // default to not having to run again
 }
 
 cmCTestRunTest::~cmCTestRunTest()
@@ -357,13 +360,50 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
     this->MemCheckPostProcess();
     this->ComputeWeightedCost();
     }
-  // Always push the current TestResult onto the
+  // If the test does not need to rerun push the current TestResult onto the
   // TestHandler vector
-  this->TestHandler->TestResults.push_back(this->TestResult);
+  if(!this->NeedsToRerun())
+    {
+    this->TestHandler->TestResults.push_back(this->TestResult);
+    }
   delete this->TestProcess;
   return passed;
 }
 
+bool cmCTestRunTest::StartAgain()
+{
+  if(!this->RunAgain)
+    {
+    return false;
+    }
+  this->RunAgain = false; // reset
+  // change to tests directory
+  std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
+  cmSystemTools::ChangeDirectory(this->TestProperties->Directory);
+  this->StartTest(this->TotalNumberOfTests);
+  // change back
+  cmSystemTools::ChangeDirectory(current_dir);
+  return true;
+}
+
+bool cmCTestRunTest::NeedsToRerun()
+{
+  this->NumberOfRunsLeft--;
+  if(this->NumberOfRunsLeft == 0)
+    {
+    return false;
+    }
+  // if number of runs left is not 0, and we are running until
+  // we find a failed test, then return true so the test can be
+  // restarted
+  if(this->RunUntilFail
+     && this->TestResult.Status == cmCTestTestHandler::COMPLETED)
+    {
+    this->RunAgain = true;
+    return true;
+    }
+  return false;
+}
 //----------------------------------------------------------------------
 void cmCTestRunTest::ComputeWeightedCost()
 {
@@ -400,6 +440,7 @@ void cmCTestRunTest::MemCheckPostProcess()
 // Starts the execution of a test.  Returns once it has started
 bool cmCTestRunTest::StartTest(size_t total)
 {
+  this->TotalNumberOfTests = total; // save for rerun case
   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(2*getNumWidth(total) + 8)
     << "Start "
     << std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
@@ -494,10 +535,10 @@ bool cmCTestRunTest::StartTest(size_t total)
 //----------------------------------------------------------------------
 void cmCTestRunTest::ComputeArguments()
 {
+  this->Arguments.clear(); // reset becaue this might be a rerun
   std::vector<std::string>::const_iterator j =
     this->TestProperties->Args.begin();
   ++j; // skip test name
-
   // find the test executable
   if(this->TestHandler->MemCheck)
     {
@@ -682,10 +723,28 @@ bool cmCTestRunTest::ForkProcess(double testTimeOut, bool explicitTimeout,
 
 void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
 {
-  cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
-             << completed << "/");
-  cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
-             << total << " ");
+  // if this is the last or only run of this test
+  // then print out completed / total
+  // Only issue is if a test fails and we are running until fail
+  // then it will never print out the completed / total, same would
+  // got for run until pass.  Trick is when this is called we don't
+  // yet know if we are passing or failing.
+  if(this->NumberOfRunsLeft == 1)
+    {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << completed << "/");
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << total << " ");
+    }
+  // if this is one of several runs of a test just print blank space
+  // to keep things neat
+  else
+    {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << " " << " ");
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+               << " " << " ");
+    }
 
   if ( this->TestHandler->MemCheck )
     {
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 476f3e1..3b5c831 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -27,6 +27,8 @@ public:
   cmCTestRunTest(cmCTestTestHandler* handler);
   ~cmCTestRunTest();
 
+  void SetNumberOfRuns(int n) {this->NumberOfRunsLeft = n;}
+  void SetRunUntilFailOn() { this->RunUntilFail = true;}
   void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties * prop)
   { this->TestProperties = prop; }
 
@@ -58,7 +60,10 @@ public:
   void ComputeArguments();
 
   void ComputeWeightedCost();
+
+  bool StartAgain();
 private:
+  bool NeedsToRerun();
   void DartProcessing();
   void ExeNotFound(std::string exe);
   // Figures out a final timeout which is min(STOP_TIME, NOW+TIMEOUT)
@@ -92,6 +97,10 @@ private:
   std::string ActualCommand;
   std::vector<std::string> Arguments;
   bool StopTimePassed;
+  bool RunUntilFail;
+  int NumberOfRunsLeft;
+  bool RunAgain;
+  size_t TotalNumberOfTests;
 };
 
 inline int getNumWidth(size_t n)
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index e6d3960..2cbdf9c 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -328,6 +328,8 @@ cmCTest::cmCTest()
   this->OutputTestOutputOnTestFailure = false;
   this->ComputedCompressTestOutput = false;
   this->ComputedCompressMemCheckOutput = false;
+  this->RepeatTests = 1; // default to run each test once
+  this->RepeatUntilFail = false;
   if(cmSystemTools::GetEnv("CTEST_OUTPUT_ON_FAILURE"))
     {
     this->OutputTestOutputOnTestFailure = true;
@@ -1983,11 +1985,11 @@ bool cmCTest::CheckArgument(const std::string& arg, const char* varg1,
 //----------------------------------------------------------------------
 // Processes one command line argument (and its arguments if any)
 // for many simple options and then returns
-void cmCTest::HandleCommandLineArguments(size_t &i,
-                                         std::vector<std::string> &args)
+bool cmCTest::HandleCommandLineArguments(size_t &i,
+                                         std::vector<std::string> &args,
+                                         std::string& errormsg)
 {
   std::string arg = args[i];
-
   if(this->CheckArgument(arg, "-F"))
     {
     this->Failover = true;
@@ -2005,6 +2007,27 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
     this->SetParallelLevel(plevel);
     this->ParallelLevelSetInCli = true;
     }
+  if(this->CheckArgument(arg, "--repeat-until-fail"))
+    {
+    if( i >= args.size() - 1)
+      {
+      errormsg = "'--repeat-until-fail' requires an argument";
+      return false;
+      }
+    i++;
+    long repeat = 1;
+    if(!cmSystemTools::StringToLong(args[i].c_str(), &repeat))
+      {
+      errormsg = "'--repeat-until-fail' given non-integer value '"
+        + args[i] + "'";
+      return false;
+      }
+    this->RepeatTests = static_cast<int>(repeat);
+    if(repeat > 1)
+      {
+      this->RepeatUntilFail = true;
+      }
+    }
 
   if(this->CheckArgument(arg, "--no-compress-output"))
     {
@@ -2190,6 +2213,7 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
     this->GetHandler("test")->SetPersistentOption("RerunFailed", "true");
     this->GetHandler("memcheck")->SetPersistentOption("RerunFailed", "true");
     }
+  return true;
 }
 
 //----------------------------------------------------------------------
@@ -2272,7 +2296,12 @@ int cmCTest::Run(std::vector<std::string> &args, std::string* output)
   for(size_t i=1; i < args.size(); ++i)
     {
     // handle the simple commandline arguments
-    this->HandleCommandLineArguments(i,args);
+    std::string errormsg;
+    if(!this->HandleCommandLineArguments(i,args, errormsg))
+      {
+      cmSystemTools::Error(errormsg.c_str());
+      return 1;
+      }
 
     // handle the script arguments -S -SR -SP
     this->HandleScriptArguments(i,args,SRArgumentSpecified);
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index 88191c4..3f033d9 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -429,8 +429,13 @@ public:
     {
     return this->Definitions;
     }
-
+  // return the number of times a test should be run
+  int GetTestRepeat() { return this->RepeatTests;}
+  // return true if test should run until fail
+  bool GetRepeatUntilFail() { return this->RepeatUntilFail;}
 private:
+  int RepeatTests;
+  bool RepeatUntilFail;
   std::string ConfigType;
   std::string ScheduleType;
   std::string StopTime;
@@ -535,8 +540,9 @@ private:
   bool AddVariableDefinition(const std::string &arg);
 
   //! parse and process most common command line arguments
-  void HandleCommandLineArguments(size_t &i,
-                                  std::vector<std::string> &args);
+  bool HandleCommandLineArguments(size_t &i,
+                                  std::vector<std::string> &args,
+                                  std::string& errormsg);
 
   //! hande the -S -SP and -SR arguments
   void HandleScriptArguments(size_t &i,
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index c0eb8ac..0fc47b7 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -75,6 +75,8 @@ static const char * cmDocumentationOptions[][2] =
    "Run a specific number of tests by number."},
   {"-U, --union", "Take the Union of -I and -R"},
   {"--rerun-failed", "Run only the tests that failed previously"},
+  {"--repeat-until-fail <n>", "Require each test to run <n> "
+   "times without failing in order to pass"},
   {"--max-width <width>", "Set the max width for a test name to output"},
   {"--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1."},
   {"--no-label-summary", "Disable timing summary information for labels."},