ctest: Add --repeat-until-pass option

Add an option to re-run tests if they fail.  This will help tolerate
sporadic failures.

Issue: #17010
Co-Author: Ben Boeckel <ben.boeckel@kitware.com>
Co-Author: Chuck Atkins <chuck.atkins@kitware.com>

3 files changed, 22 insertions, 11 deletions

diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index 7e8d548..4812f30 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -171,8 +171,8 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
   this->RunningCount += GetProcessorsUsed(test);
 
   cmCTestRunTest* testRun = new cmCTestRunTest(*this);
-  if (this->CTest->GetRepeatUntilFail()) {
-    testRun->SetRunUntilFailOn();
+  if (this->CTest->GetRerunMode() != cmCTest::Rerun::Never) {
+    testRun->SetRerunMode(this->CTest->GetRerunMode());
     testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
   }
   testRun->SetIndex(test);
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index 2e7836a..ce9e13b 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -340,10 +340,12 @@ bool cmCTestRunTest::NeedsToRerun()
     return false;
   }
   // if number of runs left is not 0, and we are running until
-  // we find a failed test, then return true so the test can be
+  // we find a failed (or passed) test, then return true so the test can be
   // restarted
-  if (this->RunUntilFail &&
-      this->TestResult.Status == cmCTestTestHandler::COMPLETED) {
+  if ((this->RerunMode == cmCTest::Rerun::UntilFail &&
+       this->TestResult.Status == cmCTestTestHandler::COMPLETED) ||
+      (this->RerunMode == cmCTest::Rerun::UntilPass &&
+       this->TestResult.Status != cmCTestTestHandler::COMPLETED)) {
     this->RunAgain = true;
     return true;
   }
@@ -743,7 +745,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
   // then it will never print out the completed / total, same would
   // got for run until pass.  Trick is when this is called we don't
   // yet know if we are passing or failing.
-  if (this->NumberOfRunsLeft == 1 || this->CTest->GetTestProgressOutput()) {
+  if ((this->RerunMode != cmCTest::Rerun::UntilPass &&
+       this->NumberOfRunsLeft == 1) ||
+      (this->RerunMode == cmCTest::Rerun::UntilPass &&
+       this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
+      this->CTest->GetTestProgressOutput()) {
     outputStream << std::setw(getNumWidth(total)) << completed << "/";
     outputStream << std::setw(getNumWidth(total)) << total << " ";
   }
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 7b202d1..881cbb6 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -13,13 +13,12 @@
 
 #include <stddef.h>
 
+#include "cmCTest.h"
 #include "cmCTestMultiProcessHandler.h"
 #include "cmCTestTestHandler.h"
 #include "cmDuration.h"
 #include "cmProcess.h"
 
-class cmCTest;
-
 /** \class cmRunTest
  * \brief represents a single test to be run
  *
@@ -30,8 +29,13 @@ class cmCTestRunTest
 public:
   explicit cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler);
 
-  void SetNumberOfRuns(int n) { this->NumberOfRunsLeft = n; }
-  void SetRunUntilFailOn() { this->RunUntilFail = true; }
+  void SetNumberOfRuns(int n)
+  {
+    this->NumberOfRunsLeft = n;
+    this->NumberOfRunsTotal = n;
+  }
+
+  void SetRerunMode(cmCTest::Rerun r) { this->RerunMode = r; }
   void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties* prop)
   {
     this->TestProperties = prop;
@@ -129,8 +133,9 @@ private:
   std::vector<std::map<
     std::string, std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>>
     AllocatedHardware;
-  bool RunUntilFail = false; // default to run the test once
+  cmCTest::Rerun RerunMode = cmCTest::Rerun::Never;
   int NumberOfRunsLeft = 1;  // default to 1 run of the test
+  int NumberOfRunsTotal = 1; // default to 1 run of the test
   bool RunAgain = false;     // default to not having to run again
   size_t TotalNumberOfTests;
 };