From 8612aa10b681fb31bcf3e4959b151fc7f3a5442e Mon Sep 17 00:00:00 2001
From: Zach Mullen <zach.mullen@kitware.com>
Date: Thu, 29 Oct 2009 15:30:12 -0400
Subject: Hook for scheduling tests in a random order
This may help statistically detect implicit dependencies among unit
tests while running in parallel.
---
Source/CTest/cmCTestTestCommand.cxx | 6 ++++++
Source/CTest/cmCTestTestCommand.h | 8 ++++++--
Source/CTest/cmCTestTestHandler.cxx | 18 +++++++++++++++++-
Source/cmCTest.cxx | 6 ++++++
Source/cmCTest.h | 4 ++++
Source/ctest.cxx | 3 +++
Tests/CTestTest3/test.cmake.in | 4 ++--
7 files changed, 44 insertions(+), 5 deletions(-)
diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx
index a719b09..23cc20e 100644
--- a/Source/CTest/cmCTestTestCommand.cxx
+++ b/Source/CTest/cmCTestTestCommand.cxx
@@ -24,6 +24,7 @@ cmCTestTestCommand::cmCTestTestCommand()
this->Arguments[ctt_EXCLUDE_LABEL] = "EXCLUDE_LABEL";
this->Arguments[ctt_INCLUDE_LABEL] = "INCLUDE_LABEL";
this->Arguments[ctt_PARALLEL_LEVEL] = "PARALLEL_LEVEL";
+ this->Arguments[ctt_SCHEDULE_RANDOM] = "SCHEDULE_RANDOM";
this->Arguments[ctt_LAST] = 0;
this->Last = ctt_LAST;
}
@@ -91,6 +92,11 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
handler->SetOption("ParallelLevel",
this->Values[ctt_PARALLEL_LEVEL]);
}
+ if(this->Values[ctt_SCHEDULE_RANDOM])
+ {
+ handler->SetOption("ScheduleRandom",
+ this->Values[ctt_SCHEDULE_RANDOM]);
+ }
return handler;
}
diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h
index 73ce913..12314df 100644
--- a/Source/CTest/cmCTestTestCommand.h
+++ b/Source/CTest/cmCTestTestCommand.h
@@ -61,7 +61,8 @@ public:
" [INCLUDE include regex] [RETURN_VALUE res] \n"
" [EXCLUDE_LABEL exclude regex] \n"
" [INCLUDE_LABEL label regex] \n"
- " [PARALLEL_LEVEL level]) \n"
+ " [PARALLEL_LEVEL level] \n"
+ " [SCHEDULE_RANDOM on]) \n"
"Tests the given build directory and stores results in Test.xml. The "
"second argument is a variable that will hold value. Optionally, "
"you can specify the starting test number START, the ending test number "
@@ -70,7 +71,9 @@ public:
"to not run EXCLUDE. EXCLUDE_LABEL and INCLUDE_LABEL are regular "
"expression for test to be included or excluded by the test "
"property LABEL. PARALLEL_LEVEL should be set to a positive number "
- "representing the number of tests to be run in parallel."
+ "representing the number of tests to be run in parallel. "
+ "SCHEDULE_RANDOM will launch tests in a random order, and is "
+ "typically used to detect implicit test dependencies."
"\n"
CTEST_COMMAND_APPEND_OPTION_DOCS;
}
@@ -92,6 +95,7 @@ protected:
ctt_EXCLUDE_LABEL,
ctt_INCLUDE_LABEL,
ctt_PARALLEL_LEVEL,
+ ctt_SCHEDULE_RANDOM,
ctt_LAST
};
};
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index 3572b11..56ddec7 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -491,11 +491,16 @@ int cmCTestTestHandler::ProcessHandler()
{
// Update internal data structure from generic one
this->SetTestsToRunInformation(this->GetOption("TestsToRunInformation"));
- this->SetUseUnion(cmSystemTools::IsOn(this->GetOption("UseUnion")));
+ this->SetUseUnion(cmSystemTools::IsOn(this->GetOption("UseUnion")));
+ if(cmSystemTools::IsOn(this->GetOption("ScheduleRandom")))
+ {
+ this->CTest->SetScheduleType("Random");
+ }
if(this->GetOption("ParallelLevel"))
{
this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel")));
}
+
const char* val;
val = this->GetOption("LabelRegularExpression");
if ( val )
@@ -1021,12 +1026,23 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
cmCTestMultiProcessHandler::TestMap tests;
cmCTestMultiProcessHandler::PropertiesMap properties;
+ bool randomSchedule = this->CTest->GetScheduleType() == "Random";
+ if(randomSchedule)
+ {
+ srand((unsigned)time(0));
+ }
+
for (ListOfTests::iterator it = this->TestList.begin();
it != this->TestList.end(); ++it)
{
cmCTestTestProperties& p = *it;
cmCTestMultiProcessHandler::TestSet depends;
+ if(randomSchedule)
+ {
+ p.Cost = rand();
+ }
+
if(p.Depends.size())
{
for(std::vector<std::string>::iterator i = p.Depends.begin();
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 6d21ab5..fee94d1 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -225,6 +225,7 @@ cmCTest::cmCTest()
this->TimeOut = 0;
this->CompressXMLFiles = false;
this->CTestConfigFile = "";
+ this->ScheduleType = "";
this->OutputLogFile = 0;
this->OutputLogFileLastTag = -1;
this->SuppressUpdatingCTestConfiguration = false;
@@ -2027,6 +2028,11 @@ int cmCTest::Run(std::vector<std::string> &args, std::string* output)
cmakeAndTest = true;
}
+ if(this->CheckArgument(arg, "--schedule-random"))
+ {
+ this->ScheduleType = "Random";
+ }
+
// pass the argument to all the handlers as well, but i may no longer be
// set to what it was originally so I'm not sure this is working as
// intended
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index f401c55..47023f1 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -191,6 +191,9 @@ public:
///! Should we only show what we would do?
bool GetShowOnly();
+ //Used for parallel ctest job scheduling
+ std::string GetScheduleType() { return this->ScheduleType; }
+ void SetScheduleType(std::string type) { this->ScheduleType = type; }
///! The max output width
int GetMaxTestNameWidth() const;
@@ -374,6 +377,7 @@ public:
bool GetLabelSummary() { return this->LabelSummary;}
private:
std::string ConfigType;
+ std::string ScheduleType;
bool Verbose;
bool ExtraVerbose;
bool ProduceXML;
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index 2f99410..893e9cb 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -208,6 +208,9 @@ static const char * cmDocumentationOptions[][3] =
"By default CTest will run child CTest instances within the same process. "
"If this behavior is not desired, this argument will enforce new "
"processes for child CTest processes." },
+ {"--schedule-random", "Use a random order for scheduling tests",
+ "This option will run the tests in a random order. It is commonly used to "
+ "detect implicit dependencies in a test suite." },
{"--submit-index", "Submit individual dashboard tests with specific index",
"This option allows performing the same CTest action (such as test) "
"multiple times and submit all stages to the same dashboard (Dart2 "
diff --git a/Tests/CTestTest3/test.cmake.in b/Tests/CTestTest3/test.cmake.in
index 734bdf4..d0a1f88 100644
--- a/Tests/CTestTest3/test.cmake.in
+++ b/Tests/CTestTest3/test.cmake.in
@@ -50,7 +50,7 @@ CTEST_UPDATE(SOURCE "${CTEST_SOURCE_DIRECTORY}" RETURN_VALUE res)
CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_READ_CUSTOM_FILES("${CTEST_BINARY_DIRECTORY}")
CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
-CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
+CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5 SCHEDULE_RANDOM ON)
CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_SUBMIT(RETURN_VALUE res)
@@ -97,7 +97,7 @@ IF(svncommand)
CTEST_UPDATE(SOURCE "${CTEST_SOURCE_DIRECTORY}" RETURN_VALUE res)
CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
- CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5)
+ CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5 SCHEDULE_RANDOM ON)
CTEST_MEMCHECK(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res PARALLEL_LEVEL 5)
CTEST_COVERAGE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res)
CTEST_SUBMIT(RETURN_VALUE res)
--
cgit v0.12