From f460bbd4c8afab56f93b9831f1f4d8336480fcc5 Mon Sep 17 00:00:00 2001 From: Betsy McPhail Date: Thu, 4 Oct 2018 11:34:27 -0400 Subject: ctest_submit: Refactor file list to use a vector instead of a set Remove duplicates in a way that preserves order. --- Source/CTest/cmCTestSubmitHandler.cxx | 51 ++++++++++++++++++++--------------- Source/CTest/cmCTestSubmitHandler.h | 12 ++++----- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/Source/CTest/cmCTestSubmitHandler.cxx b/Source/CTest/cmCTestSubmitHandler.cxx index ecf309a..38b5411 100644 --- a/Source/CTest/cmCTestSubmitHandler.cxx +++ b/Source/CTest/cmCTestSubmitHandler.cxx @@ -12,6 +12,7 @@ #include #include +#include "cmAlgorithms.h" #include "cmCTest.h" #include "cmCTestCurl.h" #include "cmCTestScriptHandler.h" @@ -152,10 +153,9 @@ void cmCTestSubmitHandler::Initialize() this->Files.clear(); } -bool cmCTestSubmitHandler::SubmitUsingFTP(const std::string& localprefix, - const std::set& files, - const std::string& remoteprefix, - const std::string& url) +bool cmCTestSubmitHandler::SubmitUsingFTP( + const std::string& localprefix, const std::vector& files, + const std::string& remoteprefix, const std::string& url) { CURL* curl; CURLcode res; @@ -299,10 +299,9 @@ bool cmCTestSubmitHandler::SubmitUsingFTP(const std::string& localprefix, } // Uploading files is simpler -bool cmCTestSubmitHandler::SubmitUsingHTTP(const std::string& localprefix, - const std::set& files, - const std::string& remoteprefix, - const std::string& url) +bool cmCTestSubmitHandler::SubmitUsingHTTP( + const std::string& localprefix, const std::vector& files, + const std::string& remoteprefix, const std::string& url) { CURL* curl; CURLcode res; @@ -662,9 +661,9 @@ void cmCTestSubmitHandler::ParseResponse( } } -bool cmCTestSubmitHandler::TriggerUsingHTTP(const std::set& files, - const std::string& remoteprefix, - const std::string& url) +bool cmCTestSubmitHandler::TriggerUsingHTTP( + const std::vector& files, const std::string& remoteprefix, + const std::string& url) { CURL* curl; char error_buffer[1024]; @@ -792,11 +791,10 @@ bool cmCTestSubmitHandler::TriggerUsingHTTP(const std::set& files, return true; } -bool cmCTestSubmitHandler::SubmitUsingSCP(const std::string& scp_command, - const std::string& localprefix, - const std::set& files, - const std::string& remoteprefix, - const std::string& url) +bool cmCTestSubmitHandler::SubmitUsingSCP( + const std::string& scp_command, const std::string& localprefix, + const std::vector& files, const std::string& remoteprefix, + const std::string& url) { if (scp_command.empty() || localprefix.empty() || files.empty() || remoteprefix.empty() || url.empty()) { @@ -890,7 +888,7 @@ bool cmCTestSubmitHandler::SubmitUsingSCP(const std::string& scp_command, } bool cmCTestSubmitHandler::SubmitUsingCP(const std::string& localprefix, - const std::set& files, + const std::vector& files, const std::string& remoteprefix, const std::string& destination) { @@ -925,7 +923,7 @@ bool cmCTestSubmitHandler::SubmitUsingCP(const std::string& localprefix, #if defined(CTEST_USE_XMLRPC) bool cmCTestSubmitHandler::SubmitUsingXMLRPC( - const std::string& localprefix, const std::set& files, + const std::string& localprefix, const std::vector& files, const std::string& remoteprefix, const std::string& url) { xmlrpc_env env; @@ -1020,7 +1018,7 @@ bool cmCTestSubmitHandler::SubmitUsingXMLRPC( } #else bool cmCTestSubmitHandler::SubmitUsingXMLRPC( - std::string const& /*unused*/, std::set const& /*unused*/, + std::string const& /*unused*/, std::vector const& /*unused*/, std::string const& /*unused*/, std::string const& /*unused*/) { return false; @@ -1351,13 +1349,13 @@ int cmCTestSubmitHandler::ProcessHandler() cmGeneratedFileStream ofs; this->StartLogFile("Submit", ofs); - cmCTest::SetOfStrings files; + std::vector files; std::string prefix = this->GetSubmitResultsPrefix(); if (!this->Files.empty()) { // Submit the explicitly selected files: // - files.insert(this->Files.begin(), this->Files.end()); + files.insert(files.end(), this->Files.begin(), this->Files.end()); } // Add to the list of files to submit from any selected, existing parts: @@ -1404,7 +1402,16 @@ int cmCTestSubmitHandler::ProcessHandler() // Submit files from this part. std::vector const& pfiles = this->CTest->GetSubmitFiles(p); - files.insert(pfiles.begin(), pfiles.end()); + files.insert(files.end(), pfiles.begin(), pfiles.end()); + } + + // Make sure files are unique, but preserve order. + { + // This endPos intermediate is needed to work around non-conformant C++11 + // standard libraries that have erase(iterator,iterator) instead of + // erase(const_iterator,const_iterator). + size_t endPos = cmRemoveDuplicates(files) - files.cbegin(); + files.erase(files.begin() + endPos, files.end()); } if (ofs) { diff --git a/Source/CTest/cmCTestSubmitHandler.h b/Source/CTest/cmCTestSubmitHandler.h index b4d0e77..66f2173 100644 --- a/Source/CTest/cmCTestSubmitHandler.h +++ b/Source/CTest/cmCTestSubmitHandler.h @@ -57,27 +57,27 @@ private: * Submit file using various ways */ bool SubmitUsingFTP(const std::string& localprefix, - const std::set& files, + const std::vector& files, const std::string& remoteprefix, const std::string& url); bool SubmitUsingHTTP(const std::string& localprefix, - const std::set& files, + const std::vector& files, const std::string& remoteprefix, const std::string& url); bool SubmitUsingSCP(const std::string& scp_command, const std::string& localprefix, - const std::set& files, + const std::vector& files, const std::string& remoteprefix, const std::string& url); bool SubmitUsingCP(const std::string& localprefix, - const std::set& files, + const std::vector& files, const std::string& remoteprefix, const std::string& url); - bool TriggerUsingHTTP(const std::set& files, + bool TriggerUsingHTTP(const std::vector& files, const std::string& remoteprefix, const std::string& url); bool SubmitUsingXMLRPC(const std::string& localprefix, - const std::set& files, + const std::vector& files, const std::string& remoteprefix, const std::string& url); -- cgit v0.12 From a6e0158712bd25256fb66cf21c047b81d65bc4fe Mon Sep 17 00:00:00 2001 From: Betsy McPhail Date: Thu, 4 Oct 2018 11:34:27 -0400 Subject: ctest_submit: Add support for a "Done" part Teach CTest to submit Done.xml. Submission of this file indicates to CDash that a build is complete and no more files will be uploaded. It contains the build id returned by CDash and the current time. This file is submitted last for a given build when using the `ctest_submit()` command. If submitting by PARTS, use `ctest_submit(PARTS Done)`. --- Help/command/ctest_submit.rst | 1 + Help/release/dev/ctest-done.rst | 5 +++++ Source/CTest/cmCTestSubmitHandler.cxx | 10 ++++++++++ Source/cmCTest.cxx | 21 +++++++++++++++++++++ Source/cmCTest.h | 10 ++++++++++ Tests/RunCMake/ctest_submit/PARTSDone-result.txt | 1 + Tests/RunCMake/ctest_submit/PARTSDone-stderr.txt | 3 +++ Tests/RunCMake/ctest_submit/RunCMakeTest.cmake | 1 + 8 files changed, 52 insertions(+) create mode 100644 Help/release/dev/ctest-done.rst create mode 100644 Tests/RunCMake/ctest_submit/PARTSDone-result.txt create mode 100644 Tests/RunCMake/ctest_submit/PARTSDone-stderr.txt diff --git a/Help/command/ctest_submit.rst b/Help/command/ctest_submit.rst index 2ba6bef..426475c 100644 --- a/Help/command/ctest_submit.rst +++ b/Help/command/ctest_submit.rst @@ -33,6 +33,7 @@ The options are: ExtraFiles = Files listed by CTEST_EXTRA_SUBMIT_FILES Upload = Files prepared for upload by ctest_upload(), in Upload.xml Submit = nothing + Done = Build is complete, in Done.xml ``FILES ...`` Specify an explicit list of specific files to be submitted. diff --git a/Help/release/dev/ctest-done.rst b/Help/release/dev/ctest-done.rst new file mode 100644 index 0000000..9ec0e24 --- /dev/null +++ b/Help/release/dev/ctest-done.rst @@ -0,0 +1,5 @@ +ctest-done +---------- + +* The :command:`ctest_submit` command learned a new ``Done`` part that can be used + to inform CDash that a build is complete and that no more parts will be uploaded. diff --git a/Source/CTest/cmCTestSubmitHandler.cxx b/Source/CTest/cmCTestSubmitHandler.cxx index 38b5411..6ad0e03 100644 --- a/Source/CTest/cmCTestSubmitHandler.cxx +++ b/Source/CTest/cmCTestSubmitHandler.cxx @@ -56,6 +56,7 @@ public: std::string Filename; std::string MD5; std::string Message; + std::string BuildID; private: std::vector CurrentValue; @@ -97,6 +98,8 @@ private: this->MD5 = this->GetCurrentValue(); } else if (name == "message") { this->Message = this->GetCurrentValue(); + } else if (name == "buildId") { + this->BuildID = this->GetCurrentValue(); } } }; @@ -645,6 +648,7 @@ void cmCTestSubmitHandler::ParseResponse( " Submission failed: " << parser.Message << std::endl); return; } + this->CTest->SetBuildID(parser.BuildID); } output = cmSystemTools::UpperCase(output); if (output.find("WARNING") != std::string::npos) { @@ -1414,6 +1418,12 @@ int cmCTestSubmitHandler::ProcessHandler() files.erase(files.begin() + endPos, files.end()); } + // Submit Done.xml last + if (this->SubmitPart[cmCTest::PartDone]) { + this->CTest->GenerateDoneFile(); + files.push_back("Done.xml"); + } + if (ofs) { ofs << "Upload files:" << std::endl; int cnt = 0; diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 908eea1..d0d5db6 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -294,6 +294,7 @@ cmCTest::cmCTest() this->SuppressUpdatingCTestConfiguration = false; this->DartVersion = 1; this->DropSiteCDash = false; + this->BuildID = ""; this->OutputTestOutputOnTestFailure = false; this->RepeatTests = 1; // default to run each test once this->RepeatUntilFail = false; @@ -320,6 +321,7 @@ cmCTest::cmCTest() this->Parts[PartNotes].SetName("Notes"); this->Parts[PartExtraFiles].SetName("ExtraFiles"); this->Parts[PartUpload].SetName("Upload"); + this->Parts[PartDone].SetName("Done"); // Fill the part name-to-id map. for (Part p = PartStart; p != PartCount; p = Part(p + 1)) { @@ -612,6 +614,7 @@ bool cmCTest::InitializeFromCommand(cmCTestStartCommand* command) std::string bld_dir = this->GetCTestConfiguration("BuildDirectory"); this->DartVersion = 1; this->DropSiteCDash = false; + this->BuildID = ""; for (Part p = PartStart; p != PartCount; p = Part(p + 1)) { this->Parts[p].SubmitFiles.clear(); } @@ -1565,6 +1568,24 @@ int cmCTest::GenerateNotesFile(const char* cfiles) return this->GenerateNotesFile(files); } +int cmCTest::GenerateDoneFile() +{ + cmGeneratedFileStream ofs; + if (!this->OpenOutputFile(this->CurrentTag, "Done.xml", ofs)) { + cmCTestLog(this, ERROR_MESSAGE, "Cannot open done file" << std::endl); + return 1; + } + cmXMLWriter xml(ofs); + xml.StartDocument(); + xml.StartElement("Done"); + xml.Element("buildId", this->BuildID); + xml.Element("time", std::chrono::system_clock::now()); + xml.EndElement(); // Done + xml.EndDocument(); + + return 0; +} + std::string cmCTest::Base64GzipEncodeFile(std::string const& file) { std::string tarFile = file + "_temp.tar.gz"; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 345b538..1ee002a 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -50,6 +50,7 @@ public: PartNotes, PartExtraFiles, PartUpload, + PartDone, PartCount // Update names in constructor when adding a part }; @@ -373,6 +374,9 @@ public: /** Create XML file that contains all the notes specified */ int GenerateNotesFile(const VectorOfStrings& files); + /** Create XML file to indicate that build is complete */ + int GenerateDoneFile(); + /** Submit extra files to the server */ bool SubmitExtraFiles(const char* files); bool SubmitExtraFiles(const VectorOfStrings& files); @@ -405,6 +409,10 @@ public: int GetDartVersion() { return this->DartVersion; } int GetDropSiteCDash() { return this->DropSiteCDash; } + /** The Build ID is assigned by CDash */ + void SetBuildID(const std::string& id) { this->BuildID = id; } + std::string GetBuildID() { return this->BuildID; } + /** Add file to be submitted */ void AddSubmitFile(Part part, const char* name); std::vector const& GetSubmitFiles(Part part) @@ -607,6 +615,8 @@ private: int DartVersion; bool DropSiteCDash; + std::string BuildID; + std::vector InitialCommandLineArguments; int SubmitIndex; diff --git a/Tests/RunCMake/ctest_submit/PARTSDone-result.txt b/Tests/RunCMake/ctest_submit/PARTSDone-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_submit/PARTSDone-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_submit/PARTSDone-stderr.txt b/Tests/RunCMake/ctest_submit/PARTSDone-stderr.txt new file mode 100644 index 0000000..0020a0f --- /dev/null +++ b/Tests/RunCMake/ctest_submit/PARTSDone-stderr.txt @@ -0,0 +1,3 @@ + *Error when uploading file: .*/Done.xml + *Error message was: ([Cc]ould *n.t resolve host:? '?-no-site-'?.*|The requested URL returned error:.*) + *Problems when submitting via HTTP diff --git a/Tests/RunCMake/ctest_submit/RunCMakeTest.cmake b/Tests/RunCMake/ctest_submit/RunCMakeTest.cmake index 952368d..178f0cb 100644 --- a/Tests/RunCMake/ctest_submit/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_submit/RunCMakeTest.cmake @@ -24,6 +24,7 @@ run_ctest_submit(BadFILES FILES bad-file) run_ctest_submit(RepeatRETURN_VALUE RETURN_VALUE res RETURN_VALUE res) run_ctest_submit(PARTSCDashUpload PARTS Configure CDASH_UPLOAD) run_ctest_submit(PARTSCDashUploadType PARTS Configure CDASH_UPLOAD_TYPE) +run_ctest_submit(PARTSDone PARTS Done) run_ctest_submit(CDashUploadPARTS CDASH_UPLOAD bad-upload PARTS) run_ctest_submit(CDashUploadFILES CDASH_UPLOAD bad-upload FILES) run_ctest_submit(CDashUploadNone CDASH_UPLOAD) -- cgit v0.12