Merge topic 'resolve/ctest-file-checksum/remove-CTestTest3'

38c762c Merge 'remove-CTestTest3' into ctest-file-checksum
46df0b4 Activate retry code on any curl submit failure.
8705497 Checksum test should use CMAKE_TESTS_CDASH_SERVER
d0d1cdd Mock checksum failure output for old CDash versions
af5ef0c Testing for CTest checksum
86e81b5 CTest should resubmit in the checksum failed case
d6b7107 Fix subscript out of range crash
082c87e Cross-platform fixes for checksum/retry code
e525649 Checksums on CTest submit files, and retry timed out submissions.

4 files changed, 232 insertions, 9 deletions

diff --git a/Source/CTest/cmCTestSubmitCommand.cxx b/Source/CTest/cmCTestSubmitCommand.cxx
index d1226da..24974e3 100644
--- a/Source/CTest/cmCTestSubmitCommand.cxx
+++ b/Source/CTest/cmCTestSubmitCommand.cxx
@@ -147,6 +147,13 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler()
     static_cast<cmCTestSubmitHandler*>(handler)->SelectParts(this->Parts);
     }
 
+  static_cast<cmCTestSubmitHandler*>(handler)->SetOption("RetryDelay",
+    this->RetryDelay.c_str());
+  static_cast<cmCTestSubmitHandler*>(handler)->SetOption("RetryCount",
+    this->RetryCount.c_str());
+  static_cast<cmCTestSubmitHandler*>(handler)->SetOption("InternalTest",
+    this->InternalTest ? "ON" : "OFF");
+
   return handler;
 }
 
@@ -169,6 +176,24 @@ bool cmCTestSubmitCommand::CheckArgumentKeyword(std::string const& arg)
     return true;
     }
 
+  if(arg == "RETRY_COUNT")
+    {
+    this->ArgumentDoing = ArgumentDoingRetryCount;
+    return true;
+    }
+
+  if(arg == "RETRY_DELAY")
+    {
+    this->ArgumentDoing = ArgumentDoingRetryDelay;
+    return true;
+    }
+
+  if(arg == "INTERNAL_TEST_CHECKSUM")
+    {
+    this->InternalTest = true;
+    return true;
+    }
+
   // Look for other arguments.
   return this->Superclass::CheckArgumentKeyword(arg);
 }
@@ -213,6 +238,18 @@ bool cmCTestSubmitCommand::CheckArgumentValue(std::string const& arg)
     return true;
     }
 
+  if(this->ArgumentDoing == ArgumentDoingRetryCount)
+    {
+    this->RetryCount = arg;
+    return true;
+    }
+
+  if(this->ArgumentDoing == ArgumentDoingRetryDelay)
+    {
+    this->RetryDelay = arg;
+    return true;
+    }
+
   // Look for other arguments.
   return this->Superclass::CheckArgumentValue(arg);
 }
diff --git a/Source/CTest/cmCTestSubmitCommand.h b/Source/CTest/cmCTestSubmitCommand.h
index ccaef7e..edc9c65 100644
--- a/Source/CTest/cmCTestSubmitCommand.h
+++ b/Source/CTest/cmCTestSubmitCommand.h
@@ -29,6 +29,9 @@ public:
     {
     this->PartsMentioned = false;
     this->FilesMentioned = false;
+    this->InternalTest = false;
+    this->RetryCount = "";
+    this->RetryDelay = "";
     }
 
   /**
@@ -61,7 +64,8 @@ public:
   virtual const char* GetFullDocumentation()
     {
     return
-      "  ctest_submit([PARTS ...] [FILES ...] [RETURN_VALUE res])\n"
+      "  ctest_submit([PARTS ...] [FILES ...] [RETRY_COUNT count] "
+      "               [RETRY_DELAY delay][RETURN_VALUE res])\n"
       "By default all available parts are submitted if no PARTS or FILES "
       "are specified.  "
       "The PARTS option lists a subset of parts to be submitted.  "
@@ -77,7 +81,11 @@ public:
       "  ExtraFiles = Files listed by CTEST_EXTRA_SUBMIT_FILES\n"
       "  Submit     = nothing\n"
       "The FILES option explicitly lists specific files to be submitted.  "
-      "Each individual file must exist at the time of the call.\n";
+      "Each individual file must exist at the time of the call.\n"
+      "The RETRY_DELAY option specifies how long in seconds to wait after "
+      "a timed-out submission before attempting to re-submit.\n"
+      "The RETRY_COUNT option specifies how many times to retry a timed-out "
+      "submission.\n";
     }
 
   cmTypeMacro(cmCTestSubmitCommand, cmCTestHandlerCommand);
@@ -92,13 +100,18 @@ protected:
   {
     ArgumentDoingParts = Superclass::ArgumentDoingLast1,
     ArgumentDoingFiles,
+    ArgumentDoingRetryDelay,
+    ArgumentDoingRetryCount,
     ArgumentDoingLast2
   };
 
   bool PartsMentioned;
   std::set<cmCTest::Part> Parts;
   bool FilesMentioned;
+  bool InternalTest;
   cmCTest::SetOfStrings Files;
+  std::string RetryCount;
+  std::string RetryDelay;
 };
 
 
diff --git a/Source/CTest/cmCTestSubmitHandler.cxx b/Source/CTest/cmCTestSubmitHandler.cxx
index 7b4f38b..1e18259 100644
--- a/Source/CTest/cmCTestSubmitHandler.cxx
+++ b/Source/CTest/cmCTestSubmitHandler.cxx
@@ -15,6 +15,7 @@
 #include "cmVersion.h"
 #include "cmGeneratedFileStream.h"
 #include "cmCTest.h"
+#include "cmXMLParser.h"
 
 #include <cmsys/Process.h>
 #include <cmsys/Base64.h>
@@ -31,6 +32,90 @@
 
 typedef std::vector<char> cmCTestSubmitHandlerVectorOfChar;
 
+//----------------------------------------------------------------------------
+class cmCTestSubmitHandler::ResponseParser: public cmXMLParser
+{
+public:
+  ResponseParser() { this->Status = STATUS_OK; }
+  ~ResponseParser() {}
+
+public:
+
+  enum StatusType
+    {
+    STATUS_OK,
+    STATUS_WARNING,
+    STATUS_ERROR
+    };
+
+  StatusType Status;
+  std::string CDashVersion;
+  std::string Filename;
+  std::string MD5;
+  std::string Message;
+
+private:
+
+  std::vector<char> CurrentValue;
+
+  std::string GetCurrentValue()
+    {
+    std::string val;
+    if(this->CurrentValue.size())
+      {
+      val.assign(&this->CurrentValue[0], this->CurrentValue.size());
+      }
+    return val;
+    }
+
+  virtual void StartElement(const char* name, const char** atts)
+    {
+    this->CurrentValue.clear();
+    if(strcmp(name, "cdash") == 0)
+      {
+      this->CDashVersion = this->FindAttribute(atts, "version");
+      }
+    }
+
+  virtual void CharacterDataHandler(const char* data, int length)
+    {
+    this->CurrentValue.insert(this->CurrentValue.end(), data, data+length);
+    }
+
+  virtual void EndElement(const char* name)
+    {
+    if(strcmp(name, "status") == 0)
+      {
+      std::string status = cmSystemTools::UpperCase(this->GetCurrentValue());
+      if(status == "OK" || status == "SUCCESS")
+        {
+        this->Status = STATUS_OK;
+        }
+      else if(status == "WARNING")
+        {
+        this->Status = STATUS_WARNING;
+        }
+      else
+        {
+        this->Status = STATUS_ERROR;
+        }
+      }
+    else if(strcmp(name, "filename") == 0)
+      {
+      this->Filename = this->GetCurrentValue();
+      }
+    else if(strcmp(name, "md5") == 0)
+      {
+      this->MD5 = this->GetCurrentValue();
+      }
+    else if(strcmp(name, "message") == 0)
+      {
+      this->Message = this->GetCurrentValue();
+      }
+    }
+};
+
+
 static size_t
 cmCTestSubmitHandlerWriteMemoryCallback(void *ptr, size_t size, size_t nmemb,
   void *data)
@@ -367,6 +452,20 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
         = url + ((url.find("?",0) == cmStdString::npos) ? "?" : "&")
         + "FileName=" + ofile;
 
+      upload_as += "&MD5=";
+
+      if(cmSystemTools::IsOn(this->GetOption("InternalTest")))
+        {
+        upload_as += "bad_md5sum";
+        }
+      else
+        {
+        char md5[33];
+        cmSystemTools::ComputeFileMD5(local_file.c_str(), md5);
+        md5[32] = 0;
+        upload_as += md5;
+        }
+
       struct stat st;
       if ( ::stat(local_file.c_str(), &st) )
         {
@@ -382,7 +481,6 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
         << local_file.c_str() << " to "
         << upload_as.c_str() << " Size: " << st.st_size << std::endl);
 
-
       // specify target
       ::curl_easy_setopt(curl,CURLOPT_URL, upload_as.c_str());
 
@@ -411,6 +509,19 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
       // Now run off and do what you've been told!
       res = ::curl_easy_perform(curl);
 
+      if(cmSystemTools::IsOn(this->GetOption("InternalTest")) &&
+         cmSystemTools::VersionCompare(cmSystemTools::OP_LESS,
+         this->CTest->GetCDashVersion().c_str(), "1.7"))
+        {
+        // mock failure output for internal test case
+        std::string mock_output = "<cdash version=\"1.7.0\">\n"
+          "  <status>ERROR</status>\n"
+          "  <message>Checksum failed for file.</message>\n"
+          "</cdash>\n";
+        chunk.clear();
+        chunk.assign(mock_output.begin(), mock_output.end());
+        }
+
       if ( chunk.size() > 0 )
         {
         cmCTestLog(this->CTest, DEBUG, "CURL output: ["
@@ -425,6 +536,60 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const cmStdString& localprefix,
           << std::endl);
         }
 
+      // If curl failed for any reason, or checksum fails, wait and retry
+      //
+      if(res != CURLE_OK || this->HasErrors)
+        {
+        std::string retryDelay = this->GetOption("RetryDelay") == NULL ?
+          "" : this->GetOption("RetryDelay");
+        std::string retryCount = this->GetOption("RetryCount") == NULL ?
+          "" : this->GetOption("RetryCount");
+
+        int delay = retryDelay == "" ? atoi(this->CTest->GetCTestConfiguration(
+          "CTestSubmitRetryDelay").c_str()) : atoi(retryDelay.c_str());
+        int count = retryCount == "" ? atoi(this->CTest->GetCTestConfiguration(
+          "CTestSubmitRetryCount").c_str()) : atoi(retryCount.c_str());
+
+        for(int i = 0; i < count; i++)
+          {
+          cmCTestLog(this->CTest, HANDLER_OUTPUT,
+            "   Submit failed, waiting " << delay << " seconds...\n");
+
+          double stop = cmSystemTools::GetTime() + delay;
+          while(cmSystemTools::GetTime() < stop)
+            {
+            cmSystemTools::Delay(100);
+            }
+
+          cmCTestLog(this->CTest, HANDLER_OUTPUT,
+            "   Retry submission: Attempt " << (i + 1) << " of "
+            << count << std::endl);
+
+          ::fclose(ftpfile);
+          ftpfile = ::fopen(local_file.c_str(), "rb");
+          ::curl_easy_setopt(curl, CURLOPT_INFILE, ftpfile);
+
+          chunk.clear();
+          chunkDebug.clear();
+          this->HasErrors = false;
+
+          res = ::curl_easy_perform(curl);
+
+          if ( chunk.size() > 0 )
+            {
+            cmCTestLog(this->CTest, DEBUG, "CURL output: ["
+              << cmCTestLogWrite(&*chunk.begin(), chunk.size()) << "]"
+              << std::endl);
+            this->ParseResponse(chunk);
+            }
+
+          if(res == CURLE_OK && !this->HasErrors)
+            {
+            break;
+            }
+          }
+        }
+
       fclose(ftpfile);
       if ( res )
         {
@@ -467,14 +632,22 @@ void cmCTestSubmitHandler
 ::ParseResponse(cmCTestSubmitHandlerVectorOfChar chunk)
 {
   std::string output = "";
+  output.append(chunk.begin(), chunk.end());
 
-  for(cmCTestSubmitHandlerVectorOfChar::iterator i = chunk.begin();
-      i != chunk.end(); ++i)
+  if(output.find("<cdash") != output.npos)
     {
-    output += *i;
+    ResponseParser parser;
+    parser.Parse(output.c_str());
+
+    if(parser.Status != ResponseParser::STATUS_OK)
+      {
+      this->HasErrors = true;
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, "   Submission failed: " <<
+        parser.Message << std::endl);
+      return;
+      }
     }
   output = cmSystemTools::UpperCase(output);
-  
   if(output.find("WARNING") != std::string::npos)
     {
     this->HasWarnings = true;
@@ -483,13 +656,12 @@ void cmCTestSubmitHandler
     {
     this->HasErrors = true;
     }
-  
+
   if(this->HasWarnings || this->HasErrors)
     {
     cmCTestLog(this->CTest, HANDLER_OUTPUT, "   Server Response:\n" <<
           cmCTestLogWrite(&*chunk.begin(), chunk.size()) << "\n");
     }
-  
 }
 
 //----------------------------------------------------------------------------
diff --git a/Source/CTest/cmCTestSubmitHandler.h b/Source/CTest/cmCTestSubmitHandler.h
index 8b011ea..e7755b1 100644
--- a/Source/CTest/cmCTestSubmitHandler.h
+++ b/Source/CTest/cmCTestSubmitHandler.h
@@ -79,6 +79,7 @@ private:
 
   std::string GetSubmitResultsPrefix();
 
+  class         ResponseParser;
   cmStdString   HTTPProxy;
   int           HTTPProxyType;
   cmStdString   HTTPProxyAuth;