diff options
author | Brad King <brad.king@kitware.com> | 2016-11-16 12:46:48 (GMT) |
---|---|---|
committer | CMake Topic Stage <kwrobot@kitware.com> | 2016-11-16 12:46:48 (GMT) |
commit | 2a663ebce10fb9d393594e28f8ecc870488f6eba (patch) | |
tree | 90f256dabded5657b3d8276776335ad5afd7b181 /Source | |
parent | b6da714af79578bfdcbfe990addc557448b51eea (diff) | |
parent | f55fcdc8598be03d0622dd5a7b96d33a6613a296 (diff) | |
download | CMake-2a663ebce10fb9d393594e28f8ecc870488f6eba.zip CMake-2a663ebce10fb9d393594e28f8ecc870488f6eba.tar.gz CMake-2a663ebce10fb9d393594e28f8ecc870488f6eba.tar.bz2 |
Merge topic 'childEncoding'
f55fcdc8 CTest: Use UTF-8 encoding for output from Git
40bd42df Add Encoding option for RunChild, RunMakeCommand and RunProcess
595feb32 Windows: Encode child process output to internally-used encoding
96103972 Add cmProcessOutput class to be used for decoding text data
92c865b8 cmCTestBuildHandler: Use size_t in ProcessBuffer length argument
Diffstat (limited to 'Source')
-rw-r--r-- | Source/CMakeLists.txt | 5 | ||||
-rw-r--r-- | Source/CTest/cmCTestBuildHandler.cxx | 23 | ||||
-rw-r--r-- | Source/CTest/cmCTestBuildHandler.h | 7 | ||||
-rw-r--r-- | Source/CTest/cmCTestGIT.cxx | 17 | ||||
-rw-r--r-- | Source/CTest/cmCTestLaunch.cxx | 23 | ||||
-rw-r--r-- | Source/CTest/cmCTestSubmitHandler.cxx | 13 | ||||
-rw-r--r-- | Source/CTest/cmCTestVC.cxx | 9 | ||||
-rw-r--r-- | Source/CTest/cmCTestVC.h | 6 | ||||
-rw-r--r-- | Source/CTest/cmProcess.cxx | 10 | ||||
-rw-r--r-- | Source/cmCTest.cxx | 62 | ||||
-rw-r--r-- | Source/cmCTest.h | 11 | ||||
-rw-r--r-- | Source/cmExecProgramCommand.cxx | 16 | ||||
-rw-r--r-- | Source/cmExecProgramCommand.h | 5 | ||||
-rw-r--r-- | Source/cmExecuteProcessCommand.cxx | 23 | ||||
-rw-r--r-- | Source/cmProcessOutput.cxx | 155 | ||||
-rw-r--r-- | Source/cmProcessOutput.h | 80 | ||||
-rw-r--r-- | Source/cmProcessTools.cxx | 23 | ||||
-rw-r--r-- | Source/cmProcessTools.h | 5 | ||||
-rw-r--r-- | Source/cmSystemTools.cxx | 45 | ||||
-rw-r--r-- | Source/cmSystemTools.h | 5 |
20 files changed, 489 insertions, 54 deletions
diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 718b022..fcda6f9 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -330,6 +330,8 @@ set(SRCS cmOrderDirectories.h cmPolicies.h cmPolicies.cxx + cmProcessOutput.cxx + cmProcessOutput.h cmProcessTools.cxx cmProcessTools.h cmProperty.cxx @@ -632,6 +634,9 @@ set(SRCS cm_codecvt.cxx ) +SET_PROPERTY(SOURCE cmProcessOutput.cxx APPEND PROPERTY COMPILE_DEFINITIONS + KWSYS_ENCODING_DEFAULT_CODEPAGE=${KWSYS_ENCODING_DEFAULT_CODEPAGE}) + # Kdevelop only works on UNIX and not windows if(UNIX) set(SRCS ${SRCS} cmGlobalKdevelopGenerator.cxx) diff --git a/Source/CTest/cmCTestBuildHandler.cxx b/Source/CTest/cmCTestBuildHandler.cxx index 7b4d994..a455908 100644 --- a/Source/CTest/cmCTestBuildHandler.cxx +++ b/Source/CTest/cmCTestBuildHandler.cxx @@ -7,6 +7,7 @@ #include "cmFileTimeComparison.h" #include "cmGeneratedFileStream.h" #include "cmMakefile.h" +#include "cmProcessOutput.h" #include "cmSystemTools.h" #include "cmXMLWriter.h" @@ -765,7 +766,7 @@ void cmCTestBuildHandler::LaunchHelper::WriteScrapeMatchers( int cmCTestBuildHandler::RunMakeCommand(const char* command, int* retVal, const char* dir, int timeout, - std::ostream& ofs) + std::ostream& ofs, Encoding encoding) { // First generate the command and arguments std::vector<std::string> args = cmSystemTools::ParseArguments(command); @@ -809,6 +810,8 @@ int cmCTestBuildHandler::RunMakeCommand(const char* command, int* retVal, char* data; int length; + cmProcessOutput processOutput(encoding); + std::string strdata; cmCTestOptionalLog( this->CTest, HANDLER_PROGRESS_OUTPUT, " Each symbol represents " << tick_len << " bytes of output." << std::endl @@ -842,13 +845,25 @@ int cmCTestBuildHandler::RunMakeCommand(const char* command, int* retVal, // Process the chunk of data if (res == cmsysProcess_Pipe_STDERR) { - this->ProcessBuffer(data, length, tick, tick_len, ofs, + processOutput.DecodeText(data, length, strdata, 1); + this->ProcessBuffer(strdata.c_str(), strdata.size(), tick, tick_len, ofs, &this->BuildProcessingErrorQueue); } else { - this->ProcessBuffer(data, length, tick, tick_len, ofs, + processOutput.DecodeText(data, length, strdata, 2); + this->ProcessBuffer(strdata.c_str(), strdata.size(), tick, tick_len, ofs, &this->BuildProcessingQueue); } } + processOutput.DecodeText(std::string(), strdata, 1); + if (!strdata.empty()) { + this->ProcessBuffer(strdata.c_str(), strdata.size(), tick, tick_len, ofs, + &this->BuildProcessingErrorQueue); + } + processOutput.DecodeText(std::string(), strdata, 2); + if (!strdata.empty()) { + this->ProcessBuffer(strdata.c_str(), strdata.size(), tick, tick_len, ofs, + &this->BuildProcessingQueue); + } this->ProcessBuffer(CM_NULLPTR, 0, tick, tick_len, ofs, &this->BuildProcessingQueue); @@ -920,7 +935,7 @@ int cmCTestBuildHandler::RunMakeCommand(const char* command, int* retVal, //###################################################################### //###################################################################### -void cmCTestBuildHandler::ProcessBuffer(const char* data, int length, +void cmCTestBuildHandler::ProcessBuffer(const char* data, size_t length, size_t& tick, size_t tick_len, std::ostream& ofs, t_BuildProcessingQueueType* queue) diff --git a/Source/CTest/cmCTestBuildHandler.h b/Source/CTest/cmCTestBuildHandler.h index 5bd1157..a2f6112 100644 --- a/Source/CTest/cmCTestBuildHandler.h +++ b/Source/CTest/cmCTestBuildHandler.h @@ -7,6 +7,7 @@ #include "cmCTestGenericHandler.h" +#include <cmProcessOutput.h> #include <cmsys/RegularExpression.hxx> #include <deque> #include <iosfwd> @@ -25,6 +26,7 @@ class cmCTestBuildHandler : public cmCTestGenericHandler { public: typedef cmCTestGenericHandler Superclass; + typedef cmProcessOutput::Encoding Encoding; /* * The main entry point for this class @@ -49,7 +51,8 @@ private: //! Run command specialized for make and configure. Returns process status // and retVal is return value or exception. int RunMakeCommand(const char* command, int* retVal, const char* dir, - int timeout, std::ostream& ofs); + int timeout, std::ostream& ofs, + Encoding encoding = cmProcessOutput::Auto); enum { @@ -107,7 +110,7 @@ private: typedef std::deque<char> t_BuildProcessingQueueType; - void ProcessBuffer(const char* data, int length, size_t& tick, + void ProcessBuffer(const char* data, size_t length, size_t& tick, size_t tick_len, std::ostream& ofs, t_BuildProcessingQueueType* queue); int ProcessSingleLine(const char* data); diff --git a/Source/CTest/cmCTestGIT.cxx b/Source/CTest/cmCTestGIT.cxx index 1bc1851..20512ea 100644 --- a/Source/CTest/cmCTestGIT.cxx +++ b/Source/CTest/cmCTestGIT.cxx @@ -91,7 +91,8 @@ std::string cmCTestGIT::FindGitDir() std::string git_dir_line; OneLineParser rev_parse_out(this, "rev-parse-out> ", git_dir_line); OutputLogger rev_parse_err(this->Log, "rev-parse-err> "); - if (this->RunChild(git_rev_parse, &rev_parse_out, &rev_parse_err)) { + if (this->RunChild(git_rev_parse, &rev_parse_out, &rev_parse_err, CM_NULLPTR, + cmProcessOutput::UTF8)) { git_dir = git_dir_line; } if (git_dir.empty()) { @@ -114,7 +115,8 @@ std::string cmCTestGIT::FindGitDir() 0 }; OneLineParser cygpath_out(this, "cygpath-out> ", git_dir_line); OutputLogger cygpath_err(this->Log, "cygpath-err> "); - if (this->RunChild(cygpath, &cygpath_out, &cygpath_err)) { + if (this->RunChild(cygpath, &cygpath_out, &cygpath_err, CM_NULLPTR, + cmProcessOutput::UTF8)) { git_dir = git_dir_line; } } @@ -134,7 +136,8 @@ std::string cmCTestGIT::FindTopDir() std::string cdup; OneLineParser rev_parse_out(this, "rev-parse-out> ", cdup); OutputLogger rev_parse_err(this->Log, "rev-parse-err> "); - if (this->RunChild(git_rev_parse, &rev_parse_out, &rev_parse_err) && + if (this->RunChild(git_rev_parse, &rev_parse_out, &rev_parse_err, CM_NULLPTR, + cmProcessOutput::UTF8) && !cdup.empty()) { top_dir += "/"; top_dir += cdup; @@ -624,7 +627,7 @@ void cmCTestGIT::LoadRevisions() CommitParser out(this, "dt-out> "); OutputLogger err(this->Log, "dt-err> "); - this->RunProcess(cp, &out, &err); + this->RunProcess(cp, &out, &err, cmProcessOutput::UTF8); // Send one extra zero-byte to terminate the last record. out.Process("", 1); @@ -641,14 +644,16 @@ void cmCTestGIT::LoadModifications() CM_NULLPTR }; OutputLogger ui_out(this->Log, "ui-out> "); OutputLogger ui_err(this->Log, "ui-err> "); - this->RunChild(git_update_index, &ui_out, &ui_err); + this->RunChild(git_update_index, &ui_out, &ui_err, CM_NULLPTR, + cmProcessOutput::UTF8); // Use 'git diff-index' to get modified files. const char* git_diff_index[] = { git, "diff-index", "-z", "HEAD", "--", CM_NULLPTR }; DiffParser out(this, "di-out> "); OutputLogger err(this->Log, "di-err> "); - this->RunChild(git_diff_index, &out, &err); + this->RunChild(git_diff_index, &out, &err, CM_NULLPTR, + cmProcessOutput::UTF8); for (std::vector<Change>::const_iterator ci = out.Changes.begin(); ci != out.Changes.end(); ++ci) { diff --git a/Source/CTest/cmCTestLaunch.cxx b/Source/CTest/cmCTestLaunch.cxx index f7a6e0b..43c0fef 100644 --- a/Source/CTest/cmCTestLaunch.cxx +++ b/Source/CTest/cmCTestLaunch.cxx @@ -8,6 +8,7 @@ #include "cmGeneratedFileStream.h" #include "cmGlobalGenerator.h" #include "cmMakefile.h" +#include "cmProcessOutput.h" #include "cmStateTypes.h" #include "cmSystemTools.h" #include "cmXMLWriter.h" @@ -225,17 +226,31 @@ void cmCTestLaunch::RunChild() if (!this->Passthru) { char* data = CM_NULLPTR; int length = 0; + cmProcessOutput processOutput; + std::string strdata; while (int p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) { if (p == cmsysProcess_Pipe_STDOUT) { - fout.write(data, length); - std::cout.write(data, length); + processOutput.DecodeText(data, length, strdata, 1); + fout.write(strdata.c_str(), strdata.size()); + std::cout.write(strdata.c_str(), strdata.size()); this->HaveOut = true; } else if (p == cmsysProcess_Pipe_STDERR) { - ferr.write(data, length); - std::cerr.write(data, length); + processOutput.DecodeText(data, length, strdata, 2); + ferr.write(strdata.c_str(), strdata.size()); + std::cerr.write(strdata.c_str(), strdata.size()); this->HaveErr = true; } } + processOutput.DecodeText(std::string(), strdata, 1); + if (!strdata.empty()) { + fout.write(strdata.c_str(), strdata.size()); + std::cout.write(strdata.c_str(), strdata.size()); + } + processOutput.DecodeText(std::string(), strdata, 2); + if (!strdata.empty()) { + ferr.write(strdata.c_str(), strdata.size()); + std::cerr.write(strdata.c_str(), strdata.size()); + } } // Wait for the real command to finish. diff --git a/Source/CTest/cmCTestSubmitHandler.cxx b/Source/CTest/cmCTestSubmitHandler.cxx index 8383132..2b2d207 100644 --- a/Source/CTest/cmCTestSubmitHandler.cxx +++ b/Source/CTest/cmCTestSubmitHandler.cxx @@ -7,6 +7,7 @@ #include "cmCTestScriptHandler.h" #include "cmCurl.h" #include "cmGeneratedFileStream.h" +#include "cmProcessOutput.h" #include "cmState.h" #include "cmStateTypes.h" #include "cmSystemTools.h" @@ -784,10 +785,20 @@ bool cmCTestSubmitHandler::SubmitUsingSCP(const std::string& scp_command, cmsysProcess_Execute(cp); char* data; int length; + cmProcessOutput processOutput; + std::string strdata; while (cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) { + processOutput.DecodeText(data, length, strdata); cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, - cmCTestLogWrite(data, length), this->Quiet); + cmCTestLogWrite(strdata.c_str(), strdata.size()), + this->Quiet); + } + processOutput.DecodeText(std::string(), strdata); + if (!strdata.empty()) { + cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + cmCTestLogWrite(strdata.c_str(), strdata.size()), + this->Quiet); } cmsysProcess_WaitForExit(cp, CM_NULLPTR); diff --git a/Source/CTest/cmCTestVC.cxx b/Source/CTest/cmCTestVC.cxx index 7a2fa69..444c43d 100644 --- a/Source/CTest/cmCTestVC.cxx +++ b/Source/CTest/cmCTestVC.cxx @@ -76,7 +76,8 @@ bool cmCTestVC::InitialCheckout(const char* command) } bool cmCTestVC::RunChild(char const* const* cmd, OutputParser* out, - OutputParser* err, const char* workDir) + OutputParser* err, const char* workDir, + Encoding encoding) { this->Log << this->ComputeCommandLine(cmd) << "\n"; @@ -84,7 +85,7 @@ bool cmCTestVC::RunChild(char const* const* cmd, OutputParser* out, cmsysProcess_SetCommand(cp, cmd); workDir = workDir ? workDir : this->SourceDirectory.c_str(); cmsysProcess_SetWorkingDirectory(cp, workDir); - this->RunProcess(cp, out, err); + this->RunProcess(cp, out, err, encoding); int result = cmsysProcess_GetExitValue(cp); cmsysProcess_Delete(cp); return result == 0; @@ -102,7 +103,7 @@ std::string cmCTestVC::ComputeCommandLine(char const* const* cmd) } bool cmCTestVC::RunUpdateCommand(char const* const* cmd, OutputParser* out, - OutputParser* err) + OutputParser* err, Encoding encoding) { // Report the command line. this->UpdateCommandLine = this->ComputeCommandLine(cmd); @@ -112,7 +113,7 @@ bool cmCTestVC::RunUpdateCommand(char const* const* cmd, OutputParser* out, } // Run the command. - return this->RunChild(cmd, out, err); + return this->RunChild(cmd, out, err, CM_NULLPTR, encoding); } std::string cmCTestVC::GetNightlyTime() diff --git a/Source/CTest/cmCTestVC.h b/Source/CTest/cmCTestVC.h index 4f2bba0..dd8b973 100644 --- a/Source/CTest/cmCTestVC.h +++ b/Source/CTest/cmCTestVC.h @@ -116,11 +116,13 @@ protected: /** Run a command line and send output to given parsers. */ bool RunChild(char const* const* cmd, OutputParser* out, OutputParser* err, - const char* workDir = CM_NULLPTR); + const char* workDir = CM_NULLPTR, + Encoding encoding = cmProcessOutput::Auto); /** Run VC update command line and send output to given parsers. */ bool RunUpdateCommand(char const* const* cmd, OutputParser* out, - OutputParser* err = CM_NULLPTR); + OutputParser* err = CM_NULLPTR, + Encoding encoding = cmProcessOutput::Auto); /** Write xml element for one file. */ void WriteXMLEntry(cmXMLWriter& xml, std::string const& path, diff --git a/Source/CTest/cmProcess.cxx b/Source/CTest/cmProcess.cxx index a24fe21..98bd3bb 100644 --- a/Source/CTest/cmProcess.cxx +++ b/Source/CTest/cmProcess.cxx @@ -3,6 +3,7 @@ #include "cmProcess.h" #include <cmConfigure.h> +#include <cmProcessOutput.h> #include <cmSystemTools.h> #include <iostream> @@ -104,6 +105,8 @@ bool cmProcess::Buffer::GetLast(std::string& line) int cmProcess::GetNextOutputLine(std::string& line, double timeout) { + cmProcessOutput processOutput; + std::string strdata; for (;;) { // Look for lines already buffered. if (this->Output.GetLine(line)) { @@ -118,12 +121,17 @@ int cmProcess::GetNextOutputLine(std::string& line, double timeout) return cmsysProcess_Pipe_Timeout; } if (p == cmsysProcess_Pipe_STDOUT) { - this->Output.insert(this->Output.end(), data, data + length); + processOutput.DecodeText(data, length, strdata); + this->Output.insert(this->Output.end(), strdata.begin(), strdata.end()); } else { // p == cmsysProcess_Pipe_None // The process will provide no more data. break; } } + processOutput.DecodeText(std::string(), strdata); + if (!strdata.empty()) { + this->Output.insert(this->Output.end(), strdata.begin(), strdata.end()); + } // Look for partial last lines. if (this->Output.GetLast(line)) { diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index f35a0b3..1527b30 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -41,6 +41,7 @@ #include "cmGeneratedFileStream.h" #include "cmGlobalGenerator.h" #include "cmMakefile.h" +#include "cmProcessOutput.h" #include "cmState.h" #include "cmStateSnapshot.h" #include "cmStateTypes.h" @@ -960,7 +961,7 @@ int cmCTest::GetTestModelFromString(const char* str) int cmCTest::RunMakeCommand(const char* command, std::string& output, int* retVal, const char* dir, int timeout, - std::ostream& ofs) + std::ostream& ofs, Encoding encoding) { // First generate the command and arguments std::vector<std::string> args = cmSystemTools::ParseArguments(command); @@ -999,16 +1000,19 @@ int cmCTest::RunMakeCommand(const char* command, std::string& output, char* data; int length; + cmProcessOutput processOutput(encoding); + std::string strdata; cmCTestLog(this, HANDLER_PROGRESS_OUTPUT, " Each . represents " << tick_len << " bytes of output" << std::endl << " " << std::flush); while (cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) { - for (int cc = 0; cc < length; ++cc) { - if (data[cc] == 0) { - data[cc] = '\n'; + processOutput.DecodeText(data, length, strdata); + for (size_t cc = 0; cc < strdata.size(); ++cc) { + if (strdata[cc] == 0) { + strdata[cc] = '\n'; } } - output.append(data, length); + output.append(strdata); while (output.size() > (tick * tick_len)) { tick++; cmCTestLog(this, HANDLER_PROGRESS_OUTPUT, "." << std::flush); @@ -1019,9 +1023,19 @@ int cmCTest::RunMakeCommand(const char* command, std::string& output, << " " << std::flush); } } - cmCTestLog(this, HANDLER_VERBOSE_OUTPUT, cmCTestLogWrite(data, length)); + cmCTestLog(this, HANDLER_VERBOSE_OUTPUT, + cmCTestLogWrite(strdata.c_str(), strdata.size())); + if (ofs) { + ofs << cmCTestLogWrite(strdata.c_str(), strdata.size()); + } + } + processOutput.DecodeText(std::string(), strdata); + if (!strdata.empty()) { + output.append(strdata); + cmCTestLog(this, HANDLER_VERBOSE_OUTPUT, + cmCTestLogWrite(strdata.c_str(), strdata.size())); if (ofs) { - ofs << cmCTestLogWrite(data, length); + ofs << cmCTestLogWrite(strdata.c_str(), strdata.size()); } } cmCTestLog(this, HANDLER_PROGRESS_OUTPUT, " Size of output: " @@ -1061,7 +1075,7 @@ int cmCTest::RunMakeCommand(const char* command, std::string& output, int cmCTest::RunTest(std::vector<const char*> argv, std::string* output, int* retVal, std::ostream* log, double testTimeOut, - std::vector<std::string>* environment) + std::vector<std::string>* environment, Encoding encoding) { bool modifyEnv = (environment && !environment->empty()); @@ -1156,17 +1170,30 @@ int cmCTest::RunTest(std::vector<const char*> argv, std::string* output, char* data; int length; + cmProcessOutput processOutput(encoding); + std::string strdata; while (cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) { + processOutput.DecodeText(data, length, strdata); if (output) { tempOutput.insert(tempOutput.end(), data, data + length); } - cmCTestLog(this, HANDLER_VERBOSE_OUTPUT, cmCTestLogWrite(data, length)); + cmCTestLog(this, HANDLER_VERBOSE_OUTPUT, + cmCTestLogWrite(strdata.c_str(), strdata.size())); + if (log) { + log->write(strdata.c_str(), strdata.size()); + } + } + processOutput.DecodeText(std::string(), strdata); + if (!strdata.empty()) { + cmCTestLog(this, HANDLER_VERBOSE_OUTPUT, + cmCTestLogWrite(strdata.c_str(), strdata.size())); if (log) { - log->write(data, length); + log->write(strdata.c_str(), strdata.size()); } } cmsysProcess_WaitForExit(cp, CM_NULLPTR); + processOutput.DecodeText(tempOutput, tempOutput); if (output && tempOutput.begin() != tempOutput.end()) { output->append(&*tempOutput.begin(), tempOutput.size()); } @@ -2496,7 +2523,7 @@ bool cmCTest::SetCTestConfigurationFromCMakeVariable( bool cmCTest::RunCommand(const char* command, std::string* stdOut, std::string* stdErr, int* retVal, const char* dir, - double timeout) + double timeout, Encoding encoding) { std::vector<std::string> args = cmSystemTools::ParseArguments(command); @@ -2527,6 +2554,8 @@ bool cmCTest::RunCommand(const char* command, std::string* stdOut, std::vector<char> tempError; char* data; int length; + cmProcessOutput processOutput(encoding); + std::string strdata; int res; bool done = false; while (!done) { @@ -2543,15 +2572,24 @@ bool cmCTest::RunCommand(const char* command, std::string* stdOut, } if ((res == cmsysProcess_Pipe_STDOUT || res == cmsysProcess_Pipe_STDERR) && this->ExtraVerbose) { - cmSystemTools::Stdout(data, length); + processOutput.DecodeText(data, length, strdata); + cmSystemTools::Stdout(strdata.c_str(), strdata.size()); + } + } + if (this->ExtraVerbose) { + processOutput.DecodeText(std::string(), strdata); + if (!strdata.empty()) { + cmSystemTools::Stdout(strdata.c_str(), strdata.size()); } } cmsysProcess_WaitForExit(cp, CM_NULLPTR); if (!tempOutput.empty()) { + processOutput.DecodeText(tempOutput, tempOutput); stdOut->append(&*tempOutput.begin(), tempOutput.size()); } if (!tempError.empty()) { + processOutput.DecodeText(tempError, tempError); stdErr->append(&*tempError.begin(), tempError.size()); } diff --git a/Source/cmCTest.h b/Source/cmCTest.h index e5b4728..4436327 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -5,6 +5,7 @@ #include <cmConfigure.h> +#include <cmProcessOutput.h> #include <cmsys/String.hxx> #include <map> #include <set> @@ -48,6 +49,7 @@ class cmCTest friend class cmCTestMultiProcessHandler; public: + typedef cmProcessOutput::Encoding Encoding; /** Enumerate parts of the testing and submission process. */ enum Part { @@ -267,7 +269,8 @@ public: */ bool RunCommand(const char* command, std::string* stdOut, std::string* stdErr, int* retVal = CM_NULLPTR, - const char* dir = CM_NULLPTR, double timeout = 0.0); + const char* dir = CM_NULLPTR, double timeout = 0.0, + Encoding encoding = cmProcessOutput::Auto); /** * Clean/make safe for xml the given value such that it may be used as @@ -286,7 +289,8 @@ public: * and retVal is return value or exception. */ int RunMakeCommand(const char* command, std::string& output, int* retVal, - const char* dir, int timeout, std::ostream& ofs); + const char* dir, int timeout, std::ostream& ofs, + Encoding encoding = cmProcessOutput::Auto); /** Return the current tag */ std::string GetCurrentTag(); @@ -333,7 +337,8 @@ public: */ int RunTest(std::vector<const char*> args, std::string* output, int* retVal, std::ostream* logfile, double testTimeOut, - std::vector<std::string>* environment); + std::vector<std::string>* environment, + Encoding encoding = cmProcessOutput::Auto); /** * Execute handler and return its result. If the handler fails, it returns diff --git a/Source/cmExecProgramCommand.cxx b/Source/cmExecProgramCommand.cxx index df92592..6a7292d 100644 --- a/Source/cmExecProgramCommand.cxx +++ b/Source/cmExecProgramCommand.cxx @@ -6,6 +6,7 @@ #include <stdio.h> #include "cmMakefile.h" +#include "cmProcessOutput.h" #include "cmSystemTools.h" class cmExecutionStatus; @@ -116,7 +117,7 @@ bool cmExecProgramCommand::InitialPass(std::vector<std::string> const& args, bool cmExecProgramCommand::RunCommand(const char* command, std::string& output, int& retVal, const char* dir, - bool verbose) + bool verbose, Encoding encoding) { if (cmSystemTools::GetRunCommandOutput()) { verbose = false; @@ -214,17 +215,28 @@ bool cmExecProgramCommand::RunCommand(const char* command, std::string& output, int length; char* data; int p; + cmProcessOutput processOutput(encoding); + std::string strdata; while ((p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR), p)) { if (p == cmsysProcess_Pipe_STDOUT || p == cmsysProcess_Pipe_STDERR) { if (verbose) { - cmSystemTools::Stdout(data, length); + processOutput.DecodeText(data, length, strdata); + cmSystemTools::Stdout(strdata.c_str(), strdata.size()); } output.append(data, length); } } + if (verbose) { + processOutput.DecodeText(std::string(), strdata); + if (!strdata.empty()) { + cmSystemTools::Stdout(strdata.c_str(), strdata.size()); + } + } + // All output has been read. Wait for the process to exit. cmsysProcess_WaitForExit(cp, CM_NULLPTR); + processOutput.DecodeText(output, output); // Check the result of running the process. std::string msg; diff --git a/Source/cmExecProgramCommand.h b/Source/cmExecProgramCommand.h index 7cdf777..58e948e 100644 --- a/Source/cmExecProgramCommand.h +++ b/Source/cmExecProgramCommand.h @@ -8,6 +8,7 @@ #include <vector> #include "cmCommand.h" +#include "cmProcessOutput.h" class cmExecutionStatus; @@ -21,6 +22,7 @@ class cmExecutionStatus; class cmExecProgramCommand : public cmCommand { public: + typedef cmProcessOutput::Encoding Encoding; /** * This is a virtual constructor for the command. */ @@ -46,7 +48,8 @@ public: private: static bool RunCommand(const char* command, std::string& output, int& retVal, const char* directory = CM_NULLPTR, - bool verbose = true); + bool verbose = true, + Encoding encoding = cmProcessOutput::Auto); }; #endif diff --git a/Source/cmExecuteProcessCommand.cxx b/Source/cmExecuteProcessCommand.cxx index c8a3a84..1562223 100644 --- a/Source/cmExecuteProcessCommand.cxx +++ b/Source/cmExecuteProcessCommand.cxx @@ -8,6 +8,7 @@ #include <stdio.h> #include "cmMakefile.h" +#include "cmProcessOutput.h" #include "cmSystemTools.h" class cmExecutionStatus; @@ -222,25 +223,43 @@ bool cmExecuteProcessCommand::InitialPass(std::vector<std::string> const& args, int length; char* data; int p; + cmProcessOutput processOutput; + std::string strdata; while ((p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR), p)) { // Put the output in the right place. if (p == cmsysProcess_Pipe_STDOUT && !output_quiet) { if (output_variable.empty()) { - cmSystemTools::Stdout(data, length); + processOutput.DecodeText(data, length, strdata, 1); + cmSystemTools::Stdout(strdata.c_str(), strdata.size()); } else { cmExecuteProcessCommandAppend(tempOutput, data, length); } } else if (p == cmsysProcess_Pipe_STDERR && !error_quiet) { if (error_variable.empty()) { - cmSystemTools::Stderr(data, length); + processOutput.DecodeText(data, length, strdata, 2); + cmSystemTools::Stderr(strdata.c_str(), strdata.size()); } else { cmExecuteProcessCommandAppend(tempError, data, length); } } } + if (!output_quiet && output_variable.empty()) { + processOutput.DecodeText(std::string(), strdata, 1); + if (!strdata.empty()) { + cmSystemTools::Stdout(strdata.c_str(), strdata.size()); + } + } + if (!error_quiet && error_variable.empty()) { + processOutput.DecodeText(std::string(), strdata, 2); + if (!strdata.empty()) { + cmSystemTools::Stderr(strdata.c_str(), strdata.size()); + } + } // All output has been read. Wait for the process to exit. cmsysProcess_WaitForExit(cp, CM_NULLPTR); + processOutput.DecodeText(tempOutput, tempOutput); + processOutput.DecodeText(tempError, tempError); // Fix the text in the output strings. cmExecuteProcessCommandFixText(tempOutput, output_strip_trailing_whitespace); diff --git a/Source/cmProcessOutput.cxx b/Source/cmProcessOutput.cxx new file mode 100644 index 0000000..1440223 --- /dev/null +++ b/Source/cmProcessOutput.cxx @@ -0,0 +1,155 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ + +#include "cmProcessOutput.h" + +#if defined(_WIN32) +#include <windows.h> +unsigned int cmProcessOutput::defaultCodepage = + KWSYS_ENCODING_DEFAULT_CODEPAGE; +#endif + +cmProcessOutput::cmProcessOutput(Encoding encoding, unsigned int maxSize) +{ +#if defined(_WIN32) + codepage = 0; + bufferSize = maxSize; + if (encoding == None) { + codepage = defaultCodepage; + } else if (encoding == Auto) { + codepage = GetConsoleCP(); + } else if (encoding == UTF8) { + codepage = CP_UTF8; + } else if (encoding == OEM) { + codepage = GetOEMCP(); + } + if (!codepage || encoding == ANSI) { + codepage = GetACP(); + } +#else + static_cast<void>(encoding); + static_cast<void>(maxSize); +#endif +} + +cmProcessOutput::~cmProcessOutput() +{ +} + +bool cmProcessOutput::DecodeText(std::string raw, std::string& decoded, + size_t id) +{ + bool success = true; + decoded = raw; +#if defined(_WIN32) + if (id > 0) { + if (rawparts.size() < id) { + rawparts.reserve(id); + while (rawparts.size() < id) + rawparts.push_back(std::string()); + } + raw = rawparts[id - 1] + raw; + rawparts[id - 1].clear(); + decoded = raw; + } + if (raw.size() > 0 && codepage != defaultCodepage) { + success = false; + CPINFOEXW cpinfo; + if (id > 0 && bufferSize > 0 && raw.size() == bufferSize && + GetCPInfoExW(codepage, 0, &cpinfo) == 1 && cpinfo.MaxCharSize > 1) { + if (cpinfo.MaxCharSize == 2 && cpinfo.LeadByte[0] != 0) { + LPSTR prevChar = + CharPrevExA(codepage, raw.c_str(), raw.c_str() + raw.size(), 0); + bool isLeadByte = + (*(prevChar + 1) == 0) && IsDBCSLeadByteEx(codepage, *prevChar); + if (isLeadByte) { + rawparts[id - 1] += *(raw.end() - 1); + raw.resize(raw.size() - 1); + } + success = DoDecodeText(raw, decoded, NULL); + } else { + bool restoreDecoded = false; + std::string firstDecoded = decoded; + wchar_t lastChar = 0; + for (UINT i = 0; i < cpinfo.MaxCharSize; i++) { + success = DoDecodeText(raw, decoded, &lastChar); + if (success && lastChar != 0) { + if (i == 0) { + firstDecoded = decoded; + } + if (lastChar == cpinfo.UnicodeDefaultChar) { + restoreDecoded = true; + rawparts[id - 1] = *(raw.end() - 1) + rawparts[id - 1]; + raw.resize(raw.size() - 1); + } else { + restoreDecoded = false; + break; + } + } else { + break; + } + } + if (restoreDecoded) { + decoded = firstDecoded; + rawparts[id - 1].clear(); + } + } + } else { + success = DoDecodeText(raw, decoded, NULL); + } + } +#else + static_cast<void>(id); +#endif + return success; +} + +bool cmProcessOutput::DecodeText(const char* data, size_t length, + std::string& decoded, size_t id) +{ + return DecodeText(std::string(data, length), decoded, id); +} + +bool cmProcessOutput::DecodeText(std::vector<char> raw, + std::vector<char>& decoded, size_t id) +{ + std::string str; + const bool success = + DecodeText(std::string(raw.begin(), raw.end()), str, id); + decoded.assign(str.begin(), str.end()); + return success; +} + +#if defined(_WIN32) +bool cmProcessOutput::DoDecodeText(std::string raw, std::string& decoded, + wchar_t* lastChar) +{ + bool success = false; + const int wlength = + MultiByteToWideChar(codepage, 0, raw.c_str(), int(raw.size()), NULL, 0); + wchar_t* wdata = new wchar_t[wlength]; + int r = MultiByteToWideChar(codepage, 0, raw.c_str(), int(raw.size()), wdata, + wlength); + if (r > 0) { + if (lastChar) { + *lastChar = 0; + if ((wlength >= 2 && wdata[wlength - 2] != wdata[wlength - 1]) || + wlength >= 1) { + *lastChar = wdata[wlength - 1]; + } + } + int length = WideCharToMultiByte(defaultCodepage, 0, wdata, wlength, NULL, + 0, NULL, NULL); + char* data = new char[length]; + r = WideCharToMultiByte(defaultCodepage, 0, wdata, wlength, data, length, + NULL, NULL); + if (r > 0) { + decoded = std::string(data, length); + success = true; + } + delete[] data; + } + delete[] wdata; + return success; +} +#endif diff --git a/Source/cmProcessOutput.h b/Source/cmProcessOutput.h new file mode 100644 index 0000000..d7a5e98 --- /dev/null +++ b/Source/cmProcessOutput.h @@ -0,0 +1,80 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ +#ifndef cmProcessOutput_h +#define cmProcessOutput_h + +#include <cmConfigure.h> // IWYU pragma: keep + +#include <string> +#include <vector> + +/** \class cmProcessOutput + * \brief Decode text data to internal encoding. + * + * cmProcessOutput is used to decode text output from external process + * using external encoding to our internal encoding. + */ +class cmProcessOutput +{ +public: + enum Encoding + { + None, + Auto, + UTF8, + ANSI, + OEM + }; + + /// The code page that is used as internal encoding to which we will encode. + static unsigned int defaultCodepage; + + /** + * A class constructor. + * \param encoding external process encoding from which we will decode. + * \param maxSize a maximal size for process output buffer. It should match + * to KWSYSPE_PIPE_BUFFER_SIZE. If text we decode is same size as \a maxSize + * then we will check for incomplete character at end of buffer and + * we will not return last incomplete character. This character will be + * returned with next DecodeText() call. To disable this behavior specify + * 0 as \a maxSize. + */ + cmProcessOutput(Encoding encoding = Auto, unsigned int maxSize = 1024); + ~cmProcessOutput(); + /** + * Decode \a raw string using external encoding to internal + * encoding in \a decoded. + * \a id specifies which internal buffer to use. This is important when we + * are decoding both stdout and stderr from process output and we need to + * keep incomplete characters in separate buffers for each stream. + * \return true if successfully decoded \a raw to \a decoded or false if not. + */ + bool DecodeText(std::string raw, std::string& decoded, size_t id = 0); + /** + * Decode \a data with \a length from external encoding to internal + * encoding in \a decoded. + * \param data a pointer to process output text data. + * \param length a size of data buffer. + * \param decoded a string which will contain decoded text. + * \param id an internal buffer id to use. + * \return true if successfully decoded \a data to \a decoded or false if + * not. + */ + bool DecodeText(const char* data, size_t length, std::string& decoded, + size_t id = 0); + /** + * \overload + */ + bool DecodeText(std::vector<char> raw, std::vector<char>& decoded, + size_t id = 0); + +private: +#if defined(_WIN32) + unsigned int codepage; + unsigned int bufferSize; + std::vector<std::string> rawparts; + bool DoDecodeText(std::string raw, std::string& decoded, wchar_t* lastChar); +#endif +}; + +#endif diff --git a/Source/cmProcessTools.cxx b/Source/cmProcessTools.cxx index e7b6051..b756650 100644 --- a/Source/cmProcessTools.cxx +++ b/Source/cmProcessTools.cxx @@ -1,29 +1,46 @@ /* Distributed under the OSI-approved BSD 3-Clause License. See accompanying file Copyright.txt or https://cmake.org/licensing for details. */ #include "cmProcessTools.h" +#include "cmProcessOutput.h" #include <cmsys/Process.h> #include <ostream> void cmProcessTools::RunProcess(struct cmsysProcess_s* cp, OutputParser* out, - OutputParser* err) + OutputParser* err, Encoding encoding) { cmsysProcess_Execute(cp); char* data = CM_NULLPTR; int length = 0; int p; + cmProcessOutput processOutput(encoding); + std::string strdata; while ((out || err) && (p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR), p)) { if (out && p == cmsysProcess_Pipe_STDOUT) { - if (!out->Process(data, length)) { + processOutput.DecodeText(data, length, strdata, 1); + if (!out->Process(strdata.c_str(), int(strdata.size()))) { out = CM_NULLPTR; } } else if (err && p == cmsysProcess_Pipe_STDERR) { - if (!err->Process(data, length)) { + processOutput.DecodeText(data, length, strdata, 2); + if (!err->Process(strdata.c_str(), int(strdata.size()))) { err = CM_NULLPTR; } } } + if (out) { + processOutput.DecodeText(std::string(), strdata, 1); + if (!strdata.empty()) { + out->Process(strdata.c_str(), int(strdata.size())); + } + } + if (err) { + processOutput.DecodeText(std::string(), strdata, 2); + if (!strdata.empty()) { + out->Process(strdata.c_str(), int(strdata.size())); + } + } cmsysProcess_WaitForExit(cp, CM_NULLPTR); } diff --git a/Source/cmProcessTools.h b/Source/cmProcessTools.h index e5a3fe9..df131b9 100644 --- a/Source/cmProcessTools.h +++ b/Source/cmProcessTools.h @@ -3,6 +3,7 @@ #ifndef cmProcessTools_h #define cmProcessTools_h +#include "cmProcessOutput.h" #include <cmConfigure.h> #include <iosfwd> @@ -16,6 +17,7 @@ class cmProcessTools { public: + typedef cmProcessOutput::Encoding Encoding; /** Abstract interface for process output parsers. */ class OutputParser { @@ -79,7 +81,8 @@ public: /** Run a process and send output to given parsers. */ static void RunProcess(struct cmsysProcess_s* cp, OutputParser* out, - OutputParser* err = CM_NULLPTR); + OutputParser* err = CM_NULLPTR, + Encoding encoding = cmProcessOutput::Auto); }; #endif diff --git a/Source/cmSystemTools.cxx b/Source/cmSystemTools.cxx index 7738ab6..029594f 100644 --- a/Source/cmSystemTools.cxx +++ b/Source/cmSystemTools.cxx @@ -3,6 +3,7 @@ #include "cmSystemTools.h" #include "cmAlgorithms.h" +#include "cmProcessOutput.h" #if defined(CMAKE_BUILD_WITH_CMAKE) #include "cmArchiveWrite.h" @@ -573,7 +574,7 @@ bool cmSystemTools::RunSingleCommand(std::vector<std::string> const& command, std::string* captureStdOut, std::string* captureStdErr, int* retVal, const char* dir, OutputOption outputflag, - double timeout) + double timeout, Encoding encoding) { std::vector<const char*> argv; for (std::vector<std::string>::const_iterator a = command.begin(); @@ -609,6 +610,8 @@ bool cmSystemTools::RunSingleCommand(std::vector<std::string> const& command, char* data; int length; int pipe; + cmProcessOutput processOutput(encoding); + std::string strdata; if (outputflag != OUTPUT_PASSTHROUGH && (captureStdOut || captureStdErr || outputflag != OUTPUT_NONE)) { while ((pipe = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) > @@ -624,28 +627,44 @@ bool cmSystemTools::RunSingleCommand(std::vector<std::string> const& command, if (pipe == cmsysProcess_Pipe_STDOUT) { if (outputflag != OUTPUT_NONE) { - cmSystemTools::Stdout(data, length); + processOutput.DecodeText(data, length, strdata, 1); + cmSystemTools::Stdout(strdata.c_str(), strdata.size()); } if (captureStdOut) { tempStdOut.insert(tempStdOut.end(), data, data + length); } } else if (pipe == cmsysProcess_Pipe_STDERR) { if (outputflag != OUTPUT_NONE) { - cmSystemTools::Stderr(data, length); + processOutput.DecodeText(data, length, strdata, 2); + cmSystemTools::Stderr(strdata.c_str(), strdata.size()); } if (captureStdErr) { tempStdErr.insert(tempStdErr.end(), data, data + length); } } } + + if (outputflag != OUTPUT_NONE) { + processOutput.DecodeText(std::string(), strdata, 1); + if (!strdata.empty()) { + cmSystemTools::Stdout(strdata.c_str(), strdata.size()); + } + processOutput.DecodeText(std::string(), strdata, 2); + if (!strdata.empty()) { + cmSystemTools::Stderr(strdata.c_str(), strdata.size()); + } + } } cmsysProcess_WaitForExit(cp, CM_NULLPTR); + if (captureStdOut) { captureStdOut->assign(tempStdOut.begin(), tempStdOut.end()); + processOutput.DecodeText(*captureStdOut, *captureStdOut); } if (captureStdErr) { captureStdErr->assign(tempStdErr.begin(), tempStdErr.end()); + processOutput.DecodeText(*captureStdErr, *captureStdErr); } bool result = true; @@ -1643,6 +1662,8 @@ int cmSystemTools::WaitForLine(cmsysProcess* process, std::string& line, line = ""; std::vector<char>::iterator outiter = out.begin(); std::vector<char>::iterator erriter = err.begin(); + cmProcessOutput processOutput; + std::string strdata; while (1) { // Check for a newline in stdout. for (; outiter != out.end(); ++outiter) { @@ -1687,17 +1708,31 @@ int cmSystemTools::WaitForLine(cmsysProcess* process, std::string& line, return pipe; } if (pipe == cmsysProcess_Pipe_STDOUT) { + processOutput.DecodeText(data, length, strdata, 1); // Append to the stdout buffer. std::vector<char>::size_type size = out.size(); - out.insert(out.end(), data, data + length); + out.insert(out.end(), strdata.begin(), strdata.end()); outiter = out.begin() + size; } else if (pipe == cmsysProcess_Pipe_STDERR) { + processOutput.DecodeText(data, length, strdata, 2); // Append to the stderr buffer. std::vector<char>::size_type size = err.size(); - err.insert(err.end(), data, data + length); + err.insert(err.end(), strdata.begin(), strdata.end()); erriter = err.begin() + size; } else if (pipe == cmsysProcess_Pipe_None) { // Both stdout and stderr pipes have broken. Return leftover data. + processOutput.DecodeText(std::string(), strdata, 1); + if (!strdata.empty()) { + std::vector<char>::size_type size = out.size(); + out.insert(out.end(), strdata.begin(), strdata.end()); + outiter = out.begin() + size; + } + processOutput.DecodeText(std::string(), strdata, 2); + if (!strdata.empty()) { + std::vector<char>::size_type size = err.size(); + err.insert(err.end(), strdata.begin(), strdata.end()); + erriter = err.begin() + size; + } if (!out.empty()) { line.append(&out[0], outiter - out.begin()); out.erase(out.begin(), out.end()); diff --git a/Source/cmSystemTools.h b/Source/cmSystemTools.h index 0801f26..10e8280 100644 --- a/Source/cmSystemTools.h +++ b/Source/cmSystemTools.h @@ -5,6 +5,7 @@ #include <cmConfigure.h> // IWYU pragma: keep +#include <cmProcessOutput.h> #include <cmsys/Process.h> #include <cmsys/SystemTools.hxx> #include <stddef.h> @@ -29,6 +30,7 @@ class cmSystemTools : public cmsys::SystemTools { public: typedef cmsys::SystemTools Superclass; + typedef cmProcessOutput::Encoding Encoding; /** Expand out any arguments in the vector that have ; separated * strings into multiple arguments. A new vector is created @@ -239,7 +241,8 @@ public: int* retVal = CM_NULLPTR, const char* dir = CM_NULLPTR, OutputOption outputflag = OUTPUT_MERGE, - double timeout = 0.0); + double timeout = 0.0, + Encoding encoding = cmProcessOutput::Auto); static std::string PrintSingleCommand(std::vector<std::string> const&); |