From 755c501084412a0896312af05cda6d4227136174 Mon Sep 17 00:00:00 2001 From: Jiri Moudry Date: Tue, 27 Mar 2012 17:39:01 +0200 Subject: Improve handling of fatal errors on Windows, support creation of minidumps --- src/ninja.cc | 35 +++++++++++++++++++++++++++ src/util.cc | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/util.h | 7 ++++++ 3 files changed, 120 insertions(+) diff --git a/src/ninja.cc b/src/ninja.cc index 04cd771..5807dcd 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -614,7 +614,42 @@ int RunBuild(Globals* globals, int argc, char** argv) { } // anonymous namespace + +#ifdef _WIN32 + +/// This handler processes fatal crashes that you can't catch +/// Test example: C++ exception in a stack-unwind-block +/// Real-world example: ninja launched a compiler to process a tricky C++ input file. +/// The compiler got itself into a state where it generated 3 GB of output and caused ninja to crash +void ninja_terminate_fct() { + Create_Win32_MiniDump(NULL); + Fatal("terminate handler called"); +} + +/// main_unsafe is called from within an exception handling block +int main_unsafe(int argc, char** argv); + +/// Windows main() uses SEH (Structured exception handling) +int main(int argc, char** argv) { + // set a handler to catch crashes not caught by the __try..__except block (e.g. an exception in a stack-unwind-block) + set_terminate(ninja_terminate_fct); + __try { + // running inside __try ... __except suppresses any Windows error dialogs for errors such as bad_alloc + return main_unsafe(argc, argv); + } + __except(exception_filter(GetExceptionCode(), GetExceptionInformation())) { + // you will land here e.g. if you run out of memory, or run inside a distribution environment that fails + fprintf(stderr, "ninja: exception, exiting with error code 2\n"); + // common error situations below return exitCode=1, 2 was chosen to indicate a more serious problem + return 2; + } +} + +int main_unsafe (int argc, char** argv) { +#else +//on Linux, we have no exception handling int main(int argc, char** argv) { +#endif Globals globals; globals.ninja_command = argv[0]; const char* input_file = "build.ninja"; diff --git a/src/util.cc b/src/util.cc index 4d9adf3..d240e1c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -34,6 +34,7 @@ #include #ifdef _WIN32 +#include // for minidump #include // _mkdir #endif @@ -292,3 +293,80 @@ string StripAnsiEscapeCodes(const string& in) { } return stripped; } + +#ifdef _WIN32 +typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( + IN HANDLE, + IN DWORD, + IN HANDLE, + IN MINIDUMP_TYPE, + IN CONST PMINIDUMP_EXCEPTION_INFORMATION, OPTIONAL + IN CONST PMINIDUMP_USER_STREAM_INFORMATION, OPTIONAL + IN CONST PMINIDUMP_CALLBACK_INFORMATION OPTIONAL + ); + +/// this function creates a windows minidump in temp folder. +void Create_Win32_MiniDump( _EXCEPTION_POINTERS* pep ) { + char tempPathBuff[MAX_PATH]; + GetTempPath(sizeof(tempPathBuff), tempPathBuff); + char tempFileName[MAX_PATH]; + sprintf(tempFileName, "%s\\ninja_crash_dump_%d.dmp", tempPathBuff, GetCurrentProcessId()); + + //delete any previous minidump of the same name + DeleteFile(tempFileName); + + // load DbgHelp.dll dynamically, as library is not present on all windows versions + HMODULE hModDbgHelp = LoadLibrary("dbghelp.dll"); + if (hModDbgHelp == NULL) { + fprintf(stderr, "ninja: failed to create minidump, failed to load dbghelp.dll, error %s \n", + GetLastErrorString().c_str()); + return; + } + + MiniDumpWriteDumpFunc pfnMiniDumpWriteDump = (MiniDumpWriteDumpFunc)GetProcAddress(hModDbgHelp, "MiniDumpWriteDump"); + if (pfnMiniDumpWriteDump == NULL) { + fprintf(stderr, "ninja: failed to create minidump, failed on GetProcAddress('MiniDumpWriteDump'), error %s \n", + GetLastErrorString().c_str()); + return; + } + + // Open the file + HANDLE hFile = CreateFileA( tempFileName, GENERIC_READ | GENERIC_WRITE, + 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL ); + if (hFile == NULL) { + fprintf(stderr, "ninja: failed to create minidump, failed on CreateFileA(%s), error %s \n", + tempFileName, GetLastErrorString().c_str()); + return; + } + + // Create the mini dump + MINIDUMP_EXCEPTION_INFORMATION mdei; + mdei.ThreadId = GetCurrentThreadId(); + mdei.ExceptionPointers = pep; + mdei.ClientPointers = FALSE; + MINIDUMP_TYPE mdt = (MINIDUMP_TYPE) (MiniDumpWithDataSegs | MiniDumpWithHandleData); + + BOOL rv = pfnMiniDumpWriteDump( GetCurrentProcess(), GetCurrentProcessId(), + hFile, mdt, (pep != 0) ? &mdei : 0, 0, 0 ); + + if ( !rv ) + fprintf(stderr, "ninja: MiniDumpWriteDump failed. Error: %s \n", GetLastErrorString().c_str() ); + else + fprintf(stderr, "ninja: Minidump created: %s\n", tempFileName ); + + // Close the file + CloseHandle( hFile ); +} + +/// On Windows, we want to prevent error dialogs in case of exceptions. +/// This function handles the exception, and writes a minidump. +int exception_filter(unsigned int code, struct _EXCEPTION_POINTERS *ep) { + fprintf(stderr, "ninja.exe fatal error: 0x%X \n", code); //e.g. EXCEPTION_ACCESS_VIOLATION + fflush(stderr); + Create_Win32_MiniDump(ep); + return EXCEPTION_EXECUTE_HANDLER; +} +#else + //on Linux, core dumps are created automatically, no code needed +#endif + diff --git a/src/util.h b/src/util.h index 9b4e745..016e6cf 100644 --- a/src/util.h +++ b/src/util.h @@ -76,6 +76,13 @@ string StripAnsiEscapeCodes(const string& in); #endif #ifdef _WIN32 + +/// Handler for __except block +int exception_filter(unsigned int code, struct _EXCEPTION_POINTERS *ep); + +/// Write a windows minidump file in temp directory. User can specify null 'pep' argument. +void Create_Win32_MiniDump( struct _EXCEPTION_POINTERS* pep ); + /// Convert the value returned by GetLastError() into a string. string GetLastErrorString(); #endif -- cgit v0.12 From 28c53397ed6876363a93e5a0ac3cd54ed66b662f Mon Sep 17 00:00:00 2001 From: Jiri Moudry Date: Thu, 26 Apr 2012 16:49:34 +0200 Subject: Changed #ifdef _WIN32 to #ifdef _MSC_VER to make sure that MinGW compiles happily --- src/ninja.cc | 2 +- src/util.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index 5807dcd..1a02fd6 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -615,7 +615,7 @@ int RunBuild(Globals* globals, int argc, char** argv) { } // anonymous namespace -#ifdef _WIN32 +#ifdef _MSC_VER /// This handler processes fatal crashes that you can't catch /// Test example: C++ exception in a stack-unwind-block diff --git a/src/util.cc b/src/util.cc index d240e1c..e59ff74 100644 --- a/src/util.cc +++ b/src/util.cc @@ -294,7 +294,7 @@ string StripAnsiEscapeCodes(const string& in) { return stripped; } -#ifdef _WIN32 +#ifdef _MSC_VER typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( IN HANDLE, IN DWORD, @@ -367,6 +367,6 @@ int exception_filter(unsigned int code, struct _EXCEPTION_POINTERS *ep) { return EXCEPTION_EXECUTE_HANDLER; } #else - //on Linux, core dumps are created automatically, no code needed + //on Linux or MinGW, core dumps are created automatically, no code needed #endif -- cgit v0.12 From 9ef56fd56f414bc6aa231b8d65cfa4ddec4cdedb Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Tue, 19 Jun 2012 13:15:41 -0700 Subject: fix win32 compile, fix BuildLogTest.WriteRead on 2nd run --- configure.py | 4 ++-- src/build_log_test.cc | 1 + src/win32port.h | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/configure.py b/configure.py index 5329708..feb8a98 100755 --- a/configure.py +++ b/configure.py @@ -150,8 +150,8 @@ else: def shell_escape(str): """Escape str such that it's interpreted as a single argument by the shell.""" # This isn't complete, but it's just enough to make NINJA_PYTHON work. - # TODO: do the appropriate thing for Windows-style cmd here, perhaps by - # just returning the input string. + if platform == 'windows': + return str if '"' in str: return "'%s'" % str.replace("'", "\\'") return str diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 9fb42c9..8e73e6b 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -64,6 +64,7 @@ TEST_F(BuildLogTest, WriteRead) { ASSERT_TRUE(*e1 == *e2); ASSERT_EQ(15, e1->start_time); ASSERT_EQ("out", e1->output); + log2.Close(); } TEST_F(BuildLogTest, FirstWriteAddsSignature) { diff --git a/src/win32port.h b/src/win32port.h index 8b42b38..3799af1 100644 --- a/src/win32port.h +++ b/src/win32port.h @@ -16,6 +16,8 @@ #define NINJA_WIN32PORT_H_ #pragma once +typedef signed short int16_t; +typedef unsigned short uint16_t; /// A 64-bit integer type typedef signed long long int64_t; typedef unsigned long long uint64_t; -- cgit v0.12 From dd91094196ba8a824cd2f12ceec7e12452c4d32c Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Tue, 19 Jun 2012 14:32:04 -0700 Subject: don't leak file handle on fail-to-upgrade case --- src/build_log.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/build_log.cc b/src/build_log.cc index 02a9fb5..5eddf8f 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -235,6 +235,7 @@ bool BuildLog::Load(const string& path, string* err) { if (log_version < kOldestSupportedVersion) { *err = "unable to extract version from build log, perhaps due to " "being too old; you must clobber your build output and rebuild"; + fclose(file); return false; } } -- cgit v0.12 From 98d1b7b718de76831ddf4da283e6c0510266322b Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Tue, 19 Jun 2012 14:32:39 -0700 Subject: remove unnecessary manual Close() --- src/build_log_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 8e73e6b..9fb42c9 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -64,7 +64,6 @@ TEST_F(BuildLogTest, WriteRead) { ASSERT_TRUE(*e1 == *e2); ASSERT_EQ(15, e1->start_time); ASSERT_EQ("out", e1->output); - log2.Close(); } TEST_F(BuildLogTest, FirstWriteAddsSignature) { -- cgit v0.12 From 904c9610fe66c4f4bd63a07d6f057c8603d24394 Mon Sep 17 00:00:00 2001 From: Thiago Farina Date: Sat, 30 Jun 2012 11:40:10 -0300 Subject: Make StringPiece data members private. Signed-off-by: Thiago Farina --- src/build_log.cc | 4 ++-- src/depfile_parser.cc | 2 +- src/edit_distance.cc | 10 +++++----- src/eval_env.cc | 2 +- src/graph.cc | 3 ++- src/hash_map.h | 2 +- src/lexer.cc | 6 +++--- src/string_piece.h | 4 ++++ 8 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/build_log.cc b/src/build_log.cc index 02a9fb5..0a44817 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -80,7 +80,7 @@ uint64_t MurmurHash64A(const void* key, int len) { h *= m; h ^= h >> r; return h; -} +} #undef BIG_CONSTANT @@ -88,7 +88,7 @@ uint64_t MurmurHash64A(const void* key, int len) { // static uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { - return MurmurHash64A(command.str_, command.len_); + return MurmurHash64A(command.str(), command.len()); } BuildLog::BuildLog() diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 54b934c..b6ea1ce 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -195,7 +195,7 @@ yy13: if (!is_target) { ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str_) { + } else if (!out_.str()) { out_ = StringPiece(filename, len); } else if (out_ != StringPiece(filename, len)) { *err = "depfile has multiple output paths."; diff --git a/src/edit_distance.cc b/src/edit_distance.cc index 22db4fe..50e641d 100644 --- a/src/edit_distance.cc +++ b/src/edit_distance.cc @@ -29,8 +29,8 @@ int EditDistance(const StringPiece& s1, // Although the algorithm is typically described using an m x n // array, only two rows are used at a time, so this implemenation // just keeps two separate vectors for those two rows. - int m = s1.len_; - int n = s2.len_; + int m = s1.len(); + int n = s2.len(); std::vector previous(n + 1); std::vector current(n + 1); @@ -44,11 +44,11 @@ int EditDistance(const StringPiece& s1, for (int x = 1; x <= n; ++x) { if (allow_replacements) { - current[x] = min(previous[x-1] + (s1.str_[y-1] == s2.str_[x-1] ? 0 : 1), - min(current[x-1], previous[x])+1); + current[x] = min(previous[x-1] + (s1.str()[y-1] == s2.str()[x-1] ? + 0 : 1), min(current[x-1], previous[x]) + 1); } else { - if (s1.str_[y-1] == s2.str_[x-1]) + if (s1.str()[y-1] == s2.str()[x-1]) current[x] = previous[x-1]; else current[x] = min(current[x-1], previous[x]) + 1; diff --git a/src/eval_env.cc b/src/eval_env.cc index 81a8765..793ea64 100644 --- a/src/eval_env.cc +++ b/src/eval_env.cc @@ -41,7 +41,7 @@ string EvalString::Evaluate(Env* env) const { void EvalString::AddText(StringPiece text) { // Add it to the end of an existing RAW token if possible. if (!parsed_.empty() && parsed_.back().second == RAW) { - parsed_.back().first.append(text.str_, text.len_); + parsed_.back().first.append(text.str(), text.len()); } else { parsed_.push_back(make_pair(text.AsString(), RAW)); } diff --git a/src/graph.cc b/src/graph.cc index 56584e3..4bf558a 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -292,7 +292,8 @@ bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, // Add all its in-edges. for (vector::iterator i = depfile.ins_.begin(); i != depfile.ins_.end(); ++i, ++implicit_dep) { - if (!CanonicalizePath(const_cast(i->str_), &i->len_, err)) + int length = i->len(); + if (!CanonicalizePath(const_cast(i->str()), &length, err)) return false; Node* node = state->GetNode(*i); diff --git a/src/hash_map.h b/src/hash_map.h index 88c2681..15e86da 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -86,7 +86,7 @@ struct hash { template<> struct hash { size_t operator()(StringPiece key) const { - return MurmurHash2(key.str_, key.len_); + return MurmurHash2(key.str(), key.len()); } }; diff --git a/src/lexer.cc b/src/lexer.cc index b3efe22..48a46b0 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -23,8 +23,8 @@ bool Lexer::Error(const string& message, string* err) { // Compute line/column. int line = 1; - const char* context = input_.str_; - for (const char* p = input_.str_; p < last_token_; ++p) { + const char* context = input_.str(); + for (const char* p = input_.str(); p < last_token_; ++p) { if (*p == '\n') { ++line; context = p + 1; @@ -66,7 +66,7 @@ Lexer::Lexer(const char* input) { void Lexer::Start(StringPiece filename, StringPiece input) { filename_ = filename; input_ = input; - ofs_ = input_.str_; + ofs_ = input_.str(); last_token_ = NULL; } diff --git a/src/string_piece.h b/src/string_piece.h index ad1153e..76679f1 100644 --- a/src/string_piece.h +++ b/src/string_piece.h @@ -46,6 +46,10 @@ struct StringPiece { return len_ ? string(str_, len_) : string(); } + const char* str() const { return str_; } + int len() const { return len_; } + + private: const char* str_; int len_; }; -- cgit v0.12 From c0c5ef9bffb0130e169aeddcb8487b670c6b56fe Mon Sep 17 00:00:00 2001 From: Peter Kuemmel Date: Fri, 29 Jun 2012 16:12:03 +0200 Subject: print edges per second prints the rate of finished edges per second to the console, for instance with NINJA_STATUS="[%s/%t %o(%c)/s] ": [132/1922 16.1(14)/s] 16.1 is the average for all processed files (here 132 since start) 14 is the average of the last n files while n is the number specifies by -j (or its default) --- doc/manual.asciidoc | 2 ++ src/build.cc | 28 +++++++++++++++++++++++++++- src/build.h | 35 +++++++++++++++++++++++++++++++++++ src/build_log_perftest.cc | 1 + src/canon_perftest.cc | 1 + src/metrics.cc | 10 ++++++++++ src/metrics.h | 25 +++++++++++++++++++++++++ src/parser_perftest.cc | 1 + src/util.cc | 10 ---------- src/util.h | 5 ----- 10 files changed, 102 insertions(+), 16 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index e7983be..94744be 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -218,6 +218,8 @@ Several placeholders are available: * `%r`: The number of currently running edges. * `%u`: The number of remaining edges to start. * `%f`: The number of finished edges. +* `%o`: Overall rate of finished edges per second +* `%c`: Current rate of finished edges per second (average over builds specified by -j or its default) * `%%`: A plain `%` character. * The default progress status is `"[%s/%t] "` (note the trailing space to separate from the build rule). Another example of possible progress status diff --git a/src/build.cc b/src/build.cc index 0fc387c..48e5c4b 100644 --- a/src/build.cc +++ b/src/build.cc @@ -37,7 +37,9 @@ BuildStatus::BuildStatus(const BuildConfig& config) : config_(config), start_time_millis_(GetTimeMillis()), started_edges_(0), finished_edges_(0), total_edges_(0), - have_blank_line_(true), progress_status_format_(NULL) { + have_blank_line_(true), progress_status_format_(NULL), + overall_rate_(), current_rate_(), + current_rate_average_count_(config.parallelism) { #ifndef _WIN32 const char* term = getenv("TERM"); smart_terminal_ = isatty(1) && term && string(term) != "dumb"; @@ -171,6 +173,25 @@ string BuildStatus::FormatProgressStatus(const char* progress_status_format) con out += buf; break; + // Overall finished edges per second. + case 'o': + overall_rate_.UpdateRate(finished_edges_, finished_edges_); + overall_rate_.snprinfRate(buf, "%.1f"); + out += buf; + break; + + // Current rate, average over the last '-j' jobs. + case 'c': + // TODO use sliding window? + if (finished_edges_ > current_rate_.last_update() && + finished_edges_ - current_rate_.last_update() == current_rate_average_count_) { + current_rate_.UpdateRate(current_rate_average_count_, finished_edges_); + current_rate_.Restart(); + } + current_rate_.snprinfRate(buf, "%.0f"); + out += buf; + break; + default: { Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s); return ""; @@ -208,6 +229,10 @@ void BuildStatus::PrintStatus(Edge* edge) { #endif } + if (finished_edges_ == 0) { + overall_rate_.Restart(); + current_rate_.Restart(); + } to_print = FormatProgressStatus(progress_status_format_) + to_print; if (smart_terminal_ && !force_full_command) { @@ -770,3 +795,4 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { if (success && log_) log_->RecordCommand(edge, start_time, end_time, restat_mtime); } + diff --git a/src/build.h b/src/build.h index c9ee4ac..4859c29 100644 --- a/src/build.h +++ b/src/build.h @@ -24,6 +24,7 @@ using namespace std; #include "exit_status.h" +#include "metrics.h" #include "util.h" // int64_t struct BuildLog; @@ -191,9 +192,43 @@ struct BuildStatus { /// The custom progress status format to use. const char* progress_status_format_; + struct RateInfo { + RateInfo() : last_update_(0), rate_(-1) {} + + double rate() const { return rate_; } + int last_update() const { return last_update_; } + void Restart() { return stopwatch_.Restart(); } + + double UpdateRate(int edges, int update_hint) { + if (update_hint != last_update_) { + rate_ = edges / stopwatch_.Elapsed() + 0.5; + last_update_ = update_hint; + } + return rate_; + } + + template + void snprinfRate(T buf, const char* format) { + if (rate_ == -1) + snprintf(buf, sizeof(buf), "?"); + else + snprintf(buf, sizeof(buf), format, rate_); + } + + private: + Stopwatch stopwatch_; + int last_update_; + double rate_; + }; + + mutable RateInfo overall_rate_; + mutable RateInfo current_rate_; + const int current_rate_average_count_; + #ifdef _WIN32 void* console_; #endif }; #endif // NINJA_BUILD_H_ + diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc index 02f4c60..5755079 100644 --- a/src/build_log_perftest.cc +++ b/src/build_log_perftest.cc @@ -20,6 +20,7 @@ #include "manifest_parser.h" #include "state.h" #include "util.h" +#include "metrics.h" const char kTestFilename[] = "BuildLogPerfTest-tempfile"; diff --git a/src/canon_perftest.cc b/src/canon_perftest.cc index 6248937..d0ed397 100644 --- a/src/canon_perftest.cc +++ b/src/canon_perftest.cc @@ -16,6 +16,7 @@ #include #include "util.h" +#include "metrics.h" const char kPath[] = "../../third_party/WebKit/Source/WebCore/" diff --git a/src/metrics.cc b/src/metrics.cc index fb44868..8407717 100644 --- a/src/metrics.cc +++ b/src/metrics.cc @@ -72,6 +72,7 @@ int64_t TimerToMicros(int64_t dt) { } // anonymous namespace + ScopedMetric::ScopedMetric(Metric* metric) { metric_ = metric; if (!metric_) @@ -113,3 +114,12 @@ void Metrics::Report() { metric->count, avg, total); } } + +uint64_t Stopwatch::Now() const { + return TimerToMicros(HighResTimer()); +} + +int64_t GetTimeMillis() { + return 1000 * TimerToMicros(HighResTimer()); +} + diff --git a/src/metrics.h b/src/metrics.h index af6e9a2..dae6b9f 100644 --- a/src/metrics.h +++ b/src/metrics.h @@ -33,6 +33,7 @@ struct Metric { int64_t sum; }; + /// A scoped object for recording a metric across the body of a function. /// Used by the METRIC_RECORD macro. struct ScopedMetric { @@ -57,6 +58,30 @@ private: vector metrics_; }; +/// Get the current time as relative to some epoch. +/// Epoch varies between platforms; only useful for measuring elapsed +/// time. +int64_t GetTimeMillis(); + + +/// A simple stopwatch which retruns the time +// in seconds since Restart() was called +class Stopwatch +{ +public: + Stopwatch() : started_(0) {} + + /// Seconds since Restart() call + double Elapsed() const { return 1e-6 * (Now() - started_); } + + void Restart() { started_ = Now(); } + +private: + uint64_t started_; + uint64_t Now() const; +}; + + /// The primary interface to metrics. Use METRIC_RECORD("foobar") at the top /// of a function to get timing stats recorded for each call of the function. #define METRIC_RECORD(name) \ diff --git a/src/parser_perftest.cc b/src/parser_perftest.cc index b1d5bf0..b215221 100644 --- a/src/parser_perftest.cc +++ b/src/parser_perftest.cc @@ -17,6 +17,7 @@ #include "depfile_parser.h" #include "util.h" +#include "metrics.h" int main(int argc, char* argv[]) { if (argc < 2) { diff --git a/src/util.cc b/src/util.cc index d8d7fb3..ab3275a 100644 --- a/src/util.cc +++ b/src/util.cc @@ -204,16 +204,6 @@ void SetCloseOnExec(int fd) { #endif // ! _WIN32 } -int64_t GetTimeMillis() { -#ifdef _WIN32 - // GetTickCount64 is only available on Vista or later. - return GetTickCount(); -#else - timeval now; - gettimeofday(&now, NULL); - return ((int64_t)now.tv_sec * 1000) + (now.tv_usec / 1000); -#endif -} const char* SpellcheckStringV(const string& text, const vector& words) { diff --git a/src/util.h b/src/util.h index 82f4850..b108c25 100644 --- a/src/util.h +++ b/src/util.h @@ -53,11 +53,6 @@ int ReadFile(const string& path, string* contents, string* err); /// Mark a file descriptor to not be inherited on exec()s. void SetCloseOnExec(int fd); -/// Get the current time as relative to some epoch. -/// Epoch varies between platforms; only useful for measuring elapsed -/// time. -int64_t GetTimeMillis(); - /// Given a misspelled string and a list of correct spellings, returns /// the closest match or NULL if there is no close enough match. const char* SpellcheckStringV(const string& text, const vector& words); -- cgit v0.12 From 5d120a0ae4b139761d731d6d0bd70a92aee0fc61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20K=C3=BCmmel?= Date: Fri, 6 Jul 2012 11:12:36 +0200 Subject: GCC 4.7 needs cstdio --- src/build.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build.h b/src/build.h index 4859c29..5e26bbb 100644 --- a/src/build.h +++ b/src/build.h @@ -21,7 +21,7 @@ #include #include #include -using namespace std; +#include #include "exit_status.h" #include "metrics.h" -- cgit v0.12 From 74fb2bec2ae6041876da654236107edfac215ea5 Mon Sep 17 00:00:00 2001 From: Oleksandr Usov Date: Mon, 16 Jul 2012 14:05:53 +0100 Subject: Add missing include of --- src/subprocess_test.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index c155012..0c10250 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -16,6 +16,11 @@ #include "test.h" +#ifdef linux +// required by SubprocessTest +#include +#endif + namespace { #ifdef _WIN32 -- cgit v0.12 From 520c1cff03e83437652d76c3ad27067355dc993c Mon Sep 17 00:00:00 2001 From: Thiago Farina Date: Tue, 17 Jul 2012 11:35:12 -0300 Subject: Remove #pragma once from our header files. https://github.com/martine/ninja/issues/358 Signed-off-by: Thiago Farina --- src/clean.h | 1 - src/state.h | 1 - src/util.h | 1 - src/win32port.h | 1 - 4 files changed, 4 deletions(-) diff --git a/src/clean.h b/src/clean.h index 4d9b4e6..5938dff 100644 --- a/src/clean.h +++ b/src/clean.h @@ -14,7 +14,6 @@ #ifndef NINJA_CLEAN_H_ #define NINJA_CLEAN_H_ -#pragma once #include #include diff --git a/src/state.h b/src/state.h index 23ca12b..9197ef8 100644 --- a/src/state.h +++ b/src/state.h @@ -14,7 +14,6 @@ #ifndef NINJA_STATE_H_ #define NINJA_STATE_H_ -#pragma once #include #include diff --git a/src/util.h b/src/util.h index 82f4850..3192ded 100644 --- a/src/util.h +++ b/src/util.h @@ -14,7 +14,6 @@ #ifndef NINJA_UTIL_H_ #define NINJA_UTIL_H_ -#pragma once #ifdef _WIN32 #include "win32port.h" diff --git a/src/win32port.h b/src/win32port.h index 8b42b38..97bf817 100644 --- a/src/win32port.h +++ b/src/win32port.h @@ -14,7 +14,6 @@ #ifndef NINJA_WIN32PORT_H_ #define NINJA_WIN32PORT_H_ -#pragma once /// A 64-bit integer type typedef signed long long int64_t; -- cgit v0.12 From 183a2d8f194116e4161ff40a74dee86a8c70c54a Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 17 Jul 2012 08:59:55 -0700 Subject: fix manual to reflect reality of depfile cleaning Fixes issue #362. --- doc/manual.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index e7983be..4db0565 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -278,9 +278,9 @@ built for them. If used like +ninja -t clean -r _rules_+ it removes all files built using the given rules. + -depfiles are not removed. Files created but not referenced in the -graph are not removed. This tool takes in account the +-v+ and the -+-n+ options (note that +-n+ implies +-v+). +Files created but not referenced in the graph are not removed. This +tool takes in account the +-v+ and the +-n+ options (note that +-n+ +implies +-v+). -- cgit v0.12 From 1e2514e5212f24234c50c775a2d2333e31dd4aa8 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 17 Jul 2012 13:26:11 -0700 Subject: ninja_syntax: escape colons in paths --- misc/ninja_syntax.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py index 3ecbcee..66babbe 100644 --- a/misc/ninja_syntax.py +++ b/misc/ninja_syntax.py @@ -10,8 +10,8 @@ use Python. import textwrap import re -def escape_spaces(word): - return word.replace('$ ','$$ ').replace(' ','$ ') +def escape_path(word): + return word.replace('$ ','$$ ').replace(' ','$ ').replace(':', '$:') class Writer(object): def __init__(self, output, width=78): @@ -53,15 +53,15 @@ class Writer(object): variables=None): outputs = self._as_list(outputs) all_inputs = self._as_list(inputs)[:] - out_outputs = list(map(escape_spaces, outputs)) - all_inputs = list(map(escape_spaces, all_inputs)) + out_outputs = list(map(escape_path, outputs)) + all_inputs = list(map(escape_path, all_inputs)) if implicit: - implicit = map(escape_spaces, self._as_list(implicit)) + implicit = map(escape_path, self._as_list(implicit)) all_inputs.append('|') all_inputs.extend(implicit) if order_only: - order_only = map(escape_spaces, self._as_list(order_only)) + order_only = map(escape_path, self._as_list(order_only)) all_inputs.append('||') all_inputs.extend(order_only) -- cgit v0.12 From 4bfc66b3a955ebc82779f8eaa7b4c56a52d4b4c0 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 17 Jul 2012 09:34:10 -0700 Subject: subprocess_test: make SetWithLots run on all non-Windows platforms I think it might have been an oversight that this was marked Linux-specific. --- src/subprocess_test.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index 0c10250..7dbaf97 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -16,9 +16,11 @@ #include "test.h" -#ifdef linux -// required by SubprocessTest +#ifndef _WIN32 +// SetWithLots need setrlimit. +#include #include +#include #endif namespace { @@ -147,7 +149,7 @@ TEST_F(SubprocessTest, SetWithMulti) { } } -#ifdef linux +#ifndef _WIN32 TEST_F(SubprocessTest, SetWithLots) { // Arbitrary big number; needs to be over 1024 to confirm we're no longer // hostage to pselect. @@ -174,4 +176,4 @@ TEST_F(SubprocessTest, SetWithLots) { } ASSERT_EQ(kNumProcs, subprocs_.finished_.size()); } -#endif // linux +#endif // _WIN32 -- cgit v0.12 From fb996074eaab5272d6cbd198d120b1a44478c8ec Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 17 Jul 2012 16:19:50 -0700 Subject: gitignore unpacked gtest Patch from Scott Graham . --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1072670..fb9a94c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ TAGS /graph.png /doc/manual.html /doc/doxygen +/gtest-1.6.0 -- cgit v0.12 From dc9759b530496a06ef5039ec232a2ebea9744e90 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 17 Jul 2012 16:32:44 -0700 Subject: windows: get correct path for ninja_syntax Patch from Scott Graham . --- configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.py b/configure.py index 22aa36a..bf2596c 100755 --- a/configure.py +++ b/configure.py @@ -379,7 +379,7 @@ if host != 'mingw': options.with_python, generator=True) n.build('build.ninja', 'configure', - implicit=['configure.py', 'misc/ninja_syntax.py']) + implicit=['configure.py', os.path.normpath('misc/ninja_syntax.py')]) n.newline() n.comment('Build only the main binary by default.') -- cgit v0.12 From 42163db5edfd6865a05226af23f99aef3622f550 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 17 Jul 2012 16:42:53 -0700 Subject: explain mode: show which input is more recent Patch from Scott Graham . --- src/graph.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/graph.cc b/src/graph.cc index 071a3b6..18adeee 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -140,7 +140,10 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, if (rule_->restat() && build_log && (entry = build_log->LookupByOutput(output->path()))) { if (entry->restat_mtime < most_recent_input) { - EXPLAIN("restat of output %s older than inputs", output->path().c_str()); + EXPLAIN("restat of output %s older than most recent input %s (%d vs %d)", + output->path().c_str(), + most_recent_node ? most_recent_node->path().c_str() : "", + entry->restat_mtime, most_recent_input); return true; } } else { -- cgit v0.12 From 8a37bfffcbea27ffbb9168b2740476336cd4963a Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 17 Jul 2012 17:38:48 -0700 Subject: disallow crlf in manifest files It turns out to be trickier than expected to process these correctly. It turns out to also be trickier than expected to give a nice error message on encountering these. But the behavior prior to this patch would just be silent failures where we attempted to examine paths that accidentally contained embedded \r. For now, fix all regexes of the form [^...] to include \r in the excluded block, then assert that we get a vague lexer error near the problem. In the future perhaps we can open manifest files in text mode on Windows or just disallow Windows-style CRLF in the manual. --- src/lexer.cc | 110 ++++++++++++++++++++++++-------------------- src/lexer.in.cc | 4 +- src/manifest_parser_test.cc | 20 ++++++++ 3 files changed, 81 insertions(+), 53 deletions(-) diff --git a/src/lexer.cc b/src/lexer.cc index b3efe22..ca6f367 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -127,7 +127,7 @@ Lexer::Token Lexer::ReadToken() { unsigned int yyaccept = 0; static const unsigned char yybm[] = { 0, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 0, 64, 64, 64, 64, 64, + 64, 64, 0, 64, 64, 0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 192, 64, 64, 64, 64, 64, 64, 64, @@ -216,7 +216,8 @@ yy3: yy4: yyaccept = 1; yych = *(q = ++p); - if (yych >= 0x01) goto yy60; + if (yych <= 0x00) goto yy5; + if (yych != '\r') goto yy60; yy5: { token = ERROR; break; } yy6: @@ -354,7 +355,9 @@ yy60: if (yybm[0+yych] & 64) { goto yy59; } - if (yych >= 0x01) goto yy62; + if (yych <= 0x00) goto yy61; + if (yych <= '\f') goto yy62; +yy61: p = q; if (yyaccept <= 0) { goto yy3; @@ -576,7 +579,7 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { unsigned char yych; static const unsigned char yybm[] = { 0, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 16, 128, 128, 128, 0, 128, 128, 128, @@ -609,24 +612,25 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { 128, 128, 128, 128, 128, 128, 128, 128, }; yych = *p; - if (yych <= '#') { + if (yych <= ' ') { if (yych <= '\n') { if (yych <= 0x00) goto yy96; if (yych >= '\n') goto yy92; } else { - if (yych == ' ') goto yy92; + if (yych == '\r') goto yy98; + if (yych >= ' ') goto yy92; } } else { - if (yych <= ':') { - if (yych <= '$') goto yy94; - if (yych >= ':') goto yy92; + if (yych <= '9') { + if (yych == '$') goto yy94; } else { + if (yych <= ':') goto yy92; if (yych == '|') goto yy92; } } ++p; yych = *p; - goto yy120; + goto yy121; yy91: { eval->AddText(StringPiece(start, p - start)); @@ -649,39 +653,40 @@ yy94: ++p; if ((yych = *p) <= '/') { if (yych <= ' ') { - if (yych == '\n') goto yy109; - if (yych <= 0x1F) goto yy98; - goto yy100; + if (yych == '\n') goto yy110; + if (yych <= 0x1F) goto yy99; + goto yy101; } else { if (yych <= '$') { - if (yych <= '#') goto yy98; - goto yy102; + if (yych <= '#') goto yy99; + goto yy103; } else { - if (yych == '-') goto yy104; - goto yy98; + if (yych == '-') goto yy105; + goto yy99; } } } else { if (yych <= '^') { if (yych <= ':') { - if (yych <= '9') goto yy104; - goto yy106; + if (yych <= '9') goto yy105; + goto yy107; } else { - if (yych <= '@') goto yy98; - if (yych <= 'Z') goto yy104; - goto yy98; + if (yych <= '@') goto yy99; + if (yych <= 'Z') goto yy105; + goto yy99; } } else { if (yych <= '`') { - if (yych <= '_') goto yy104; - goto yy98; + if (yych <= '_') goto yy105; + goto yy99; } else { - if (yych <= 'z') goto yy104; - if (yych <= '{') goto yy108; - goto yy98; + if (yych <= 'z') goto yy105; + if (yych <= '{') goto yy109; + goto yy99; } } } +yy95: { last_token_ = start; return Error("lexing error", err); @@ -693,83 +698,86 @@ yy96: return Error("unexpected EOF", err); } yy98: - ++p; + yych = *++p; + goto yy95; yy99: + ++p; +yy100: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy100: +yy101: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy102: +yy103: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy104: +yy105: ++p; yych = *p; - goto yy118; -yy105: + goto yy119; +yy106: { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy106: +yy107: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy108: +yy109: yych = *(q = ++p); if (yybm[0+yych] & 32) { - goto yy112; + goto yy113; } - goto yy99; -yy109: + goto yy100; +yy110: ++p; yych = *p; if (yybm[0+yych] & 16) { - goto yy109; + goto yy110; } { continue; } -yy112: +yy113: ++p; yych = *p; if (yybm[0+yych] & 32) { - goto yy112; + goto yy113; } - if (yych == '}') goto yy115; + if (yych == '}') goto yy116; p = q; - goto yy99; -yy115: + goto yy100; +yy116: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy117: +yy118: ++p; yych = *p; -yy118: +yy119: if (yybm[0+yych] & 64) { - goto yy117; + goto yy118; } - goto yy105; -yy119: + goto yy106; +yy120: ++p; yych = *p; -yy120: +yy121: if (yybm[0+yych] & 128) { - goto yy119; + goto yy120; } goto yy91; } diff --git a/src/lexer.in.cc b/src/lexer.in.cc index e478921..852d6e9 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -130,7 +130,7 @@ Lexer::Token Lexer::ReadToken() { simple_varname = [a-zA-Z0-9_-]+; varname = [a-zA-Z0-9_.-]+; - [ ]*"#"[^\000\n]*"\n" { continue; } + [ ]*"#"[^\000\r\n]*"\n" { continue; } [ ]*[\n] { token = NEWLINE; break; } [ ]+ { token = INDENT; break; } "build" { token = BUILD; break; } @@ -200,7 +200,7 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { for (;;) { start = p; /*!re2c - [^$ :\n|\000]+ { + [^$ :\r\n|\000]+ { eval->AddText(StringPiece(start, p - start)); continue; } diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 9c6644c..3261d39 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -682,3 +682,23 @@ TEST_F(ParserTest, UTF8) { " command = true\n" " description = compilaci\xC3\xB3\n")); } + +// We might want to eventually allow CRLF to be nice to Windows developers, +// but for now just verify we error out with a nice message. +TEST_F(ParserTest, CRLF) { + State state; + ManifestParser parser(&state, NULL); + string err; + + EXPECT_FALSE(parser.ParseTest("# comment with crlf\r\n", + &err)); + EXPECT_EQ("input:1: lexing error\n", + err); + + EXPECT_FALSE(parser.ParseTest("foo = foo\nbar = bar\r\n", + &err)); + EXPECT_EQ("input:2: lexing error\n" + "bar = bar\r\n" + " ^ near here", + err); +} -- cgit v0.12 From 73b8aa33f295aa64a8c115d886048a0a06ceee81 Mon Sep 17 00:00:00 2001 From: Maxim Kalaev Date: Thu, 19 Jul 2012 08:49:34 +0300 Subject: dep file parsing: allow '@' and '=' in paths --- src/depfile_parser.cc | 36 ++++++++++++++++++++++-------------- src/depfile_parser.in.cc | 2 +- src/depfile_parser_test.cc | 9 +++++++-- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 54b934c..03dad92 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -56,8 +56,8 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 0, 0, 0, 0, 0, - 0, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 0, 0, 128, 0, 0, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 128, @@ -84,27 +84,35 @@ bool DepfileParser::Parse(string* content, string* err) { }; yych = *in; - if (yych <= '[') { + if (yych <= 'Z') { if (yych <= '*') { if (yych <= 0x00) goto yy6; if (yych <= '\'') goto yy8; if (yych <= ')') goto yy4; goto yy8; } else { - if (yych <= ':') goto yy4; - if (yych <= '@') goto yy8; - if (yych <= 'Z') goto yy4; - goto yy8; + if (yych <= '<') { + if (yych <= ':') goto yy4; + goto yy8; + } else { + if (yych <= '=') goto yy4; + if (yych <= '?') goto yy8; + goto yy4; + } } } else { - if (yych <= '`') { - if (yych <= '\\') goto yy2; - if (yych == '_') goto yy4; - goto yy8; + if (yych <= '_') { + if (yych == '\\') goto yy2; + if (yych <= '^') goto yy8; + goto yy4; } else { - if (yych <= 'z') goto yy4; - if (yych == '~') goto yy4; - goto yy8; + if (yych <= 'z') { + if (yych <= '`') goto yy8; + goto yy4; + } else { + if (yych == '~') goto yy4; + goto yy8; + } } } yy2: diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 8c415b9..68b6a95 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -68,7 +68,7 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = yych; continue; } - [a-zA-Z0-9+,/_:.~()-]+ { + [a-zA-Z0-9+,/_:.~()@=-]+ { // Got a span of plain text. int len = in - start; // Need to shift it over if we're overwriting backslashes. diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index fd76ae7..2bb9105 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -106,12 +106,17 @@ TEST_F(DepfileParserTest, Escapes) { TEST_F(DepfileParserTest, SpecialChars) { string err; EXPECT_TRUE(Parse( -"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h:", +"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n" +" en@quot.header~ t+t-x=1", &err)); ASSERT_EQ("", err); EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", parser_.out_.AsString()); - ASSERT_EQ(0u, parser_.ins_.size()); + ASSERT_EQ(2u, parser_.ins_.size()); + EXPECT_EQ("en@quot.header~", + parser_.ins_[0].AsString()); + EXPECT_EQ("t+t-x=1", + parser_.ins_[1].AsString()); } TEST_F(DepfileParserTest, UnifyMultipleOutputs) { -- cgit v0.12 From fc554c2290ba894a056b371db6527547f488ae6c Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Thu, 19 Jul 2012 15:17:38 -0700 Subject: windows: use WriteConsoleOutput instead of printf to avoid moving cursor --- src/build.cc | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/build.cc b/src/build.cc index 0fc387c..c6f5eac 100644 --- a/src/build.cc +++ b/src/build.cc @@ -236,22 +236,36 @@ void BuildStatus::PrintStatus(Edge* edge) { #endif } - printf("%s", to_print.c_str()); - if (smart_terminal_ && !force_full_command) { #ifndef _WIN32 + printf("%s", to_print.c_str()); printf("\x1B[K"); // Clear to end of line. fflush(stdout); have_blank_line_ = false; #else - // Clear to end of line. + // We don't want to have the cursor spamming back and forth, so + // use WriteConsoleOutput instead which updates the contents of + // the buffer, but doesn't move the cursor position. GetConsoleScreenBufferInfo(console_, &csbi); - int num_spaces = csbi.dwSize.X - 1 - csbi.dwCursorPosition.X; - printf("%*s", num_spaces, ""); + COORD buf_size = { csbi.dwSize.X, 1 }; + COORD zero_zero = { 0, 0 }; + SMALL_RECT target = { csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y, + csbi.dwCursorPosition.X + csbi.dwSize.X - 1, + csbi.dwCursorPosition.Y }; + CHAR_INFO* char_data = new CHAR_INFO[csbi.dwSize.X]; + memset(char_data, 0, sizeof(CHAR_INFO) * csbi.dwSize.X); + for (int i = 0; i < csbi.dwSize.X; ++i) { + char_data[i].Char.AsciiChar = ' '; + char_data[i].Attributes = csbi.wAttributes; + + } + for (size_t i = 0; i < to_print.size(); ++i) + char_data[i].Char.AsciiChar = to_print[i]; + WriteConsoleOutput(console_, char_data, buf_size, zero_zero, &target); have_blank_line_ = false; #endif } else { - printf("\n"); + printf("%s\n", to_print.c_str()); } } -- cgit v0.12 From 2b410db3cc3efed1800c39fd01f6a0af69fc7539 Mon Sep 17 00:00:00 2001 From: Thiago Farina Date: Fri, 20 Jul 2012 12:19:18 -0300 Subject: Use lowercase consistently. https://github.com/martine/ninja/issues/360 Signed-off-by: Thiago Farina --- src/util.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util.cc b/src/util.cc index d8d7fb3..72553dc 100644 --- a/src/util.cc +++ b/src/util.cc @@ -43,7 +43,7 @@ void Fatal(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: FATAL: "); + fprintf(stderr, "ninja: fatal: "); va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); @@ -61,7 +61,7 @@ void Fatal(const char* msg, ...) { void Warning(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: WARNING: "); + fprintf(stderr, "ninja: warning: "); va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); @@ -70,7 +70,7 @@ void Warning(const char* msg, ...) { void Error(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: ERROR: "); + fprintf(stderr, "ninja: error: "); va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); -- cgit v0.12 From a6fee5d01df65dff1a7b104d40c7a35e0d0a06b5 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Wed, 25 Jul 2012 10:34:14 -0700 Subject: Don't print 'Entering directory' when running tools. --- src/ninja.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ninja.cc b/src/ninja.cc index 809768c..619c3bf 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -707,7 +707,10 @@ int main(int argc, char** argv) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for // subsequent commands. - printf("ninja: Entering directory `%s'\n", working_dir); + // Don't print this if a tool is being used, so that tool output + // can be piped into a file without this string showing up. + if (tool == "") + printf("ninja: Entering directory `%s'\n", working_dir); #ifdef _WIN32 if (_chdir(working_dir) < 0) { #else -- cgit v0.12 From 0cd552c0dc7b2aa1da692737f0f7d257b08ecdbc Mon Sep 17 00:00:00 2001 From: Maxim Kalaev Date: Thu, 26 Jul 2012 15:21:30 +0300 Subject: packaging: added basic RPM building --- configure.py | 20 ++++++++++++++++++++ misc/packaging/ninja.spec | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 misc/packaging/ninja.spec diff --git a/configure.py b/configure.py index bf2596c..b8c6d86 100755 --- a/configure.py +++ b/configure.py @@ -386,6 +386,26 @@ n.comment('Build only the main binary by default.') n.default(ninja) n.newline() +if host == 'linux': + n.comment('Packaging') + n.rule('rpmbuild', + command="rpmbuild \ + --define 'ver git' \ + --define \"rel `git rev-parse --short HEAD`\" \ + --define '_topdir %(pwd)/rpm-build' \ + --define '_builddir %{_topdir}' \ + --define '_rpmdir %{_topdir}' \ + --define '_srcrpmdir %{_topdir}' \ + --define '_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm' \ + --define '_specdir %{_topdir}' \ + --define '_sourcedir %{_topdir}' \ + --quiet \ + -bb misc/packaging/ninja.spec", + description='Building RPM..') + n.build('rpm', 'rpmbuild', + implicit=['ninja','README', 'COPYING', doc('manual.html')]) + n.newline() + n.build('all', 'phony', all_targets) print 'wrote %s.' % BUILD_FILENAME diff --git a/misc/packaging/ninja.spec b/misc/packaging/ninja.spec new file mode 100644 index 0000000..d513c6d --- /dev/null +++ b/misc/packaging/ninja.spec @@ -0,0 +1,33 @@ +Summary: Ninja is a small build system with a focus on speed. +Name: ninja +Version: %{ver} +Release: %{rel}%{?dist} +Group: Development/Tools +License: Apache 2.0 +URL: https://github.com/martine/ninja + +%description +Ninja is yet another build system. It takes as input the interdependencies of files (typically source code and output executables) and +orchestrates building them, quickly. + +Ninja joins a sea of other build systems. Its distinguishing goal is to be fast. It is born from my work on the Chromium browser project, +which has over 30,000 source files and whose other build systems (including one built from custom non-recursive Makefiles) can take ten +seconds to start building after changing one file. Ninja is under a second. + +%build +# Assuming we've bootstrapped already.. +../ninja manual ninja -C .. + +%install +mkdir -p %{buildroot}%{_bindir} %{buildroot}%{_docdir} +cp -p ../ninja %{buildroot}%{_bindir}/ +git log --oneline --pretty=format:'%h: %s (%an, %cd)' --abbrev-commit --all > GITLOG + +%files +%defattr(-, root, root) +%doc GITLOG ../COPYING ../README ../doc/manual.html +%{_bindir}/* + +%clean +mv %{_topdir}/*.rpm .. +rm -rf %{_topdir} -- cgit v0.12 From adadf71e8f7df63371e5362bd65f1f1adc616e61 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 09:53:40 -0700 Subject: wrap at 80 columns --- bootstrap.py | 3 ++- src/subprocess-win32.cc | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index 9ac46ba..86d38d9 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -74,7 +74,8 @@ if sys.platform.startswith('win32'): vcdir = os.environ.get('VCINSTALLDIR') if vcdir: - args = [os.path.join(vcdir, 'bin', 'cl.exe'), '/nologo', '/EHsc', '/DNOMINMAX'] + args = [os.path.join(vcdir, 'bin', 'cl.exe'), + '/nologo', '/EHsc', '/DNOMINMAX'] else: args = shlex.split(os.environ.get('CXX', 'g++')) args.extend(['-Wno-deprecated', diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 3484e5f..7dd8809 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -271,9 +271,12 @@ Subprocess* SubprocessSet::NextFinished() { void SubprocessSet::Clear() { for (vector::iterator i = running_.begin(); i != running_.end(); ++i) { - if ((*i)->child_) - if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, GetProcessId((*i)->child_))) + if ((*i)->child_) { + if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, + GetProcessId((*i)->child_))) { Win32Fatal("GenerateConsoleCtrlEvent"); + } + } } for (vector::iterator i = running_.begin(); i != running_.end(); ++i) -- cgit v0.12 From ce7e2b2171c82fe2b8649afea852d272d1abcbcd Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 10:01:53 -0700 Subject: bootstrap: set _WIN32_WINNT to WinXP for mingw From a patch from Peter Kuemmel . --- bootstrap.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index 86d38d9..33acc5d 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -78,9 +78,11 @@ if vcdir: '/nologo', '/EHsc', '/DNOMINMAX'] else: args = shlex.split(os.environ.get('CXX', 'g++')) - args.extend(['-Wno-deprecated', - '-DNINJA_PYTHON="' + sys.executable + '"', - '-DNINJA_BOOTSTRAP']) + cflags.extend(['-Wno-deprecated', + '-DNINJA_PYTHON="' + sys.executable + '"', + '-DNINJA_BOOTSTRAP']) + if sys.platform.startswith('win32'): + cflags.append('-D_WIN32_WINNT=0x0501') args.extend(cflags) args.extend(ldflags) binary = 'ninja.bootstrap' -- cgit v0.12 From 238b5616c54a0b702eec36dd8168d6ec1cbdde80 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 10:05:20 -0700 Subject: mingw: fix quoting of -DNINJA_PYTHON --- configure.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.py b/configure.py index b8c6d86..df1d6bb 100755 --- a/configure.py +++ b/configure.py @@ -111,7 +111,7 @@ if platform == 'windows': cflags = ['/nologo', '/Zi', '/W4', '/WX', '/wd4530', '/wd4100', '/wd4706', '/wd4512', '/wd4800', '/wd4702', '/wd4819', '/GR-', '/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS', - "/DNINJA_PYTHON=\"%s\"" % (options.with_python,)] + '/DNINJA_PYTHON="%s"' % options.with_python] ldflags = ['/DEBUG', '/libpath:$builddir'] if not options.debug: cflags += ['/Ox', '/DNDEBUG', '/GL'] @@ -123,7 +123,7 @@ else: '-fno-rtti', '-fno-exceptions', '-fvisibility=hidden', '-pipe', - "-DNINJA_PYTHON=\"%s\"" % options.with_python] + '-DNINJA_PYTHON="%s"' % options.with_python] if options.debug: cflags += ['-D_GLIBCXX_DEBUG', '-D_GLIBCXX_DEBUG_PEDANTIC'] else: @@ -150,7 +150,7 @@ else: def shell_escape(str): """Escape str such that it's interpreted as a single argument by the shell.""" # This isn't complete, but it's just enough to make NINJA_PYTHON work. - if platform == 'windows': + if platform in ('windows', 'mingw'): return str if '"' in str: return "'%s'" % str.replace("'", "\\'") -- cgit v0.12 From 2c7354b47d7719b9ca758bc98084cb204aa3d7a9 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 10:08:06 -0700 Subject: windows: fix printf format for process id --- src/subprocess-win32.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 7dd8809..38b6957 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -44,7 +44,7 @@ Subprocess::~Subprocess() { HANDLE Subprocess::SetupPipe(HANDLE ioport) { char pipe_name[100]; snprintf(pipe_name, sizeof(pipe_name), - "\\\\.\\pipe\\ninja_pid%u_sp%p", GetCurrentProcessId(), this); + "\\\\.\\pipe\\ninja_pid%lu_sp%p", GetCurrentProcessId(), this); pipe_ = ::CreateNamedPipeA(pipe_name, PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED, -- cgit v0.12 From 00d68972b13ec20fd09301784ae330f7a1921568 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 10:15:34 -0700 Subject: fix some mingw warnings in getopt.c From a patch from Claus Klein . --- src/getopt.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/getopt.c b/src/getopt.c index 1e3c09d..75ef99c 100644 --- a/src/getopt.c +++ b/src/getopt.c @@ -91,10 +91,6 @@ gpietsch@comcast.net #include "getopt.h" #endif -#ifdef _WIN32 -#pragma warning(disable: 4701) -#endif - /* macros */ /* types */ @@ -159,7 +155,7 @@ getopt_internal (int argc, char **argv, char *shortopts, char *possible_arg = NULL; int longopt_match = -1; int has_arg = -1; - char *cp; + char *cp = NULL; int arg_next = 0; /* first, deal with silly parameters and easy stuff */ @@ -255,7 +251,7 @@ getopt_internal (int argc, char **argv, char *shortopts, longopts[optindex].name, match_chars) == 0) { /* do we have an exact match? */ - if (match_chars == (int) (strlen (longopts[optindex].name))) + if (match_chars == strlen (longopts[optindex].name)) { longopt_match = optindex; break; -- cgit v0.12 From 69d94732ad63fe5d73fe18972eb48225bcaadf98 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 10:17:20 -0700 Subject: mingw: specify _WIN32_WINNT also in configure.py --- configure.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.py b/configure.py index df1d6bb..e52262d 100755 --- a/configure.py +++ b/configure.py @@ -130,6 +130,8 @@ else: cflags += ['-O2', '-DNDEBUG'] if 'clang' in os.path.basename(CXX): cflags += ['-fcolor-diagnostics'] + if platform == 'mingw': + cflags += ['-D_WIN32_WINNT=0x0501'] ldflags = ['-L$builddir'] libs = [] -- cgit v0.12 From 547998bc63640abc71573d94bc6afb11f74c13bc Mon Sep 17 00:00:00 2001 From: Maxim Kalaev Date: Thu, 28 Jun 2012 09:34:19 +0300 Subject: Minor: zero log buffer before reading data/parsing it Not a real problem, mainly to make valgrind happy --- src/build_log.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/build_log.cc b/src/build_log.cc index 5eddf8f..1b27be3 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -164,7 +164,9 @@ void BuildLog::Close() { class LineReader { public: explicit LineReader(FILE* file) - : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {} + : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) { + memset(buf_, 0, sizeof(buf_)); + } // Reads a \n-terminated line from the file passed to the constructor. // On return, *line_start points to the beginning of the next line, and -- cgit v0.12 From 34fdf02dd57fc7b92e61eeee2c2e6e031ed14477 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 10:38:44 -0700 Subject: make the build log test match the indended build log logic As described on https://github.com/martine/ninja/pull/349 , this test shouldn't have been passing. Fixing a Valgrind issue showed that the test was wrong. Thankfully it wasn't a critical issue. The new code verifies that we either succeed or report an error when parsing -- the only behavior we really care about is not crashing. --- src/build_log_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 9fb42c9..0225684 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -131,7 +131,7 @@ TEST_F(BuildLogTest, Truncate) { ASSERT_GT(statbuf.st_size, 0); // For all possible truncations of the input file, assert that we don't - // crash or report an error when parsing. + // crash when parsing. for (off_t size = statbuf.st_size; size > 0; --size) { #ifndef _WIN32 ASSERT_EQ(0, truncate(kTestFilename, size)); @@ -143,8 +143,8 @@ TEST_F(BuildLogTest, Truncate) { #endif BuildLog log2; - EXPECT_TRUE(log2.Load(kTestFilename, &err)); - ASSERT_EQ("", err); + err.clear(); + ASSERT_TRUE(log2.Load(kTestFilename, &err) || !err.empty()); } } -- cgit v0.12 From bb6da9afc166e61ea60dacfc4da94b9181ff887b Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 12:02:59 -0700 Subject: rearrange minidump patch to match ninja code style --- configure.py | 4 ++- src/minidump-win32.cc | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/ninja.cc | 79 ++++++++++++++++++++++++++-------------------- src/util.cc | 79 ---------------------------------------------- 4 files changed, 135 insertions(+), 114 deletions(-) create mode 100644 src/minidump-win32.cc diff --git a/configure.py b/configure.py index e52262d..41b3cf7 100755 --- a/configure.py +++ b/configure.py @@ -243,8 +243,10 @@ for name in ['build', 'state', 'util']: objs += cxx(name) -if platform == 'mingw' or platform == 'windows': +if platform in ('mingw', 'windows'): objs += cxx('subprocess-win32') + if platform == 'windows': + objs += cxx('minidump-win32') objs += cc('getopt') else: objs += cxx('subprocess') diff --git a/src/minidump-win32.cc b/src/minidump-win32.cc new file mode 100644 index 0000000..f7067f3 --- /dev/null +++ b/src/minidump-win32.cc @@ -0,0 +1,87 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_BOOTSTRAP + +#include +#include + +#include "util.h" + +typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( + IN HANDLE, + IN DWORD, + IN HANDLE, + IN MINIDUMP_TYPE, + IN CONST PMINIDUMP_EXCEPTION_INFORMATION, OPTIONAL + IN CONST PMINIDUMP_USER_STREAM_INFORMATION, OPTIONAL + IN CONST PMINIDUMP_CALLBACK_INFORMATION OPTIONAL + ); + +/// Creates a windows minidump in temp folder. +void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep) { + char temp_path[MAX_PATH]; + GetTempPath(sizeof(temp_path), temp_path); + char temp_file[MAX_PATH]; + sprintf(temp_file, "%s\\ninja_crash_dump_%d.dmp", + temp_path, GetCurrentProcessId()); + + // Delete any previous minidump of the same name. + DeleteFile(temp_file); + + // Load DbgHelp.dll dynamically, as library is not present on all + // Windows versions. + HMODULE dbghelp = LoadLibrary("dbghelp.dll"); + if (dbghelp == NULL) { + Error("failed to create minidump: LoadLibrary('dbghelp.dll'): %s", + GetLastErrorString().c_str()); + return; + } + + MiniDumpWriteDumpFunc mini_dump_write_dump = + (MiniDumpWriteDumpFunc)GetProcAddress(dbghelp, "MiniDumpWriteDump"); + if (mini_dump_write_dump == NULL) { + Error("failed to create minidump: GetProcAddress('MiniDumpWriteDump'): %s", + GetLastErrorString().c_str()); + return; + } + + HANDLE hFile = CreateFileA(temp_file, GENERIC_READ | GENERIC_WRITE, 0, NULL, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == NULL) { + Error("failed to create minidump: CreateFileA(%s): %s", + temp_file, GetLastErrorString().c_str()); + return; + } + + MINIDUMP_EXCEPTION_INFORMATION mdei; + mdei.ThreadId = GetCurrentThreadId(); + mdei.ExceptionPointers = pep; + mdei.ClientPointers = FALSE; + MINIDUMP_TYPE mdt = (MINIDUMP_TYPE) (MiniDumpWithDataSegs | + MiniDumpWithHandleData); + + BOOL rv = mini_dump_write_dump(GetCurrentProcess(), GetCurrentProcessId(), + hFile, mdt, (pep != 0) ? &mdei : 0, 0, 0); + CloseHandle(hFile); + + if (!rv) { + Error("MiniDumpWriteDump failed: %s", GetLastErrorString().c_str()); + return; + } + + Warning("minidump created: %s", temp_file); +} + +#endif // NINJA_BOOTSTRAP diff --git a/src/ninja.cc b/src/ninja.cc index 393f717..c1caf37 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -626,44 +626,33 @@ int RunBuild(Globals* globals, int argc, char** argv) { return 0; } -} // anonymous namespace - - #ifdef _MSC_VER -/// This handler processes fatal crashes that you can't catch -/// Test example: C++ exception in a stack-unwind-block -/// Real-world example: ninja launched a compiler to process a tricky C++ input file. -/// The compiler got itself into a state where it generated 3 GB of output and caused ninja to crash -void ninja_terminate_fct() { - Create_Win32_MiniDump(NULL); - Fatal("terminate handler called"); +// Defined in minidump-win32.cc. +void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep); + +/// This handler processes fatal crashes that you can't catch +/// Test example: C++ exception in a stack-unwind-block +/// Real-world example: ninja launched a compiler to process a tricky +/// C++ input file. The compiler got itself into a state where it +/// generated 3 GB of output and caused ninja to crash. +void TerminateHandler() { + CreateWin32MiniDump(NULL); + Fatal("terminate handler called"); } -/// main_unsafe is called from within an exception handling block -int main_unsafe(int argc, char** argv); - -/// Windows main() uses SEH (Structured exception handling) -int main(int argc, char** argv) { - // set a handler to catch crashes not caught by the __try..__except block (e.g. an exception in a stack-unwind-block) - set_terminate(ninja_terminate_fct); - __try { - // running inside __try ... __except suppresses any Windows error dialogs for errors such as bad_alloc - return main_unsafe(argc, argv); - } - __except(exception_filter(GetExceptionCode(), GetExceptionInformation())) { - // you will land here e.g. if you run out of memory, or run inside a distribution environment that fails - fprintf(stderr, "ninja: exception, exiting with error code 2\n"); - // common error situations below return exitCode=1, 2 was chosen to indicate a more serious problem - return 2; - } -} - -int main_unsafe (int argc, char** argv) { -#else -//on Linux, we have no exception handling -int main(int argc, char** argv) { -#endif +/// On Windows, we want to prevent error dialogs in case of exceptions. +/// This function handles the exception, and writes a minidump. +int ExceptionFilter(unsigned int code, struct _EXCEPTION_POINTERS *ep) { + Error("exception: 0x%X", code); // e.g. EXCEPTION_ACCESS_VIOLATION + fflush(stderr); + CreateWin32MiniDump(ep); + return EXCEPTION_EXECUTE_HANDLER; +} + +#endif // _MSC_VER + +int NinjaMain(int argc, char** argv) { Globals globals; globals.ninja_command = argv[0]; const char* input_file = "build.ninja"; @@ -824,3 +813,25 @@ reload: } return result; } + +} // anonymous namespace + +int main(int argc, char** argv) { +#if !defined(NINJA_BOOTSTRAP) && defined(_MSC_VER) + // Set a handler to catch crashes not caught by the __try..__except + // block (e.g. an exception in a stack-unwind-block). + set_terminate(ninja_terminate_fct); + __try { + // Running inside __try ... __except suppresses any Windows error + // dialogs for errors such as bad_alloc. + return NinjaMain(argc, argv); + } + __except(ExceptionFilter(GetExceptionCode(), GetExceptionInformation())) { + // Common error situations return exitCode=1. 2 was chosen to + // indicate a more serious problem. + return 2; + } +#else + return NinjaMain(argc, argv); +#endif +} diff --git a/src/util.cc b/src/util.cc index be74c4d..ca05292 100644 --- a/src/util.cc +++ b/src/util.cc @@ -35,7 +35,6 @@ #include #ifdef _WIN32 -#include // for minidump #include // _mkdir #endif @@ -312,81 +311,3 @@ double GetLoadAverage() return GetLoadAverage_unix(); #endif // _WIN32 } - -#ifdef _MSC_VER -typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( - IN HANDLE, - IN DWORD, - IN HANDLE, - IN MINIDUMP_TYPE, - IN CONST PMINIDUMP_EXCEPTION_INFORMATION, OPTIONAL - IN CONST PMINIDUMP_USER_STREAM_INFORMATION, OPTIONAL - IN CONST PMINIDUMP_CALLBACK_INFORMATION OPTIONAL - ); - -/// this function creates a windows minidump in temp folder. -void Create_Win32_MiniDump( _EXCEPTION_POINTERS* pep ) { - char tempPathBuff[MAX_PATH]; - GetTempPath(sizeof(tempPathBuff), tempPathBuff); - char tempFileName[MAX_PATH]; - sprintf(tempFileName, "%s\\ninja_crash_dump_%d.dmp", tempPathBuff, GetCurrentProcessId()); - - //delete any previous minidump of the same name - DeleteFile(tempFileName); - - // load DbgHelp.dll dynamically, as library is not present on all windows versions - HMODULE hModDbgHelp = LoadLibrary("dbghelp.dll"); - if (hModDbgHelp == NULL) { - fprintf(stderr, "ninja: failed to create minidump, failed to load dbghelp.dll, error %s \n", - GetLastErrorString().c_str()); - return; - } - - MiniDumpWriteDumpFunc pfnMiniDumpWriteDump = (MiniDumpWriteDumpFunc)GetProcAddress(hModDbgHelp, "MiniDumpWriteDump"); - if (pfnMiniDumpWriteDump == NULL) { - fprintf(stderr, "ninja: failed to create minidump, failed on GetProcAddress('MiniDumpWriteDump'), error %s \n", - GetLastErrorString().c_str()); - return; - } - - // Open the file - HANDLE hFile = CreateFileA( tempFileName, GENERIC_READ | GENERIC_WRITE, - 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL ); - if (hFile == NULL) { - fprintf(stderr, "ninja: failed to create minidump, failed on CreateFileA(%s), error %s \n", - tempFileName, GetLastErrorString().c_str()); - return; - } - - // Create the mini dump - MINIDUMP_EXCEPTION_INFORMATION mdei; - mdei.ThreadId = GetCurrentThreadId(); - mdei.ExceptionPointers = pep; - mdei.ClientPointers = FALSE; - MINIDUMP_TYPE mdt = (MINIDUMP_TYPE) (MiniDumpWithDataSegs | MiniDumpWithHandleData); - - BOOL rv = pfnMiniDumpWriteDump( GetCurrentProcess(), GetCurrentProcessId(), - hFile, mdt, (pep != 0) ? &mdei : 0, 0, 0 ); - - if ( !rv ) - fprintf(stderr, "ninja: MiniDumpWriteDump failed. Error: %s \n", GetLastErrorString().c_str() ); - else - fprintf(stderr, "ninja: Minidump created: %s\n", tempFileName ); - - // Close the file - CloseHandle( hFile ); -} - -/// On Windows, we want to prevent error dialogs in case of exceptions. -/// This function handles the exception, and writes a minidump. -int exception_filter(unsigned int code, struct _EXCEPTION_POINTERS *ep) { - fprintf(stderr, "ninja.exe fatal error: 0x%X \n", code); //e.g. EXCEPTION_ACCESS_VIOLATION - fflush(stderr); - Create_Win32_MiniDump(ep); - return EXCEPTION_EXECUTE_HANDLER; -} -#else - //on Linux or MinGW, core dumps are created automatically, no code needed -#endif - - -- cgit v0.12 From f4b8c7510029f3f5898c2fb4a9c04fb2c6ecaeb2 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 12:24:40 -0700 Subject: rename subprocess.cc to reflex its posixness --- bootstrap.py | 2 +- configure.py | 2 +- src/subprocess-posix.cc | 300 ++++++++++++++++++++++++++++++++++++++++++++++++ src/subprocess.cc | 300 ------------------------------------------------ 4 files changed, 302 insertions(+), 302 deletions(-) create mode 100644 src/subprocess-posix.cc delete mode 100644 src/subprocess.cc diff --git a/bootstrap.py b/bootstrap.py index 33acc5d..ab03cf8 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -61,7 +61,7 @@ for src in glob.glob('src/*.cc'): continue if sys.platform.startswith('win32'): - if filename == 'subprocess.cc': + if src.endswith('-posix.cc'): continue else: if src.endswith('-win32.cc'): diff --git a/configure.py b/configure.py index 41b3cf7..95f88b1 100755 --- a/configure.py +++ b/configure.py @@ -249,7 +249,7 @@ if platform in ('mingw', 'windows'): objs += cxx('minidump-win32') objs += cc('getopt') else: - objs += cxx('subprocess') + objs += cxx('subprocess-posix') if platform == 'windows': ninja_lib = n.build(built('ninja.lib'), 'ar', objs) else: diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc new file mode 100644 index 0000000..1c47fd1 --- /dev/null +++ b/src/subprocess-posix.cc @@ -0,0 +1,300 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "subprocess.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Older versions of glibc (like 2.4) won't find this in . glibc +// 2.4 keeps it in , though attempting to include that +// will redefine the pollfd structure. +#ifndef POLLRDHUP +#define POLLRDHUP 0x2000 +#endif + +#include "util.h" + +Subprocess::Subprocess() : fd_(-1), pid_(-1) { +} +Subprocess::~Subprocess() { + if (fd_ >= 0) + close(fd_); + // Reap child if forgotten. + if (pid_ != -1) + Finish(); +} + +bool Subprocess::Start(SubprocessSet* set, const string& command) { + int output_pipe[2]; + if (pipe(output_pipe) < 0) + Fatal("pipe: %s", strerror(errno)); + fd_ = output_pipe[0]; +#if !defined(linux) + // On linux we use ppoll in DoWork(); elsewhere we use pselect and so must + // avoid overly-large FDs. + if (fd_ >= static_cast(FD_SETSIZE)) + Fatal("pipe: %s", strerror(EMFILE)); +#endif // !linux + SetCloseOnExec(fd_); + + pid_ = fork(); + if (pid_ < 0) + Fatal("fork: %s", strerror(errno)); + + if (pid_ == 0) { + close(output_pipe[0]); + + // Track which fd we use to report errors on. + int error_pipe = output_pipe[1]; + do { + if (setpgid(0, 0) < 0) + break; + + if (sigaction(SIGINT, &set->old_act_, 0) < 0) + break; + if (sigprocmask(SIG_SETMASK, &set->old_mask_, 0) < 0) + break; + + // Open /dev/null over stdin. + int devnull = open("/dev/null", O_WRONLY); + if (devnull < 0) + break; + if (dup2(devnull, 0) < 0) + break; + close(devnull); + + if (dup2(output_pipe[1], 1) < 0 || + dup2(output_pipe[1], 2) < 0) + break; + + // Now can use stderr for errors. + error_pipe = 2; + close(output_pipe[1]); + + execl("/bin/sh", "/bin/sh", "-c", command.c_str(), (char *) NULL); + } while (false); + + // If we get here, something went wrong; the execl should have + // replaced us. + char* err = strerror(errno); + if (write(error_pipe, err, strlen(err)) < 0) { + // If the write fails, there's nothing we can do. + // But this block seems necessary to silence the warning. + } + _exit(1); + } + + close(output_pipe[1]); + return true; +} + +void Subprocess::OnPipeReady() { + char buf[4 << 10]; + ssize_t len = read(fd_, buf, sizeof(buf)); + if (len > 0) { + buf_.append(buf, len); + } else { + if (len < 0) + Fatal("read: %s", strerror(errno)); + close(fd_); + fd_ = -1; + } +} + +ExitStatus Subprocess::Finish() { + assert(pid_ != -1); + int status; + if (waitpid(pid_, &status, 0) < 0) + Fatal("waitpid(%d): %s", pid_, strerror(errno)); + pid_ = -1; + + if (WIFEXITED(status)) { + int exit = WEXITSTATUS(status); + if (exit == 0) + return ExitSuccess; + } else if (WIFSIGNALED(status)) { + if (WTERMSIG(status) == SIGINT) + return ExitInterrupted; + } + return ExitFailure; +} + +bool Subprocess::Done() const { + return fd_ == -1; +} + +const string& Subprocess::GetOutput() const { + return buf_; +} + +bool SubprocessSet::interrupted_; + +void SubprocessSet::SetInterruptedFlag(int signum) { + (void) signum; + interrupted_ = true; +} + +SubprocessSet::SubprocessSet() { + interrupted_ = false; + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_BLOCK, &set, &old_mask_) < 0) + Fatal("sigprocmask: %s", strerror(errno)); + + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = SetInterruptedFlag; + if (sigaction(SIGINT, &act, &old_act_) < 0) + Fatal("sigaction: %s", strerror(errno)); +} + +SubprocessSet::~SubprocessSet() { + Clear(); + + if (sigaction(SIGINT, &old_act_, 0) < 0) + Fatal("sigaction: %s", strerror(errno)); + if (sigprocmask(SIG_SETMASK, &old_mask_, 0) < 0) + Fatal("sigprocmask: %s", strerror(errno)); +} + +Subprocess *SubprocessSet::Add(const string& command) { + Subprocess *subprocess = new Subprocess; + if (!subprocess->Start(this, command)) { + delete subprocess; + return 0; + } + running_.push_back(subprocess); + return subprocess; +} + +#ifdef linux +bool SubprocessSet::DoWork() { + vector fds; + nfds_t nfds = 0; + + for (vector::iterator i = running_.begin(); + i != running_.end(); ++i) { + int fd = (*i)->fd_; + if (fd < 0) + continue; + pollfd pfd = { fd, POLLIN | POLLPRI | POLLRDHUP, 0 }; + fds.push_back(pfd); + ++nfds; + } + + int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_); + if (ret == -1) { + if (errno != EINTR) { + perror("ninja: ppoll"); + return false; + } + bool interrupted = interrupted_; + interrupted_ = false; + return interrupted; + } + + nfds_t cur_nfd = 0; + for (vector::iterator i = running_.begin(); + i != running_.end(); ) { + int fd = (*i)->fd_; + if (fd < 0) + continue; + assert(fd == fds[cur_nfd].fd); + if (fds[cur_nfd++].revents) { + (*i)->OnPipeReady(); + if ((*i)->Done()) { + finished_.push(*i); + i = running_.erase(i); + continue; + } + } + ++i; + } + + return false; +} + +#else // linux +bool SubprocessSet::DoWork() { + fd_set set; + int nfds = 0; + FD_ZERO(&set); + + for (vector::iterator i = running_.begin(); + i != running_.end(); ++i) { + int fd = (*i)->fd_; + if (fd >= 0) { + FD_SET(fd, &set); + if (nfds < fd+1) + nfds = fd+1; + } + } + + int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_); + if (ret == -1) { + if (errno != EINTR) { + perror("ninja: pselect"); + return false; + } + bool interrupted = interrupted_; + interrupted_ = false; + return interrupted; + } + + for (vector::iterator i = running_.begin(); + i != running_.end(); ) { + int fd = (*i)->fd_; + if (fd >= 0 && FD_ISSET(fd, &set)) { + (*i)->OnPipeReady(); + if ((*i)->Done()) { + finished_.push(*i); + i = running_.erase(i); + continue; + } + } + ++i; + } + + return false; +} +#endif // linux + +Subprocess* SubprocessSet::NextFinished() { + if (finished_.empty()) + return NULL; + Subprocess* subproc = finished_.front(); + finished_.pop(); + return subproc; +} + +void SubprocessSet::Clear() { + for (vector::iterator i = running_.begin(); + i != running_.end(); ++i) + kill(-(*i)->pid_, SIGINT); + for (vector::iterator i = running_.begin(); + i != running_.end(); ++i) + delete *i; + running_.clear(); +} diff --git a/src/subprocess.cc b/src/subprocess.cc deleted file mode 100644 index 1c47fd1..0000000 --- a/src/subprocess.cc +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2012 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "subprocess.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Older versions of glibc (like 2.4) won't find this in . glibc -// 2.4 keeps it in , though attempting to include that -// will redefine the pollfd structure. -#ifndef POLLRDHUP -#define POLLRDHUP 0x2000 -#endif - -#include "util.h" - -Subprocess::Subprocess() : fd_(-1), pid_(-1) { -} -Subprocess::~Subprocess() { - if (fd_ >= 0) - close(fd_); - // Reap child if forgotten. - if (pid_ != -1) - Finish(); -} - -bool Subprocess::Start(SubprocessSet* set, const string& command) { - int output_pipe[2]; - if (pipe(output_pipe) < 0) - Fatal("pipe: %s", strerror(errno)); - fd_ = output_pipe[0]; -#if !defined(linux) - // On linux we use ppoll in DoWork(); elsewhere we use pselect and so must - // avoid overly-large FDs. - if (fd_ >= static_cast(FD_SETSIZE)) - Fatal("pipe: %s", strerror(EMFILE)); -#endif // !linux - SetCloseOnExec(fd_); - - pid_ = fork(); - if (pid_ < 0) - Fatal("fork: %s", strerror(errno)); - - if (pid_ == 0) { - close(output_pipe[0]); - - // Track which fd we use to report errors on. - int error_pipe = output_pipe[1]; - do { - if (setpgid(0, 0) < 0) - break; - - if (sigaction(SIGINT, &set->old_act_, 0) < 0) - break; - if (sigprocmask(SIG_SETMASK, &set->old_mask_, 0) < 0) - break; - - // Open /dev/null over stdin. - int devnull = open("/dev/null", O_WRONLY); - if (devnull < 0) - break; - if (dup2(devnull, 0) < 0) - break; - close(devnull); - - if (dup2(output_pipe[1], 1) < 0 || - dup2(output_pipe[1], 2) < 0) - break; - - // Now can use stderr for errors. - error_pipe = 2; - close(output_pipe[1]); - - execl("/bin/sh", "/bin/sh", "-c", command.c_str(), (char *) NULL); - } while (false); - - // If we get here, something went wrong; the execl should have - // replaced us. - char* err = strerror(errno); - if (write(error_pipe, err, strlen(err)) < 0) { - // If the write fails, there's nothing we can do. - // But this block seems necessary to silence the warning. - } - _exit(1); - } - - close(output_pipe[1]); - return true; -} - -void Subprocess::OnPipeReady() { - char buf[4 << 10]; - ssize_t len = read(fd_, buf, sizeof(buf)); - if (len > 0) { - buf_.append(buf, len); - } else { - if (len < 0) - Fatal("read: %s", strerror(errno)); - close(fd_); - fd_ = -1; - } -} - -ExitStatus Subprocess::Finish() { - assert(pid_ != -1); - int status; - if (waitpid(pid_, &status, 0) < 0) - Fatal("waitpid(%d): %s", pid_, strerror(errno)); - pid_ = -1; - - if (WIFEXITED(status)) { - int exit = WEXITSTATUS(status); - if (exit == 0) - return ExitSuccess; - } else if (WIFSIGNALED(status)) { - if (WTERMSIG(status) == SIGINT) - return ExitInterrupted; - } - return ExitFailure; -} - -bool Subprocess::Done() const { - return fd_ == -1; -} - -const string& Subprocess::GetOutput() const { - return buf_; -} - -bool SubprocessSet::interrupted_; - -void SubprocessSet::SetInterruptedFlag(int signum) { - (void) signum; - interrupted_ = true; -} - -SubprocessSet::SubprocessSet() { - interrupted_ = false; - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_BLOCK, &set, &old_mask_) < 0) - Fatal("sigprocmask: %s", strerror(errno)); - - struct sigaction act; - memset(&act, 0, sizeof(act)); - act.sa_handler = SetInterruptedFlag; - if (sigaction(SIGINT, &act, &old_act_) < 0) - Fatal("sigaction: %s", strerror(errno)); -} - -SubprocessSet::~SubprocessSet() { - Clear(); - - if (sigaction(SIGINT, &old_act_, 0) < 0) - Fatal("sigaction: %s", strerror(errno)); - if (sigprocmask(SIG_SETMASK, &old_mask_, 0) < 0) - Fatal("sigprocmask: %s", strerror(errno)); -} - -Subprocess *SubprocessSet::Add(const string& command) { - Subprocess *subprocess = new Subprocess; - if (!subprocess->Start(this, command)) { - delete subprocess; - return 0; - } - running_.push_back(subprocess); - return subprocess; -} - -#ifdef linux -bool SubprocessSet::DoWork() { - vector fds; - nfds_t nfds = 0; - - for (vector::iterator i = running_.begin(); - i != running_.end(); ++i) { - int fd = (*i)->fd_; - if (fd < 0) - continue; - pollfd pfd = { fd, POLLIN | POLLPRI | POLLRDHUP, 0 }; - fds.push_back(pfd); - ++nfds; - } - - int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_); - if (ret == -1) { - if (errno != EINTR) { - perror("ninja: ppoll"); - return false; - } - bool interrupted = interrupted_; - interrupted_ = false; - return interrupted; - } - - nfds_t cur_nfd = 0; - for (vector::iterator i = running_.begin(); - i != running_.end(); ) { - int fd = (*i)->fd_; - if (fd < 0) - continue; - assert(fd == fds[cur_nfd].fd); - if (fds[cur_nfd++].revents) { - (*i)->OnPipeReady(); - if ((*i)->Done()) { - finished_.push(*i); - i = running_.erase(i); - continue; - } - } - ++i; - } - - return false; -} - -#else // linux -bool SubprocessSet::DoWork() { - fd_set set; - int nfds = 0; - FD_ZERO(&set); - - for (vector::iterator i = running_.begin(); - i != running_.end(); ++i) { - int fd = (*i)->fd_; - if (fd >= 0) { - FD_SET(fd, &set); - if (nfds < fd+1) - nfds = fd+1; - } - } - - int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_); - if (ret == -1) { - if (errno != EINTR) { - perror("ninja: pselect"); - return false; - } - bool interrupted = interrupted_; - interrupted_ = false; - return interrupted; - } - - for (vector::iterator i = running_.begin(); - i != running_.end(); ) { - int fd = (*i)->fd_; - if (fd >= 0 && FD_ISSET(fd, &set)) { - (*i)->OnPipeReady(); - if ((*i)->Done()) { - finished_.push(*i); - i = running_.erase(i); - continue; - } - } - ++i; - } - - return false; -} -#endif // linux - -Subprocess* SubprocessSet::NextFinished() { - if (finished_.empty()) - return NULL; - Subprocess* subproc = finished_.front(); - finished_.pop(); - return subproc; -} - -void SubprocessSet::Clear() { - for (vector::iterator i = running_.begin(); - i != running_.end(); ++i) - kill(-(*i)->pid_, SIGINT); - for (vector::iterator i = running_.begin(); - i != running_.end(); ++i) - delete *i; - running_.clear(); -} -- cgit v0.12 From 6a410a3d5b652f12e6f755c3bc1d0fc52b54d207 Mon Sep 17 00:00:00 2001 From: Claus Klein Date: Fri, 27 Jul 2012 13:06:03 -0700 Subject: fix error and prevent warning in GetTimeMillis() --- src/metrics.cc | 2 +- src/metrics.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/metrics.cc b/src/metrics.cc index 8407717..ca4f97a 100644 --- a/src/metrics.cc +++ b/src/metrics.cc @@ -120,6 +120,6 @@ uint64_t Stopwatch::Now() const { } int64_t GetTimeMillis() { - return 1000 * TimerToMicros(HighResTimer()); + return TimerToMicros(HighResTimer()) / 1000; } diff --git a/src/metrics.h b/src/metrics.h index dae6b9f..f5ac0de 100644 --- a/src/metrics.h +++ b/src/metrics.h @@ -72,7 +72,7 @@ public: Stopwatch() : started_(0) {} /// Seconds since Restart() call - double Elapsed() const { return 1e-6 * (Now() - started_); } + double Elapsed() const { return 1e-6 * static_cast(Now() - started_); } void Restart() { started_ = Now(); } -- cgit v0.12 From fa8b3003d350d7dc22d2b722401134eeb0dcfff5 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 27 Jul 2012 13:46:46 -0700 Subject: fix a warning and leak in windows-specific console output code --- src/build.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build.cc b/src/build.cc index 90a84c2..09d7f65 100644 --- a/src/build.cc +++ b/src/build.cc @@ -275,18 +275,18 @@ void BuildStatus::PrintStatus(Edge* edge) { COORD buf_size = { csbi.dwSize.X, 1 }; COORD zero_zero = { 0, 0 }; SMALL_RECT target = { csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y, - csbi.dwCursorPosition.X + csbi.dwSize.X - 1, + (SHORT)(csbi.dwCursorPosition.X + csbi.dwSize.X - 1), csbi.dwCursorPosition.Y }; CHAR_INFO* char_data = new CHAR_INFO[csbi.dwSize.X]; memset(char_data, 0, sizeof(CHAR_INFO) * csbi.dwSize.X); for (int i = 0; i < csbi.dwSize.X; ++i) { char_data[i].Char.AsciiChar = ' '; char_data[i].Attributes = csbi.wAttributes; - } for (size_t i = 0; i < to_print.size(); ++i) char_data[i].Char.AsciiChar = to_print[i]; WriteConsoleOutput(console_, char_data, buf_size, zero_zero, &target); + delete[] char_data; have_blank_line_ = false; #endif } else { -- cgit v0.12 From 44105a941b1b973fe4554024e34e09b8aeab64b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20K=C3=BCmmel?= Date: Fri, 27 Jul 2012 23:09:52 +0200 Subject: also build with msvc --- src/hash_map.h | 6 +++--- src/minidump-win32.cc | 3 ++- src/ninja.cc | 6 +++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/hash_map.h b/src/hash_map.h index 15e86da..ff2f7ba 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -55,16 +55,16 @@ using stdext::hash_compare; struct StringPieceCmp : public hash_compare { size_t operator()(const StringPiece& key) const { - return MurmurHash2(key.str_, key.len_); + return MurmurHash2(key.str(), key.len()); } bool operator()(const StringPiece& a, const StringPiece& b) const { - int cmp = strncmp(a.str_, b.str_, min(a.len_, b.len_)); + int cmp = strncmp(a.str(), b.str(), min(a.len(), b.len())); if (cmp < 0) { return true; } else if (cmp > 0) { return false; } else { - return a.len_ < b.len_; + return a.len() < b.len(); } } }; diff --git a/src/minidump-win32.cc b/src/minidump-win32.cc index f7067f3..c79ec0e 100644 --- a/src/minidump-win32.cc +++ b/src/minidump-win32.cc @@ -14,8 +14,9 @@ #ifndef NINJA_BOOTSTRAP -#include #include +#include + #include "util.h" diff --git a/src/ninja.cc b/src/ninja.cc index c1caf37..538d0d7 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -628,9 +628,13 @@ int RunBuild(Globals* globals, int argc, char** argv) { #ifdef _MSC_VER +} // anonymous namespace + // Defined in minidump-win32.cc. void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep); +namespace { + /// This handler processes fatal crashes that you can't catch /// Test example: C++ exception in a stack-unwind-block /// Real-world example: ninja launched a compiler to process a tricky @@ -820,7 +824,7 @@ int main(int argc, char** argv) { #if !defined(NINJA_BOOTSTRAP) && defined(_MSC_VER) // Set a handler to catch crashes not caught by the __try..__except // block (e.g. an exception in a stack-unwind-block). - set_terminate(ninja_terminate_fct); + set_terminate(TerminateHandler); __try { // Running inside __try ... __except suppresses any Windows error // dialogs for errors such as bad_alloc. -- cgit v0.12 From 3ab35e5403066f48117f442f9ba8853e70edab13 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Sat, 28 Jul 2012 09:56:55 -0700 Subject: Add a regression test for issue #380 (which fails at the moment) --- src/graph_test.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/graph_test.cc b/src/graph_test.cc index 07a1936..38ff28a 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -140,3 +140,22 @@ TEST_F(GraphTest, VarInOutQuoteSpaces) { EXPECT_EQ("cat nospace \"with space\" nospace2 > \"a b\"", edge->EvaluateCommand()); } + +// Regression test for https://github.com/martine/ninja/issues/380 +TEST_F(GraphTest, DepfileWithCanonicalizablePath) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule catdep\n" +" depfile = $out.d\n" +" command = cat $in > $out\n" +"build ./out.o: catdep ./foo.cc\n")); + fs_.Create("foo.cc", 1, ""); + fs_.Create("out.o.d", 1, "out.o: bar/../foo.cc\n"); + fs_.Create("out.o", 1, ""); + + Edge* edge = GetNode("out.o")->in_edge(); + string err; + EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + ASSERT_EQ("", err); + + EXPECT_FALSE(GetNode("out.o")->dirty()); +} -- cgit v0.12 From 957a801ace27426f3ceb544449f882dba1924161 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Sat, 28 Jul 2012 09:56:59 -0700 Subject: Revert "Make StringPiece data members private." This reverts commit 904c9610fe66c4f4bd63a07d6f057c8603d24394. The commit caused issue #380, this revert fixes it. The revert also makes the test from the previous commit pass. --- src/build_log.cc | 4 ++-- src/depfile_parser.cc | 2 +- src/edit_distance.cc | 10 +++++----- src/eval_env.cc | 2 +- src/graph.cc | 3 +-- src/hash_map.h | 2 +- src/lexer.cc | 6 +++--- src/string_piece.h | 4 ---- 8 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/build_log.cc b/src/build_log.cc index c35b0e4..1b27be3 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -80,7 +80,7 @@ uint64_t MurmurHash64A(const void* key, int len) { h *= m; h ^= h >> r; return h; -} +} #undef BIG_CONSTANT @@ -88,7 +88,7 @@ uint64_t MurmurHash64A(const void* key, int len) { // static uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { - return MurmurHash64A(command.str(), command.len()); + return MurmurHash64A(command.str_, command.len_); } BuildLog::BuildLog() diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 47b64d1..03dad92 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -203,7 +203,7 @@ yy13: if (!is_target) { ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str()) { + } else if (!out_.str_) { out_ = StringPiece(filename, len); } else if (out_ != StringPiece(filename, len)) { *err = "depfile has multiple output paths."; diff --git a/src/edit_distance.cc b/src/edit_distance.cc index 50e641d..22db4fe 100644 --- a/src/edit_distance.cc +++ b/src/edit_distance.cc @@ -29,8 +29,8 @@ int EditDistance(const StringPiece& s1, // Although the algorithm is typically described using an m x n // array, only two rows are used at a time, so this implemenation // just keeps two separate vectors for those two rows. - int m = s1.len(); - int n = s2.len(); + int m = s1.len_; + int n = s2.len_; std::vector previous(n + 1); std::vector current(n + 1); @@ -44,11 +44,11 @@ int EditDistance(const StringPiece& s1, for (int x = 1; x <= n; ++x) { if (allow_replacements) { - current[x] = min(previous[x-1] + (s1.str()[y-1] == s2.str()[x-1] ? - 0 : 1), min(current[x-1], previous[x]) + 1); + current[x] = min(previous[x-1] + (s1.str_[y-1] == s2.str_[x-1] ? 0 : 1), + min(current[x-1], previous[x])+1); } else { - if (s1.str()[y-1] == s2.str()[x-1]) + if (s1.str_[y-1] == s2.str_[x-1]) current[x] = previous[x-1]; else current[x] = min(current[x-1], previous[x]) + 1; diff --git a/src/eval_env.cc b/src/eval_env.cc index 793ea64..81a8765 100644 --- a/src/eval_env.cc +++ b/src/eval_env.cc @@ -41,7 +41,7 @@ string EvalString::Evaluate(Env* env) const { void EvalString::AddText(StringPiece text) { // Add it to the end of an existing RAW token if possible. if (!parsed_.empty() && parsed_.back().second == RAW) { - parsed_.back().first.append(text.str(), text.len()); + parsed_.back().first.append(text.str_, text.len_); } else { parsed_.push_back(make_pair(text.AsString(), RAW)); } diff --git a/src/graph.cc b/src/graph.cc index caf2aca..18adeee 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -295,8 +295,7 @@ bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, // Add all its in-edges. for (vector::iterator i = depfile.ins_.begin(); i != depfile.ins_.end(); ++i, ++implicit_dep) { - int length = i->len(); - if (!CanonicalizePath(const_cast(i->str()), &length, err)) + if (!CanonicalizePath(const_cast(i->str_), &i->len_, err)) return false; Node* node = state->GetNode(*i); diff --git a/src/hash_map.h b/src/hash_map.h index ff2f7ba..658f699 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -86,7 +86,7 @@ struct hash { template<> struct hash { size_t operator()(StringPiece key) const { - return MurmurHash2(key.str(), key.len()); + return MurmurHash2(key.str_, key.len_); } }; diff --git a/src/lexer.cc b/src/lexer.cc index 45bf4ef..ca6f367 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -23,8 +23,8 @@ bool Lexer::Error(const string& message, string* err) { // Compute line/column. int line = 1; - const char* context = input_.str(); - for (const char* p = input_.str(); p < last_token_; ++p) { + const char* context = input_.str_; + for (const char* p = input_.str_; p < last_token_; ++p) { if (*p == '\n') { ++line; context = p + 1; @@ -66,7 +66,7 @@ Lexer::Lexer(const char* input) { void Lexer::Start(StringPiece filename, StringPiece input) { filename_ = filename; input_ = input; - ofs_ = input_.str(); + ofs_ = input_.str_; last_token_ = NULL; } diff --git a/src/string_piece.h b/src/string_piece.h index 76679f1..ad1153e 100644 --- a/src/string_piece.h +++ b/src/string_piece.h @@ -46,10 +46,6 @@ struct StringPiece { return len_ ? string(str_, len_) : string(); } - const char* str() const { return str_; } - int len() const { return len_; } - - private: const char* str_; int len_; }; -- cgit v0.12 From db7066551ae75c0a304dd8cd365cf811ac345a53 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Sat, 28 Jul 2012 11:17:08 -0700 Subject: Disable SubprocessTest.TestWithLots on OS X. It fails on OS X, see issue #384. --- src/subprocess_test.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index 7dbaf97..d89525e 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -149,7 +149,9 @@ TEST_F(SubprocessTest, SetWithMulti) { } } -#ifndef _WIN32 +// OS X's process limit is less than 1025 by default +// (|sysctl kern.maxprocperuid| is 709 on 10.7 and 10.8 and less prior to that). +#ifdef linux TEST_F(SubprocessTest, SetWithLots) { // Arbitrary big number; needs to be over 1024 to confirm we're no longer // hostage to pselect. @@ -176,4 +178,4 @@ TEST_F(SubprocessTest, SetWithLots) { } ASSERT_EQ(kNumProcs, subprocs_.finished_.size()); } -#endif // _WIN32 +#endif // linux -- cgit v0.12 From 809b8d7e04af7811c3be49987ff6c680fe289b57 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sat, 28 Jul 2012 13:21:39 -0700 Subject: delete vestigal declarations --- src/util.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/util.h b/src/util.h index fd91772..5caf644 100644 --- a/src/util.h +++ b/src/util.h @@ -75,13 +75,6 @@ double GetLoadAverage(); #endif #ifdef _WIN32 - -/// Handler for __except block -int exception_filter(unsigned int code, struct _EXCEPTION_POINTERS *ep); - -/// Write a windows minidump file in temp directory. User can specify null 'pep' argument. -void Create_Win32_MiniDump( struct _EXCEPTION_POINTERS* pep ); - /// Convert the value returned by GetLastError() into a string. string GetLastErrorString(); #endif -- cgit v0.12 From 48bd2f2b997b26ac187757a65936619969d15ddb Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sat, 28 Jul 2012 13:27:27 -0700 Subject: simplify load-average code --- src/util.cc | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/util.cc b/src/util.cc index ca05292..2277a82 100644 --- a/src/util.cc +++ b/src/util.cc @@ -284,17 +284,14 @@ string StripAnsiEscapeCodes(const string& in) { } #ifdef _WIN32 -static double GetLoadAverage_win32() -{ +double GetLoadAverage() { // TODO(nicolas.despres@gmail.com): Find a way to implement it on Windows. return -0.0f; } #else -static double GetLoadAverage_unix() -{ +double GetLoadAverage() { double loadavg[3] = { 0.0f, 0.0f, 0.0f }; - if (getloadavg(loadavg, 3) < 0) - { + if (getloadavg(loadavg, 3) < 0) { // Maybe we should return an error here or the availability of // getloadavg(3) should be checked when ninja is configured. return -0.0f; @@ -302,12 +299,3 @@ static double GetLoadAverage_unix() return loadavg[0]; } #endif // _WIN32 - -double GetLoadAverage() -{ -#ifdef _WIN32 - return GetLoadAverage_win32(); -#else - return GetLoadAverage_unix(); -#endif // _WIN32 -} -- cgit v0.12 From 0f56b0e5ebe20eccefe064875d15acb3b9378e2a Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sat, 28 Jul 2012 13:28:56 -0700 Subject: note that -l doesn't work on windows Filed issue #386 about it too. --- src/ninja.cc | 3 +++ src/util.cc | 1 + 2 files changed, 4 insertions(+) diff --git a/src/ninja.cc b/src/ninja.cc index 538d0d7..7929e68 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -91,6 +91,9 @@ void Usage(const BuildConfig& config) { "\n" " -j N run N jobs in parallel [default=%d]\n" " -l N do not start new jobs if the load average is greater than N\n" +#ifdef _WIN32 +" (not yet implemented on Windows)\n" +#endif " -k N keep going until N jobs fail [default=1]\n" " -n dry run (don't run commands but pretend they succeeded)\n" " -v show all command lines while building\n" diff --git a/src/util.cc b/src/util.cc index 2277a82..10c9dc6 100644 --- a/src/util.cc +++ b/src/util.cc @@ -286,6 +286,7 @@ string StripAnsiEscapeCodes(const string& in) { #ifdef _WIN32 double GetLoadAverage() { // TODO(nicolas.despres@gmail.com): Find a way to implement it on Windows. + // Remember to also update Usage() when this is fixed. return -0.0f; } #else -- cgit v0.12 From dcaa573cc2e6f30d346262b3c42d4497a539d90d Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sat, 28 Jul 2012 13:32:23 -0700 Subject: move processor-count code to util.cc --- src/ninja.cc | 26 +------------------------- src/util.cc | 30 ++++++++++++++++++++++++++++++ src/util.h | 4 ++++ 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index 7929e68..ca45ab1 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -19,12 +19,6 @@ #include #include -#if defined(__APPLE__) || defined(__FreeBSD__) -#include -#elif defined(linux) -#include -#endif - #ifdef _WIN32 #include "getopt.h" #include @@ -107,25 +101,7 @@ void Usage(const BuildConfig& config) { /// Choose a default value for the -j (parallelism) flag. int GuessParallelism() { - int processors = 0; - -#if defined(linux) - processors = get_nprocs(); -#elif defined(__APPLE__) || defined(__FreeBSD__) - size_t processors_size = sizeof(processors); - int name[] = {CTL_HW, HW_NCPU}; - if (sysctl(name, sizeof(name) / sizeof(int), - &processors, &processors_size, - NULL, 0) < 0) { - processors = 1; - } -#elif defined(_WIN32) - SYSTEM_INFO info; - GetSystemInfo(&info); - processors = info.dwNumberOfProcessors; -#endif - - switch (processors) { + switch (int processors = GetProcessorCount()) { case 0: case 1: return 2; diff --git a/src/util.cc b/src/util.cc index 10c9dc6..4ce610b 100644 --- a/src/util.cc +++ b/src/util.cc @@ -38,6 +38,12 @@ #include // _mkdir #endif +#if defined(__APPLE__) || defined(__FreeBSD__) +#include +#elif defined(linux) +#include +#endif + #include "edit_distance.h" #include "metrics.h" @@ -283,6 +289,30 @@ string StripAnsiEscapeCodes(const string& in) { return stripped; } +#if defined(linux) +int GetProcessorCount() { + return get_nprocs(); +} +#elif defined(__APPLE__) || defined(__FreeBSD__) +int GetProcessorCount() { + int processors; + size_t processors_size = sizeof(processors); + int name[] = {CTL_HW, HW_NCPU}; + if (sysctl(name, sizeof(name) / sizeof(int), + &processors, &processors_size, + NULL, 0) < 0) { + return 0; + } + return processors; +} +#elif defined(_WIN32) +int GetProcessorCount() { + SYSTEM_INFO info; + GetSystemInfo(&info); + return info.dwNumberOfProcessors; +} +#endif + #ifdef _WIN32 double GetLoadAverage() { // TODO(nicolas.despres@gmail.com): Find a way to implement it on Windows. diff --git a/src/util.h b/src/util.h index 5caf644..9267091 100644 --- a/src/util.h +++ b/src/util.h @@ -62,6 +62,10 @@ const char* SpellcheckString(const string& text, ...); /// Removes all Ansi escape codes (http://www.termsys.demon.co.uk/vtansi.htm). string StripAnsiEscapeCodes(const string& in); +/// @return the number of processors on the machine. Useful for an initial +/// guess for how many jobs to run in parallel. @return 0 on error. +int GetProcessorCount(); + /// @return the load average of the machine. A negative value is returned /// on error. double GetLoadAverage(); -- cgit v0.12 From 080314c22ff73fd9e6bc085e0e7053a09a3a6592 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sat, 28 Jul 2012 13:35:13 -0700 Subject: use chdir() define from util.h --- src/ninja.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index ca45ab1..778eb53 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -718,11 +718,7 @@ int NinjaMain(int argc, char** argv) { // can be piped into a file without this string showing up. if (tool == "") printf("ninja: Entering directory `%s'\n", working_dir); -#ifdef _WIN32 - if (_chdir(working_dir) < 0) { -#else if (chdir(working_dir) < 0) { -#endif Fatal("chdir to '%s' - %s", working_dir, strerror(errno)); } } -- cgit v0.12 From 8b590881013a05fd5017aa94185a02f6b7d09758 Mon Sep 17 00:00:00 2001 From: Nicolas Despres Date: Sat, 28 Jul 2012 14:41:20 +0200 Subject: Re-factor elide code and test it. --- src/build.cc | 15 ++------------- src/util.cc | 13 +++++++++++++ src/util.h | 4 ++++ src/util_test.cc | 11 +++++++++++ 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/build.cc b/src/build.cc index 09d7f65..a1c94e4 100644 --- a/src/build.cc +++ b/src/build.cc @@ -236,28 +236,17 @@ void BuildStatus::PrintStatus(Edge* edge) { to_print = FormatProgressStatus(progress_status_format_) + to_print; if (smart_terminal_ && !force_full_command) { - const int kMargin = 3; // Space for "...". #ifndef _WIN32 // Limit output to width of the terminal if provided so we don't cause // line-wrapping. winsize size; if ((ioctl(0, TIOCGWINSZ, &size) == 0) && size.ws_col) { - if (to_print.size() + kMargin > size.ws_col) { - int elide_size = (size.ws_col - kMargin) / 2; - to_print = to_print.substr(0, elide_size) - + "..." - + to_print.substr(to_print.size() - elide_size, elide_size); - } + to_print = ElideMiddle(to_print, size.ws_col); } #else // Don't use the full width or console will move to next line. size_t width = static_cast(csbi.dwSize.X) - 1; - if (to_print.size() + kMargin > width) { - int elide_size = (width - kMargin) / 2; - to_print = to_print.substr(0, elide_size) - + "..." - + to_print.substr(to_print.size() - elide_size, elide_size); - } + to_print = ElideMiddle(to_print, width); #endif } diff --git a/src/util.cc b/src/util.cc index ca05292..cb3e141 100644 --- a/src/util.cc +++ b/src/util.cc @@ -311,3 +311,16 @@ double GetLoadAverage() return GetLoadAverage_unix(); #endif // _WIN32 } + +string ElideMiddle(const string& str, size_t width) +{ + const int kMargin = 3; // Space for "...". + string result = str; + if (result.size() + kMargin > width) { + int elide_size = (width - kMargin) / 2; + result = result.substr(0, elide_size) + + "..." + + result.substr(result.size() - elide_size, elide_size); + } + return result; +} diff --git a/src/util.h b/src/util.h index fd91772..4c0f4bb 100644 --- a/src/util.h +++ b/src/util.h @@ -66,6 +66,10 @@ string StripAnsiEscapeCodes(const string& in); /// on error. double GetLoadAverage(); +/// Elide the given string @a str with '...' in the middle if the length +/// exceeds @a width. +string ElideMiddle(const string& str, size_t width); + #ifdef _MSC_VER #define snprintf _snprintf #define fileno _fileno diff --git a/src/util_test.cc b/src/util_test.cc index 23d4b83..5ace5e7 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -136,3 +136,14 @@ TEST(StripAnsiEscapeCodes, StripColors) { EXPECT_EQ("affixmgr.cxx:286:15: warning: using the result... [-Wparentheses]", stripped); } + +TEST(ElideMiddle, NothingToElide) { + string input = "Nothing to elide in this short string."; + EXPECT_EQ(input, ElideMiddle(input, 80)); +} + +TEST(ElideMiddle, ElideInTheMiddle) { + string input = "01234567890123456789"; + string elided = ElideMiddle(input, 10); + EXPECT_EQ("012...789", elided); +} -- cgit v0.12 From 142fb71161f241e684e1a8ee4ecef22b61b14e0e Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Tue, 31 Jul 2012 10:05:34 -0700 Subject: fix hash_map StringPiece comparator --- src/hash_map.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hash_map.h b/src/hash_map.h index 658f699..88c2681 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -55,16 +55,16 @@ using stdext::hash_compare; struct StringPieceCmp : public hash_compare { size_t operator()(const StringPiece& key) const { - return MurmurHash2(key.str(), key.len()); + return MurmurHash2(key.str_, key.len_); } bool operator()(const StringPiece& a, const StringPiece& b) const { - int cmp = strncmp(a.str(), b.str(), min(a.len(), b.len())); + int cmp = strncmp(a.str_, b.str_, min(a.len_, b.len_)); if (cmp < 0) { return true; } else if (cmp > 0) { return false; } else { - return a.len() < b.len(); + return a.len_ < b.len_; } } }; -- cgit v0.12 From ed07eb9f2f25ddee464e786f0f2f82e9e8a33e0a Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Thu, 2 Aug 2012 15:24:32 -0700 Subject: reject tabs (and CRs) in input files more aggressively --- src/lexer.cc | 37 +++++++++++++++++++------------------ src/lexer.h | 7 +++++-- src/lexer.in.cc | 37 +++++++++++++++++++------------------ src/lexer_test.cc | 10 ++++++++++ src/manifest_parser.cc | 5 +++-- src/manifest_parser_test.cc | 2 +- 6 files changed, 57 insertions(+), 41 deletions(-) diff --git a/src/lexer.cc b/src/lexer.cc index ca6f367..f4036d4 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -90,24 +90,25 @@ const char* Lexer::TokenName(Token t) { return NULL; // not reached } -const char* Lexer::TokenErrorHint(Token t) { - switch (t) { - case ERROR: return ""; - case BUILD: return ""; - case COLON: return " ($ also escapes ':')"; - case DEFAULT: return ""; - case EQUALS: return ""; - case IDENT: return ""; - case INCLUDE: return ""; - case INDENT: return ""; - case NEWLINE: return ""; - case PIPE2: return ""; - case PIPE: return ""; - case RULE: return ""; - case SUBNINJA: return ""; - case TEOF: return ""; +const char* Lexer::TokenErrorHint(Token expected) { + switch (expected) { + case COLON: + return " ($ also escapes ':')"; + default: + return ""; + } +} + +string Lexer::DescribeLastError() { + if (last_token_) { + switch (last_token_[0]) { + case '\r': + return "carriage returns are not allowed, use newlines"; + case '\t': + return "tabs are not allowed, use spaces"; + } } - return ""; + return "lexing error"; } void Lexer::UnreadToken() { @@ -689,7 +690,7 @@ yy94: yy95: { last_token_ = start; - return Error("lexing error", err); + return Error(DescribeLastError(), err); } yy96: ++p; diff --git a/src/lexer.h b/src/lexer.h index 19008d7..03c59f2 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -49,9 +49,12 @@ struct Lexer { /// Return a human-readable form of a token, used in error messages. static const char* TokenName(Token t); - /// Return a human-readable token hint, used in error messages. - static const char* TokenErrorHint(Token t); + static const char* TokenErrorHint(Token expected); + + /// If the last token read was an ERROR token, provide more info + /// or the empty string. + string DescribeLastError(); /// Start parsing some input. void Start(StringPiece filename, StringPiece input); diff --git a/src/lexer.in.cc b/src/lexer.in.cc index 852d6e9..ec3ad6b 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -89,24 +89,25 @@ const char* Lexer::TokenName(Token t) { return NULL; // not reached } -const char* Lexer::TokenErrorHint(Token t) { - switch (t) { - case ERROR: return ""; - case BUILD: return ""; - case COLON: return " ($ also escapes ':')"; - case DEFAULT: return ""; - case EQUALS: return ""; - case IDENT: return ""; - case INCLUDE: return ""; - case INDENT: return ""; - case NEWLINE: return ""; - case PIPE2: return ""; - case PIPE: return ""; - case RULE: return ""; - case SUBNINJA: return ""; - case TEOF: return ""; +const char* Lexer::TokenErrorHint(Token expected) { + switch (expected) { + case COLON: + return " ($ also escapes ':')"; + default: + return ""; + } +} + +string Lexer::DescribeLastError() { + if (last_token_) { + switch (last_token_[0]) { + case '\r': + return "carriage returns are not allowed, use newlines"; + case '\t': + return "tabs are not allowed, use spaces"; + } } - return ""; + return "lexing error"; } void Lexer::UnreadToken() { @@ -248,7 +249,7 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { } [^] { last_token_ = start; - return Error("lexing error", err); + return Error(DescribeLastError(), err); } */ } diff --git a/src/lexer_test.cc b/src/lexer_test.cc index 5795e5e..e8a1642 100644 --- a/src/lexer_test.cc +++ b/src/lexer_test.cc @@ -85,3 +85,13 @@ TEST(Lexer, CommentEOF) { Lexer::Token token = lexer.ReadToken(); EXPECT_EQ(Lexer::ERROR, token); } + +TEST(Lexer, Tabs) { + // Verify we print a useful error on a disallowed character. + Lexer lexer(" \tfoobar"); + Lexer::Token token = lexer.ReadToken(); + EXPECT_EQ(Lexer::INDENT, token); + token = lexer.ReadToken(); + EXPECT_EQ(Lexer::ERROR, token); + EXPECT_EQ("tabs are not allowed, use spaces", lexer.DescribeLastError()); +} diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 057e12c..405e244 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -76,8 +76,9 @@ bool ManifestParser::Parse(const string& filename, const string& input, if (!ParseFileInclude(true, err)) return false; break; - case Lexer::ERROR: - return lexer_.Error("lexing error", err); + case Lexer::ERROR: { + return lexer_.Error(lexer_.DescribeLastError(), err); + } case Lexer::TEOF: return true; case Lexer::NEWLINE: diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 3261d39..a48c99e 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -697,7 +697,7 @@ TEST_F(ParserTest, CRLF) { EXPECT_FALSE(parser.ParseTest("foo = foo\nbar = bar\r\n", &err)); - EXPECT_EQ("input:2: lexing error\n" + EXPECT_EQ("input:2: carriage returns are not allowed, use newlines\n" "bar = bar\r\n" " ^ near here", err); -- cgit v0.12 From de8d5080c592f6e1cf59689e321d4bd66ee73290 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Sun, 5 Aug 2012 12:02:01 -0700 Subject: Include unistd.h in files that use unlink() on OS X. `man unlink` says this is necessary, and according to a report by Claus Klein, omitting them breaks the build on OS X 10.5 with gcc 4.7 (see issue #396). (On Windows, ninja's util.h includes a define for unlink.) --- src/build_log_perftest.cc | 4 ++++ src/build_log_test.cc | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc index 5755079..a09beb8 100644 --- a/src/build_log_perftest.cc +++ b/src/build_log_perftest.cc @@ -22,6 +22,10 @@ #include "util.h" #include "metrics.h" +#ifndef _WIN32 +#include +#endif + const char kTestFilename[] = "BuildLogPerfTest-tempfile"; bool WriteTestData(string* err) { diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 0225684..186dad9 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -20,9 +20,7 @@ #ifdef _WIN32 #include #include -#endif - -#ifdef linux +#else #include #include #include -- cgit v0.12 From cc222d3f6ba769fd800b1dfdb1699fbb84516c7b Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 7 Aug 2012 13:58:55 -0700 Subject: use DiskInterface to create the build directory Fixes issue #392 (didn't handle creating nested build dirs right). Moves MakeDir out of util.h; all code should go through DiskInterface to simplify testing. Moves ownership of the DiskInterface into the client of the Builder, which also allows removing some code that reached inside the object as well as a minor leak. --- src/build.cc | 6 +++--- src/build.h | 6 ++++-- src/build_test.cc | 8 +++----- src/disk_interface.cc | 10 ++++++++++ src/ninja.cc | 19 +++++++++++-------- src/util.cc | 12 ------------ src/util.h | 4 ---- 7 files changed, 31 insertions(+), 34 deletions(-) diff --git a/src/build.cc b/src/build.cc index a1c94e4..6d1318c 100644 --- a/src/build.cc +++ b/src/build.cc @@ -552,9 +552,9 @@ struct DryRunCommandRunner : public CommandRunner { queue finished_; }; -Builder::Builder(State* state, const BuildConfig& config) - : state_(state), config_(config) { - disk_interface_ = new RealDiskInterface; +Builder::Builder(State* state, const BuildConfig& config, + DiskInterface* disk_interface) + : state_(state), config_(config), disk_interface_(disk_interface) { status_ = new BuildStatus(config); log_ = state->build_log_; } diff --git a/src/build.h b/src/build.h index 5e26bbb..986e8a9 100644 --- a/src/build.h +++ b/src/build.h @@ -119,7 +119,8 @@ struct BuildConfig { /// Builder wraps the build process: starting commands, updating status. struct Builder { - Builder(State* state, const BuildConfig& config); + Builder(State* state, const BuildConfig& config, + DiskInterface* disk_interface); ~Builder(); /// Clean up after interrupted commands by deleting output files. @@ -144,12 +145,13 @@ struct Builder { State* state_; const BuildConfig& config_; Plan plan_; - DiskInterface* disk_interface_; auto_ptr command_runner_; BuildStatus* status_; BuildLog* log_; private: + DiskInterface* disk_interface_; + // Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr. Builder(const Builder &other); // DO NOT IMPLEMENT void operator=(const Builder &other); // DO NOT IMPLEMENT diff --git a/src/build_test.cc b/src/build_test.cc index a3f345b..574ffb4 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -178,9 +178,8 @@ TEST_F(PlanTest, DependencyCycle) { struct BuildTest : public StateTestWithBuiltinRules, public CommandRunner { - BuildTest() : config_(MakeConfig()), builder_(&state_, config_), now_(1), - last_command_(NULL), status_(config_) { - builder_.disk_interface_ = &fs_; + BuildTest() : config_(MakeConfig()), builder_(&state_, config_, &fs_), + now_(1), last_command_(NULL), status_(config_) { builder_.command_runner_.reset(this); AssertParse(&state_, "build cat1: cat in1\n" @@ -212,11 +211,10 @@ struct BuildTest : public StateTestWithBuiltinRules, } BuildConfig config_; + VirtualFileSystem fs_; Builder builder_; int now_; - VirtualFileSystem fs_; - vector commands_ran_; Edge* last_command_; BuildStatus status_; diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 74f33c4..515ff59 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -18,9 +18,11 @@ #include #include #include +#include #ifdef _WIN32 #include +#include // _mkdir #endif #include "util.h" @@ -42,6 +44,14 @@ string DirName(const string& path) { return path.substr(0, slash_pos); } +int MakeDir(const string& path) { +#ifdef _WIN32 + return _mkdir(path.c_str()); +#else + return mkdir(path.c_str(), 0777); +#endif +} + } // namespace // DiskInterface --------------------------------------------------------------- diff --git a/src/ninja.cc b/src/ninja.cc index 778eb53..d212579 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -32,6 +32,7 @@ #include "build.h" #include "build_log.h" #include "clean.h" +#include "disk_interface.h" #include "edit_distance.h" #include "explain.h" #include "graph.h" @@ -66,6 +67,8 @@ struct Globals { BuildConfig config; /// Loaded state (rules, nodes). This is a pointer so it can be reset. State* state; + /// Functions for interacting with the disk. + RealDiskInterface disk_interface; }; /// Print usage information. @@ -122,16 +125,16 @@ struct RealFileReader : public ManifestParser::FileReader { /// Rebuild the build manifest, if necessary. /// Returns true if the manifest was rebuilt. -bool RebuildManifest(State* state, const BuildConfig& config, - const char* input_file, string* err) { +bool RebuildManifest(Globals* globals, const char* input_file, string* err) { string path = input_file; if (!CanonicalizePath(&path, err)) return false; - Node* node = state->LookupNode(path); + Node* node = globals->state->LookupNode(path); if (!node) return false; - Builder manifest_builder(state, config); + Builder manifest_builder(globals->state, globals->config, + &globals->disk_interface); if (!manifest_builder.AddTarget(node, err)) return false; @@ -579,7 +582,7 @@ int RunBuild(Globals* globals, int argc, char** argv) { return 1; } - Builder builder(globals->state, globals->config); + Builder builder(globals->state, globals->config, &globals->disk_interface); for (size_t i = 0; i < targets.size(); ++i) { if (!builder.AddTarget(targets[i], &err)) { if (!err.empty()) { @@ -745,12 +748,12 @@ reload: const char* kLogPath = ".ninja_log"; string log_path = kLogPath; if (!build_dir.empty()) { - if (MakeDir(build_dir) < 0 && errno != EEXIST) { + log_path = build_dir + "/" + kLogPath; + if (globals.disk_interface.MakeDirs(log_path) < 0 && errno != EEXIST) { Error("creating build directory %s: %s", build_dir.c_str(), strerror(errno)); return 1; } - log_path = build_dir + "/" + kLogPath; } if (!build_log.Load(log_path, &err)) { @@ -765,7 +768,7 @@ reload: if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild // target that is never up to date. - if (RebuildManifest(globals.state, globals.config, input_file, &err)) { + if (RebuildManifest(&globals, input_file, &err)) { rebuilt_manifest = true; globals.ResetState(); goto reload; diff --git a/src/util.cc b/src/util.cc index be2347c..7c2f895 100644 --- a/src/util.cc +++ b/src/util.cc @@ -34,10 +34,6 @@ #include -#ifdef _WIN32 -#include // _mkdir -#endif - #if defined(__APPLE__) || defined(__FreeBSD__) #include #elif defined(linux) @@ -163,14 +159,6 @@ bool CanonicalizePath(char* path, int* len, string* err) { return true; } -int MakeDir(const string& path) { -#ifdef _WIN32 - return _mkdir(path.c_str()); -#else - return mkdir(path.c_str(), 0777); -#endif -} - int ReadFile(const string& path, string* contents, string* err) { FILE* f = fopen(path.c_str(), "r"); if (!f) { diff --git a/src/util.h b/src/util.h index fc701cd..bfcbfa8 100644 --- a/src/util.h +++ b/src/util.h @@ -41,10 +41,6 @@ bool CanonicalizePath(string* path, string* err); bool CanonicalizePath(char* path, int* len, string* err); -/// Create a directory (mode 0777 on Unix). -/// Portability abstraction. -int MakeDir(const string& path); - /// Read a file to a string. /// Returns -errno and fills in \a err on error. int ReadFile(const string& path, string* contents, string* err); -- cgit v0.12 From a9b7ae497faec86d5e413b46bb0b98c16a361c79 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Wed, 8 Aug 2012 09:32:50 -0700 Subject: fix windows build --- src/ninja.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ninja.cc b/src/ninja.cc index d212579..b645b90 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -749,7 +749,7 @@ reload: string log_path = kLogPath; if (!build_dir.empty()) { log_path = build_dir + "/" + kLogPath; - if (globals.disk_interface.MakeDirs(log_path) < 0 && errno != EEXIST) { + if (!globals.disk_interface.MakeDirs(log_path) && errno != EEXIST) { Error("creating build directory %s: %s", build_dir.c_str(), strerror(errno)); return 1; -- cgit v0.12 From 49dd59d707c4e79804835c9cb5a49d1516e7eeff Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 10 Aug 2012 10:58:36 -0700 Subject: document that \r and \t are disallowed Finishes #394. --- doc/manual.asciidoc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 52cce3b..03d27df 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -524,7 +524,9 @@ Lexical syntax Ninja is mostly encoding agnostic, as long as the bytes Ninja cares about (like slashes in paths) are ASCII. This means e.g. UTF-8 or -ISO-8859-1 input files ought to work. +ISO-8859-1 input files ought to work. (To simplify some code, tabs +and carriage returns are currently disallowed; this could be fixed if +it really mattered to you.) Comments begin with `#` and extend to the end of the line. -- cgit v0.12 From 40ac1f6e58ca6b6c4c49fe582c30390ba4012945 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Wed, 1 Aug 2012 15:30:41 -0700 Subject: windows bootstrap: clean up object files --- bootstrap.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bootstrap.py b/bootstrap.py index ab03cf8..a5df22a 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -108,4 +108,8 @@ run([sys.executable, 'configure.py'] + conf_args) run(['./' + binary] + verbose) os.unlink(binary) +if sys.platform.startswith('win32'): + for obj in glob.glob('*.obj'): + os.unlink(obj) + print 'Done!' -- cgit v0.12 From b00557194b2730bb755f39c79755db885a6afec9 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 10 Aug 2012 11:39:35 -0700 Subject: windows: ignore pdb files --- .gitignore | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index fb9a94c..b7a7ab8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,16 @@ -*.pyc -*.exe -TAGS -/build -/build.ninja -/ninja -/build_log_perftest -/canon_perftest -/hash_collision_bench -/ninja_test -/parser_perftest -/graph.png -/doc/manual.html -/doc/doxygen -/gtest-1.6.0 +*.pyc +*.exe +*.pdb +TAGS +/build +/build.ninja +/ninja +/build_log_perftest +/canon_perftest +/hash_collision_bench +/ninja_test +/parser_perftest +/graph.png +/doc/manual.html +/doc/doxygen +/gtest-1.6.0 -- cgit v0.12 From 45ca47e10bb4ef3b220a097f324a83daa4f63e94 Mon Sep 17 00:00:00 2001 From: Matthew Woehlke Date: Fri, 3 Aug 2012 18:42:44 -0400 Subject: add bootstrap option to force 64-bit --- bootstrap.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index a5df22a..e2aa866 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -26,6 +26,8 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) parser = OptionParser() parser.add_option('--verbose', action='store_true', help='enable verbose build',) +parser.add_option('--x64', action='store_true', + help='force 64-bit build (Windows)',) (options, conf_args) = parser.parse_args() def run(*args, **kwargs): @@ -74,8 +76,12 @@ if sys.platform.startswith('win32'): vcdir = os.environ.get('VCINSTALLDIR') if vcdir: - args = [os.path.join(vcdir, 'bin', 'cl.exe'), - '/nologo', '/EHsc', '/DNOMINMAX'] + if options.x64: + args = [os.path.join(vcdir, 'bin', 'amd64', 'cl.exe'), + '/nologo', '/EHsc', '/DNOMINMAX'] + else: + args = [os.path.join(vcdir, 'bin', 'cl.exe'), + '/nologo', '/EHsc', '/DNOMINMAX'] else: args = shlex.split(os.environ.get('CXX', 'g++')) cflags.extend(['-Wno-deprecated', @@ -83,6 +89,8 @@ else: '-DNINJA_BOOTSTRAP']) if sys.platform.startswith('win32'): cflags.append('-D_WIN32_WINNT=0x0501') + if options.x64: + cflags.append('-m64') args.extend(cflags) args.extend(ldflags) binary = 'ninja.bootstrap' -- cgit v0.12 From d98ba72ef8adfb1698b8de197de6751c8b14d5c0 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 10 Aug 2012 12:45:39 -0700 Subject: refactor --- bootstrap.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index e2aa866..7b6e781 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -77,11 +77,10 @@ if sys.platform.startswith('win32'): vcdir = os.environ.get('VCINSTALLDIR') if vcdir: if options.x64: - args = [os.path.join(vcdir, 'bin', 'amd64', 'cl.exe'), - '/nologo', '/EHsc', '/DNOMINMAX'] + cl = os.path.join(vcdir, 'bin', 'amd64', 'cl.exe'), else: - args = [os.path.join(vcdir, 'bin', 'cl.exe'), - '/nologo', '/EHsc', '/DNOMINMAX'] + cl = [os.path.join(vcdir, 'bin', 'cl.exe') + args = cl + ['/nologo', '/EHsc', '/DNOMINMAX'] else: args = shlex.split(os.environ.get('CXX', 'g++')) cflags.extend(['-Wno-deprecated', -- cgit v0.12 From d58a10965610eb6b338e411ce816b810d76ab59e Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 10 Aug 2012 13:27:08 -0700 Subject: windows: fix size_t<->int conversions in ninja.exe --- src/build_log.cc | 12 ++++++------ src/depfile_parser.cc | 4 ++-- src/depfile_parser.in.cc | 4 ++-- src/graph.h | 8 ++++---- src/hash_map.h | 2 +- src/lexer.cc | 2 +- src/lexer.in.cc | 2 +- src/string_piece.h | 4 ++-- src/util.cc | 6 +++--- src/util.h | 2 +- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/build_log.cc b/src/build_log.cc index 1b27be3..e72a93e 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -49,7 +49,7 @@ const int kCurrentVersion = 5; #define BIG_CONSTANT(x) (x##LLU) #endif // !defined(_MSC_VER) inline -uint64_t MurmurHash64A(const void* key, int len) { +uint64_t MurmurHash64A(const void* key, size_t len) { static const uint64_t seed = 0xDECAFBADDECAFBADull; const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); const int r = 47; @@ -58,11 +58,11 @@ uint64_t MurmurHash64A(const void* key, int len) { const uint64_t * end = data + (len/8); while(data != end) { uint64_t k = *data++; - k *= m; - k ^= k >> r; - k *= m; + k *= m; + k ^= k >> r; + k *= m; h ^= k; - h *= m; + h *= m; } const unsigned char* data2 = (const unsigned char*)data; switch(len & 7) @@ -80,7 +80,7 @@ uint64_t MurmurHash64A(const void* key, int len) { h *= m; h ^= h >> r; return h; -} +} #undef BIG_CONSTANT diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 03dad92..6887c91 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -149,7 +149,7 @@ yy4: yy5: { // Got a span of plain text. - int len = in - start; + int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. if (out < start) memmove(out, start, len); @@ -191,7 +191,7 @@ yy13: } - int len = out - filename; + int len = (int)(out - filename); const bool is_target = parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 68b6a95..1d4a177 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -70,7 +70,7 @@ bool DepfileParser::Parse(string* content, string* err) { } [a-zA-Z0-9+,/_:.~()@=-]+ { // Got a span of plain text. - int len = in - start; + int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. if (out < start) memmove(out, start, len); @@ -88,7 +88,7 @@ bool DepfileParser::Parse(string* content, string* err) { */ } - int len = out - filename; + int len = (int)(out - filename); const bool is_target = parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. diff --git a/src/graph.h b/src/graph.h index 0ef4f3f..f487a22 100644 --- a/src/graph.h +++ b/src/graph.h @@ -199,12 +199,12 @@ struct Edge { // pointer...) int implicit_deps_; int order_only_deps_; - bool is_implicit(int index) { - return index >= ((int)inputs_.size()) - order_only_deps_ - implicit_deps_ && + bool is_implicit(size_t index) { + return index >= inputs_.size() - order_only_deps_ - implicit_deps_ && !is_order_only(index); } - bool is_order_only(int index) { - return index >= ((int)inputs_.size()) - order_only_deps_; + bool is_order_only(size_t index) { + return index >= inputs_.size() - order_only_deps_; } bool is_phony() const; diff --git a/src/hash_map.h b/src/hash_map.h index 88c2681..9904fb8 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -19,7 +19,7 @@ // MurmurHash2, by Austin Appleby static inline -unsigned int MurmurHash2(const void* key, int len) { +unsigned int MurmurHash2(const void* key, size_t len) { static const unsigned int seed = 0xDECAFBAD; const unsigned int m = 0x5bd1e995; const int r = 24; diff --git a/src/lexer.cc b/src/lexer.cc index f4036d4..5d7d185 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -30,7 +30,7 @@ bool Lexer::Error(const string& message, string* err) { context = p + 1; } } - int col = last_token_ ? last_token_ - context : 0; + int col = last_token_ ? (int)(last_token_ - context) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); diff --git a/src/lexer.in.cc b/src/lexer.in.cc index ec3ad6b..7ae9c61 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -29,7 +29,7 @@ bool Lexer::Error(const string& message, string* err) { context = p + 1; } } - int col = last_token_ ? last_token_ - context : 0; + int col = last_token_ ? (int)(last_token_ - context) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); diff --git a/src/string_piece.h b/src/string_piece.h index ad1153e..b1bf105 100644 --- a/src/string_piece.h +++ b/src/string_piece.h @@ -31,7 +31,7 @@ struct StringPiece { StringPiece(const string& str) : str_(str.data()), len_(str.size()) {} StringPiece(const char* str) : str_(str), len_(strlen(str)) {} - StringPiece(const char* str, int len) : str_(str), len_(len) {} + StringPiece(const char* str, size_t len) : str_(str), len_(len) {} bool operator==(const StringPiece& other) const { return len_ == other.len_ && memcmp(str_, other.str_, len_) == 0; @@ -47,7 +47,7 @@ struct StringPiece { } const char* str_; - int len_; + size_t len_; }; #endif // NINJA_STRINGPIECE_H_ diff --git a/src/util.cc b/src/util.cc index 7c2f895..90a1d45 100644 --- a/src/util.cc +++ b/src/util.cc @@ -81,7 +81,7 @@ void Error(const char* msg, ...) { bool CanonicalizePath(string* path, string* err) { METRIC_RECORD("canonicalize str"); - int len = path->size(); + size_t len = path->size(); char* str = 0; if (len > 0) str = &(*path)[0]; @@ -91,7 +91,7 @@ bool CanonicalizePath(string* path, string* err) { return true; } -bool CanonicalizePath(char* path, int* len, string* err) { +bool CanonicalizePath(char* path, size_t* len, string* err) { // WARNING: this function is performance-critical; please benchmark // any changes you make to it. METRIC_RECORD("canonicalize path"); @@ -323,7 +323,7 @@ string ElideMiddle(const string& str, size_t width) { const int kMargin = 3; // Space for "...". string result = str; if (result.size() + kMargin > width) { - int elide_size = (width - kMargin) / 2; + size_t elide_size = (width - kMargin) / 2; result = result.substr(0, elide_size) + "..." + result.substr(result.size() - elide_size, elide_size); diff --git a/src/util.h b/src/util.h index bfcbfa8..ab1882b 100644 --- a/src/util.h +++ b/src/util.h @@ -39,7 +39,7 @@ void Error(const char* msg, ...); /// Canonicalize a path like "foo/../bar.h" into just "bar.h". bool CanonicalizePath(string* path, string* err); -bool CanonicalizePath(char* path, int* len, string* err); +bool CanonicalizePath(char* path, size_t* len, string* err); /// Read a file to a string. /// Returns -errno and fills in \a err on error. -- cgit v0.12 From 09775f02209a07a4e1032b9ab9d8ce5f0ff8d583 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 10 Aug 2012 13:39:25 -0700 Subject: windows: fix integer truncation issues for helper binaries Disable the size_t truncation warning. (Note that this leaves on the other truncation-related warnings, like int->char.) --- configure.py | 3 +++ src/canon_perftest.cc | 2 +- src/hash_collision_bench.cc | 9 +++++++-- src/util_test.cc | 6 +++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/configure.py b/configure.py index 95f88b1..76abc72 100755 --- a/configure.py +++ b/configure.py @@ -110,6 +110,9 @@ else: if platform == 'windows': cflags = ['/nologo', '/Zi', '/W4', '/WX', '/wd4530', '/wd4100', '/wd4706', '/wd4512', '/wd4800', '/wd4702', '/wd4819', '/GR-', + # Disable size_t -> int truncation warning. + # We never have strings or arrays larger than 2**31. + '/wd4267', '/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS', '/DNINJA_PYTHON="%s"' % options.with_python] ldflags = ['/DEBUG', '/libpath:$builddir'] diff --git a/src/canon_perftest.cc b/src/canon_perftest.cc index d0ed397..59bd18f 100644 --- a/src/canon_perftest.cc +++ b/src/canon_perftest.cc @@ -27,7 +27,7 @@ int main() { string err; char buf[200]; - int len = strlen(kPath); + size_t len = strlen(kPath); strcpy(buf, kPath); for (int j = 0; j < 5; ++j) { diff --git a/src/hash_collision_bench.cc b/src/hash_collision_bench.cc index 6736109..d0eabde 100644 --- a/src/hash_collision_bench.cc +++ b/src/hash_collision_bench.cc @@ -14,6 +14,11 @@ #include "build_log.h" +#include +using namespace std; + +#include + int random(int low, int high) { return int(low + (rand() / double(RAND_MAX)) * (high - low) + 0.5); } @@ -22,7 +27,7 @@ void RandomCommand(char** s) { int len = random(5, 100); *s = new char[len]; for (int i = 0; i < len; ++i) - (*s)[i] = random(32, 127); + (*s)[i] = (char)random(32, 127); } int main() { @@ -32,7 +37,7 @@ int main() { char** commands = new char*[N]; pair* hashes = new pair[N]; - srand(time(NULL)); + srand((int)time(NULL)); for (int i = 0; i < N; ++i) { RandomCommand(&commands[i]); diff --git a/src/util_test.cc b/src/util_test.cc index 5ace5e7..7a34671 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -105,18 +105,18 @@ TEST(CanonicalizePath, AbsolutePath) { TEST(CanonicalizePath, NotNullTerminated) { string path; string err; - int len; + size_t len; path = "foo/. bar/."; len = strlen("foo/."); // Canonicalize only the part before the space. EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); - EXPECT_EQ(strlen("foo"), static_cast(len)); + EXPECT_EQ(strlen("foo"), len); EXPECT_EQ("foo/. bar/.", string(path)); path = "foo/../file bar/."; len = strlen("foo/../file"); EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); - EXPECT_EQ(strlen("file"), static_cast(len)); + EXPECT_EQ(strlen("file"), len); EXPECT_EQ("file ./file bar/.", string(path)); } -- cgit v0.12 From ee625b0b8a16a3182bf1e0531c81f846ef9f4df9 Mon Sep 17 00:00:00 2001 From: Matthew Woehlke Date: Fri, 10 Aug 2012 17:22:34 -0400 Subject: fix syntax errors in bootstrap.py --- bootstrap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index 7b6e781..8c0687e 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -77,9 +77,9 @@ if sys.platform.startswith('win32'): vcdir = os.environ.get('VCINSTALLDIR') if vcdir: if options.x64: - cl = os.path.join(vcdir, 'bin', 'amd64', 'cl.exe'), + cl = [os.path.join(vcdir, 'bin', 'amd64', 'cl.exe')] else: - cl = [os.path.join(vcdir, 'bin', 'cl.exe') + cl = [os.path.join(vcdir, 'bin', 'cl.exe')] args = cl + ['/nologo', '/EHsc', '/DNOMINMAX'] else: args = shlex.split(os.environ.get('CXX', 'g++')) -- cgit v0.12 From 07bd1e2f088c55dfb01609e73febc08cedf854ad Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 11:59:35 -0700 Subject: doc that ReadFile reads in text mode on Windows --- src/util.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util.h b/src/util.h index ab1882b..7e30368 100644 --- a/src/util.h +++ b/src/util.h @@ -41,7 +41,8 @@ bool CanonicalizePath(string* path, string* err); bool CanonicalizePath(char* path, size_t* len, string* err); -/// Read a file to a string. +/// Read a file to a string (in text mode: with CRLF conversion +/// on Windows). /// Returns -errno and fills in \a err on error. int ReadFile(const string& path, string* contents, string* err); -- cgit v0.12 From edaf99e9bda420696b2173b44f0abfc63184aaae Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 12:01:38 -0700 Subject: use correct path separator for --with-gtest source --- configure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.py b/configure.py index 76abc72..f4881a7 100755 --- a/configure.py +++ b/configure.py @@ -291,10 +291,10 @@ if options.with_gtest: else: gtest_cflags = '-fvisibility=hidden ' + gtest_all_incs objs += n.build(built('gtest-all' + objext), 'cxx', - os.path.join(path, 'src/gtest-all.cc'), + os.path.join(path, 'src', 'gtest-all.cc'), variables=[('cflags', gtest_cflags)]) objs += n.build(built('gtest_main' + objext), 'cxx', - os.path.join(path, 'src/gtest_main.cc'), + os.path.join(path, 'src', 'gtest_main.cc'), variables=[('cflags', gtest_cflags)]) test_cflags = cflags + ['-DGTEST_HAS_RTTI=0', -- cgit v0.12 From 5cc5615211b1cf83605a0fb7565cc640f4d7698e Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 12:12:01 -0700 Subject: add a module for working with MSVC (cl.exe) behavior This will be needed for performant builds on Windows. --- configure.py | 3 +++ src/msvc_helper-win32.cc | 35 +++++++++++++++++++++++++++++++++++ src/msvc_helper.h | 29 +++++++++++++++++++++++++++++ src/msvc_helper_test.cc | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 src/msvc_helper-win32.cc create mode 100644 src/msvc_helper.h create mode 100644 src/msvc_helper_test.cc diff --git a/configure.py b/configure.py index f4881a7..331e027 100755 --- a/configure.py +++ b/configure.py @@ -249,6 +249,7 @@ for name in ['build', if platform in ('mingw', 'windows'): objs += cxx('subprocess-win32') if platform == 'windows': + objs += cxx('msvc_helper-win32') objs += cxx('minidump-win32') objs += cc('getopt') else: @@ -318,6 +319,8 @@ for name in ['build_log_test', 'test', 'util_test']: objs += cxx(name, variables=[('cflags', test_cflags)]) +if platform == 'windows': + objs += cxx('msvc_helper_test', variables=[('cflags', test_cflags)]) if platform != 'mingw' and platform != 'windows': test_libs.append('-lpthread') diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc new file mode 100644 index 0000000..ff18e80 --- /dev/null +++ b/src/msvc_helper-win32.cc @@ -0,0 +1,35 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "msvc_helper.h" + +#include + +#include "string_piece.h" + +// static +string CLWrapper::FilterShowIncludes(const string& line) { + static const char kMagicPrefix[] = "Note: including file: "; + const char* in = line.c_str(); + const char* end = in + line.size(); + + if (end - in > (int)sizeof(kMagicPrefix) - 1 && + memcmp(in, kMagicPrefix, sizeof(kMagicPrefix) - 1) == 0) { + in += sizeof(kMagicPrefix) - 1; + while (*in == ' ') + ++in; + return line.substr(in - line.c_str()); + } + return ""; +} diff --git a/src/msvc_helper.h b/src/msvc_helper.h new file mode 100644 index 0000000..e7c2b1a --- /dev/null +++ b/src/msvc_helper.h @@ -0,0 +1,29 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +using namespace std; + +struct StringPiece; + +/// Visual Studio's cl.exe requires some massaging to work with Ninja; +/// for example, it emits include information on stderr in a funny +/// format when building with /showIncludes. This class wraps a CL +/// process and parses that output to extract the file list. +struct CLWrapper { + /// Parse a line of cl.exe output and extract /showIncludes info. + /// If a dependency is extracted, returns a nonempty string. + static string FilterShowIncludes(const string& line); +}; diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc new file mode 100644 index 0000000..9eb3c89 --- /dev/null +++ b/src/msvc_helper_test.cc @@ -0,0 +1,32 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "msvc_helper.h" + +#include + +#include "test.h" +#include "util.h" + +TEST(MSVCHelperTest, ShowIncludes) { + ASSERT_EQ("", CLWrapper::FilterShowIncludes("")); + + ASSERT_EQ("", CLWrapper::FilterShowIncludes("Sample compiler output")); + ASSERT_EQ("c:\\Some Files\\foobar.h", + CLWrapper::FilterShowIncludes("Note: including file: " + "c:\\Some Files\\foobar.h")); + ASSERT_EQ("c:\\initspaces.h", + CLWrapper::FilterShowIncludes("Note: including file: " + "c:\\initspaces.h")); +} -- cgit v0.12 From 5024ada1786143f4946a574f4f0bbe3da7b93520 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 13:57:35 -0700 Subject: move Win32Fatal into util --- src/subprocess-win32.cc | 8 -------- src/util.cc | 4 ++++ src/util.h | 3 +++ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 38b6957..4b103a5 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -20,14 +20,6 @@ #include "util.h" -namespace { - -void Win32Fatal(const char* function) { - Fatal("%s: %s", function, GetLastErrorString().c_str()); -} - -} // anonymous namespace - Subprocess::Subprocess() : child_(NULL) , overlapped_(), is_reading_(false) { } diff --git a/src/util.cc b/src/util.cc index 90a1d45..14f6265 100644 --- a/src/util.cc +++ b/src/util.cc @@ -247,6 +247,10 @@ string GetLastErrorString() { LocalFree(msg_buf); return msg; } + +void Win32Fatal(const char* function) { + Fatal("%s: %s", function, GetLastErrorString().c_str()); +} #endif static bool islatinalpha(int c) { diff --git a/src/util.h b/src/util.h index 7e30368..3a658fb 100644 --- a/src/util.h +++ b/src/util.h @@ -82,6 +82,9 @@ string ElideMiddle(const string& str, size_t width); #ifdef _WIN32 /// Convert the value returned by GetLastError() into a string. string GetLastErrorString(); + +/// Calls Fatal() with a function name and GetLastErrorString. +void Win32Fatal(const char* function); #endif #endif // NINJA_UTIL_H_ -- cgit v0.12 From 031237133e33ee4eb5e9641396d6b0f566b575c8 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 14:07:38 -0700 Subject: doc some cl.exe flags in the configure script --- configure.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/configure.py b/configure.py index 331e027..1cdab49 100755 --- a/configure.py +++ b/configure.py @@ -108,8 +108,13 @@ else: n.variable('ar', configure_env.get('AR', 'ar')) if platform == 'windows': - cflags = ['/nologo', '/Zi', '/W4', '/WX', '/wd4530', '/wd4100', '/wd4706', - '/wd4512', '/wd4800', '/wd4702', '/wd4819', '/GR-', + cflags = ['/nologo', # Don't print startup banner. + '/Zi', # Create pdb with debug info. + '/W4', # Highest warning level. + '/WX', # Warnings as errors. + '/wd4530', '/wd4100', '/wd4706', + '/wd4512', '/wd4800', '/wd4702', '/wd4819', + '/GR-', # Disable RTTI. # Disable size_t -> int truncation warning. # We never have strings or arrays larger than 2**31. '/wd4267', -- cgit v0.12 From 1843f550d9b8b6d271cefdfb5fffd150bb8ef069 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 14:52:18 -0700 Subject: add subprocess-spawning to msvc_helper Rather than using subprocess.h, reimplement the subprocess code. This allows: 1) using anonymous (instead of named) pipes 2) not using all the completion port craziness 3) printing the output as it happens 4) further variation, like adjusting the environment (in a forthcoming change) without affecting the main subprocess code --- src/msvc_helper-win32.cc | 98 +++++++++++++++++++++++++++++++++++++++++++++++- src/msvc_helper.h | 9 +++++ src/msvc_helper_test.cc | 10 +++++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index ff18e80..ac957e4 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -15,8 +15,9 @@ #include "msvc_helper.h" #include +#include -#include "string_piece.h" +#include "util.h" // static string CLWrapper::FilterShowIncludes(const string& line) { @@ -33,3 +34,98 @@ string CLWrapper::FilterShowIncludes(const string& line) { } return ""; } + +int CLWrapper::Run(const string& command, string* extra_output) { + SECURITY_ATTRIBUTES security_attributes = {}; + security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); + security_attributes.bInheritHandle = TRUE; + + // Must be inheritable so subprocesses can dup to children. + HANDLE nul = CreateFile("NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | + FILE_SHARE_DELETE, + &security_attributes, OPEN_EXISTING, 0, NULL); + if (nul == INVALID_HANDLE_VALUE) + Fatal("couldn't open nul"); + + HANDLE stdout_read, stdout_write; + if (!CreatePipe(&stdout_read, &stdout_write, &security_attributes, 0)) + Win32Fatal("CreatePipe"); + + if (!SetHandleInformation(stdout_read, HANDLE_FLAG_INHERIT, 0)) + Win32Fatal("SetHandleInformation"); + + PROCESS_INFORMATION process_info = {}; + STARTUPINFO startup_info = {}; + startup_info.cb = sizeof(STARTUPINFO); + startup_info.hStdInput = nul; + startup_info.hStdError = stdout_write; + startup_info.hStdOutput = stdout_write; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + if (!CreateProcessA(NULL, (char*)command.c_str(), NULL, NULL, + /* inherit handles */ TRUE, 0, + NULL, NULL, + &startup_info, &process_info)) { + Win32Fatal("CreateProcess"); + } + + if (!CloseHandle(nul) || + !CloseHandle(stdout_write)) { + Win32Fatal("CloseHandle"); + } + + // Read output of the subprocess and parse it. + string output; + DWORD read_len = 1; + while (read_len) { + char buf[64 << 10]; + read_len = 0; + if (!::ReadFile(stdout_read, buf, sizeof(buf), &read_len, NULL) && + GetLastError() != ERROR_BROKEN_PIPE) { + Win32Fatal("ReadFile"); + } + output.append(buf, read_len); + + // Loop over all lines in the output and process them. + for (;;) { + size_t ofs = output.find_first_of("\r\n"); + if (ofs == string::npos) + break; + string line = output.substr(0, ofs); + + string include = FilterShowIncludes(line); + if (!include.empty()) { + includes_.push_back(include); + } else { + if (extra_output) { + extra_output->append(line); + extra_output->append("\n"); + } else { + printf("%s\n", line.c_str()); + } + } + + if (ofs < output.size() && output[ofs] == '\r') + ++ofs; + if (ofs < output.size() && output[ofs] == '\n') + ++ofs; + output = output.substr(ofs); + } + } + + if (WaitForSingleObject(process_info.hProcess, INFINITE) == WAIT_FAILED) + Win32Fatal("WaitForSingleObject"); + + DWORD exit_code = 0; + if (!GetExitCodeProcess(process_info.hProcess, &exit_code)) + Win32Fatal("GetExitCodeProcess"); + + if (!CloseHandle(stdout_read) || + !CloseHandle(process_info.hProcess) || + !CloseHandle(process_info.hThread)) { + Win32Fatal("CloseHandle"); + } + + return exit_code; +} diff --git a/src/msvc_helper.h b/src/msvc_helper.h index e7c2b1a..5a657be 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -23,7 +23,16 @@ struct StringPiece; /// format when building with /showIncludes. This class wraps a CL /// process and parses that output to extract the file list. struct CLWrapper { + /// Start a process and parse its output. Returns its exit code. + /// Any non-parsed output is buffered into \a extra_output if provided, + /// otherwise it is printed to stdout while the process runs. + /// Crashes (calls Fatal()) on error. + int Run(const string& command, string* extra_output=NULL); + /// Parse a line of cl.exe output and extract /showIncludes info. /// If a dependency is extracted, returns a nonempty string. + /// Exposed for testing. static string FilterShowIncludes(const string& line); + + vector includes_; }; diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index 9eb3c89..8db4c69 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -30,3 +30,13 @@ TEST(MSVCHelperTest, ShowIncludes) { CLWrapper::FilterShowIncludes("Note: including file: " "c:\\initspaces.h")); } + +TEST(MSVCHelperTest, Run) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo foo&& echo Note: including file: foo.h&&echo bar\"", + &output); + ASSERT_EQ("foo\nbar\n", output); + ASSERT_EQ(1u, cl.includes_.size()); + ASSERT_EQ("foo.h", cl.includes_[0]); +} -- cgit v0.12 From c963c4834d0ab05ce8ecf341c74db6ded379fa8a Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 15:09:33 -0700 Subject: msvc helper: attempt to filter out when it prints the input filename This is a heuristic but it appears to work for the Chrome build. --- src/msvc_helper-win32.cc | 23 +++++++++++++++++++++++ src/msvc_helper.h | 6 ++++++ src/msvc_helper_test.cc | 18 ++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index ac957e4..f6b2a22 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -19,6 +19,16 @@ #include "util.h" +namespace { + +/// Return true if \a input ends with \a needle. +bool EndsWith(const string& input, const string& needle) { + return (input.size() >= needle.size() && + input.substr(input.size() - needle.size()) == needle); +} + +} // anonymous namespace + // static string CLWrapper::FilterShowIncludes(const string& line) { static const char kMagicPrefix[] = "Note: including file: "; @@ -35,6 +45,15 @@ string CLWrapper::FilterShowIncludes(const string& line) { return ""; } +// static +bool CLWrapper::FilterInputFilename(const string& line) { + // TODO: other extensions, like .asm? + return EndsWith(line, ".c") || + EndsWith(line, ".cc") || + EndsWith(line, ".cxx") || + EndsWith(line, ".cpp"); +} + int CLWrapper::Run(const string& command, string* extra_output) { SECURITY_ATTRIBUTES security_attributes = {}; security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); @@ -97,6 +116,10 @@ int CLWrapper::Run(const string& command, string* extra_output) { string include = FilterShowIncludes(line); if (!include.empty()) { includes_.push_back(include); + } else if (FilterInputFilename(line)) { + // Drop it. + // TODO: if we support compiling multiple output files in a single + // cl.exe invocation, we should stash the filename. } else { if (extra_output) { extra_output->append(line); diff --git a/src/msvc_helper.h b/src/msvc_helper.h index 5a657be..5bf9787 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -34,5 +34,11 @@ struct CLWrapper { /// Exposed for testing. static string FilterShowIncludes(const string& line); + /// Parse a line of cl.exe output and return true if it looks like + /// it's printing an input filename. This is a heuristic but it appears + /// to be the best we can do. + /// Exposed for testing. + static bool FilterInputFilename(const string& line); + vector includes_; }; diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index 8db4c69..a0bab90 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -31,6 +31,16 @@ TEST(MSVCHelperTest, ShowIncludes) { "c:\\initspaces.h")); } +TEST(MSVCHelperTest, FilterInputFilename) { + ASSERT_TRUE(CLWrapper::FilterInputFilename("foobar.cc")); + ASSERT_TRUE(CLWrapper::FilterInputFilename("foo bar.cc")); + ASSERT_TRUE(CLWrapper::FilterInputFilename("baz.c")); + + ASSERT_FALSE(CLWrapper::FilterInputFilename( + "src\\cl_helper.cc(166) : fatal error C1075: end " + "of file found ...")); +} + TEST(MSVCHelperTest, Run) { CLWrapper cl; string output; @@ -40,3 +50,11 @@ TEST(MSVCHelperTest, Run) { ASSERT_EQ(1u, cl.includes_.size()); ASSERT_EQ("foo.h", cl.includes_[0]); } + +TEST(MSVCHelperTest, RunFilenameFilter) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo foo.cc&& echo cl: warning\"", + &output); + ASSERT_EQ("cl: warning\n", output); +} -- cgit v0.12 From c08766e86d10f0ef5417f6c6accbff37706b08c4 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 15:23:31 -0700 Subject: add .ilk (incremental linker) to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b7a7ab8..1872263 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pyc *.exe *.pdb +*.ilk TAGS /build /build.ninja -- cgit v0.12 From 0af67f1babe08c7e00ba194ccb47c4a0c60fa52a Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Sun, 12 Aug 2012 15:51:21 -0700 Subject: add functions for normalizing win32 include paths (Note from Evan: this is landing Scott's code more or less verbatim without a lot of analysis; it could maybe be simplified and reduced, but it's only intended to be used in the MSVC helper so it's fine to be experimental.) --- configure.py | 4 +- src/includes_normalize-win32.cc | 116 ++++++++++++++++++++++++++++++++++++++++ src/includes_normalize.h | 35 ++++++++++++ src/includes_normalize_test.cc | 98 +++++++++++++++++++++++++++++++++ 4 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 src/includes_normalize-win32.cc create mode 100644 src/includes_normalize.h create mode 100644 src/includes_normalize_test.cc diff --git a/configure.py b/configure.py index 1cdab49..ed05031 100755 --- a/configure.py +++ b/configure.py @@ -254,6 +254,7 @@ for name in ['build', if platform in ('mingw', 'windows'): objs += cxx('subprocess-win32') if platform == 'windows': + objs += cxx('includes_normalize-win32') objs += cxx('msvc_helper-win32') objs += cxx('minidump-win32') objs += cc('getopt') @@ -325,7 +326,8 @@ for name in ['build_log_test', 'util_test']: objs += cxx(name, variables=[('cflags', test_cflags)]) if platform == 'windows': - objs += cxx('msvc_helper_test', variables=[('cflags', test_cflags)]) + for name in ['includes_normalize_test', 'msvc_helper_test']: + objs += cxx(name, variables=[('cflags', test_cflags)]) if platform != 'mingw' and platform != 'windows': test_libs.append('-lpthread') diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc new file mode 100644 index 0000000..cf398ae --- /dev/null +++ b/src/includes_normalize-win32.cc @@ -0,0 +1,116 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "includes_normalize.h" + +#include "string_piece.h" +#include "util.h" + +#include +#include +#include + +#include + +namespace { + +/// Return true if paths a and b are on the same Windows drive. +bool SameDrive(StringPiece a, StringPiece b) { + char a_absolute[_MAX_PATH]; + char b_absolute[_MAX_PATH]; + GetFullPathName(a.AsString().c_str(), sizeof(a_absolute), a_absolute, NULL); + GetFullPathName(b.AsString().c_str(), sizeof(b_absolute), b_absolute, NULL); + char a_drive[_MAX_DIR]; + char b_drive[_MAX_DIR]; + _splitpath(a_absolute, a_drive, NULL, NULL, NULL); + _splitpath(b_absolute, b_drive, NULL, NULL, NULL); + return _stricmp(a_drive, b_drive) == 0; +} + +} // anonymous namespace + +string IncludesNormalize::Join(const vector& list, char sep) { + string ret; + for (size_t i = 0; i < list.size(); ++i) { + ret += list[i]; + if (i != list.size() - 1) + ret += sep; + } + return ret; +} + +vector IncludesNormalize::Split(const string& input, char sep) { + vector elems; + stringstream ss(input); + string item; + while (getline(ss, item, sep)) + elems.push_back(item); + return elems; +} + +string IncludesNormalize::ToLower(const string& s) { + string ret; + transform(s.begin(), s.end(), back_inserter(ret), tolower); + return ret; +} + +string IncludesNormalize::AbsPath(StringPiece s) { + char result[_MAX_PATH]; + GetFullPathName(s.AsString().c_str(), sizeof(result), result, NULL); + return result; +} + +string IncludesNormalize::Relativize(StringPiece path, const string& start) { + vector start_list = Split(AbsPath(start), '\\'); + vector path_list = Split(AbsPath(path), '\\'); + int i; + for (i = 0; i < static_cast(min(start_list.size(), path_list.size())); + ++i) { + if (ToLower(start_list[i]) != ToLower(path_list[i])) + break; + } + + vector rel_list; + for (int j = 0; j < static_cast(start_list.size() - i); ++j) + rel_list.push_back(".."); + for (int j = i; j < static_cast(path_list.size()); ++j) + rel_list.push_back(path_list[j]); + if (rel_list.size() == 0) + return "."; + return Join(rel_list, '\\'); +} + +string IncludesNormalize::Normalize(StringPiece input, + const char* relative_to) { + char copy[_MAX_PATH]; + size_t len = input.len_; + strncpy(copy, input.str_, input.len_ + 1); + for (size_t j = 0; j < len; ++j) + if (copy[j] == '/') + copy[j] = '\\'; + string err; + if (!CanonicalizePath(copy, &len, &err)) { + Warning("couldn't canonicalize '%*s': %s\n", input.len_, input.str_, + err.c_str()); + } + string curdir; + if (!relative_to) { + curdir = AbsPath("."); + relative_to = curdir.c_str(); + } + StringPiece partially_fixed(copy, len); + if (!SameDrive(partially_fixed, relative_to)) + return partially_fixed.AsString(); + return ToLower(Relativize(partially_fixed, relative_to)); +} diff --git a/src/includes_normalize.h b/src/includes_normalize.h new file mode 100644 index 0000000..458613a --- /dev/null +++ b/src/includes_normalize.h @@ -0,0 +1,35 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +using namespace std; + +struct StringPiece; + +/// Utility functions for normalizing include paths on Windows. +/// TODO: this likely duplicates functionality of CanonicalizePath; refactor. +struct IncludesNormalize { + // Internal utilities made available for testing, maybe useful otherwise. + static string Join(const vector& list, char sep); + static vector Split(const string& input, char sep); + static string ToLower(const string& s); + static string AbsPath(StringPiece s); + static string Relativize(StringPiece path, const string& start); + + /// Normalize by fixing slashes style, fixing redundant .. and . and makes the + /// path relative to |relative_to|. Case is normalized to lowercase on + /// Windows too. + static string Normalize(StringPiece input, const char* relative_to); +}; diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc new file mode 100644 index 0000000..97f7174 --- /dev/null +++ b/src/includes_normalize_test.cc @@ -0,0 +1,98 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "includes_normalize.h" + +#include + +#include "test.h" +#include "util.h" + +TEST(IncludesNormalize, Simple) { + EXPECT_EQ("b", IncludesNormalize::Normalize("a\\..\\b", NULL)); + EXPECT_EQ("b", IncludesNormalize::Normalize("a\\../b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\.\\b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\./b", NULL)); +} + +namespace { + +string GetCurDir() { + char buf[_MAX_PATH]; + _getcwd(buf, sizeof(buf)); + vector parts = IncludesNormalize::Split(string(buf), '\\'); + return parts[parts.size() - 1]; +} + +} // namespace + +TEST(IncludesNormalize, WithRelative) { + string currentdir = IncludesNormalize::ToLower(GetCurDir()); + EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b")); + EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), NULL)); + EXPECT_EQ(string("..\\") + currentdir + string("\\a"), + IncludesNormalize::Normalize("a", "../b")); + EXPECT_EQ(string("..\\") + currentdir + string("\\a\\b"), + IncludesNormalize::Normalize("a/b", "../c")); + EXPECT_EQ("..\\..\\a", IncludesNormalize::Normalize("a", "b/c")); + EXPECT_EQ(".", IncludesNormalize::Normalize("a", "a")); +} + +TEST(IncludesNormalize, Case) { + EXPECT_EQ("b", IncludesNormalize::Normalize("Abc\\..\\b", NULL)); + EXPECT_EQ("bdef", IncludesNormalize::Normalize("Abc\\..\\BdEf", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\.\\b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\./b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\.\\B", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\./B", NULL)); +} + +TEST(IncludesNormalize, Join) { + vector x; + EXPECT_EQ("", IncludesNormalize::Join(x, ':')); + x.push_back("alpha"); + EXPECT_EQ("alpha", IncludesNormalize::Join(x, ':')); + x.push_back("beta"); + x.push_back("gamma"); + EXPECT_EQ("alpha:beta:gamma", IncludesNormalize::Join(x, ':')); +} + +TEST(IncludesNormalize, Split) { + EXPECT_EQ("", IncludesNormalize::Join(IncludesNormalize::Split("", '/'), ':')); + EXPECT_EQ("a", IncludesNormalize::Join(IncludesNormalize::Split("a", '/'), ':')); + EXPECT_EQ("a:b:c", IncludesNormalize::Join(IncludesNormalize::Split("a/b/c", '/'), ':')); +} + +TEST(IncludesNormalize, ToLower) { + EXPECT_EQ("", IncludesNormalize::ToLower("")); + EXPECT_EQ("stuff", IncludesNormalize::ToLower("Stuff")); + EXPECT_EQ("stuff and things", IncludesNormalize::ToLower("Stuff AND thINGS")); + EXPECT_EQ("stuff 3and thin43gs", IncludesNormalize::ToLower("Stuff 3AND thIN43GS")); +} + +TEST(IncludesNormalize, DifferentDrive) { + EXPECT_EQ("stuff.h", + IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "p:\\vs08")); + EXPECT_EQ("stuff.h", + IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "p:\\vs08")); + EXPECT_EQ("P:\\vs08\\stuff.h", + IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "c:\\vs08")); + EXPECT_EQ("P:\\vs08\\stuff.h", + IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "D:\\stuff/things")); + EXPECT_EQ("P:\\vs08\\stuff.h", + IncludesNormalize::Normalize("P:/vs08\\stuff.h", "D:\\stuff/things")); + // TODO: this fails; fix it. + //EXPECT_EQ("P:\\wee\\stuff.h", + // IncludesNormalize::Normalize("P:/vs08\\../wee\\stuff.h", "D:\\stuff/things")); +} -- cgit v0.12 From de3f570cfe0bfdd564913a1431dc95fd17ff44eb Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 16:27:58 -0700 Subject: includes_normalize: also lowercase cross-drive includes It seems to me inconsistent to normalize one but not the other. --- src/includes_normalize-win32.cc | 11 +++++------ src/includes_normalize.h | 2 +- src/includes_normalize_test.cc | 6 +++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index cf398ae..134cfb8 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -91,18 +91,17 @@ string IncludesNormalize::Relativize(StringPiece path, const string& start) { return Join(rel_list, '\\'); } -string IncludesNormalize::Normalize(StringPiece input, +string IncludesNormalize::Normalize(const string& input, const char* relative_to) { char copy[_MAX_PATH]; - size_t len = input.len_; - strncpy(copy, input.str_, input.len_ + 1); + size_t len = input.size(); + strncpy(copy, input.c_str(), input.size() + 1); for (size_t j = 0; j < len; ++j) if (copy[j] == '/') copy[j] = '\\'; string err; if (!CanonicalizePath(copy, &len, &err)) { - Warning("couldn't canonicalize '%*s': %s\n", input.len_, input.str_, - err.c_str()); + Warning("couldn't canonicalize '%s: %s\n", input.c_str(), err.c_str()); } string curdir; if (!relative_to) { @@ -111,6 +110,6 @@ string IncludesNormalize::Normalize(StringPiece input, } StringPiece partially_fixed(copy, len); if (!SameDrive(partially_fixed, relative_to)) - return partially_fixed.AsString(); + return ToLower(partially_fixed.AsString()); return ToLower(Relativize(partially_fixed, relative_to)); } diff --git a/src/includes_normalize.h b/src/includes_normalize.h index 458613a..43527af 100644 --- a/src/includes_normalize.h +++ b/src/includes_normalize.h @@ -31,5 +31,5 @@ struct IncludesNormalize { /// Normalize by fixing slashes style, fixing redundant .. and . and makes the /// path relative to |relative_to|. Case is normalized to lowercase on /// Windows too. - static string Normalize(StringPiece input, const char* relative_to); + static string Normalize(const string& input, const char* relative_to); }; diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc index 97f7174..77b5b3b 100644 --- a/src/includes_normalize_test.cc +++ b/src/includes_normalize_test.cc @@ -86,11 +86,11 @@ TEST(IncludesNormalize, DifferentDrive) { IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "p:\\vs08")); EXPECT_EQ("stuff.h", IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "p:\\vs08")); - EXPECT_EQ("P:\\vs08\\stuff.h", + EXPECT_EQ("p:\\vs08\\stuff.h", IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "c:\\vs08")); - EXPECT_EQ("P:\\vs08\\stuff.h", + EXPECT_EQ("p:\\vs08\\stuff.h", IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "D:\\stuff/things")); - EXPECT_EQ("P:\\vs08\\stuff.h", + EXPECT_EQ("p:\\vs08\\stuff.h", IncludesNormalize::Normalize("P:/vs08\\stuff.h", "D:\\stuff/things")); // TODO: this fails; fix it. //EXPECT_EQ("P:\\wee\\stuff.h", -- cgit v0.12 From 449b62075c81431c58ab9fef0054d61513d9d656 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 16:29:43 -0700 Subject: msvc helper: drop system includes Drop any #includes that look like they're referencing system headers. This reduces the dependency information considerably. --- src/msvc_helper-win32.cc | 12 +++++++++++- src/msvc_helper.h | 7 +++++-- src/msvc_helper_test.cc | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index f6b2a22..624bc2d 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -17,6 +17,7 @@ #include #include +#include "includes_normalize.h" #include "util.h" namespace { @@ -46,6 +47,13 @@ string CLWrapper::FilterShowIncludes(const string& line) { } // static +bool CLWrapper::IsSystemInclude(const string& path) { + // TODO: this is a heuristic, perhaps there's a better way? + return (path.find("program files") != string::npos || + path.find("microsoft visual studio") != string::npos); +} + +// static bool CLWrapper::FilterInputFilename(const string& line) { // TODO: other extensions, like .asm? return EndsWith(line, ".c") || @@ -115,7 +123,9 @@ int CLWrapper::Run(const string& command, string* extra_output) { string include = FilterShowIncludes(line); if (!include.empty()) { - includes_.push_back(include); + include = IncludesNormalize::Normalize(include, NULL); + if (!IsSystemInclude(include)) + includes_.push_back(include); } else if (FilterInputFilename(line)) { // Drop it. // TODO: if we support compiling multiple output files in a single diff --git a/src/msvc_helper.h b/src/msvc_helper.h index 5bf9787..41b9f9b 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -16,8 +16,6 @@ #include using namespace std; -struct StringPiece; - /// Visual Studio's cl.exe requires some massaging to work with Ninja; /// for example, it emits include information on stderr in a funny /// format when building with /showIncludes. This class wraps a CL @@ -34,6 +32,11 @@ struct CLWrapper { /// Exposed for testing. static string FilterShowIncludes(const string& line); + /// Return true if a mentioned include file is a system path. + /// Expects the path to already by normalized (including lower case). + /// Filtering these out reduces dependency information considerably. + static bool IsSystemInclude(const string& path); + /// Parse a line of cl.exe output and return true if it looks like /// it's printing an input filename. This is a heuristic but it appears /// to be the best we can do. diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index a0bab90..b65d66f 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -58,3 +58,17 @@ TEST(MSVCHelperTest, RunFilenameFilter) { &output); ASSERT_EQ("cl: warning\n", output); } + +TEST(MSVCHelperTest, RunSystemInclude) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo Note: including file: c:\\Program Files\\foo.h&&" + "echo Note: including file: d:\\Microsoft Visual Studio\\bar.h&&" + "echo Note: including file: path.h\"", + &output); + // We should have dropped the first two includes because they look like + // system headers. + ASSERT_EQ("", output); + ASSERT_EQ(1u, cl.includes_.size()); + ASSERT_EQ("path.h", cl.includes_[0]); +} -- cgit v0.12 From 356f31abb92c87b3fa32700d3b44e49f1321cdb2 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 12 Aug 2012 18:43:42 -0700 Subject: create phony rules for all binaries --- configure.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/configure.py b/configure.py index ed05031..bdf4613 100755 --- a/configure.py +++ b/configure.py @@ -97,7 +97,9 @@ def cxx(name, **kwargs): return n.build(built(name + objext), 'cxx', src(name + '.cc'), **kwargs) def binary(name): if platform in ('mingw', 'windows'): - return name + '.exe' + exe = name + '.exe' + n.build(name, 'phony', exe) + return exe return name n.variable('builddir', 'build') @@ -277,8 +279,6 @@ n.comment('Main executable is library plus main() function.') objs = cxx('ninja') ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) -if 'ninja' not in ninja: - n.build('ninja', 'phony', ninja) n.newline() all_targets += ninja @@ -334,8 +334,6 @@ if platform != 'mingw' and platform != 'windows': ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib, variables=[('ldflags', test_ldflags), ('libs', test_libs)]) -if 'ninja_test' not in ninja_test: - n.build('ninja_test', 'phony', ninja_test) n.newline() all_targets += ninja_test -- cgit v0.12 From 77b078b01f0a0262f5f0abe61ca73cd981fcd506 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Mon, 13 Aug 2012 11:39:32 -0700 Subject: rename -V to --version and clean up --help output --- src/ninja.cc | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index b645b90..00bd104 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -77,14 +77,12 @@ void Usage(const BuildConfig& config) { "usage: ninja [options] [targets...]\n" "\n" "if targets are unspecified, builds the 'default' target (see manual).\n" -"targets are paths, with additional special syntax:\n" -" 'target^' means 'the first output that uses target'.\n" -" example: 'ninja foo.cc^' will likely build foo.o.\n" "\n" "options:\n" +" --version print ninja version (\"%s\")\n" +"\n" " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" -" -V print ninja version (\"%s\")\n" "\n" " -j N run N jobs in parallel [default=%d]\n" " -l N do not start new jobs if the load average is greater than N\n" @@ -92,13 +90,12 @@ void Usage(const BuildConfig& config) { " (not yet implemented on Windows)\n" #endif " -k N keep going until N jobs fail [default=1]\n" -" -n dry run (don't run commands but pretend they succeeded)\n" +" -n dry run (don't run commands but act like they succeeded)\n" " -v show all command lines while building\n" "\n" " -d MODE enable debugging (use -d list to list modes)\n" -" -t TOOL run a subtool\n" -" use '-t list' to list subtools.\n" -" terminates toplevel options; further flags are passed to the tool.\n", +" -t TOOL run a subtool (use -t list to list subtools)\n" +" terminates toplevel options; further flags are passed to the tool\n", kVersion, config.parallelism); } @@ -649,8 +646,10 @@ int NinjaMain(int argc, char** argv) { globals.config.parallelism = GuessParallelism(); + enum { OPT_VERSION = 1 }; const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, OPT_VERSION }, { NULL, 0, NULL, 0 } }; @@ -701,7 +700,7 @@ int NinjaMain(int argc, char** argv) { case 'C': working_dir = optarg; break; - case 'V': + case OPT_VERSION: printf("%s\n", kVersion); return 0; case 'h': -- cgit v0.12 From 59e0d69ec2775f1aa46d87ad7d14e6985e5187b6 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 14 Aug 2012 20:54:59 -0700 Subject: pass env block to cl helper --- src/msvc_helper-win32.cc | 2 +- src/msvc_helper.h | 7 +++++++ src/msvc_helper_test.cc | 9 +++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index 624bc2d..8e440fe 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -92,7 +92,7 @@ int CLWrapper::Run(const string& command, string* extra_output) { if (!CreateProcessA(NULL, (char*)command.c_str(), NULL, NULL, /* inherit handles */ TRUE, 0, - NULL, NULL, + env_block_, NULL, &startup_info, &process_info)) { Win32Fatal("CreateProcess"); } diff --git a/src/msvc_helper.h b/src/msvc_helper.h index 41b9f9b..f623520 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -21,6 +21,12 @@ using namespace std; /// format when building with /showIncludes. This class wraps a CL /// process and parses that output to extract the file list. struct CLWrapper { + CLWrapper() : env_block_(NULL) {} + + /// Set the environment block (as suitable for CreateProcess) to be used + /// by Run(). + void SetEnvBlock(void* env_block) { env_block_ = env_block; } + /// Start a process and parse its output. Returns its exit code. /// Any non-parsed output is buffered into \a extra_output if provided, /// otherwise it is printed to stdout while the process runs. @@ -43,5 +49,6 @@ struct CLWrapper { /// Exposed for testing. static bool FilterInputFilename(const string& line); + void* env_block_; vector includes_; }; diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index b65d66f..29fefd4 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -72,3 +72,12 @@ TEST(MSVCHelperTest, RunSystemInclude) { ASSERT_EQ(1u, cl.includes_.size()); ASSERT_EQ("path.h", cl.includes_[0]); } + +TEST(MSVCHelperTest, EnvBlock) { + char env_block[] = "foo=bar\0"; + CLWrapper cl; + cl.SetEnvBlock(env_block); + string output; + cl.Run("cmd /c \"echo foo is %foo%", &output); + ASSERT_EQ("foo is bar\n", output); +} -- cgit v0.12 From ac04abe2f9c87afe4e4d43ac63e5af2dd10376fb Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 14 Aug 2012 20:59:21 -0700 Subject: add a helper binary for wrapping cl.exe Modify bootstrap etc. to make use of this binary. --- bootstrap.py | 42 ++++++++++++--- configure.py | 18 ++++++- src/msvc_helper_main-win32.cc | 115 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 src/msvc_helper_main-win32.cc diff --git a/bootstrap.py b/bootstrap.py index 8c0687e..abd2528 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -68,6 +68,8 @@ for src in glob.glob('src/*.cc'): else: if src.endswith('-win32.cc'): continue + if '_main' in src: + continue sources.append(src) @@ -110,13 +112,41 @@ verbose = [] if options.verbose: verbose = ['-v'] -print 'Building ninja using itself...' -run([sys.executable, 'configure.py'] + conf_args) -run(['./' + binary] + verbose) -os.unlink(binary) - if sys.platform.startswith('win32'): + # Build ninja-msvc-helper using ninja without an msvc-helper. + print 'Building ninja-msvc-helper...' + run([sys.executable, 'configure.py', '--with-msvc-helper='] + conf_args) + run(['./' + binary] + verbose + ['ninja-msvc-helper']) + + # Rename the helper to the same name + .bootstrap. + helper_binary = 'ninja-msvc-helper.bootstrap.exe' + try: + os.unlink(helper_binary) + except: + pass + os.rename('ninja-msvc-helper.exe', helper_binary) + + # Build ninja using the newly-built msvc-helper. + print 'Building ninja using itself...' + run([sys.executable, 'configure.py', + '--with-msvc-helper=%s' % helper_binary] + conf_args) + run(['./' + binary] + verbose) + + # Clean up. for obj in glob.glob('*.obj'): os.unlink(obj) -print 'Done!' + print """ +Done! + +Note: to work around Windows file locking, where you can't rebuild an +in-use binary, to run ninja after making any changes to build ninja itself +you should run ninja.bootstrap instead. Your build is also configured to +use ninja-msvc-helper.bootstrap.exe instead of the ninja-msvc-helper.exe +that it builds; see the --help output of configure.py.""" +else: + print 'Building ninja using itself...' + run([sys.executable, 'configure.py'] + conf_args) + run(['./' + binary] + verbose) + os.unlink(binary) + print 'Done!' diff --git a/configure.py b/configure.py index bdf4613..981d401 100755 --- a/configure.py +++ b/configure.py @@ -45,6 +45,8 @@ parser.add_option('--with-gtest', metavar='PATH', parser.add_option('--with-python', metavar='EXE', help='use EXE as the Python interpreter', default=os.path.basename(sys.executable)) +parser.add_option('--with-msvc-helper', metavar='NAME', + help="name for ninja-msvc-helper binary (MSVC only)") (options, args) = parser.parse_args() if args: print 'ERROR: extra unparsed command-line arguments:', args @@ -177,8 +179,11 @@ n.variable('ldflags', ' '.join(shell_escape(flag) for flag in ldflags)) n.newline() if platform == 'windows': + compiler = '$cxx' + if options.with_msvc_helper: + compiler = '%s -o $out -- $cxx /showIncludes' % options.with_msvc_helper n.rule('cxx', - command='$cxx $cflags -c $in /Fo$out', + command='%s $cflags -c $in /Fo$out' % compiler, depfile='$out.d', description='CXX $out') else: @@ -282,6 +287,16 @@ ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib, n.newline() all_targets += ninja +if platform == 'windows': + n.comment('Helper for working with MSVC.') + msvc_helper = n.build(binary('ninja-msvc-helper'), 'link', + cxx('msvc_helper_main-win32'), + implicit=ninja_lib, + variables=[('libs', libs)]) + n.default(msvc_helper) + n.newline() + all_targets += msvc_helper + n.comment('Tests all build into ninja_test executable.') variables = [] @@ -397,7 +412,6 @@ if host != 'mingw': implicit=['configure.py', os.path.normpath('misc/ninja_syntax.py')]) n.newline() -n.comment('Build only the main binary by default.') n.default(ninja) n.newline() diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc new file mode 100644 index 0000000..f265010 --- /dev/null +++ b/src/msvc_helper_main-win32.cc @@ -0,0 +1,115 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "msvc_helper.h" + +#include + +#include "util.h" + +#include "getopt.h" + +namespace { + +void Usage() { + printf( +"ninja-msvc-helper: adjust msvc command-line tools for use by ninja.\n" +"\n" +"usage: ninja-mvsc-helper [options] -- command args\n" +"options:\n" +" -e ENVFILE load environment block from ENVFILE as environment\n" +" -r BASE normalize paths and make relative to BASE before output\n" +" -o FILE write output dependency information to FILE.d\n" + ); +} + +void PushPathIntoEnvironment(const string& env_block) { + const char* as_str = env_block.c_str(); + while (as_str[0]) { + if (_strnicmp(as_str, "path=", 5) == 0) { + _putenv(as_str); + return; + } else { + as_str = &as_str[strlen(as_str) + 1]; + } + } +} + +} // anonymous namespace + +int main(int argc, char** argv) { + const char* output_filename = NULL; + const char* relative_to = NULL; + const char* envfile = NULL; + + const option kLongOptions[] = { + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + int opt; + while ((opt = getopt_long(argc, argv, "e:o:r:h", kLongOptions, NULL)) != -1) { + switch (opt) { + case 'e': + envfile = optarg; + break; + case 'o': + output_filename = optarg; + break; + case 'r': + relative_to = optarg; + break; + case 'h': + default: + Usage(); + return 0; + } + } + + if (!output_filename) + Fatal("-o required"); + + string env; + if (envfile) { + string err; + if (ReadFile(envfile, &env, &err) != 0) + Fatal("couldn't open %s: %s", envfile, err.c_str()); + PushPathIntoEnvironment(env); + } + + char* command = GetCommandLine(); + command = strstr(command, " -- "); + if (!command) { + Fatal("expected command line to end with \" -- command args\""); + } + command += 4; + + CLWrapper cl; + if (!env.empty()) + cl.SetEnvBlock((void*)env.data()); + int exit_code = cl.Run(command); + + string depfile = string(output_filename) + ".d"; + FILE* output = fopen(depfile.c_str(), "w"); + if (!output) { + Fatal("opening %s: %s", depfile.c_str(), GetLastErrorString().c_str()); + } + fprintf(output, "%s: ", output_filename); + for (vector::iterator i = cl.includes_.begin(); + i != cl.includes_.end(); ++i) { + fprintf(output, "%s\n", i->c_str()); + } + fclose(output); + + return exit_code; +} -- cgit v0.12 From b56fe8082bfd59205efb55f6b1e7862045e005f2 Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Tue, 14 Aug 2012 21:09:19 -0700 Subject: if a file is missing in the log, count it as dirty This could cause overbuilding (if the log is missing an entry and the right file is already in place) but is otherwise necessary for correctness (if a file is already in place but we don't have a log entry for it). --- src/build_test.cc | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- src/graph.cc | 13 +++++++++---- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/build_test.cc b/src/build_test.cc index 574ffb4..d4673ae 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -723,6 +723,31 @@ struct BuildWithLogTest : public BuildTest { BuildLog build_log_; }; +TEST_F(BuildWithLogTest, NotInLogButOnDisk) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc\n" +" command = cc\n" +"build out1: cc in\n")); + + // Create input/output that would be considered up to date when + // not considering the command line hash. + fs_.Create("in", now_, ""); + fs_.Create("out1", now_, ""); + string err; + + // Because it's not in the log, it should not be up-to-date until + // we build again. + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + commands_ran_.clear(); + state_.Reset(); + + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); +} + TEST_F(BuildWithLogTest, RestatTest) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule true\n" @@ -743,9 +768,22 @@ TEST_F(BuildWithLogTest, RestatTest) { fs_.Create("in", now_, ""); + // Do a pre-build so that there's commands in the log for the outputs, + // otherwise, the lack of an entry in the build log will cause out3 to rebuild + // regardless of restat. + string err; + EXPECT_TRUE(builder_.AddTarget("out3", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + commands_ran_.clear(); + state_.Reset(); + + now_++; + + fs_.Create("in", now_, ""); // "cc" touches out1, so we should build out2. But because "true" does not // touch out2, we should cancel the build of out3. - string err; EXPECT_TRUE(builder_.AddTarget("out3", &err)); ASSERT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); @@ -790,10 +828,24 @@ TEST_F(BuildWithLogTest, RestatMissingFile) { fs_.Create("in", now_, ""); fs_.Create("out2", now_, ""); + // Do a pre-build so that there's commands in the log for the outputs, + // otherwise, the lack of an entry in the build log will cause out2 to rebuild + // regardless of restat. + string err; + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + commands_ran_.clear(); + state_.Reset(); + + now_++; + fs_.Create("in", now_, ""); + fs_.Create("out2", now_, ""); + // Run a build, expect only the first command to run. // It doesn't touch its output (due to being the "true" command), so // we shouldn't run the dependent build. - string err; EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); diff --git a/src/graph.cc b/src/graph.cc index 18adeee..9654c1a 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -158,10 +158,15 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, // May also be dirty due to the command changing since the last build. // But if this is a generator rule, the command changing does not make us // dirty. - if (!rule_->generator() && build_log && - (entry || (entry = build_log->LookupByOutput(output->path())))) { - if (BuildLog::LogEntry::HashCommand(command) != entry->command_hash) { - EXPLAIN("command line changed for %s", output->path().c_str()); + if (!rule_->generator() && build_log) { + if (entry || (entry = build_log->LookupByOutput(output->path()))) { + if (BuildLog::LogEntry::HashCommand(command) != entry->command_hash) { + EXPLAIN("command line changed for %s", output->path().c_str()); + return true; + } + } + if (!entry) { + EXPLAIN("command line not found in log for %s", output->path().c_str()); return true; } } -- cgit v0.12 From c3928c1c0df0fab93d7b47392ba593d9e25ade49 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Wed, 15 Aug 2012 14:46:57 -0700 Subject: convert HACKING to markdown, add MSVC section --- HACKING | 76 ----------------------------------------- HACKING.md | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 76 deletions(-) delete mode 100644 HACKING create mode 100644 HACKING.md diff --git a/HACKING b/HACKING deleted file mode 100644 index 4e01d49..0000000 --- a/HACKING +++ /dev/null @@ -1,76 +0,0 @@ -Adjusting build flags: - CFLAGS=-O3 ./configure.py - and rebuild. - -Building tests requires gtest, to get it: - - On older Ubuntus you can apt-get install libgtest. - - On newer Ubuntus it's only distributed as source; - 1) apt-get install libgtest-dev - 2) ./configure --with-gtest=/usr/src/gtest - - Otherwise you need to download it, unpack it, and pass --with-gtest - as appropriate. - -Test-driven development: - Set your build command to - ./ninja ninja_test && ./ninja_test --gtest_filter=MyTest.Name - now you can repeatedly run that while developing until the tests pass. - Remember to build "all" before committing to verify the other source - still works! - -Testing performance impact of changes: - If you have a Chrome build handy, it's a good test case. - Otherwise, https://github.com/martine/ninja/downloads has a copy of - the Chrome build files (and depfiles). You can untar that, then run - path/to/my/ninja chrome - and compare that against a baseline Ninja. - - There's a script at misc/measure.py that repeatedly runs a command like - the above (to address variance) and summarizes its runtime. E.g. - path/to/misc/measure.py path/to/my/ninja chrome - - For changing the depfile parser, you can also build 'parser_perftest' - and run that directly on some representative input files. - -Coding guidelines: -- Function name are camelcase. -- Member methods are camelcase, expect for trivial getters which are - underscore separated. -- Local variables are underscore separated. -- Member variables are underscore separated and suffixed by an extra underscore. -- Two spaces indentation. -- Opening braces is at the end of line. -- Lines are 80 columns maximum. -- All source files should have the Google Inc. license header. -- Also follow this style: - http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml - -Documentation guidelines: -- Use /// for doxygen. -- Use \a to refer to arguments. -- It's not necessary to document each argument, especially when they're - relatively self-evident (e.g. in CanonicalizePath(string* path, string* err), - the arguments are hopefully obvious) - -Generating the manual: - sudo apt-get install asciidoc --no-install-recommends - ./ninja manual - -Windows development on Linux (this is kind of hacky right now): -- Get the gtest source, unpack it into your source dir -- sudo apt-get install gcc-mingw32 wine -- export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar -- ./configure.py --platform=mingw --host=linux --with-gtest=gtest-1.6.0 -- Build ninja: /path/to/linux/ninja -- Run: ./ninja.exe (implicitly runs through wine(!)) - -Windows development on Windows: -- install mingw, msys, and python -- in the mingw shell, put Python in your path, and: python bootstrap.py -- to reconfigure, run 'python configure.py' -- remember to strip the resulting executable if size matters to you -- you'll need to rename ninja.exe into my-ninja.exe during development, - otherwise ninja won't be able to overwrite itself when building - -Using clang: -- Enable colors manually: - CXX='/path/to/llvm/Release+Asserts/bin/clang++ -fcolor-diagnostics' ./configure.py diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..aa965ca --- /dev/null +++ b/HACKING.md @@ -0,0 +1,113 @@ + +### Adjusting build flags + + CFLAGS=-O3 ./configure.py + +### Testing + +#### Installing gtest + +* On older Ubuntus it'll install as libraries into `/usr/lib`: + + apt-get install libgtest + +* On newer Ubuntus it's only distributed as source + + apt-get install libgtest-dev + ./configure --with-gtest=/usr/src/gtest + +* Otherwise you need to download it, unpack it, and pass --with-gtest + as appropriate. + +#### Test-driven development + +Set your build command to + + ./ninja ninja_test && ./ninja_test --gtest_filter=MyTest.Name + +now you can repeatedly run that while developing until the tests pass. +Remember to build "all" before committing to verify the other source +still works! + +### Testing performance impact of changes + +If you have a Chrome build handy, it's a good test case. +Otherwise, https://github.com/martine/ninja/downloads has a copy of +the Chrome build files (and depfiles). You can untar that, then run + + path/to/my/ninja chrome + +and compare that against a baseline Ninja. + +There's a script at `misc/measure.py` that repeatedly runs a command like +the above (to address variance) and summarizes its runtime. E.g. + + path/to/misc/measure.py path/to/my/ninja chrome + +For changing the depfile parser, you can also build `parser_perftest` +and run that directly on some representative input files. + +## Coding guidelines + +Generally it's the [Google C++ coding style][], but in brief: + +* Function name are camelcase. +* Member methods are camelcase, expect for trivial getters which are + underscore separated. +* Local variables are underscore separated. +* Member variables are underscore separated and suffixed by an extra underscore. +* Two spaces indentation. +* Opening braces is at the end of line. +* Lines are 80 columns maximum. +* All source files should have the Google Inc. license header. + +[Google C++ coding style]: http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml + +## Documentation + +### Style guidelines + +* Use `///` for doxygen. +* Use `\a` to refer to arguments. +* It's not necessary to document each argument, especially when they're + relatively self-evident (e.g. in `CanonicalizePath(string* path, string* err)`, + the arguments are hopefully obvious) + +### Building the manual + + sudo apt-get install asciidoc --no-install-recommends + ./ninja manual + +## Building on Windows + +While developing, it's helpful to copy `ninja.exe` to another name like +`n.exe`; otherwise, rebuilds will be unable to write `ninja.exe` because +it's locked while in use. + +### Via Visual Studio + +* Install Visual Studio (Express is fine), [Python for Windows][], + and (if making changes) googletest (see above instructions) +* In a Visual Studio command prompt: `python bootstrap.py` + +[Python for Windows]: http://www.python.org/getit/windows/ + +### Via mingw on Linux (not well supported) + +* `sudo apt-get install gcc-mingw32 wine` +* `export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar` +* `./configure.py --platform=mingw --host=linux` +* Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja` +* Run: `./ninja.exe` (implicitly runs through wine(!)) + +### Via mingw on Windows (not well supported) +* Install mingw, msys, and python +* In the mingw shell, put Python in your path, and: python bootstrap.py +* To reconfigure, run `python configure.py` +* Remember to strip the resulting executable if size matters to you + +## Clang + +Enable colors manually via `-fcolor-diagnostics`: + + CXX='/path/to/llvm/Release+Asserts/bin/clang++ -fcolor-diagnostics' ./configure.py -- cgit v0.12 From 8dcf8cd9f517360ff60be52ca938b1b786e21982 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Wed, 15 Aug 2012 17:34:20 -0700 Subject: Remove -fcolor-diagnostics reference from HACKING configure.py adds that flag automatically if CXX is set to clang. --- HACKING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HACKING.md b/HACKING.md index aa965ca..8a477f3 100644 --- a/HACKING.md +++ b/HACKING.md @@ -108,6 +108,6 @@ it's locked while in use. ## Clang -Enable colors manually via `-fcolor-diagnostics`: +To use clang, set `CXX`: - CXX='/path/to/llvm/Release+Asserts/bin/clang++ -fcolor-diagnostics' ./configure.py + CXX=clang++ ./configure.py -- cgit v0.12 From 6c7fae2af8e5c2d34bdbed5d1c3c5739bb548ee8 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Wed, 15 Aug 2012 20:11:27 -0700 Subject: Only write re2c rules if a re2c binary is found in the PATH. --- configure.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/configure.py b/configure.py index 981d401..287f58a 100755 --- a/configure.py +++ b/configure.py @@ -233,12 +233,22 @@ if platform not in ('mingw', 'windows'): n.newline() n.comment('the depfile parser and ninja lexers are generated using re2c.') -n.rule('re2c', - command='re2c -b -i --no-generation-date -o $out $in', - description='RE2C $out') -# Generate the .cc files in the source directory so we can check them in. -n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) -n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) +def has_re2c(): + import subprocess + try: + subprocess.call(['re2c', '-v'], stdout=subprocess.PIPE) + return True + except OSError: + return False +if has_re2c(): + n.rule('re2c', + command='re2c -b -i --no-generation-date -o $out $in', + description='RE2C $out') + # Generate the .cc files in the source directory so we can check them in. + n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) + n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) +else: + n.comment('(re2c not found, using checked-in cc files instead.)') n.newline() n.comment('Core source files all build into ninja library.') -- cgit v0.12 From ee063d100b5d93a0ae0127a7b3f86e1d5a37a660 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Thu, 16 Aug 2012 11:25:55 -0700 Subject: many updates for HACKING.md In particular, describe a policy for good patches. --- HACKING.md | 104 +++++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 78 insertions(+), 26 deletions(-) diff --git a/HACKING.md b/HACKING.md index 8a477f3..bc23773 100644 --- a/HACKING.md +++ b/HACKING.md @@ -1,11 +1,59 @@ +## Basic overview + +`./configure.py` generates the `build.ninja` files used to build +ninja. It accepts various flags to adjust build parameters. + +The primary build target of interest is `ninja`, but when hacking on +Ninja your changes should be testable so it's more useful to build +and run `ninja_test` when developing. + +(`./bootstrap.py` creates a bootstrap `ninja` and runs the above +process; it's only necessary to run if you don't have a copy of +`ninja` to build with.) ### Adjusting build flags - CFLAGS=-O3 ./configure.py +Build in "debug" mode while developing (disables optimizations and builds +way faster on Windows): + + ./configure.py --debug + +To use clang, set `CXX`: + + CXX=clang++ ./configure.py + +## How to successfully make changes to Ninja + +Github pull requests are convenient for me to merge (I can just click +a button and it's all handled server-side), but I'm also comfortable +accepting pre-github git patches (via `send-email` etc.). + +Good pull requests have all of these attributes: + +* Are scoped to one specific issue +* Include a test to demonstrate their correctness +* Update the docs where relevant +* Match the Ninja coding style (see below) +* Don't include a mess of "oops, fix typo" commits -### Testing +These are typically merged without hesitation. If a change is lacking +any of the above I usually will ask you to fix it, though there are +obvious exceptions (fixing typos in comments don't need tests). -#### Installing gtest +I am very wary of changes that increase the complexity of Ninja (in +particular, new build file syntax or command-line flags) or increase +the maintenance burden of Ninja. Ninja is already successfully in use +by hundreds of developers for large projects and it already achieves +(most of) the goals I set out for it to do. It's probably best to +discuss new feature ideas on the mailing list before I shoot down your +patch. + +## Testing + +### Installing gtest + +The `ninja_test` binary, containing all the tests, depends on the +googletest (gtest) library. * On older Ubuntus it'll install as libraries into `/usr/lib`: @@ -16,24 +64,26 @@ apt-get install libgtest-dev ./configure --with-gtest=/usr/src/gtest -* Otherwise you need to download it, unpack it, and pass --with-gtest - as appropriate. +* Otherwise you need to download it, unpack it, and pass + `--with-gtest` to `configure.py`. Get it from [its downloads + page](http://code.google.com/p/googletest/downloads/list). -#### Test-driven development +### Test-driven development Set your build command to ./ninja ninja_test && ./ninja_test --gtest_filter=MyTest.Name -now you can repeatedly run that while developing until the tests pass. -Remember to build "all" before committing to verify the other source -still works! +now you can repeatedly run that while developing until the tests pass +(I frequently set it as my compilation command in Emacs). Remember to +build "all" before committing to verify the other source still works! -### Testing performance impact of changes +## Testing performance impact of changes -If you have a Chrome build handy, it's a good test case. -Otherwise, https://github.com/martine/ninja/downloads has a copy of -the Chrome build files (and depfiles). You can untar that, then run +If you have a Chrome build handy, it's a good test case. Otherwise, +[the github downoads page](https://github.com/martine/ninja/downloads) +has a copy of the Chrome build files (and depfiles). You can untar +that, then run path/to/my/ninja chrome @@ -55,7 +105,8 @@ Generally it's the [Google C++ coding style][], but in brief: * Member methods are camelcase, expect for trivial getters which are underscore separated. * Local variables are underscore separated. -* Member variables are underscore separated and suffixed by an extra underscore. +* Member variables are underscore separated and suffixed by an extra + underscore. * Two spaces indentation. * Opening braces is at the end of line. * Lines are 80 columns maximum. @@ -78,7 +129,12 @@ Generally it's the [Google C++ coding style][], but in brief: sudo apt-get install asciidoc --no-install-recommends ./ninja manual -## Building on Windows +### Building the code documentation + + sudo apt-get install doxygen + ./ninja doxygen + +## Building for Windows While developing, it's helpful to copy `ninja.exe` to another name like `n.exe`; otherwise, rebuilds will be unable to write `ninja.exe` because @@ -92,6 +148,13 @@ it's locked while in use. [Python for Windows]: http://www.python.org/getit/windows/ +### Via mingw on Windows (not well supported) + +* Install mingw, msys, and python +* In the mingw shell, put Python in your path, and `python bootstrap.py` +* To reconfigure, run `python configure.py` +* Remember to strip the resulting executable if size matters to you + ### Via mingw on Linux (not well supported) * `sudo apt-get install gcc-mingw32 wine` @@ -100,14 +163,3 @@ it's locked while in use. * Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja` * Run: `./ninja.exe` (implicitly runs through wine(!)) -### Via mingw on Windows (not well supported) -* Install mingw, msys, and python -* In the mingw shell, put Python in your path, and: python bootstrap.py -* To reconfigure, run `python configure.py` -* Remember to strip the resulting executable if size matters to you - -## Clang - -To use clang, set `CXX`: - - CXX=clang++ ./configure.py -- cgit v0.12 From 0b531af962d01009621fb98ce533fdaf86c48d3a Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Thu, 16 Aug 2012 11:28:03 -0700 Subject: use 4 space tabs in configure.py, warn on re2c missing --- configure.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/configure.py b/configure.py index 287f58a..85661b7 100755 --- a/configure.py +++ b/configure.py @@ -234,21 +234,22 @@ if platform not in ('mingw', 'windows'): n.comment('the depfile parser and ninja lexers are generated using re2c.') def has_re2c(): - import subprocess - try: - subprocess.call(['re2c', '-v'], stdout=subprocess.PIPE) - return True - except OSError: - return False + import subprocess + try: + subprocess.call(['re2c', '-v'], stdout=subprocess.PIPE) + return True + except OSError: + return False if has_re2c(): - n.rule('re2c', - command='re2c -b -i --no-generation-date -o $out $in', - description='RE2C $out') - # Generate the .cc files in the source directory so we can check them in. - n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) - n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) + n.rule('re2c', + command='re2c -b -i --no-generation-date -o $out $in', + description='RE2C $out') + # Generate the .cc files in the source directory so we can check them in. + n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) + n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) else: - n.comment('(re2c not found, using checked-in cc files instead.)') + print ("warning: re2c not found; changes to src/*.in.cc will not affect " + "your build.") n.newline() n.comment('Core source files all build into ninja library.') -- cgit v0.12 From 5fe8c1308cf1785b1903eb11cf32909df3be113b Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Thu, 16 Aug 2012 11:31:37 -0700 Subject: drop HACKING from doxygen I tried just fixing the code to pull in HACKING.md but it didn't show up in the doxygen output; it's maybe too long to include anyway. --- configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.py b/configure.py index 85661b7..e4b9eb4 100755 --- a/configure.py +++ b/configure.py @@ -407,7 +407,7 @@ n.rule('doxygen_mainpage', command='$doxygen_mainpage_generator $in > $out', description='DOXYGEN_MAINPAGE $out') mainpage = n.build(built('doxygen_mainpage'), 'doxygen_mainpage', - ['README', 'HACKING', 'COPYING'], + ['README', 'COPYING'], implicit=['$doxygen_mainpage_generator']) n.build('doxygen', 'doxygen', doc('doxygen.config'), implicit=mainpage) -- cgit v0.12 From ee44a510f448d130948380bd26926d3b17f5f11d Mon Sep 17 00:00:00 2001 From: Peter Kuemmel Date: Thu, 16 Aug 2012 15:26:33 +0200 Subject: on windows a network path starts with two backslashes --- src/util.cc | 11 +++++++++++ src/util_test.cc | 16 ++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/util.cc b/src/util.cc index 7c2f895..4bd34e6 100644 --- a/src/util.cc +++ b/src/util.cc @@ -110,8 +110,19 @@ bool CanonicalizePath(char* path, int* len, string* err) { const char* end = start + *len; if (*src == '/') { +#ifdef _WIN32 + // network path starts with // + if (*len > 1 && *(src + 1) == '/') { + src += 2; + dst += 2; + } else { + ++src; + ++dst; + } +#else ++src; ++dst; +#endif } while (src < end) { diff --git a/src/util_test.cc b/src/util_test.cc index 5ace5e7..17e2704 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -66,6 +66,22 @@ TEST(CanonicalizePath, PathSamples) { path = "foo/.hidden_bar"; EXPECT_TRUE(CanonicalizePath(&path, &err)); EXPECT_EQ("foo/.hidden_bar", path); + + path = "/foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("/foo", path); + + path = "//foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); +#ifdef _WIN32 + EXPECT_EQ("//foo", path); +#else + EXPECT_EQ("/foo", path); +#endif + + path = "/"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("", path); } TEST(CanonicalizePath, EmptyResult) { -- cgit v0.12 From 697350d1220f50c32043875b49c71c8a294098a4 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 17 Aug 2012 10:25:21 -0700 Subject: make it more explicit that a bad build log causes us to rebuild (Committing this on top of b56fe80 since they're related, but I may end up reverting both.) --- src/build_log.cc | 9 ++++++--- src/build_log_test.cc | 2 +- src/ninja.cc | 5 +++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/build_log.cc b/src/build_log.cc index e72a93e..73c2b5d 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -235,10 +235,13 @@ bool BuildLog::Load(const string& path, string* err) { sscanf(line_start, kFileSignature, &log_version); if (log_version < kOldestSupportedVersion) { - *err = "unable to extract version from build log, perhaps due to " - "being too old; you must clobber your build output and rebuild"; + *err = ("build log version invalid, perhaps due to being too old; " + "starting over"); fclose(file); - return false; + unlink(path.c_str()); + // Don't report this as a failure. An empty build log will cause + // us to rebuild the outputs anyway. + return true; } } diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 186dad9..3005333 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -154,7 +154,7 @@ TEST_F(BuildLogTest, ObsoleteOldVersion) { string err; BuildLog log; - EXPECT_FALSE(log.Load(kTestFilename, &err)); + EXPECT_TRUE(log.Load(kTestFilename, &err)); ASSERT_NE(err.find("version"), string::npos); } diff --git a/src/ninja.cc b/src/ninja.cc index 00bd104..19438e5 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -759,6 +759,11 @@ reload: Error("loading build log %s: %s", log_path.c_str(), err.c_str()); return 1; } + if (!err.empty()) { + // Hack: Load() can return a warning via err by returning true. + Warning("%s", err.c_str()); + err.clear(); + } if (!build_log.OpenForWrite(log_path, &err)) { Error("opening build log: %s", err.c_str()); -- cgit v0.12 From 9726ee77ede7713b64ea387487a989937d59848f Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 17 Aug 2012 10:41:39 -0700 Subject: fix test broken in 697350d That's what I get for making last-second adjustments before checking in! --- src/build_log_test.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 3005333..c465abc 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -131,6 +131,14 @@ TEST_F(BuildLogTest, Truncate) { // For all possible truncations of the input file, assert that we don't // crash when parsing. for (off_t size = statbuf.st_size; size > 0; --size) { + BuildLog log2; + string err; + EXPECT_TRUE(log2.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + log2.RecordCommand(state_.edges_[0], 15, 18); + log2.RecordCommand(state_.edges_[1], 20, 25); + log2.Close(); + #ifndef _WIN32 ASSERT_EQ(0, truncate(kTestFilename, size)); #else @@ -140,9 +148,9 @@ TEST_F(BuildLogTest, Truncate) { _close(fh); #endif - BuildLog log2; + BuildLog log3; err.clear(); - ASSERT_TRUE(log2.Load(kTestFilename, &err) || !err.empty()); + ASSERT_TRUE(log3.Load(kTestFilename, &err) || !err.empty()); } } -- cgit v0.12 From 73f646f122f6bd9a09a105dc995e14161a027345 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 24 Aug 2012 11:14:52 -0700 Subject: remove crlfs from .gitignore These were introduced by a change of mine on Windows, whoops. --- .gitignore | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 1872263..19a08ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,17 @@ -*.pyc -*.exe -*.pdb -*.ilk -TAGS -/build -/build.ninja -/ninja -/build_log_perftest -/canon_perftest -/hash_collision_bench -/ninja_test -/parser_perftest -/graph.png -/doc/manual.html -/doc/doxygen -/gtest-1.6.0 +*.pyc +*.exe +*.pdb +*.ilk +TAGS +/build +/build.ninja +/ninja +/build_log_perftest +/canon_perftest +/hash_collision_bench +/ninja_test +/parser_perftest +/graph.png +/doc/manual.html +/doc/doxygen +/gtest-1.6.0 -- cgit v0.12 From 50b122e5743a738ddf636fdccaa4b6712508ea7d Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 24 Aug 2012 11:20:55 -0700 Subject: windows: pass /Zi to gtest compile as well --- configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.py b/configure.py index e4b9eb4..bd50e67 100755 --- a/configure.py +++ b/configure.py @@ -320,7 +320,7 @@ if options.with_gtest: gtest_all_incs = '-I%s -I%s' % (path, os.path.join(path, 'include')) if platform == 'windows': - gtest_cflags = '/nologo /EHsc ' + gtest_all_incs + gtest_cflags = '/nologo /EHsc /Zi ' + gtest_all_incs else: gtest_cflags = '-fvisibility=hidden ' + gtest_all_incs objs += n.build(built('gtest-all' + objext), 'cxx', -- cgit v0.12 From c90bdaa9d999e0c9c0180a8982f99d086b9aa5f9 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 6 Jan 2012 13:50:29 -0800 Subject: add test that checks attributes on rules --- src/manifest_parser_test.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index a48c99e..0be489e 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -64,6 +64,17 @@ TEST_F(ParserTest, Rules) { EXPECT_EQ("[cat ][$in][ > ][$out]", rule->command().Serialize()); } +TEST_F(ParserTest, RuleAttributes) { + // Check that all of the allowed rule attributes are parsed ok. + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = a\n" +" depfile = a\n" +" description = a\n" +" generator = a\n" +" restat = a\n")); +} + TEST_F(ParserTest, IgnoreIndentedComments) { ASSERT_NO_FATAL_FAILURE(AssertParse( " #indented comment\n" -- cgit v0.12 From 336efcd0c93c6b7ee469d48210b65b5e114e061e Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 24 Aug 2012 11:30:13 -0700 Subject: update old test to cover newer rule attributes --- src/manifest_parser_test.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 0be489e..8b00efb 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -72,7 +72,10 @@ TEST_F(ParserTest, RuleAttributes) { " depfile = a\n" " description = a\n" " generator = a\n" -" restat = a\n")); +" restat = a\n" +" rspfile = a\n" +" rspfile_content = a\n" +)); } TEST_F(ParserTest, IgnoreIndentedComments) { -- cgit v0.12 From d033269a5d430247c2efe002d2d334c2b9a4875e Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 24 Aug 2012 11:53:48 -0700 Subject: link directly to gtest zip --- HACKING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/HACKING.md b/HACKING.md index bc23773..a777b57 100644 --- a/HACKING.md +++ b/HACKING.md @@ -66,7 +66,9 @@ googletest (gtest) library. * Otherwise you need to download it, unpack it, and pass `--with-gtest` to `configure.py`. Get it from [its downloads - page](http://code.google.com/p/googletest/downloads/list). + page](http://code.google.com/p/googletest/downloads/list); [this + direct download link might work + too](http://googletest.googlecode.com/files/gtest-1.6.0.zip). ### Test-driven development -- cgit v0.12 From ca6c366c639a8ff8811b07c687e327bcb2694419 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 24 Aug 2012 12:01:00 -0700 Subject: drop DepfileParserTest.Tilde, as it's covered by .SpecialChars The SpecialChars test covers a bunch of different special characters, including tilde. --- src/depfile_parser_test.cc | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index 2bb9105..93d42db 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -135,16 +135,3 @@ TEST_F(DepfileParserTest, RejectMultipleDifferentOutputs) { string err; EXPECT_FALSE(Parse("foo bar: x y z", &err)); } - -TEST_F(DepfileParserTest, Tilde) { - string err; - EXPECT_TRUE(Parse( -"foo~.o: foo~.c", - &err)); - ASSERT_EQ("", err); - EXPECT_EQ("foo~.o", - parser_.out_.AsString()); - ASSERT_EQ(1u, parser_.ins_.size()); - EXPECT_EQ("foo~.c", - parser_.ins_[0].AsString()); -} -- cgit v0.12 From 1c81d488bf39c1c5cad0d49477eedc2980747f99 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 24 Aug 2012 12:27:48 -0700 Subject: point to HACKING.md in README --- README | 1 + 1 file changed, 1 insertion(+) diff --git a/README b/README index a8fe582..733ccb3 100644 --- a/README +++ b/README @@ -13,3 +13,4 @@ help. There is no installation step. The only file of interest to a user is the resulting ninja binary. +If you're interested in making changes to Ninja, read HACKING.md first. -- cgit v0.12 From 070d618bdb66cf29ef3e66d8561d2aec7862228d Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Tue, 28 Aug 2012 12:11:27 -0700 Subject: Remove unused macro NINJA_UNUSED_ARG. --- src/util.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/util.h b/src/util.h index 3a658fb..6c142c6 100644 --- a/src/util.h +++ b/src/util.h @@ -25,8 +25,6 @@ #include using namespace std; -#define NINJA_UNUSED_ARG(arg_name) (void)arg_name; - /// Log a fatal message and exit. void Fatal(const char* msg, ...); -- cgit v0.12 From bcc8ad1369ae3d90631729d29cc83c377f44535e Mon Sep 17 00:00:00 2001 From: Maxim Kalaev Date: Wed, 29 Aug 2012 22:25:44 +0200 Subject: safer build: consider target dirty if depfile is missing --- src/graph.cc | 12 +++++++++--- src/graph_test.cc | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/graph.cc b/src/graph.cc index 9654c1a..3567bfa 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -38,8 +38,13 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, outputs_ready_ = true; if (!rule_->depfile().empty()) { - if (!LoadDepFile(state, disk_interface, err)) - return false; + if (!LoadDepFile(state, disk_interface, err)) { + if (!err->empty()) + return false; + EXPLAIN("Edge targets are dirty because depfile '%s' is missing", + EvaluateDepFile().c_str()); + dirty = true; + } } // Visit all inputs; we're dirty if any of the inputs are dirty. @@ -274,8 +279,9 @@ bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, string content = disk_interface->ReadFile(path, err); if (!err->empty()) return false; + // On a missing depfile: return false and empty *err. if (content.empty()) - return true; + return false; DepfileParser depfile; string depfile_err; diff --git a/src/graph_test.cc b/src/graph_test.cc index 38ff28a..4b41f8a 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -159,3 +159,28 @@ TEST_F(GraphTest, DepfileWithCanonicalizablePath) { EXPECT_FALSE(GetNode("out.o")->dirty()); } + +// Regression test for https://github.com/martine/ninja/issues/404 +TEST_F(GraphTest, DepfileRemoved) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule catdep\n" +" depfile = $out.d\n" +" command = cat $in > $out\n" +"build ./out.o: catdep ./foo.cc\n")); + fs_.Create("foo.h", 1, ""); + fs_.Create("foo.cc", 1, ""); + fs_.Create("out.o.d", 2, "out.o: foo.h\n"); + fs_.Create("out.o", 2, ""); + + Edge* edge = GetNode("out.o")->in_edge(); + string err; + EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + ASSERT_EQ("", err); + EXPECT_FALSE(GetNode("out.o")->dirty()); + + state_.Reset(); + fs_.RemoveFile("out.o.d"); + EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(GetNode("out.o")->dirty()); +} -- cgit v0.12 From e744221d48bdda031390a9a37c6e96ca3c658aa6 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Fri, 31 Aug 2012 22:56:11 -0700 Subject: Make sure no stale test file exists before running build log tests. --- src/build_log_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/build_log_test.cc b/src/build_log_test.cc index c465abc..1d42273 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -30,6 +30,8 @@ const char kTestFilename[] = "BuildLogTest-tempfile"; struct BuildLogTest : public StateTestWithBuiltinRules { virtual void SetUp() { + // In case a crashing test left a stale file behind. + unlink(kTestFilename); } virtual void TearDown() { unlink(kTestFilename); -- cgit v0.12 From fd91e0dc26d18209f7625730bb0e653f8321fff9 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 2 Sep 2012 11:03:01 -0700 Subject: split out dirty recomputation logic from Edge class Rather than passing States and DiskInterfaces through all the calls, put the necessary ambient information in a new DependencyScan object and move the code accordingly. Note: I didn't move the source location of the functions to preserve history, though this does result in a sort of weird order for the functions in graph.cc. --- src/build.cc | 15 ++++--- src/build.h | 4 +- src/disk_interface_test.cc | 11 +++-- src/graph.cc | 107 ++++++++++++++++++++++++--------------------- src/graph.h | 49 ++++++++++++++------- src/graph_test.cc | 19 ++++---- 6 files changed, 117 insertions(+), 88 deletions(-) diff --git a/src/build.cc b/src/build.cc index 6d1318c..18bba9e 100644 --- a/src/build.cc +++ b/src/build.cc @@ -406,7 +406,7 @@ void Plan::NodeFinished(Node* node) { } } -void Plan::CleanNode(BuildLog* build_log, Node* node) { +void Plan::CleanNode(DependencyScan* scan, Node* node) { node->set_dirty(false); for (vector::const_iterator ei = node->out_edges().begin(); @@ -435,12 +435,12 @@ void Plan::CleanNode(BuildLog* build_log, Node* node) { if (!(*ni)->dirty()) continue; - if ((*ei)->RecomputeOutputDirty(build_log, most_recent_input, NULL, command, - *ni)) { + if (scan->RecomputeOutputDirty(*ei, most_recent_input, NULL, + command, *ni)) { (*ni)->MarkDirty(); all_outputs_clean = false; } else { - CleanNode(build_log, *ni); + CleanNode(scan, *ni); } } @@ -554,7 +554,8 @@ struct DryRunCommandRunner : public CommandRunner { Builder::Builder(State* state, const BuildConfig& config, DiskInterface* disk_interface) - : state_(state), config_(config), disk_interface_(disk_interface) { + : state_(state), config_(config), disk_interface_(disk_interface), + scan_(state, disk_interface) { status_ = new BuildStatus(config); log_ = state->build_log_; } @@ -604,7 +605,7 @@ Node* Builder::AddTarget(const string& name, string* err) { bool Builder::AddTarget(Node* node, string* err) { node->StatIfNecessary(disk_interface_); if (Edge* in_edge = node->in_edge()) { - if (!in_edge->RecomputeDirty(state_, disk_interface_, err)) + if (!scan_.RecomputeDirty(in_edge, err)) return false; if (in_edge->outputs_ready()) return true; // Nothing to do. @@ -756,7 +757,7 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { // The rule command did not change the output. Propagate the clean // state through the build graph. // Note that this also applies to nonexistent outputs (mtime == 0). - plan_.CleanNode(log_, *i); + plan_.CleanNode(&scan_, *i); node_cleaned = true; } } diff --git a/src/build.h b/src/build.h index 986e8a9..9152eac 100644 --- a/src/build.h +++ b/src/build.h @@ -23,6 +23,7 @@ #include #include +#include "graph.h" // XXX needed for DependencyScan; should rearrange. #include "exit_status.h" #include "metrics.h" #include "util.h" // int64_t @@ -59,7 +60,7 @@ struct Plan { void EdgeFinished(Edge* edge); /// Clean the given node during the build. - void CleanNode(BuildLog* build_log, Node* node); + void CleanNode(DependencyScan* scan, Node* node); /// Number of edges with commands to run. int command_edge_count() const { return command_edges_; } @@ -151,6 +152,7 @@ struct Builder { private: DiskInterface* disk_interface_; + DependencyScan scan_; // Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr. Builder(const Builder &other); // DO NOT IMPLEMENT diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 985e991..68a6ca9 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -105,6 +105,8 @@ TEST_F(DiskInterfaceTest, RemoveFile) { struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { + StatTest() : scan_(&state_, this) {} + // DiskInterface implementation. virtual TimeStamp Stat(const string& path); virtual bool WriteFile(const string& path, const string& contents) { @@ -124,6 +126,7 @@ struct StatTest : public StateTestWithBuiltinRules, return 0; } + DependencyScan scan_; map mtimes_; vector stats_; }; @@ -143,7 +146,7 @@ TEST_F(StatTest, Simple) { Node* out = GetNode("out"); out->Stat(this); ASSERT_EQ(1u, stats_.size()); - out->in_edge()->RecomputeDirty(NULL, this, NULL); + scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_EQ(2u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_EQ("in", stats_[1]); @@ -157,7 +160,7 @@ TEST_F(StatTest, TwoStep) { Node* out = GetNode("out"); out->Stat(this); ASSERT_EQ(1u, stats_.size()); - out->in_edge()->RecomputeDirty(NULL, this, NULL); + scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_EQ(3u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_TRUE(GetNode("out")->dirty()); @@ -175,7 +178,7 @@ TEST_F(StatTest, Tree) { Node* out = GetNode("out"); out->Stat(this); ASSERT_EQ(1u, stats_.size()); - out->in_edge()->RecomputeDirty(NULL, this, NULL); + scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_EQ(1u + 6u, stats_.size()); ASSERT_EQ("mid1", stats_[1]); ASSERT_TRUE(GetNode("mid1")->dirty()); @@ -194,7 +197,7 @@ TEST_F(StatTest, Middle) { Node* out = GetNode("out"); out->Stat(this); ASSERT_EQ(1u, stats_.size()); - out->in_edge()->RecomputeDirty(NULL, this, NULL); + scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_FALSE(GetNode("in")->dirty()); ASSERT_TRUE(GetNode("mid")->dirty()); ASSERT_TRUE(GetNode("out")->dirty()); diff --git a/src/graph.cc b/src/graph.cc index 3567bfa..d73e38e 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -32,17 +32,16 @@ bool Node::Stat(DiskInterface* disk_interface) { return mtime_ > 0; } -bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, - string* err) { +bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { bool dirty = false; - outputs_ready_ = true; + edge->outputs_ready_ = true; - if (!rule_->depfile().empty()) { - if (!LoadDepFile(state, disk_interface, err)) { + if (!edge->rule_->depfile().empty()) { + if (!LoadDepFile(edge, err)) { if (!err->empty()) return false; EXPLAIN("Edge targets are dirty because depfile '%s' is missing", - EvaluateDepFile().c_str()); + edge->EvaluateDepFile().c_str()); dirty = true; } } @@ -50,10 +49,11 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, // Visit all inputs; we're dirty if any of the inputs are dirty. TimeStamp most_recent_input = 1; Node* most_recent_node = NULL; - for (vector::iterator i = inputs_.begin(); i != inputs_.end(); ++i) { - if ((*i)->StatIfNecessary(disk_interface)) { - if (Edge* edge = (*i)->in_edge()) { - if (!edge->RecomputeDirty(state, disk_interface, err)) + for (vector::iterator i = edge->inputs_.begin(); + i != edge->inputs_.end(); ++i) { + if ((*i)->StatIfNecessary(disk_interface_)) { + if (Edge* in_edge = (*i)->in_edge()) { + if (!RecomputeDirty(in_edge, err)) return false; } else { // This input has no in-edge; it is dirty if it is missing. @@ -64,12 +64,12 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, } // If an input is not ready, neither are our outputs. - if (Edge* edge = (*i)->in_edge()) { - if (!edge->outputs_ready_) - outputs_ready_ = false; + if (Edge* in_edge = (*i)->in_edge()) { + if (!in_edge->outputs_ready_) + edge->outputs_ready_ = false; } - if (!is_order_only(i - inputs_.begin())) { + if (!edge->is_order_only(i - edge->inputs_.begin())) { // If a regular input is dirty (or missing), we're dirty. // Otherwise consider mtime. if ((*i)->dirty()) { @@ -87,13 +87,13 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, // We may also be dirty due to output state: missing outputs, out of // date outputs, etc. Visit all outputs and determine whether they're dirty. if (!dirty) { - BuildLog* build_log = state ? state->build_log_ : 0; - string command = EvaluateCommand(true); + string command = edge->EvaluateCommand(true); - for (vector::iterator i = outputs_.begin(); - i != outputs_.end(); ++i) { - (*i)->StatIfNecessary(disk_interface); - if (RecomputeOutputDirty(build_log, most_recent_input, most_recent_node, command, *i)) { + for (vector::iterator i = edge->outputs_.begin(); + i != edge->outputs_.end(); ++i) { + (*i)->StatIfNecessary(disk_interface_); + if (RecomputeOutputDirty(edge, most_recent_input, most_recent_node, + command, *i)) { dirty = true; break; } @@ -102,30 +102,33 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, // Finally, visit each output to mark off that we've visited it, and update // their dirty state if necessary. - for (vector::iterator i = outputs_.begin(); i != outputs_.end(); ++i) { - (*i)->StatIfNecessary(disk_interface); + for (vector::iterator i = edge->outputs_.begin(); + i != edge->outputs_.end(); ++i) { + (*i)->StatIfNecessary(disk_interface_); if (dirty) (*i)->MarkDirty(); } - // If we're dirty, our outputs are normally not ready. (It's possible to be - // clean but still not be ready in the presence of order-only inputs.) - // But phony edges with no inputs have nothing to do, so are always ready. - if (dirty && !(is_phony() && inputs_.empty())) - outputs_ready_ = false; + // If an edge is dirty, its outputs are normally not ready. (It's + // possible to be clean but still not be ready in the presence of + // order-only inputs.) + // But phony edges with no inputs have nothing to do, so are always + // ready. + if (dirty && !(edge->is_phony() && edge->inputs_.empty())) + edge->outputs_ready_ = false; return true; } -bool Edge::RecomputeOutputDirty(BuildLog* build_log, - TimeStamp most_recent_input, - Node* most_recent_node, - const string& command, - Node* output) { - if (is_phony()) { +bool DependencyScan::RecomputeOutputDirty(Edge* edge, + TimeStamp most_recent_input, + Node* most_recent_node, + const string& command, + Node* output) { + if (edge->is_phony()) { // Phony edges don't write any output. Outputs are only dirty if // there are no inputs and we're missing the output. - return inputs_.empty() && !output->exists(); + return edge->inputs_.empty() && !output->exists(); } BuildLog::LogEntry* entry = 0; @@ -142,8 +145,8 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, // rule in a previous run and stored the most recent input mtime in the // build log. Use that mtime instead, so that the file will only be // considered dirty if an input was modified since the previous run. - if (rule_->restat() && build_log && - (entry = build_log->LookupByOutput(output->path()))) { + if (edge->rule_->restat() && build_log() && + (entry = build_log()->LookupByOutput(output->path()))) { if (entry->restat_mtime < most_recent_input) { EXPLAIN("restat of output %s older than most recent input %s (%d vs %d)", output->path().c_str(), @@ -163,8 +166,8 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, // May also be dirty due to the command changing since the last build. // But if this is a generator rule, the command changing does not make us // dirty. - if (!rule_->generator() && build_log) { - if (entry || (entry = build_log->LookupByOutput(output->path()))) { + if (!edge->rule_->generator() && build_log()) { + if (entry || (entry = build_log()->LookupByOutput(output->path()))) { if (BuildLog::LogEntry::HashCommand(command) != entry->command_hash) { EXPLAIN("command line changed for %s", output->path().c_str()); return true; @@ -272,11 +275,10 @@ string Edge::GetRspFileContent() { return rule_->rspfile_content().Evaluate(&env); } -bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, - string* err) { +bool DependencyScan::LoadDepFile(Edge* edge, string* err) { METRIC_RECORD("depfile load"); - string path = EvaluateDepFile(); - string content = disk_interface->ReadFile(path, err); + string path = edge->EvaluateDepFile(); + string content = disk_interface_->ReadFile(path, err); if (!err->empty()) return false; // On a missing depfile: return false and empty *err. @@ -290,18 +292,21 @@ bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, return false; } - // Check that this depfile matches our output. - StringPiece opath = StringPiece(outputs_[0]->path()); + // Check that this depfile matches the edge's output. + Node* first_output = edge->outputs_[0]; + StringPiece opath = StringPiece(first_output->path()); if (opath != depfile.out_) { *err = "expected depfile '" + path + "' to mention '" + - outputs_[0]->path() + "', got '" + depfile.out_.AsString() + "'"; + first_output->path() + "', got '" + depfile.out_.AsString() + "'"; return false; } - inputs_.insert(inputs_.end() - order_only_deps_, depfile.ins_.size(), 0); - implicit_deps_ += depfile.ins_.size(); + // Preallocate space in edge->inputs_ to be filled in below. + edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_, + depfile.ins_.size(), 0); + edge->implicit_deps_ += depfile.ins_.size(); vector::iterator implicit_dep = - inputs_.end() - order_only_deps_ - depfile.ins_.size(); + edge->inputs_.end() - edge->order_only_deps_ - depfile.ins_.size(); // Add all its in-edges. for (vector::iterator i = depfile.ins_.begin(); @@ -309,15 +314,15 @@ bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, if (!CanonicalizePath(const_cast(i->str_), &i->len_, err)) return false; - Node* node = state->GetNode(*i); + Node* node = state_->GetNode(*i); *implicit_dep = node; - node->AddOutEdge(this); + node->AddOutEdge(edge); // If we don't have a edge that generates this input already, // create one; this makes us not abort if the input is missing, // but instead will rebuild in that circumstance. if (!node->in_edge()) { - Edge* phony_edge = state->AddEdge(&State::kPhonyRule); + Edge* phony_edge = state_->AddEdge(&State::kPhonyRule); node->set_in_edge(phony_edge); phony_edge->outputs_.push_back(node); diff --git a/src/graph.h b/src/graph.h index f487a22..3e2e57a 100644 --- a/src/graph.h +++ b/src/graph.h @@ -20,6 +20,7 @@ using namespace std; #include "eval_env.h" +#include "state.h" #include "timestamp.h" struct DiskInterface; @@ -142,39 +143,25 @@ struct Edge { Edge() : rule_(NULL), env_(NULL), outputs_ready_(false), implicit_deps_(0), order_only_deps_(0) {} - /// Examine inputs, outputs, and command lines to judge whether this edge - /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| - /// state accordingly. - /// Returns false on failure. - bool RecomputeDirty(State* state, DiskInterface* disk_interface, string* err); - - /// Recompute whether a given single output should be marked dirty. - /// Returns true if so. - bool RecomputeOutputDirty(BuildLog* build_log, TimeStamp most_recent_input, - Node* most_recent_node, const string& command, - Node* output); - /// Return true if all inputs' in-edges are ready. bool AllInputsReady() const; /// Expand all variables in a command and return it as a string. - /// If incl_rsp_file is enabled, the string will also contain the + /// If incl_rsp_file is enabled, the string will also contain the /// full contents of a response file (if applicable) string EvaluateCommand(bool incl_rsp_file = false); // XXX move to env, take env ptr string EvaluateDepFile(); string GetDescription(); - + /// Does the edge use a response file? bool HasRspFile(); - + /// Get the path to the response file string GetRspFile(); /// Get the contents of the response file string GetRspFileContent(); - bool LoadDepFile(State* state, DiskInterface* disk_interface, string* err); - void Dump(const char* prefix="") const; const Rule* rule_; @@ -210,4 +197,32 @@ struct Edge { bool is_phony() const; }; + +/// DependencyScan manages the process of scanning the files in a graph +/// and updating the dirty/outputs_ready state of all the nodes and edges. +struct DependencyScan { + DependencyScan(State* state, DiskInterface* disk_interface) + : state_(state), disk_interface_(disk_interface) {} + + /// Examine inputs, outputs, and command lines to judge whether an edge + /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| + /// state accordingly. + /// Returns false on failure. + bool RecomputeDirty(Edge* edge, string* err); + + /// Recompute whether a given single output should be marked dirty. + /// Returns true if so. + bool RecomputeOutputDirty(Edge* edge, TimeStamp most_recent_input, + Node* most_recent_node, + const string& command, Node* output); + + bool LoadDepFile(Edge* edge, string* err); + + State* state_; + DiskInterface* disk_interface_; + BuildLog* build_log() const { + return state_->build_log_; + } +}; + #endif // NINJA_GRAPH_H_ diff --git a/src/graph_test.cc b/src/graph_test.cc index 4b41f8a..2bc0c17 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -17,7 +17,10 @@ #include "test.h" struct GraphTest : public StateTestWithBuiltinRules { + GraphTest() : scan_(&state_, &fs_) {} + VirtualFileSystem fs_; + DependencyScan scan_; }; TEST_F(GraphTest, MissingImplicit) { @@ -28,7 +31,7 @@ TEST_F(GraphTest, MissingImplicit) { Edge* edge = GetNode("out")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); // A missing implicit dep *should* make the output dirty. @@ -46,7 +49,7 @@ TEST_F(GraphTest, ModifiedImplicit) { Edge* edge = GetNode("out")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); // A modified implicit dep should make the output dirty. @@ -66,7 +69,7 @@ TEST_F(GraphTest, FunkyMakefilePath) { Edge* edge = GetNode("out.o")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); // implicit.h has changed, though our depfile refers to it with a @@ -89,7 +92,7 @@ TEST_F(GraphTest, ExplicitImplicit) { Edge* edge = GetNode("out.o")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); // We have both an implicit and an explicit dep on implicit.h. @@ -110,7 +113,7 @@ TEST_F(GraphTest, PathWithCurrentDirectory) { Edge* edge = GetNode("out.o")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); @@ -154,7 +157,7 @@ TEST_F(GraphTest, DepfileWithCanonicalizablePath) { Edge* edge = GetNode("out.o")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); @@ -174,13 +177,13 @@ TEST_F(GraphTest, DepfileRemoved) { Edge* edge = GetNode("out.o")->in_edge(); string err; - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); state_.Reset(); fs_.RemoveFile("out.o.d"); - EXPECT_TRUE(edge->RecomputeDirty(&state_, &fs_, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.o")->dirty()); } -- cgit v0.12 From 3220e626da48bc0fd69bee5a3dfae3b55bd0b761 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 2 Sep 2012 12:27:46 -0700 Subject: remove a redundant arg to RecomputeOutputDirty --- src/build.cc | 11 ++++++----- src/graph.cc | 29 ++++++++++++----------------- src/graph.h | 3 +-- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/build.cc b/src/build.cc index 18bba9e..4102209 100644 --- a/src/build.cc +++ b/src/build.cc @@ -422,10 +422,11 @@ void Plan::CleanNode(DependencyScan* scan, Node* node) { end = (*ei)->inputs_.end() - (*ei)->order_only_deps_; if (find_if(begin, end, mem_fun(&Node::dirty)) == end) { // Recompute most_recent_input and command. - TimeStamp most_recent_input = 1; - for (vector::iterator ni = begin; ni != end; ++ni) - if ((*ni)->mtime() > most_recent_input) - most_recent_input = (*ni)->mtime(); + Node* most_recent_input = NULL; + for (vector::iterator ni = begin; ni != end; ++ni) { + if (!most_recent_input || (*ni)->mtime() > most_recent_input->mtime()) + most_recent_input = *ni; + } string command = (*ei)->EvaluateCommand(true); // Now, recompute the dirty state of each output. @@ -435,7 +436,7 @@ void Plan::CleanNode(DependencyScan* scan, Node* node) { if (!(*ni)->dirty()) continue; - if (scan->RecomputeOutputDirty(*ei, most_recent_input, NULL, + if (scan->RecomputeOutputDirty(*ei, most_recent_input, command, *ni)) { (*ni)->MarkDirty(); all_outputs_clean = false; diff --git a/src/graph.cc b/src/graph.cc index d73e38e..6ae324d 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -47,8 +47,7 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { } // Visit all inputs; we're dirty if any of the inputs are dirty. - TimeStamp most_recent_input = 1; - Node* most_recent_node = NULL; + Node* most_recent_input = NULL; for (vector::iterator i = edge->inputs_.begin(); i != edge->inputs_.end(); ++i) { if ((*i)->StatIfNecessary(disk_interface_)) { @@ -76,9 +75,8 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { EXPLAIN("%s is dirty", (*i)->path().c_str()); dirty = true; } else { - if ((*i)->mtime() > most_recent_input) { - most_recent_input = (*i)->mtime(); - most_recent_node = *i; + if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime()) { + most_recent_input = *i; } } } @@ -92,8 +90,7 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { for (vector::iterator i = edge->outputs_.begin(); i != edge->outputs_.end(); ++i) { (*i)->StatIfNecessary(disk_interface_); - if (RecomputeOutputDirty(edge, most_recent_input, most_recent_node, - command, *i)) { + if (RecomputeOutputDirty(edge, most_recent_input, command, *i)) { dirty = true; break; } @@ -121,8 +118,7 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { } bool DependencyScan::RecomputeOutputDirty(Edge* edge, - TimeStamp most_recent_input, - Node* most_recent_node, + Node* most_recent_input, const string& command, Node* output) { if (edge->is_phony()) { @@ -140,25 +136,24 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, } // Dirty if the output is older than the input. - if (output->mtime() < most_recent_input) { + if (most_recent_input && output->mtime() < most_recent_input->mtime()) { // If this is a restat rule, we may have cleaned the output with a restat // rule in a previous run and stored the most recent input mtime in the // build log. Use that mtime instead, so that the file will only be // considered dirty if an input was modified since the previous run. + TimeStamp most_recent_stamp = most_recent_input->mtime(); if (edge->rule_->restat() && build_log() && (entry = build_log()->LookupByOutput(output->path()))) { - if (entry->restat_mtime < most_recent_input) { + if (entry->restat_mtime < most_recent_stamp) { EXPLAIN("restat of output %s older than most recent input %s (%d vs %d)", - output->path().c_str(), - most_recent_node ? most_recent_node->path().c_str() : "", - entry->restat_mtime, most_recent_input); + output->path().c_str(), most_recent_input->path().c_str(), + entry->restat_mtime, most_recent_stamp); return true; } } else { EXPLAIN("output %s older than most recent input %s (%d vs %d)", - output->path().c_str(), - most_recent_node ? most_recent_node->path().c_str() : "", - output->mtime(), most_recent_input); + output->path().c_str(), most_recent_input->path().c_str(), + output->mtime(), most_recent_stamp); return true; } } diff --git a/src/graph.h b/src/graph.h index 3e2e57a..ced1f4f 100644 --- a/src/graph.h +++ b/src/graph.h @@ -212,8 +212,7 @@ struct DependencyScan { /// Recompute whether a given single output should be marked dirty. /// Returns true if so. - bool RecomputeOutputDirty(Edge* edge, TimeStamp most_recent_input, - Node* most_recent_node, + bool RecomputeOutputDirty(Edge* edge, Node* most_recent_input, const string& command, Node* output); bool LoadDepFile(Edge* edge, string* err); -- cgit v0.12 From 4d9bf949e0fd6976725dea12bcc254fd39da6490 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Sun, 2 Sep 2012 15:53:59 -0400 Subject: remove config from BuildLog, rename members --- src/build_log.cc | 23 ++++++++++------------- src/build_log.h | 18 +++++++----------- src/build_log_test.cc | 4 ++-- src/ninja.cc | 9 +++++---- 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/src/build_log.cc b/src/build_log.cc index 73c2b5d..a633892 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -92,16 +92,13 @@ uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { } BuildLog::BuildLog() - : log_file_(NULL), config_(NULL), needs_recompaction_(false) {} + : log_file_(NULL), needs_recompaction_(false) {} BuildLog::~BuildLog() { Close(); } bool BuildLog::OpenForWrite(const string& path, string* err) { - if (config_ && config_->dry_run) - return true; // Do nothing, report success. - if (needs_recompaction_) { Close(); if (!Recompact(path, err)) @@ -136,14 +133,14 @@ void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, for (vector::iterator out = edge->outputs_.begin(); out != edge->outputs_.end(); ++out) { const string& path = (*out)->path(); - Log::iterator i = log_.find(path); + Entries::iterator i = entries_.find(path); LogEntry* log_entry; - if (i != log_.end()) { + if (i != entries_.end()) { log_entry = i->second; } else { log_entry = new LogEntry; log_entry->output = path; - log_.insert(Log::value_type(log_entry->output, log_entry)); + entries_.insert(Entries::value_type(log_entry->output, log_entry)); } log_entry->command_hash = LogEntry::HashCommand(command); log_entry->start_time = start_time; @@ -286,13 +283,13 @@ bool BuildLog::Load(const string& path, string* err) { end = line_end; LogEntry* entry; - Log::iterator i = log_.find(output); - if (i != log_.end()) { + Entries::iterator i = entries_.find(output); + if (i != entries_.end()) { entry = i->second; } else { entry = new LogEntry; entry->output = output; - log_.insert(Log::value_type(entry->output, entry)); + entries_.insert(Entries::value_type(entry->output, entry)); ++unique_entry_count; } ++total_entry_count; @@ -331,8 +328,8 @@ bool BuildLog::Load(const string& path, string* err) { } BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { - Log::iterator i = log_.find(path); - if (i != log_.end()) + Entries::iterator i = entries_.find(path); + if (i != entries_.end()) return i->second; return NULL; } @@ -359,7 +356,7 @@ bool BuildLog::Recompact(const string& path, string* err) { return false; } - for (Log::iterator i = log_.begin(); i != log_.end(); ++i) { + for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { WriteEntry(f, *i->second); } diff --git a/src/build_log.h b/src/build_log.h index d3994ff..4141ff3 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -22,24 +22,21 @@ using namespace std; #include "hash_map.h" #include "timestamp.h" -#include "util.h" +#include "util.h" // uint64_t -struct BuildConfig; struct Edge; /// Store a log of every command ran for every build. /// It has a few uses: /// -/// 1) historical command lines for output files, so we know +/// 1) (hashes of) command lines for existing output files, so we know /// when we need to rebuild due to the command changing -/// 2) historical timing information -/// 3) maybe we can generate some sort of build overview output -/// from it +/// 2) timing information, perhaps for generating reports +/// 3) restat information struct BuildLog { BuildLog(); ~BuildLog(); - void SetConfig(BuildConfig* config) { config_ = config; } bool OpenForWrite(const string& path, string* err); void RecordCommand(Edge* edge, int start_time, int end_time, TimeStamp restat_mtime = 0); @@ -74,13 +71,12 @@ struct BuildLog { /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, string* err); - typedef ExternalStringHashMap::Type Log; - const Log& log() const { return log_; } + typedef ExternalStringHashMap::Type Entries; + const Entries& entries() const { return entries_; } private: - Log log_; + Entries entries_; FILE* log_file_; - BuildConfig* config_; bool needs_recompaction_; }; diff --git a/src/build_log_test.cc b/src/build_log_test.cc index c465abc..9af95a3 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -53,8 +53,8 @@ TEST_F(BuildLogTest, WriteRead) { EXPECT_TRUE(log2.Load(kTestFilename, &err)); ASSERT_EQ("", err); - ASSERT_EQ(2u, log1.log().size()); - ASSERT_EQ(2u, log2.log().size()); + ASSERT_EQ(2u, log1.entries().size()); + ASSERT_EQ(2u, log2.entries().size()); BuildLog::LogEntry* e1 = log1.LookupByOutput("out"); ASSERT_TRUE(e1); BuildLog::LogEntry* e2 = log2.LookupByOutput("out"); diff --git a/src/ninja.cc b/src/ninja.cc index 19438e5..13c0592 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -740,7 +740,6 @@ reload: return RunTool(tool, &globals, argc, argv); BuildLog build_log; - build_log.SetConfig(&globals.config); globals.state->build_log_ = &build_log; const string build_dir = globals.state->bindings_.LookupVariable("builddir"); @@ -765,9 +764,11 @@ reload: err.clear(); } - if (!build_log.OpenForWrite(log_path, &err)) { - Error("opening build log: %s", err.c_str()); - return 1; + if (!globals.config.dry_run) { + if (!build_log.OpenForWrite(log_path, &err)) { + Error("opening build log: %s", err.c_str()); + return 1; + } } if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild -- cgit v0.12 From 81a5d01bac7bc3302c8ab1c625d357a625f90dcf Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 18:08:41 -0400 Subject: pass Builder as arg to build-running functions Reducing use of globals. --- src/ninja.cc | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index 13c0592..799af02 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -122,22 +122,20 @@ struct RealFileReader : public ManifestParser::FileReader { /// Rebuild the build manifest, if necessary. /// Returns true if the manifest was rebuilt. -bool RebuildManifest(Globals* globals, const char* input_file, string* err) { +bool RebuildManifest(Builder* builder, const char* input_file, string* err) { string path = input_file; if (!CanonicalizePath(&path, err)) return false; - Node* node = globals->state->LookupNode(path); + Node* node = builder->state_->LookupNode(path); if (!node) return false; - Builder manifest_builder(globals->state, globals->config, - &globals->disk_interface); - if (!manifest_builder.AddTarget(node, err)) + if (!builder->AddTarget(node, err)) return false; - if (manifest_builder.AlreadyUpToDate()) + if (builder->AlreadyUpToDate()) return false; // Not an error, but we didn't rebuild. - if (!manifest_builder.Build(err)) + if (!builder->Build(err)) return false; // The manifest was only rebuilt if it is now dirty (it may have been cleaned @@ -571,17 +569,16 @@ bool DebugEnable(const string& name, Globals* globals) { } } -int RunBuild(Globals* globals, int argc, char** argv) { +int RunBuild(Builder* builder, int argc, char** argv) { string err; vector targets; - if (!CollectTargetsFromArgs(globals->state, argc, argv, &targets, &err)) { + if (!CollectTargetsFromArgs(builder->state_, argc, argv, &targets, &err)) { Error("%s", err.c_str()); return 1; } - Builder builder(globals->state, globals->config, &globals->disk_interface); for (size_t i = 0; i < targets.size(); ++i) { - if (!builder.AddTarget(targets[i], &err)) { + if (!builder->AddTarget(targets[i], &err)) { if (!err.empty()) { Error("%s", err.c_str()); return 1; @@ -592,12 +589,12 @@ int RunBuild(Globals* globals, int argc, char** argv) { } } - if (builder.AlreadyUpToDate()) { + if (builder->AlreadyUpToDate()) { printf("ninja: no work to do.\n"); return 0; } - if (!builder.Build(&err)) { + if (!builder->Build(&err)) { printf("ninja: build stopped: %s.\n", err.c_str()); return 1; } @@ -773,7 +770,9 @@ reload: if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild // target that is never up to date. - if (RebuildManifest(&globals, input_file, &err)) { + Builder manifest_builder(globals.state, globals.config, + &globals.disk_interface); + if (RebuildManifest(&manifest_builder, input_file, &err)) { rebuilt_manifest = true; globals.ResetState(); goto reload; @@ -783,7 +782,8 @@ reload: } } - int result = RunBuild(&globals, argc, argv); + Builder builder(globals.state, globals.config, &globals.disk_interface); + int result = RunBuild(&builder, argc, argv); if (g_metrics) { g_metrics->Report(); -- cgit v0.12 From 703dcffd6d2e4b64411eb44d3025093abfd1d737 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 18:21:52 -0400 Subject: fix --debug for glibc pedantic mode --- configure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.py b/configure.py index bd50e67..69e00d5 100755 --- a/configure.py +++ b/configure.py @@ -138,6 +138,7 @@ else: '-DNINJA_PYTHON="%s"' % options.with_python] if options.debug: cflags += ['-D_GLIBCXX_DEBUG', '-D_GLIBCXX_DEBUG_PEDANTIC'] + cflags.remove('-fno-rtti') # Needed for above pedanticness. else: cflags += ['-O2', '-DNDEBUG'] if 'clang' in os.path.basename(CXX): -- cgit v0.12 From 1e11c8a6bd119025efd8725370ffa42354f92f88 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 18:39:04 -0400 Subject: move BuildLog to DependencyScan The build log is needed in computing whether an edge is dirty, so I think it belongs here. (It's a bit weird that Builder needs to reach into it to record completed commands, maybe it will become cleaner with more thought.) --- src/build.cc | 9 ++++----- src/build.h | 8 ++++++-- src/build_test.cc | 5 +++-- src/disk_interface_test.cc | 2 +- src/graph.h | 18 +++++++++++++----- src/graph_test.cc | 2 +- src/ninja.cc | 6 +++--- src/state.cc | 2 +- src/state.h | 2 -- 9 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/build.cc b/src/build.cc index 4102209..efc05b0 100644 --- a/src/build.cc +++ b/src/build.cc @@ -554,11 +554,10 @@ struct DryRunCommandRunner : public CommandRunner { }; Builder::Builder(State* state, const BuildConfig& config, - DiskInterface* disk_interface) + BuildLog* log, DiskInterface* disk_interface) : state_(state), config_(config), disk_interface_(disk_interface), - scan_(state, disk_interface) { + scan_(state, log, disk_interface) { status_ = new BuildStatus(config); - log_ = state->build_log_; } Builder::~Builder() { @@ -797,7 +796,7 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { int start_time, end_time; status_->BuildEdgeFinished(edge, success, output, &start_time, &end_time); - if (success && log_) - log_->RecordCommand(edge, start_time, end_time, restat_mtime); + if (success && scan_.build_log()) + scan_.build_log()->RecordCommand(edge, start_time, end_time, restat_mtime); } diff --git a/src/build.h b/src/build.h index 9152eac..3e7a144 100644 --- a/src/build.h +++ b/src/build.h @@ -121,7 +121,7 @@ struct BuildConfig { /// Builder wraps the build process: starting commands, updating status. struct Builder { Builder(State* state, const BuildConfig& config, - DiskInterface* disk_interface); + BuildLog* log, DiskInterface* disk_interface); ~Builder(); /// Clean up after interrupted commands by deleting output files. @@ -143,12 +143,16 @@ struct Builder { bool StartEdge(Edge* edge, string* err); void FinishEdge(Edge* edge, bool success, const string& output); + /// Used for tests. + void SetBuildLog(BuildLog* log) { + scan_.set_build_log(log); + } + State* state_; const BuildConfig& config_; Plan plan_; auto_ptr command_runner_; BuildStatus* status_; - BuildLog* log_; private: DiskInterface* disk_interface_; diff --git a/src/build_test.cc b/src/build_test.cc index d4673ae..859e758 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -178,7 +178,8 @@ TEST_F(PlanTest, DependencyCycle) { struct BuildTest : public StateTestWithBuiltinRules, public CommandRunner { - BuildTest() : config_(MakeConfig()), builder_(&state_, config_, &fs_), + BuildTest() : config_(MakeConfig()), + builder_(&state_, config_, NULL, &fs_), now_(1), last_command_(NULL), status_(config_) { builder_.command_runner_.reset(this); AssertParse(&state_, @@ -717,7 +718,7 @@ TEST_F(BuildTest, SwallowFailuresLimit) { struct BuildWithLogTest : public BuildTest { BuildWithLogTest() { - state_.build_log_ = builder_.log_ = &build_log_; + builder_.SetBuildLog(&build_log_); } BuildLog build_log_; diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 68a6ca9..32fe9cb 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -105,7 +105,7 @@ TEST_F(DiskInterfaceTest, RemoveFile) { struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { - StatTest() : scan_(&state_, this) {} + StatTest() : scan_(&state_, NULL, this) {} // DiskInterface implementation. virtual TimeStamp Stat(const string& path); diff --git a/src/graph.h b/src/graph.h index ced1f4f..893ec09 100644 --- a/src/graph.h +++ b/src/graph.h @@ -201,8 +201,10 @@ struct Edge { /// DependencyScan manages the process of scanning the files in a graph /// and updating the dirty/outputs_ready state of all the nodes and edges. struct DependencyScan { - DependencyScan(State* state, DiskInterface* disk_interface) - : state_(state), disk_interface_(disk_interface) {} + DependencyScan(State* state, BuildLog* build_log, + DiskInterface* disk_interface) + : state_(state), build_log_(build_log), + disk_interface_(disk_interface) {} /// Examine inputs, outputs, and command lines to judge whether an edge /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| @@ -217,11 +219,17 @@ struct DependencyScan { bool LoadDepFile(Edge* edge, string* err); - State* state_; - DiskInterface* disk_interface_; BuildLog* build_log() const { - return state_->build_log_; + return build_log_; } + void set_build_log(BuildLog* log) { + build_log_ = log; + } + + private: + State* state_; + BuildLog* build_log_; + DiskInterface* disk_interface_; }; #endif // NINJA_GRAPH_H_ diff --git a/src/graph_test.cc b/src/graph_test.cc index 2bc0c17..5b25c2f 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -17,7 +17,7 @@ #include "test.h" struct GraphTest : public StateTestWithBuiltinRules { - GraphTest() : scan_(&state_, &fs_) {} + GraphTest() : scan_(&state_, NULL, &fs_) {} VirtualFileSystem fs_; DependencyScan scan_; diff --git a/src/ninja.cc b/src/ninja.cc index 799af02..fb2bb3c 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -737,7 +737,6 @@ reload: return RunTool(tool, &globals, argc, argv); BuildLog build_log; - globals.state->build_log_ = &build_log; const string build_dir = globals.state->bindings_.LookupVariable("builddir"); const char* kLogPath = ".ninja_log"; @@ -770,7 +769,7 @@ reload: if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild // target that is never up to date. - Builder manifest_builder(globals.state, globals.config, + Builder manifest_builder(globals.state, globals.config, &build_log, &globals.disk_interface); if (RebuildManifest(&manifest_builder, input_file, &err)) { rebuilt_manifest = true; @@ -782,7 +781,8 @@ reload: } } - Builder builder(globals.state, globals.config, &globals.disk_interface); + Builder builder(globals.state, globals.config, &build_log, + &globals.disk_interface); int result = RunBuild(&builder, argc, argv); if (g_metrics) { g_metrics->Report(); diff --git a/src/state.cc b/src/state.cc index 9f38fe9..4c7168b 100644 --- a/src/state.cc +++ b/src/state.cc @@ -24,7 +24,7 @@ const Rule State::kPhonyRule("phony"); -State::State() : build_log_(NULL) { +State::State() { AddRule(&kPhonyRule); } diff --git a/src/state.h b/src/state.h index 9197ef8..026acf3 100644 --- a/src/state.h +++ b/src/state.h @@ -23,7 +23,6 @@ using namespace std; #include "eval_env.h" #include "hash_map.h" -struct BuildLog; struct Edge; struct Node; struct Rule; @@ -71,7 +70,6 @@ struct State { BindingEnv bindings_; vector defaults_; - BuildLog* build_log_; }; #endif // NINJA_STATE_H_ -- cgit v0.12 From c9f4a275a5794b60bdb2d8a4a4e9358b060d550c Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 18:43:45 -0400 Subject: clarify setter --- src/graph.h | 4 +++- src/state_test.cc | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/graph.h b/src/graph.h index 893ec09..0e203b4 100644 --- a/src/graph.h +++ b/src/graph.h @@ -112,12 +112,14 @@ struct Rule { bool restat() const { return restat_; } const EvalString& command() const { return command_; } - EvalString& command() { return command_; } const EvalString& description() const { return description_; } const EvalString& depfile() const { return depfile_; } const EvalString& rspfile() const { return rspfile_; } const EvalString& rspfile_content() const { return rspfile_content_; } + /// Used by a test. + void set_command(const EvalString& command) { command_ = command; } + private: // Allow the parsers to reach into this object and fill out its fields. friend struct ManifestParser; diff --git a/src/state_test.cc b/src/state_test.cc index 354468b..bc24edd 100644 --- a/src/state_test.cc +++ b/src/state_test.cc @@ -22,11 +22,14 @@ namespace { TEST(State, Basic) { State state; + EvalString command; + command.AddText("cat "); + command.AddSpecial("in"); + command.AddText(" > "); + command.AddSpecial("out"); + Rule* rule = new Rule("cat"); - rule->command().AddText("cat "); - rule->command().AddSpecial("in"); - rule->command().AddText(" > "); - rule->command().AddSpecial("out"); + rule->set_command(command); state.AddRule(rule); Edge* edge = state.AddEdge(rule); -- cgit v0.12 From e7014fdbdab612535bc52880c7a2a9d0b45e35fd Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 18:48:49 -0400 Subject: remove unfortunate header dependency This was temporarily added, and now it can be removed. --- src/graph.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/graph.h b/src/graph.h index 0e203b4..272fcb9 100644 --- a/src/graph.h +++ b/src/graph.h @@ -20,7 +20,6 @@ using namespace std; #include "eval_env.h" -#include "state.h" #include "timestamp.h" struct DiskInterface; -- cgit v0.12 From 3d9c93205ec429d544887fa148b3ae050dfd74b5 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 18:52:22 -0400 Subject: remove DiskInterface from globals --- src/ninja.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index fb2bb3c..d7248ac 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -67,8 +67,6 @@ struct Globals { BuildConfig config; /// Loaded state (rules, nodes). This is a pointer so it can be reset. State* state; - /// Functions for interacting with the disk. - RealDiskInterface disk_interface; }; /// Print usage information. @@ -725,6 +723,7 @@ int NinjaMain(int argc, char** argv) { bool rebuilt_manifest = false; reload: + RealDiskInterface disk_interface; RealFileReader file_reader; ManifestParser parser(globals.state, &file_reader); string err; @@ -743,7 +742,7 @@ reload: string log_path = kLogPath; if (!build_dir.empty()) { log_path = build_dir + "/" + kLogPath; - if (!globals.disk_interface.MakeDirs(log_path) && errno != EEXIST) { + if (!disk_interface.MakeDirs(log_path) && errno != EEXIST) { Error("creating build directory %s: %s", build_dir.c_str(), strerror(errno)); return 1; @@ -770,7 +769,7 @@ reload: if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild // target that is never up to date. Builder manifest_builder(globals.state, globals.config, &build_log, - &globals.disk_interface); + &disk_interface); if (RebuildManifest(&manifest_builder, input_file, &err)) { rebuilt_manifest = true; globals.ResetState(); @@ -782,7 +781,7 @@ reload: } Builder builder(globals.state, globals.config, &build_log, - &globals.disk_interface); + &disk_interface); int result = RunBuild(&builder, argc, argv); if (g_metrics) { g_metrics->Report(); -- cgit v0.12 From a737983697db99a27b4a2a5d8b25da2d54c03ea2 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 18:57:16 -0400 Subject: move BuildConfig out of globals --- src/ninja.cc | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index d7248ac..52a470f 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -63,8 +63,8 @@ struct Globals { /// Command line used to run Ninja. const char* ninja_command; - /// Build configuration (e.g. parallelism). - BuildConfig config; + /// Build configuration set from flags (e.g. parallelism). + BuildConfig* config; /// Loaded state (rules, nodes). This is a pointer so it can be reset. State* state; }; @@ -456,7 +456,7 @@ int ToolClean(Globals* globals, int argc, char* argv[]) { return 1; } - Cleaner cleaner(globals->state, globals->config); + Cleaner cleaner(globals->state, *globals->config); if (argc >= 1) { if (clean_rules) return cleaner.CleanRules(argc, argv); @@ -631,15 +631,17 @@ int ExceptionFilter(unsigned int code, struct _EXCEPTION_POINTERS *ep) { #endif // _MSC_VER int NinjaMain(int argc, char** argv) { + BuildConfig config; Globals globals; globals.ninja_command = argv[0]; + globals.config = &config; const char* input_file = "build.ninja"; const char* working_dir = NULL; string tool; setvbuf(stdout, NULL, _IOLBF, BUFSIZ); - globals.config.parallelism = GuessParallelism(); + config.parallelism = GuessParallelism(); enum { OPT_VERSION = 1 }; const option kLongOptions[] = { @@ -661,14 +663,14 @@ int NinjaMain(int argc, char** argv) { input_file = optarg; break; case 'j': - globals.config.parallelism = atoi(optarg); + config.parallelism = atoi(optarg); break; case 'l': { char* end; double value = strtod(optarg, &end); if (end == optarg) Fatal("-l parameter not numeric: did you mean -l 0.0?"); - globals.config.max_load_average = value; + config.max_load_average = value; break; } case 'k': { @@ -680,14 +682,14 @@ int NinjaMain(int argc, char** argv) { // We want to go until N jobs fail, which means we should allow // N failures and then stop. For N <= 0, INT_MAX is close enough // to infinite for most sane builds. - globals.config.failures_allowed = value > 0 ? value : INT_MAX; + config.failures_allowed = value > 0 ? value : INT_MAX; break; } case 'n': - globals.config.dry_run = true; + config.dry_run = true; break; case 'v': - globals.config.verbosity = BuildConfig::VERBOSE; + config.verbosity = BuildConfig::VERBOSE; break; case 't': tool = optarg; @@ -700,7 +702,7 @@ int NinjaMain(int argc, char** argv) { return 0; case 'h': default: - Usage(globals.config); + Usage(config); return 1; } } @@ -759,7 +761,7 @@ reload: err.clear(); } - if (!globals.config.dry_run) { + if (!config.dry_run) { if (!build_log.OpenForWrite(log_path, &err)) { Error("opening build log: %s", err.c_str()); return 1; @@ -768,7 +770,7 @@ reload: if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild // target that is never up to date. - Builder manifest_builder(globals.state, globals.config, &build_log, + Builder manifest_builder(globals.state, config, &build_log, &disk_interface); if (RebuildManifest(&manifest_builder, input_file, &err)) { rebuilt_manifest = true; @@ -780,8 +782,7 @@ reload: } } - Builder builder(globals.state, globals.config, &build_log, - &disk_interface); + Builder builder(globals.state, config, &build_log, &disk_interface); int result = RunBuild(&builder, argc, argv); if (g_metrics) { g_metrics->Report(); -- cgit v0.12 From f9d9daded0a351ac4311d57d9e806ae1e4208ce3 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 19:00:07 -0400 Subject: reduce indent --- src/ninja.cc | 77 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index 52a470f..8d1c710 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -145,51 +145,50 @@ bool CollectTargetsFromArgs(State* state, int argc, char* argv[], vector* targets, string* err) { if (argc == 0) { *targets = state->DefaultNodes(err); - if (!err->empty()) + return err->empty(); + } + + for (int i = 0; i < argc; ++i) { + string path = argv[i]; + if (!CanonicalizePath(&path, err)) return false; - } else { - for (int i = 0; i < argc; ++i) { - string path = argv[i]; - if (!CanonicalizePath(&path, err)) - return false; - - // Special syntax: "foo.cc^" means "the first output of foo.cc". - bool first_dependent = false; - if (!path.empty() && path[path.size() - 1] == '^') { - path.resize(path.size() - 1); - first_dependent = true; - } - Node* node = state->LookupNode(path); - if (node) { - if (first_dependent) { - if (node->out_edges().empty()) { - *err = "'" + path + "' has no out edge"; - return false; - } - Edge* edge = node->out_edges()[0]; - if (edge->outputs_.empty()) { - edge->Dump(); - Fatal("edge has no outputs"); - } - node = edge->outputs_[0]; + // Special syntax: "foo.cc^" means "the first output of foo.cc". + bool first_dependent = false; + if (!path.empty() && path[path.size() - 1] == '^') { + path.resize(path.size() - 1); + first_dependent = true; + } + + Node* node = state->LookupNode(path); + if (node) { + if (first_dependent) { + if (node->out_edges().empty()) { + *err = "'" + path + "' has no out edge"; + return false; + } + Edge* edge = node->out_edges()[0]; + if (edge->outputs_.empty()) { + edge->Dump(); + Fatal("edge has no outputs"); } - targets->push_back(node); + node = edge->outputs_[0]; + } + targets->push_back(node); + } else { + *err = "unknown target '" + path + "'"; + + if (path == "clean") { + *err += ", did you mean 'ninja -t clean'?"; + } else if (path == "help") { + *err += ", did you mean 'ninja -h'?"; } else { - *err = "unknown target '" + path + "'"; - - if (path == "clean") { - *err += ", did you mean 'ninja -t clean'?"; - } else if (path == "help") { - *err += ", did you mean 'ninja -h'?"; - } else { - Node* suggestion = state->SpellcheckNode(path); - if (suggestion) { - *err += ", did you mean '" + suggestion->path() + "'?"; - } + Node* suggestion = state->SpellcheckNode(path); + if (suggestion) { + *err += ", did you mean '" + suggestion->path() + "'?"; } - return false; } + return false; } } return true; -- cgit v0.12 From d442b1c850236bde6e7a206d22c3a295744393ad Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 19:03:35 -0400 Subject: show all in graph --- configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.py b/configure.py index 69e00d5..ec26a37 100755 --- a/configure.py +++ b/configure.py @@ -382,7 +382,7 @@ n.newline() n.comment('Generate a graph using the "graph" tool.') n.rule('gendot', - command='./ninja -t graph > $out') + command='./ninja -t graph all > $out') n.rule('gengraph', command='dot -Tpng $in > $out') dot = n.build(built('graph.dot'), 'gendot', ['ninja', 'build.ninja']) -- cgit v0.12 From d6ff278e22c3392704eec71250584e21966534fe Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 4 Sep 2012 21:41:13 -0400 Subject: rearrange tool-picking logic Now "ninja -t list" works from any directory. --- src/ninja.cc | 55 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index 8d1c710..d194699 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -69,6 +69,9 @@ struct Globals { State* state; }; +/// The type of functions that are the entry points to tools (subcommands). +typedef int (*ToolFunc)(Globals*, int, char**); + /// Print usage information. void Usage(const BuildConfig& config) { fprintf(stderr, @@ -491,13 +494,14 @@ void ToolUrtle() { } } -int RunTool(const string& tool, Globals* globals, int argc, char** argv) { - typedef int (*ToolFunc)(Globals*, int, char**); - struct Tool { +/// Find the function to execute for \a tool_name and return it via \a func. +/// If there is no tool to run (e.g.: unknown tool), returns an exit code. +int ChooseTool(const string& tool_name, ToolFunc* func) { + const struct Tool { const char* name; const char* desc; ToolFunc func; - } tools[] = { + } kTools[] = { #if !defined(_WIN32) && !defined(NINJA_BOOTSTRAP) { "browse", "browse dependency graph in a web browser", ToolBrowse }, @@ -517,30 +521,33 @@ int RunTool(const string& tool, Globals* globals, int argc, char** argv) { { NULL, NULL, NULL } }; - if (tool == "list") { + if (tool_name == "list") { printf("ninja subtools:\n"); - for (int i = 0; tools[i].name; ++i) { - printf("%10s %s\n", tools[i].name, tools[i].desc); + for (const Tool* tool = &kTools[0]; tool->name; ++tool) { + printf("%10s %s\n", tool->name, tool->desc); } return 0; - } else if (tool == "urtle") { + } else if (tool_name == "urtle") { ToolUrtle(); return 0; } - for (int i = 0; tools[i].name; ++i) { - if (tool == tools[i].name) - return tools[i].func(globals, argc, argv); + for (const Tool* tool = &kTools[0]; tool->name; ++tool) { + if (tool->name == tool_name) { + *func = tool->func; + return 0; + } } vector words; - for (int i = 0; tools[i].name; ++i) - words.push_back(tools[i].name); - const char* suggestion = SpellcheckStringV(tool, words); + for (const Tool* tool = &kTools[0]; tool->name; ++tool) + words.push_back(tool->name); + const char* suggestion = SpellcheckStringV(tool_name, words); if (suggestion) { - Error("unknown tool '%s', did you mean '%s'?", tool.c_str(), suggestion); + Error("unknown tool '%s', did you mean '%s'?", + tool_name.c_str(), suggestion); } else { - Error("unknown tool '%s'", tool.c_str()); + Error("unknown tool '%s'", tool_name.c_str()); } return 1; } @@ -708,6 +715,15 @@ int NinjaMain(int argc, char** argv) { argv += optind; argc -= optind; + // If specified, select a tool as early as possible, so commands like + // -t list can succeed before we attempt to load build.ninja etc. + ToolFunc tool_func = NULL; + if (!tool.empty()) { + int exit_code = ChooseTool(tool, &tool_func); + if (!tool_func) + return exit_code; + } + if (working_dir) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for @@ -733,8 +749,11 @@ reload: return 1; } - if (!tool.empty()) - return RunTool(tool, &globals, argc, argv); + // TODO: some tools want to run before we load the above manifest. + // A tool probably needs to be able to specify where in the startup + // process it runs. + if (tool_func) + return tool_func(&globals, argc, argv); BuildLog build_log; -- cgit v0.12 From c84ee51b5a8f4587db2141f04e5da45f2f0ada9b Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Wed, 5 Sep 2012 09:14:39 -0700 Subject: factor out build log load from main --- src/ninja.cc | 65 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index d194699..c55ba38 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -573,6 +573,42 @@ bool DebugEnable(const string& name, Globals* globals) { } } +bool OpenLog(BuildLog* build_log, Globals* globals, + DiskInterface* disk_interface) { + const string build_dir = + globals->state->bindings_.LookupVariable("builddir"); + const char* kLogPath = ".ninja_log"; + string log_path = kLogPath; + if (!build_dir.empty()) { + log_path = build_dir + "/" + kLogPath; + if (!disk_interface->MakeDirs(log_path) && errno != EEXIST) { + Error("creating build directory %s: %s", + build_dir.c_str(), strerror(errno)); + return false; + } + } + + string err; + if (!build_log->Load(log_path, &err)) { + Error("loading build log %s: %s", log_path.c_str(), err.c_str()); + return false; + } + if (!err.empty()) { + // Hack: Load() can return a warning via err by returning true. + Warning("%s", err.c_str()); + err.clear(); + } + + if (!globals->config->dry_run) { + if (!build_log->OpenForWrite(log_path, &err)) { + Error("opening build log: %s", err.c_str()); + return false; + } + } + + return true; +} + int RunBuild(Builder* builder, int argc, char** argv) { string err; vector targets; @@ -756,35 +792,8 @@ reload: return tool_func(&globals, argc, argv); BuildLog build_log; - - const string build_dir = globals.state->bindings_.LookupVariable("builddir"); - const char* kLogPath = ".ninja_log"; - string log_path = kLogPath; - if (!build_dir.empty()) { - log_path = build_dir + "/" + kLogPath; - if (!disk_interface.MakeDirs(log_path) && errno != EEXIST) { - Error("creating build directory %s: %s", - build_dir.c_str(), strerror(errno)); - return 1; - } - } - - if (!build_log.Load(log_path, &err)) { - Error("loading build log %s: %s", log_path.c_str(), err.c_str()); + if (!OpenLog(&build_log, &globals, &disk_interface)) return 1; - } - if (!err.empty()) { - // Hack: Load() can return a warning via err by returning true. - Warning("%s", err.c_str()); - err.clear(); - } - - if (!config.dry_run) { - if (!build_log.OpenForWrite(log_path, &err)) { - Error("opening build log: %s", err.c_str()); - return 1; - } - } if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild // target that is never up to date. -- cgit v0.12 From 14863bbc3337a64099f8509daf407f8ec1570c38 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Wed, 5 Sep 2012 09:19:26 -0700 Subject: factor out metrics dump from ninja main --- src/ninja.cc | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index c55ba38..582f38e 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -609,6 +609,22 @@ bool OpenLog(BuildLog* build_log, Globals* globals, return true; } +/// Dump the output requested by '-d stats'. +void DumpMetrics(Globals* globals) { + g_metrics->Report(); + + printf("\n"); + int count = (int)globals->state->paths_.size(); + int buckets = +#ifdef _MSC_VER + (int)globals->state->paths_.comp.bucket_size; +#else + (int)globals->state->paths_.bucket_count(); +#endif + printf("path->node hash load %.2f (%d entries / %d buckets)\n", + count / (double) buckets, count, buckets); +} + int RunBuild(Builder* builder, int argc, char** argv) { string err; vector targets; @@ -811,20 +827,8 @@ reload: Builder builder(globals.state, config, &build_log, &disk_interface); int result = RunBuild(&builder, argc, argv); - if (g_metrics) { - g_metrics->Report(); - - printf("\n"); - int count = (int)globals.state->paths_.size(); - int buckets = -#ifdef _MSC_VER - (int)globals.state->paths_.comp.bucket_size; -#else - (int)globals.state->paths_.bucket_count(); -#endif - printf("path->node hash load %.2f (%d entries / %d buckets)\n", - count / (double) buckets, count, buckets); - } + if (g_metrics) + DumpMetrics(&globals); return result; } -- cgit v0.12 From d82206d7eac1b360034f92875ecce455e0701cc9 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Thu, 6 Sep 2012 11:13:24 -0700 Subject: allow tools to specify when they run (before/after build.ninja load) This will be necessary to inline msvc-helper as well as so -t graph can get depfiles. --- src/ninja.cc | 86 ++++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 34 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index 582f38e..7585628 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -72,6 +72,27 @@ struct Globals { /// The type of functions that are the entry points to tools (subcommands). typedef int (*ToolFunc)(Globals*, int, char**); +/// Subtools, accessible via "-t foo". +struct Tool { + /// Short name of the tool. + const char* name; + + /// Description (shown in "-t list"). + const char* desc; + + /// When to run the tool. + enum { + /// Run after parsing the command-line flags (as early as possible). + RUN_AFTER_FLAGS, + + /// Run after loading build.ninja. + RUN_AFTER_LOAD, + } when; + + /// Implementation of the tool. + ToolFunc func; +}; + /// Print usage information. void Usage(const BuildConfig& config) { fprintf(stderr, @@ -469,7 +490,7 @@ int ToolClean(Globals* globals, int argc, char* argv[]) { } } -void ToolUrtle() { +int ToolUrtle(Globals* globals, int argc, char** argv) { // RLE encoded. const char* urtle = " 13 ,3;2!2;\n8 ,;<11!;\n5 `'<10!(2`'2!\n11 ,6;, `\\. `\\9 .,c13$ec,.\n6 " @@ -492,49 +513,46 @@ void ToolUrtle() { count = 0; } } + return 0; } /// Find the function to execute for \a tool_name and return it via \a func. /// If there is no tool to run (e.g.: unknown tool), returns an exit code. -int ChooseTool(const string& tool_name, ToolFunc* func) { - const struct Tool { - const char* name; - const char* desc; - ToolFunc func; - } kTools[] = { +int ChooseTool(const string& tool_name, const Tool** tool_out) { + static const Tool kTools[] = { #if !defined(_WIN32) && !defined(NINJA_BOOTSTRAP) { "browse", "browse dependency graph in a web browser", - ToolBrowse }, + Tool::RUN_AFTER_LOAD, ToolBrowse }, #endif { "clean", "clean built files", - ToolClean }, + Tool::RUN_AFTER_LOAD, ToolClean }, { "commands", "list all commands required to rebuild given targets", - ToolCommands }, + Tool::RUN_AFTER_LOAD, ToolCommands }, { "graph", "output graphviz dot file for targets", - ToolGraph }, + Tool::RUN_AFTER_LOAD, ToolGraph }, { "query", "show inputs/outputs for a path", - ToolQuery }, + Tool::RUN_AFTER_LOAD, ToolQuery }, { "rules", "list all rules", - ToolRules }, + Tool::RUN_AFTER_LOAD, ToolRules }, { "targets", "list targets by their rule or depth in the DAG", - ToolTargets }, - { NULL, NULL, NULL } + Tool::RUN_AFTER_LOAD, ToolTargets }, + { "urtle", NULL, + Tool::RUN_AFTER_FLAGS, ToolUrtle }, + { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL } }; if (tool_name == "list") { printf("ninja subtools:\n"); for (const Tool* tool = &kTools[0]; tool->name; ++tool) { - printf("%10s %s\n", tool->name, tool->desc); + if (tool->desc) + printf("%10s %s\n", tool->name, tool->desc); } return 0; - } else if (tool_name == "urtle") { - ToolUrtle(); - return 0; } for (const Tool* tool = &kTools[0]; tool->name; ++tool) { if (tool->name == tool_name) { - *func = tool->func; + *tool_out = tool; return 0; } } @@ -695,7 +713,7 @@ int NinjaMain(int argc, char** argv) { globals.config = &config; const char* input_file = "build.ninja"; const char* working_dir = NULL; - string tool; + string tool_name; setvbuf(stdout, NULL, _IOLBF, BUFSIZ); @@ -709,7 +727,7 @@ int NinjaMain(int argc, char** argv) { }; int opt; - while (tool.empty() && + while (tool_name.empty() && (opt = getopt_long(argc, argv, "d:f:hj:k:l:nt:vC:V", kLongOptions, NULL)) != -1) { switch (opt) { @@ -750,7 +768,7 @@ int NinjaMain(int argc, char** argv) { config.verbosity = BuildConfig::VERBOSE; break; case 't': - tool = optarg; + tool_name = optarg; break; case 'C': working_dir = optarg; @@ -768,21 +786,24 @@ int NinjaMain(int argc, char** argv) { argc -= optind; // If specified, select a tool as early as possible, so commands like - // -t list can succeed before we attempt to load build.ninja etc. - ToolFunc tool_func = NULL; - if (!tool.empty()) { - int exit_code = ChooseTool(tool, &tool_func); - if (!tool_func) + // -t list can run before we attempt to load build.ninja etc. + const Tool* tool = NULL; + if (!tool_name.empty()) { + int exit_code = ChooseTool(tool_name, &tool); + if (!tool) return exit_code; } + if (tool && tool->when == Tool::RUN_AFTER_FLAGS) + return tool->func(&globals, argc, argv); + if (working_dir) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for // subsequent commands. // Don't print this if a tool is being used, so that tool output // can be piped into a file without this string showing up. - if (tool == "") + if (!tool) printf("ninja: Entering directory `%s'\n", working_dir); if (chdir(working_dir) < 0) { Fatal("chdir to '%s' - %s", working_dir, strerror(errno)); @@ -801,11 +822,8 @@ reload: return 1; } - // TODO: some tools want to run before we load the above manifest. - // A tool probably needs to be able to specify where in the startup - // process it runs. - if (tool_func) - return tool_func(&globals, argc, argv); + if (tool && tool->when == Tool::RUN_AFTER_LOAD) + return tool->func(&globals, argc, argv); BuildLog build_log; if (!OpenLog(&build_log, &globals, &disk_interface)) -- cgit v0.12 From 31330231f1955279a6296cf9425a1ea9221919b1 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 7 Sep 2012 12:36:16 -0700 Subject: drop special case for msvc bucket count Calling bucket_count() works locally with MSVC. I wonder if some other change in the code fixed this. --- src/ninja.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index 7585628..fb273be 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -633,12 +633,7 @@ void DumpMetrics(Globals* globals) { printf("\n"); int count = (int)globals->state->paths_.size(); - int buckets = -#ifdef _MSC_VER - (int)globals->state->paths_.comp.bucket_size; -#else - (int)globals->state->paths_.bucket_count(); -#endif + int buckets = (int)globals->state->paths_.bucket_count(); printf("path->node hash load %.2f (%d entries / %d buckets)\n", count / (double) buckets, count, buckets); } -- cgit v0.12 From 2c145bbf2ebd80e9d6223a0672600e37a4589d32 Mon Sep 17 00:00:00 2001 From: Alex Caudill Date: Fri, 7 Sep 2012 12:46:17 -0700 Subject: add GetProcessorCount() implementation for Solaris --- src/util.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/util.cc b/src/util.cc index 580f22a..0feb99d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -36,6 +36,9 @@ #if defined(__APPLE__) || defined(__FreeBSD__) #include +#elif defined(__SVR4) && defined(__sun) +#include +#include #elif defined(linux) #include #endif @@ -314,6 +317,12 @@ int GetProcessorCount() { GetSystemInfo(&info); return info.dwNumberOfProcessors; } +#else +// This is what get_nprocs() should be doing in the Linux implementation +// above, but in a more standard way. +int GetProcessorCount() { + return sysconf(_SC_NPROCESSORS_ONLN); +} #endif #ifdef _WIN32 -- cgit v0.12 From c840cf6e7ead218828f1e24097b5de07169b83b7 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 7 Sep 2012 13:10:48 -0700 Subject: disable a new warning that popped up on Windows This pattern is safe as long as you're careful; we don't use it very much. --- configure.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.py b/configure.py index ec26a37..85f3689 100755 --- a/configure.py +++ b/configure.py @@ -118,6 +118,8 @@ if platform == 'windows': '/WX', # Warnings as errors. '/wd4530', '/wd4100', '/wd4706', '/wd4512', '/wd4800', '/wd4702', '/wd4819', + # Disable warnings about passing "this" during initialization. + '/wd4355', '/GR-', # Disable RTTI. # Disable size_t -> int truncation warning. # We never have strings or arrays larger than 2**31. -- cgit v0.12 From 241b29087a0c5c47dc216af0fa7a4697359fcc0d Mon Sep 17 00:00:00 2001 From: Alex Caudill Date: Fri, 7 Sep 2012 19:56:57 +0000 Subject: add solaris platform definitions --- configure.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/configure.py b/configure.py index 85f3689..c2daf0a 100755 --- a/configure.py +++ b/configure.py @@ -27,7 +27,7 @@ sys.path.insert(0, 'misc') import ninja_syntax parser = OptionParser() -platforms = ['linux', 'freebsd', 'mingw', 'windows'] +platforms = ['linux', 'freebsd', 'solaris', 'mingw', 'windows'] profilers = ['gmon', 'pprof'] parser.add_option('--platform', help='target platform (' + '/'.join(platforms) + ')', @@ -59,6 +59,8 @@ if platform is None: platform = 'linux' elif platform.startswith('freebsd'): platform = 'freebsd' + elif platform.startswith('solaris'): + platform = 'solaris' elif platform.startswith('mingw'): platform = 'mingw' elif platform.startswith('win'): @@ -222,7 +224,7 @@ n.newline() objs = [] -if platform not in ('mingw', 'windows'): +if platform not in ('solaris', 'mingw', 'windows'): n.comment('browse_py.h is used to inline browse.py.') n.rule('inline', command='src/inline.sh $varname < $in > $out', -- cgit v0.12 From 0c64d16e8945040cb02376a23d9e521b0382cc98 Mon Sep 17 00:00:00 2001 From: Alex Caudill Date: Fri, 7 Sep 2012 13:24:26 -0700 Subject: include termios for solaris --- src/build.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/build.cc b/src/build.cc index efc05b0..e1aaad1 100644 --- a/src/build.cc +++ b/src/build.cc @@ -26,6 +26,10 @@ #include #endif +#if defined(__SVR4) && defined(__sun) +#include +#endif + #include "build_log.h" #include "disk_interface.h" #include "graph.h" -- cgit v0.12 From fcbfc161e758f22e8dfee4a8b5b3feaca1db86ff Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 7 Sep 2012 13:26:52 -0700 Subject: scoping workaround for gcc on Windows From https://github.com/martine/ninja/issues/410. --- src/includes_normalize-win32.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 134cfb8..4bc8756 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -61,7 +61,7 @@ vector IncludesNormalize::Split(const string& input, char sep) { string IncludesNormalize::ToLower(const string& s) { string ret; - transform(s.begin(), s.end(), back_inserter(ret), tolower); + transform(s.begin(), s.end(), back_inserter(ret), ::tolower); return ret; } -- cgit v0.12 From 547f959c567fae22a4e2acf1c782b6857b657f8c Mon Sep 17 00:00:00 2001 From: "Robert A. Iannucci Jr" Date: Mon, 10 Sep 2012 20:42:32 -0700 Subject: Fix re2c detection which breaks win32 w/ gnuwin32 --- configure.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.py b/configure.py index c2daf0a..ca6615c 100755 --- a/configure.py +++ b/configure.py @@ -241,8 +241,8 @@ n.comment('the depfile parser and ninja lexers are generated using re2c.') def has_re2c(): import subprocess try: - subprocess.call(['re2c', '-v'], stdout=subprocess.PIPE) - return True + proc = subprocess.Popen(['re2c', '-V'], stdout=subprocess.PIPE) + return int(proc.communicate()[0], 10) >= 1103 except OSError: return False if has_re2c(): @@ -253,8 +253,8 @@ if has_re2c(): n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) else: - print ("warning: re2c not found; changes to src/*.in.cc will not affect " - "your build.") + print ("warning: A compatible version of re2c (>= 0.11.3) was not found; " + "changes to src/*.in.cc will not affect your build.") n.newline() n.comment('Core source files all build into ninja library.') -- cgit v0.12 From c57f8a4258998a185f961ce12fd7c905a3e86180 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Thu, 13 Sep 2012 11:07:45 -0700 Subject: windows: merge msvc-helper into ninja.exe itself Now "ninja -t msvc ..." passes the "..." to the msvc helper main. This drastically simplifies bootstrap and makes ninja a single binary again. --- bootstrap.py | 28 ++++++++-------------------- configure.py | 21 +++++++-------------- src/msvc_helper_main-win32.cc | 10 +++++----- src/ninja.cc | 17 +++++++++++++++++ 4 files changed, 37 insertions(+), 39 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index abd2528..3032a9b 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -19,6 +19,7 @@ import os import glob import errno import shlex +import shutil import subprocess os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -68,8 +69,6 @@ for src in glob.glob('src/*.cc'): else: if src.endswith('-win32.cc'): continue - if '_main' in src: - continue sources.append(src) @@ -113,25 +112,14 @@ if options.verbose: verbose = ['-v'] if sys.platform.startswith('win32'): - # Build ninja-msvc-helper using ninja without an msvc-helper. - print 'Building ninja-msvc-helper...' - run([sys.executable, 'configure.py', '--with-msvc-helper='] + conf_args) - run(['./' + binary] + verbose + ['ninja-msvc-helper']) - - # Rename the helper to the same name + .bootstrap. - helper_binary = 'ninja-msvc-helper.bootstrap.exe' - try: - os.unlink(helper_binary) - except: - pass - os.rename('ninja-msvc-helper.exe', helper_binary) - - # Build ninja using the newly-built msvc-helper. print 'Building ninja using itself...' - run([sys.executable, 'configure.py', - '--with-msvc-helper=%s' % helper_binary] + conf_args) + run([sys.executable, 'configure.py', '--with-ninja=%s' % binary] + + conf_args) run(['./' + binary] + verbose) + # Copy the new executable over the bootstrap one. + shutil.copyfile('ninja.exe', binary) + # Clean up. for obj in glob.glob('*.obj'): os.unlink(obj) @@ -142,8 +130,8 @@ Done! Note: to work around Windows file locking, where you can't rebuild an in-use binary, to run ninja after making any changes to build ninja itself you should run ninja.bootstrap instead. Your build is also configured to -use ninja-msvc-helper.bootstrap.exe instead of the ninja-msvc-helper.exe -that it builds; see the --help output of configure.py.""" +use ninja.bootstrap.exe as the MSVC helper; see the --with-ninja flag of +the --help output of configure.py.""" else: print 'Building ninja using itself...' run([sys.executable, 'configure.py'] + conf_args) diff --git a/configure.py b/configure.py index ca6615c..98274e6 100755 --- a/configure.py +++ b/configure.py @@ -45,8 +45,9 @@ parser.add_option('--with-gtest', metavar='PATH', parser.add_option('--with-python', metavar='EXE', help='use EXE as the Python interpreter', default=os.path.basename(sys.executable)) -parser.add_option('--with-msvc-helper', metavar='NAME', - help="name for ninja-msvc-helper binary (MSVC only)") +parser.add_option('--with-ninja', metavar='NAME', + help="name for ninja binary for -t msvc (MSVC only)", + default="ninja") (options, args) = parser.parse_args() if args: print 'ERROR: extra unparsed command-line arguments:', args @@ -185,8 +186,9 @@ n.newline() if platform == 'windows': compiler = '$cxx' - if options.with_msvc_helper: - compiler = '%s -o $out -- $cxx /showIncludes' % options.with_msvc_helper + if options.with_ninja: + compiler = ('%s -t msvc -o $out -- $cxx /showIncludes' % + options.with_ninja) n.rule('cxx', command='%s $cflags -c $in /Fo$out' % compiler, depfile='$out.d', @@ -279,6 +281,7 @@ if platform in ('mingw', 'windows'): if platform == 'windows': objs += cxx('includes_normalize-win32') objs += cxx('msvc_helper-win32') + objs += cxx('msvc_helper_main-win32') objs += cxx('minidump-win32') objs += cc('getopt') else: @@ -303,16 +306,6 @@ ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib, n.newline() all_targets += ninja -if platform == 'windows': - n.comment('Helper for working with MSVC.') - msvc_helper = n.build(binary('ninja-msvc-helper'), 'link', - cxx('msvc_helper_main-win32'), - implicit=ninja_lib, - variables=[('libs', libs)]) - n.default(msvc_helper) - n.newline() - all_targets += msvc_helper - n.comment('Tests all build into ninja_test executable.') variables = [] diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index f265010..0c8db37 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -24,9 +24,7 @@ namespace { void Usage() { printf( -"ninja-msvc-helper: adjust msvc command-line tools for use by ninja.\n" -"\n" -"usage: ninja-mvsc-helper [options] -- command args\n" +"usage: ninja -t msvc [options] -- cl.exe /showIncludes /otherArgs\n" "options:\n" " -e ENVFILE load environment block from ENVFILE as environment\n" " -r BASE normalize paths and make relative to BASE before output\n" @@ -48,7 +46,7 @@ void PushPathIntoEnvironment(const string& env_block) { } // anonymous namespace -int main(int argc, char** argv) { +int MSVCHelperMain(int argc, char** argv) { const char* output_filename = NULL; const char* relative_to = NULL; const char* envfile = NULL; @@ -76,8 +74,10 @@ int main(int argc, char** argv) { } } - if (!output_filename) + if (!output_filename) { + Usage(); Fatal("-o required"); + } string env; if (envfile) { diff --git a/src/ninja.cc b/src/ninja.cc index fb273be..089dbc1 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -42,6 +42,9 @@ #include "state.h" #include "util.h" +// Defined in msvc_helper_main-win32.cc. +int MSVCHelperMain(int argc, char** argv); + namespace { /// The version number of the current Ninja release. This will always @@ -289,6 +292,16 @@ int ToolBrowse(Globals* globals, int argc, char* argv[]) { } #endif // _WIN32 +#if defined(WIN32) +int ToolMSVC(Globals* globals, int argc, char* argv[]) { + // Reset getopt: push one argument onto the front of argv, reset optind. + argc++; + argv--; + optind = 0; + return MSVCHelperMain(argc, argv); +} +#endif + int ToolTargetsList(const vector& nodes, int depth, int indent) { for (vector::const_iterator n = nodes.begin(); n != nodes.end(); @@ -524,6 +537,10 @@ int ChooseTool(const string& tool_name, const Tool** tool_out) { { "browse", "browse dependency graph in a web browser", Tool::RUN_AFTER_LOAD, ToolBrowse }, #endif +#if defined(WIN32) + { "msvc", "build helper for MSVC cl.exe", + Tool::RUN_AFTER_FLAGS, ToolMSVC }, +#endif { "clean", "clean built files", Tool::RUN_AFTER_LOAD, ToolClean }, { "commands", "list all commands required to rebuild given targets", -- cgit v0.12 From 06fa62352d1e9868409b299ffc8abc8f4cd9a39d Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Thu, 13 Sep 2012 15:31:02 -0700 Subject: mark msvc-helper as experimental --- src/ninja.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ninja.cc b/src/ninja.cc index 089dbc1..ad56f1c 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -538,7 +538,7 @@ int ChooseTool(const string& tool_name, const Tool** tool_out) { Tool::RUN_AFTER_LOAD, ToolBrowse }, #endif #if defined(WIN32) - { "msvc", "build helper for MSVC cl.exe", + { "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)", Tool::RUN_AFTER_FLAGS, ToolMSVC }, #endif { "clean", "clean built files", -- cgit v0.12