diff options
Diffstat (limited to 'src')
45 files changed, 1414 insertions, 340 deletions
diff --git a/src/build.cc b/src/build.cc index 44c1df6..90a84c2 100644 --- a/src/build.cc +++ b/src/build.cc @@ -16,13 +16,14 @@ #include <assert.h> #include <stdio.h> +#include <stdlib.h> #ifdef _WIN32 #include <windows.h> #else +#include <unistd.h> #include <sys/ioctl.h> #include <sys/time.h> -#include <sys/termios.h> #endif #include "build_log.h" @@ -32,47 +33,13 @@ #include "subprocess.h" #include "util.h" -/// Tracks the status of a build: completion fraction, printing updates. -struct BuildStatus { - BuildStatus(const BuildConfig& config); - void PlanHasTotalEdges(int total); - void BuildEdgeStarted(Edge* edge); - void BuildEdgeFinished(Edge* edge, bool success, const string& output, - int* start_time, int* end_time); - void BuildFinished(); - - private: - void PrintStatus(Edge* edge); - - const BuildConfig& config_; - - /// Time the build started. - int64_t start_time_millis_; - /// Time we last printed an update. - int64_t last_update_millis_; - - int started_edges_, finished_edges_, total_edges_; - - bool have_blank_line_; - - /// Map of running edge to time the edge started running. - typedef map<Edge*, int> RunningEdgeMap; - RunningEdgeMap running_edges_; - - /// Whether we can do fancy terminal control codes. - bool smart_terminal_; - -#ifdef _WIN32 - HANDLE console_; -#endif -}; - BuildStatus::BuildStatus(const BuildConfig& config) : config_(config), start_time_millis_(GetTimeMillis()), - last_update_millis_(start_time_millis_), started_edges_(0), finished_edges_(0), total_edges_(0), - have_blank_line_(true) { + have_blank_line_(true), progress_status_format_(NULL), + overall_rate_(), current_rate_(), + current_rate_average_count_(config.parallelism) { #ifndef _WIN32 const char* term = getenv("TERM"); smart_terminal_ = isatty(1) && term && string(term) != "dumb"; @@ -90,6 +57,10 @@ BuildStatus::BuildStatus(const BuildConfig& config) // Don't do anything fancy in verbose mode. if (config_.verbosity != BuildConfig::NORMAL) smart_terminal_ = false; + + progress_status_format_ = getenv("NINJA_STATUS"); + if (!progress_status_format_) + progress_status_format_ = "[%s/%t] "; } void BuildStatus::PlanHasTotalEdges(int total) { @@ -115,7 +86,6 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, RunningEdgeMap::iterator i = running_edges_.find(edge); *start_time = i->second; *end_time = (int)(now - start_time_millis_); - int total_time = end_time - start_time; running_edges_.erase(i); if (config_.verbosity == BuildConfig::QUIET) @@ -124,15 +94,7 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, if (smart_terminal_) PrintStatus(edge); - if (success && output.empty()) { - if (!smart_terminal_) { - if (total_time > 5*1000) { - printf("%.1f%% %d/%d\n", finished_edges_ * 100 / (float)total_edges_, - finished_edges_, total_edges_); - last_update_millis_ = now; - } - } - } else { + if (!success || !output.empty()) { if (smart_terminal_) printf("\n"); @@ -170,6 +132,79 @@ void BuildStatus::BuildFinished() { printf("\n"); } +string BuildStatus::FormatProgressStatus(const char* progress_status_format) const { + string out; + char buf[32]; + for (const char* s = progress_status_format; *s != '\0'; ++s) { + if (*s == '%') { + ++s; + switch (*s) { + case '%': + out.push_back('%'); + break; + + // Started edges. + case 's': + snprintf(buf, sizeof(buf), "%d", started_edges_); + out += buf; + break; + + // Total edges. + case 't': + snprintf(buf, sizeof(buf), "%d", total_edges_); + out += buf; + break; + + // Running edges. + case 'r': + snprintf(buf, sizeof(buf), "%d", started_edges_ - finished_edges_); + out += buf; + break; + + // Unstarted edges. + case 'u': + snprintf(buf, sizeof(buf), "%d", total_edges_ - started_edges_); + out += buf; + break; + + // Finished edges. + case 'f': + snprintf(buf, sizeof(buf), "%d", finished_edges_); + out += buf; + break; + + // Overall finished edges per second. + case 'o': + overall_rate_.UpdateRate(finished_edges_, finished_edges_); + overall_rate_.snprinfRate(buf, "%.1f"); + out += buf; + break; + + // Current rate, average over the last '-j' jobs. + case 'c': + // TODO use sliding window? + if (finished_edges_ > current_rate_.last_update() && + finished_edges_ - current_rate_.last_update() == current_rate_average_count_) { + current_rate_.UpdateRate(current_rate_average_count_, finished_edges_); + current_rate_.Restart(); + } + current_rate_.snprinfRate(buf, "%.0f"); + out += buf; + break; + + default: { + Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s); + return ""; + } + } + } else { + out.push_back(*s); + } + } + + return out; +} + void BuildStatus::PrintStatus(Edge* edge) { if (config_.verbosity == BuildConfig::QUIET) return; @@ -194,15 +229,19 @@ void BuildStatus::PrintStatus(Edge* edge) { #endif } - int progress_chars = printf("[%d/%d] ", started_edges_, total_edges_); + if (finished_edges_ == 0) { + overall_rate_.Restart(); + current_rate_.Restart(); + } + to_print = FormatProgressStatus(progress_status_format_) + to_print; if (smart_terminal_ && !force_full_command) { + const int kMargin = 3; // Space for "...". #ifndef _WIN32 // Limit output to width of the terminal if provided so we don't cause // line-wrapping. winsize size; if ((ioctl(0, TIOCGWINSZ, &size) == 0) && size.ws_col) { - const int kMargin = progress_chars + 3; // Space for [xx/yy] and "...". if (to_print.size() + kMargin > size.ws_col) { int elide_size = (size.ws_col - kMargin) / 2; to_print = to_print.substr(0, elide_size) @@ -211,8 +250,7 @@ void BuildStatus::PrintStatus(Edge* edge) { } } #else - const int kMargin = progress_chars + 3; // Space for [xx/yy] and "...". - // Don't use the full width or console with move to next line. + // Don't use the full width or console will move to next line. size_t width = static_cast<size_t>(csbi.dwSize.X) - 1; if (to_print.size() + kMargin > width) { int elide_size = (width - kMargin) / 2; @@ -223,22 +261,36 @@ void BuildStatus::PrintStatus(Edge* edge) { #endif } - printf("%s", to_print.c_str()); - if (smart_terminal_ && !force_full_command) { #ifndef _WIN32 + printf("%s", to_print.c_str()); printf("\x1B[K"); // Clear to end of line. fflush(stdout); have_blank_line_ = false; #else - // Clear to end of line. + // We don't want to have the cursor spamming back and forth, so + // use WriteConsoleOutput instead which updates the contents of + // the buffer, but doesn't move the cursor position. GetConsoleScreenBufferInfo(console_, &csbi); - int num_spaces = csbi.dwSize.X - 1 - csbi.dwCursorPosition.X; - printf("%*s", num_spaces, ""); + COORD buf_size = { csbi.dwSize.X, 1 }; + COORD zero_zero = { 0, 0 }; + SMALL_RECT target = { csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y, + csbi.dwCursorPosition.X + csbi.dwSize.X - 1, + csbi.dwCursorPosition.Y }; + CHAR_INFO* char_data = new CHAR_INFO[csbi.dwSize.X]; + memset(char_data, 0, sizeof(CHAR_INFO) * csbi.dwSize.X); + for (int i = 0; i < csbi.dwSize.X; ++i) { + char_data[i].Char.AsciiChar = ' '; + char_data[i].Attributes = csbi.wAttributes; + + } + for (size_t i = 0; i < to_print.size(); ++i) + char_data[i].Char.AsciiChar = to_print[i]; + WriteConsoleOutput(console_, char_data, buf_size, zero_zero, &target); have_blank_line_ = false; #endif } else { - printf("\n"); + printf("%s\n", to_print.c_str()); } } @@ -386,7 +438,7 @@ void Plan::CleanNode(BuildLog* build_log, Node* node) { if ((*ni)->mtime() > most_recent_input) most_recent_input = (*ni)->mtime(); string command = (*ei)->EvaluateCommand(true); - + // Now, recompute the dirty state of each output. bool all_outputs_clean = true; for (vector<Node*>::iterator ni = (*ei)->outputs_.begin(); @@ -394,7 +446,7 @@ void Plan::CleanNode(BuildLog* build_log, Node* node) { if (!(*ni)->dirty()) continue; - if ((*ei)->RecomputeOutputDirty(build_log, most_recent_input, command, + if ((*ei)->RecomputeOutputDirty(build_log, most_recent_input, NULL, command, *ni)) { (*ni)->MarkDirty(); all_outputs_clean = false; @@ -425,7 +477,7 @@ void Plan::Dump() { } struct RealCommandRunner : public CommandRunner { - RealCommandRunner(const BuildConfig& config) : config_(config) {} + explicit RealCommandRunner(const BuildConfig& config) : config_(config) {} virtual ~RealCommandRunner() {} virtual bool CanRunMore(); virtual bool StartCommand(Edge* edge); @@ -451,7 +503,9 @@ void RealCommandRunner::Abort() { } bool RealCommandRunner::CanRunMore() { - return ((int)subprocs_.running_.size()) < config_.parallelism; + return ((int)subprocs_.running_.size()) < config_.parallelism + && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f) + || GetLoadAverage() < config_.max_load_average); } bool RealCommandRunner::StartCommand(Edge* edge) { @@ -460,7 +514,7 @@ bool RealCommandRunner::StartCommand(Edge* edge) { if (!subproc) return false; subproc_to_edge_.insert(make_pair(subproc, edge)); - + return true; } @@ -512,10 +566,6 @@ struct DryRunCommandRunner : public CommandRunner { Builder::Builder(State* state, const BuildConfig& config) : state_(state), config_(config) { disk_interface_ = new RealDiskInterface; - if (config.dry_run) - command_runner_.reset(new DryRunCommandRunner); - else - command_runner_.reset(new RealCommandRunner(config)); status_ = new BuildStatus(config); log_ = state->build_log_; } @@ -588,6 +638,14 @@ bool Builder::Build(string* err) { int pending_commands = 0; int failures_allowed = config_.failures_allowed; + // Set up the command runner if we haven't done so already. + if (!command_runner_.get()) { + if (config_.dry_run) + command_runner_.reset(new DryRunCommandRunner); + else + command_runner_.reset(new RealCommandRunner(config_)); + } + // This main loop runs the entire build process. // It is structured like this: // First, we attempt to start as many commands as allowed by the @@ -655,7 +713,7 @@ bool Builder::Build(string* err) { *err = "subcommand failed"; } else if (failures_allowed < config_.failures_allowed) *err = "cannot make progress due to previous errors"; - else + else *err = "stuck [this is a bug]"; return false; @@ -678,11 +736,11 @@ bool Builder::StartEdge(Edge* edge, string* err) { if (!disk_interface_->MakeDirs((*i)->path())) return false; } - + // Create response file, if needed // XXX: this may also block; do we care? if (edge->HasRspFile()) { - if (!disk_interface_->WriteFile(edge->GetRspFile(), edge->GetRspFileContent())) + if (!disk_interface_->WriteFile(edge->GetRspFile(), edge->GetRspFileContent())) return false; } @@ -720,19 +778,13 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { for (vector<Node*>::iterator i = edge->inputs_.begin(); i != edge->inputs_.end() - edge->order_only_deps_; ++i) { TimeStamp input_mtime = disk_interface_->Stat((*i)->path()); - if (input_mtime == 0) { - restat_mtime = 0; - break; - } if (input_mtime > restat_mtime) restat_mtime = input_mtime; } if (restat_mtime != 0 && !edge->rule().depfile().empty()) { TimeStamp depfile_mtime = disk_interface_->Stat(edge->EvaluateDepFile()); - if (depfile_mtime == 0) - restat_mtime = 0; - else if (depfile_mtime > restat_mtime) + if (depfile_mtime > restat_mtime) restat_mtime = depfile_mtime; } @@ -743,7 +795,7 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { } // delete the response file on success (if exists) - if (edge->HasRspFile()) + if (edge->HasRspFile()) disk_interface_->RemoveFile(edge->GetRspFile()); plan_.EdgeFinished(edge); @@ -757,3 +809,4 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { if (success && log_) log_->RecordCommand(edge, start_time, end_time, restat_mtime); } + diff --git a/src/build.h b/src/build.h index 179fca6..5e26bbb 100644 --- a/src/build.h +++ b/src/build.h @@ -21,13 +21,16 @@ #include <queue> #include <vector> #include <memory> -using namespace std; +#include <cstdio> #include "exit_status.h" +#include "metrics.h" +#include "util.h" // int64_t struct BuildLog; -struct Edge; +struct BuildStatus; struct DiskInterface; +struct Edge; struct Node; struct State; @@ -98,7 +101,7 @@ struct CommandRunner { /// Options (e.g. verbosity, parallelism) passed to a build. struct BuildConfig { BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), - failures_allowed(1) {} + failures_allowed(1), max_load_average(-0.0f) {} enum Verbosity { NORMAL, @@ -109,6 +112,9 @@ struct BuildConfig { bool dry_run; int parallelism; int failures_allowed; + /// The maximum load average we must not exceed. A negative value + /// means that we do not have any limit. + double max_load_average; }; /// Builder wraps the build process: starting commands, updating status. @@ -140,13 +146,89 @@ struct Builder { Plan plan_; DiskInterface* disk_interface_; auto_ptr<CommandRunner> command_runner_; - struct BuildStatus* status_; - struct BuildLog* log_; + BuildStatus* status_; + BuildLog* log_; -private: + private: // Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr. Builder(const Builder &other); // DO NOT IMPLEMENT void operator=(const Builder &other); // DO NOT IMPLEMENT }; +/// Tracks the status of a build: completion fraction, printing updates. +struct BuildStatus { + explicit BuildStatus(const BuildConfig& config); + void PlanHasTotalEdges(int total); + void BuildEdgeStarted(Edge* edge); + void BuildEdgeFinished(Edge* edge, bool success, const string& output, + int* start_time, int* end_time); + void BuildFinished(); + + /// Format the progress status string by replacing the placeholders. + /// See the user manual for more information about the available + /// placeholders. + /// @param progress_status_format_ The format of the progress status. + string FormatProgressStatus(const char* progress_status_format) const; + + private: + void PrintStatus(Edge* edge); + + const BuildConfig& config_; + + /// Time the build started. + int64_t start_time_millis_; + + int started_edges_, finished_edges_, total_edges_; + + bool have_blank_line_; + + /// Map of running edge to time the edge started running. + typedef map<Edge*, int> RunningEdgeMap; + RunningEdgeMap running_edges_; + + /// Whether we can do fancy terminal control codes. + bool smart_terminal_; + + /// The custom progress status format to use. + const char* progress_status_format_; + + struct RateInfo { + RateInfo() : last_update_(0), rate_(-1) {} + + double rate() const { return rate_; } + int last_update() const { return last_update_; } + void Restart() { return stopwatch_.Restart(); } + + double UpdateRate(int edges, int update_hint) { + if (update_hint != last_update_) { + rate_ = edges / stopwatch_.Elapsed() + 0.5; + last_update_ = update_hint; + } + return rate_; + } + + template<class T> + void snprinfRate(T buf, const char* format) { + if (rate_ == -1) + snprintf(buf, sizeof(buf), "?"); + else + snprintf(buf, sizeof(buf), format, rate_); + } + + private: + Stopwatch stopwatch_; + int last_update_; + double rate_; + }; + + mutable RateInfo overall_rate_; + mutable RateInfo current_rate_; + const int current_rate_average_count_; + +#ifdef _WIN32 + void* console_; +#endif +}; + #endif // NINJA_BUILD_H_ + diff --git a/src/build_log.cc b/src/build_log.cc index fd93ea8..1b27be3 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -18,8 +18,15 @@ #include <stdlib.h> #include <string.h> +#ifndef _WIN32 +#define __STDC_FORMAT_MACROS +#include <inttypes.h> +#include <unistd.h> +#endif + #include "build.h" #include "graph.h" +#include "metrics.h" #include "util.h" // Implementation details: @@ -32,10 +39,58 @@ namespace { const char kFileSignature[] = "# ninja log v%d\n"; -const int kCurrentVersion = 4; +const int kOldestSupportedVersion = 4; +const int kCurrentVersion = 5; + +// 64bit MurmurHash2, by Austin Appleby +#if defined(_MSC_VER) +#define BIG_CONSTANT(x) (x) +#else // defined(_MSC_VER) +#define BIG_CONSTANT(x) (x##LLU) +#endif // !defined(_MSC_VER) +inline +uint64_t MurmurHash64A(const void* key, int len) { + static const uint64_t seed = 0xDECAFBADDECAFBADull; + const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); + const int r = 47; + uint64_t h = seed ^ (len * m); + const uint64_t * data = (const uint64_t *)key; + const uint64_t * end = data + (len/8); + while(data != end) { + uint64_t k = *data++; + k *= m; + k ^= k >> r; + k *= m; + h ^= k; + h *= m; + } + const unsigned char* data2 = (const unsigned char*)data; + switch(len & 7) + { + case 7: h ^= uint64_t(data2[6]) << 48; + case 6: h ^= uint64_t(data2[5]) << 40; + case 5: h ^= uint64_t(data2[4]) << 32; + case 4: h ^= uint64_t(data2[3]) << 24; + case 3: h ^= uint64_t(data2[2]) << 16; + case 2: h ^= uint64_t(data2[1]) << 8; + case 1: h ^= uint64_t(data2[0]); + h *= m; + }; + h ^= h >> r; + h *= m; + h ^= h >> r; + return h; +} +#undef BIG_CONSTANT + } // namespace +// static +uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { + return MurmurHash64A(command.str_, command.len_); +} + BuildLog::BuildLog() : log_file_(NULL), config_(NULL), needs_recompaction_(false) {} @@ -61,6 +116,10 @@ bool BuildLog::OpenForWrite(const string& path, string* err) { setvbuf(log_file_, NULL, _IOLBF, BUFSIZ); SetCloseOnExec(fileno(log_file_)); + // Opening a file in append mode doesn't set the file pointer to the file's + // end on Windows. Do that explicitly. + fseek(log_file_, 0, SEEK_END); + if (ftell(log_file_) == 0) { if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) { *err = strerror(errno); @@ -86,7 +145,7 @@ void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, log_entry->output = path; log_.insert(Log::value_type(log_entry->output, log_entry)); } - log_entry->command = command; + log_entry->command_hash = LogEntry::HashCommand(command); log_entry->start_time = start_time; log_entry->end_time = end_time; log_entry->restat_mtime = restat_mtime; @@ -102,7 +161,60 @@ void BuildLog::Close() { log_file_ = NULL; } +class LineReader { + public: + explicit LineReader(FILE* file) + : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) { + memset(buf_, 0, sizeof(buf_)); + } + + // Reads a \n-terminated line from the file passed to the constructor. + // On return, *line_start points to the beginning of the next line, and + // *line_end points to the \n at the end of the line. If no newline is seen + // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF. + bool ReadLine(char** line_start, char** line_end) { + if (line_start_ >= buf_end_ || !line_end_) { + // Buffer empty, refill. + size_t size_read = fread(buf_, 1, sizeof(buf_), file_); + if (!size_read) + return false; + line_start_ = buf_; + buf_end_ = buf_ + size_read; + } else { + // Advance to next line in buffer. + line_start_ = line_end_ + 1; + } + + line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); + if (!line_end_) { + // No newline. Move rest of data to start of buffer, fill rest. + size_t already_consumed = line_start_ - buf_; + size_t size_rest = (buf_end_ - buf_) - already_consumed; + memmove(buf_, line_start_, size_rest); + + size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_); + buf_end_ = buf_ + size_rest + read; + line_start_ = buf_; + line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); + } + + *line_start = line_start_; + *line_end = line_end_; + return true; + } + + private: + FILE* file_; + char buf_[256 << 10]; + char* buf_end_; // Points one past the last valid byte in |buf_|. + + char* line_start_; + // Points at the next \n in buf_ after line_start, or NULL. + char* line_end_; +}; + bool BuildLog::Load(const string& path, string* err) { + METRIC_RECORD(".ninja_log load"); FILE* file = fopen(path.c_str(), "r"); if (!file) { if (errno == ENOENT) @@ -115,18 +227,29 @@ bool BuildLog::Load(const string& path, string* err) { int unique_entry_count = 0; int total_entry_count = 0; - char buf[256 << 10]; - while (fgets(buf, sizeof(buf), file)) { + LineReader reader(file); + char* line_start = 0; + char* line_end = 0; + while (reader.ReadLine(&line_start, &line_end)) { if (!log_version) { - log_version = 1; // Assume by default. - if (sscanf(buf, kFileSignature, &log_version) > 0) - continue; + sscanf(line_start, kFileSignature, &log_version); + + if (log_version < kOldestSupportedVersion) { + *err = "unable to extract version from build log, perhaps due to " + "being too old; you must clobber your build output and rebuild"; + fclose(file); + return false; + } } - char field_separator = log_version >= 4 ? '\t' : ' '; + // If no newline was found in this chunk, read the next. + if (!line_end) + continue; - char* start = buf; - char* end = strchr(start, field_separator); + const char kFieldSeparator = '\t'; + + char* start = line_start; + char* end = (char*)memchr(start, kFieldSeparator, line_end - start); if (!end) continue; *end = 0; @@ -137,29 +260,27 @@ bool BuildLog::Load(const string& path, string* err) { start_time = atoi(start); start = end + 1; - end = strchr(start, field_separator); + end = (char*)memchr(start, kFieldSeparator, line_end - start); if (!end) continue; *end = 0; end_time = atoi(start); start = end + 1; - end = strchr(start, field_separator); + end = (char*)memchr(start, kFieldSeparator, line_end - start); if (!end) continue; *end = 0; restat_mtime = atol(start); start = end + 1; - end = strchr(start, field_separator); + end = (char*)memchr(start, kFieldSeparator, line_end - start); if (!end) continue; string output = string(start, end - start); start = end + 1; - end = strchr(start, '\n'); - if (!end) - continue; + end = line_end; LogEntry* entry; Log::iterator i = log_.find(output); @@ -176,7 +297,19 @@ bool BuildLog::Load(const string& path, string* err) { entry->start_time = start_time; entry->end_time = end_time; entry->restat_mtime = restat_mtime; - entry->command = string(start, end - start); + if (log_version >= 5) { + char c = *end; *end = '\0'; + entry->command_hash = (uint64_t)strtoull(start, NULL, 16); + *end = c; + } else { + entry->command_hash = LogEntry::HashCommand(StringPiece(start, + end - start)); + } + } + fclose(file); + + if (!line_start) { + return true; // file was empty } // Decide whether it's time to rebuild the log: @@ -191,8 +324,6 @@ bool BuildLog::Load(const string& path, string* err) { needs_recompaction_ = true; } - fclose(file); - return true; } @@ -204,9 +335,9 @@ BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { } void BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { - fprintf(f, "%d\t%d\t%ld\t%s\t%s\n", - entry.start_time, entry.end_time, (long) entry.restat_mtime, - entry.output.c_str(), entry.command.c_str()); + fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n", + entry.start_time, entry.end_time, entry.restat_mtime, + entry.output.c_str(), entry.command_hash); } bool BuildLog::Recompact(const string& path, string* err) { diff --git a/src/build_log.h b/src/build_log.h index 040609d..d3994ff 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -22,6 +22,7 @@ using namespace std; #include "hash_map.h" #include "timestamp.h" +#include "util.h" struct BuildConfig; struct Edge; @@ -49,14 +50,16 @@ struct BuildLog { struct LogEntry { string output; - string command; + uint64_t command_hash; int start_time; int end_time; TimeStamp restat_mtime; + static uint64_t HashCommand(StringPiece command); + // Used by tests. bool operator==(const LogEntry& o) { - return output == o.output && command == o.command && + return output == o.output && command_hash == o.command_hash && start_time == o.start_time && end_time == o.end_time && restat_mtime == o.restat_mtime; } @@ -71,10 +74,11 @@ struct BuildLog { /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, string* err); - // TODO: make these private. typedef ExternalStringHashMap<LogEntry*>::Type Log; + const Log& log() const { return log_; } + + private: Log log_; -private: FILE* log_file_; BuildConfig* config_; bool needs_recompaction_; diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc new file mode 100644 index 0000000..5755079 --- /dev/null +++ b/src/build_log_perftest.cc @@ -0,0 +1,140 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stdio.h> +#include <stdlib.h> + +#include "build_log.h" +#include "graph.h" +#include "manifest_parser.h" +#include "state.h" +#include "util.h" +#include "metrics.h" + +const char kTestFilename[] = "BuildLogPerfTest-tempfile"; + +bool WriteTestData(string* err) { + BuildLog log; + + if (!log.OpenForWrite(kTestFilename, err)) + return false; + + /* + A histogram of command lengths in chromium. For example, 407 builds, + 1.4% of all builds, had commands longer than 32 bytes but shorter than 64. + 32 407 1.4% + 64 183 0.6% + 128 1461 5.1% + 256 791 2.8% + 512 1314 4.6% + 1024 6114 21.3% + 2048 11759 41.0% + 4096 2056 7.2% + 8192 4567 15.9% + 16384 13 0.0% + 32768 4 0.0% + 65536 5 0.0% + The average command length is 4.1 kB and there were 28674 commands in total, + which makes for a total log size of ~120 MB (also counting output filenames). + + Based on this, write 30000 many 4 kB long command lines. + */ + + // ManifestParser is the only object allowed to create Rules. + const size_t kRuleSize = 4000; + string long_rule_command = "gcc "; + for (int i = 0; long_rule_command.size() < kRuleSize; ++i) { + char buf[80]; + sprintf(buf, "-I../../and/arbitrary/but/fairly/long/path/suffixed/%d ", i); + long_rule_command += buf; + } + long_rule_command += "$in -o $out\n"; + + State state; + ManifestParser parser(&state, NULL); + if (!parser.ParseTest("rule cxx\n command = " + long_rule_command, err)) + return false; + + // Create build edges. Using ManifestParser is as fast as using the State api + // for edge creation, so just use that. + const int kNumCommands = 30000; + string build_rules; + for (int i = 0; i < kNumCommands; ++i) { + char buf[80]; + sprintf(buf, "build input%d.o: cxx input%d.cc\n", i, i); + build_rules += buf; + } + + if (!parser.ParseTest(build_rules, err)) + return false; + + for (int i = 0; i < kNumCommands; ++i) { + log.RecordCommand(state.edges_[i], + /*start_time=*/100 * i, + /*end_time=*/100 * i + 1, + /*restat_mtime=*/0); + } + + return true; +} + +int main() { + vector<int> times; + string err; + + if (!WriteTestData(&err)) { + fprintf(stderr, "Failed to write test data: %s\n", err.c_str()); + return 1; + } + + { + // Read once to warm up disk cache. + BuildLog log; + if (!log.Load(kTestFilename, &err)) { + fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); + return 1; + } + } + const int kNumRepetitions = 5; + for (int i = 0; i < kNumRepetitions; ++i) { + int64_t start = GetTimeMillis(); + BuildLog log; + if (!log.Load(kTestFilename, &err)) { + fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); + return 1; + } + int delta = (int)(GetTimeMillis() - start); + printf("%dms\n", delta); + times.push_back(delta); + } + + int min = times[0]; + int max = times[0]; + float total = 0; + for (size_t i = 0; i < times.size(); ++i) { + total += times[i]; + if (times[i] < min) + min = times[i]; + else if (times[i] > max) + max = times[i]; + } + + printf("min %dms max %dms avg %.1fms\n", + min, max, total / times.size()); + + unlink(kTestFilename); + + return 0; +} + diff --git a/src/build_log_test.cc b/src/build_log_test.cc index c6d6bc3..0225684 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -14,6 +14,7 @@ #include "build_log.h" +#include "util.h" #include "test.h" #ifdef _WIN32 @@ -27,7 +28,7 @@ #include <unistd.h> #endif -static const char kTestFilename[] = "BuildLogTest-tempfile"; +const char kTestFilename[] = "BuildLogTest-tempfile"; struct BuildLogTest : public StateTestWithBuiltinRules { virtual void SetUp() { @@ -54,8 +55,8 @@ TEST_F(BuildLogTest, WriteRead) { EXPECT_TRUE(log2.Load(kTestFilename, &err)); ASSERT_EQ("", err); - ASSERT_EQ(2u, log1.log_.size()); - ASSERT_EQ(2u, log2.log_.size()); + ASSERT_EQ(2u, log1.log().size()); + ASSERT_EQ(2u, log2.log().size()); BuildLog::LogEntry* e1 = log1.LookupByOutput("out"); ASSERT_TRUE(e1); BuildLog::LogEntry* e2 = log2.LookupByOutput("out"); @@ -65,11 +66,41 @@ TEST_F(BuildLogTest, WriteRead) { ASSERT_EQ("out", e1->output); } +TEST_F(BuildLogTest, FirstWriteAddsSignature) { + const char kExpectedVersion[] = "# ninja log vX\n"; + const size_t kVersionPos = strlen(kExpectedVersion) - 2; // Points at 'X'. + + BuildLog log; + string contents, err; + + EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + log.Close(); + + ASSERT_EQ(0, ReadFile(kTestFilename, &contents, &err)); + ASSERT_EQ("", err); + if (contents.size() >= kVersionPos) + contents[kVersionPos] = 'X'; + EXPECT_EQ(kExpectedVersion, contents); + + // Opening the file anew shouldn't add a second version string. + EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + log.Close(); + + contents.clear(); + ASSERT_EQ(0, ReadFile(kTestFilename, &contents, &err)); + ASSERT_EQ("", err); + if (contents.size() >= kVersionPos) + contents[kVersionPos] = 'X'; + EXPECT_EQ(kExpectedVersion, contents); +} + TEST_F(BuildLogTest, DoubleEntry) { FILE* f = fopen(kTestFilename, "wb"); - fprintf(f, "# ninja log v3\n"); - fprintf(f, "0 1 2 out command abc\n"); - fprintf(f, "3 4 5 out command def\n"); + fprintf(f, "# ninja log v4\n"); + fprintf(f, "0\t1\t2\tout\tcommand abc\n"); + fprintf(f, "3\t4\t5\tout\tcommand def\n"); fclose(f); string err; @@ -79,7 +110,7 @@ TEST_F(BuildLogTest, DoubleEntry) { BuildLog::LogEntry* e = log.LookupByOutput("out"); ASSERT_TRUE(e); - ASSERT_EQ("command def", e->command); + ASSERT_NO_FATAL_FAILURE(AssertHash("command def", e->command_hash)); } TEST_F(BuildLogTest, Truncate) { @@ -100,7 +131,7 @@ TEST_F(BuildLogTest, Truncate) { ASSERT_GT(statbuf.st_size, 0); // For all possible truncations of the input file, assert that we don't - // crash or report an error when parsing. + // crash when parsing. for (off_t size = statbuf.st_size; size > 0; --size) { #ifndef _WIN32 ASSERT_EQ(0, truncate(kTestFilename, size)); @@ -112,12 +143,12 @@ TEST_F(BuildLogTest, Truncate) { #endif BuildLog log2; - EXPECT_TRUE(log2.Load(kTestFilename, &err)); - ASSERT_EQ("", err); + err.clear(); + ASSERT_TRUE(log2.Load(kTestFilename, &err) || !err.empty()); } } -TEST_F(BuildLogTest, UpgradeV3) { +TEST_F(BuildLogTest, ObsoleteOldVersion) { FILE* f = fopen(kTestFilename, "wb"); fprintf(f, "# ninja log v3\n"); fprintf(f, "123 456 0 out command\n"); @@ -125,21 +156,38 @@ TEST_F(BuildLogTest, UpgradeV3) { string err; BuildLog log; + EXPECT_FALSE(log.Load(kTestFilename, &err)); + ASSERT_NE(err.find("version"), string::npos); +} + +TEST_F(BuildLogTest, SpacesInOutputV4) { + FILE* f = fopen(kTestFilename, "wb"); + fprintf(f, "# ninja log v4\n"); + fprintf(f, "123\t456\t456\tout with space\tcommand\n"); + fclose(f); + + string err; + BuildLog log; EXPECT_TRUE(log.Load(kTestFilename, &err)); ASSERT_EQ("", err); - BuildLog::LogEntry* e = log.LookupByOutput("out"); + BuildLog::LogEntry* e = log.LookupByOutput("out with space"); ASSERT_TRUE(e); ASSERT_EQ(123, e->start_time); ASSERT_EQ(456, e->end_time); - ASSERT_EQ(0, e->restat_mtime); - ASSERT_EQ("command", e->command); + ASSERT_EQ(456, e->restat_mtime); + ASSERT_NO_FATAL_FAILURE(AssertHash("command", e->command_hash)); } -TEST_F(BuildLogTest, SpacesInOutputV4) { +TEST_F(BuildLogTest, DuplicateVersionHeader) { + // Old versions of ninja accidentally wrote multiple version headers to the + // build log on Windows. This shouldn't crash, and the second version header + // should be ignored. FILE* f = fopen(kTestFilename, "wb"); fprintf(f, "# ninja log v4\n"); - fprintf(f, "123\t456\t456\tout with space\tcommand\n"); + fprintf(f, "123\t456\t456\tout\tcommand\n"); + fprintf(f, "# ninja log v4\n"); + fprintf(f, "456\t789\t789\tout2\tcommand2\n"); fclose(f); string err; @@ -147,10 +195,45 @@ TEST_F(BuildLogTest, SpacesInOutputV4) { EXPECT_TRUE(log.Load(kTestFilename, &err)); ASSERT_EQ("", err); - BuildLog::LogEntry* e = log.LookupByOutput("out with space"); + BuildLog::LogEntry* e = log.LookupByOutput("out"); ASSERT_TRUE(e); ASSERT_EQ(123, e->start_time); ASSERT_EQ(456, e->end_time); ASSERT_EQ(456, e->restat_mtime); - ASSERT_EQ("command", e->command); + ASSERT_NO_FATAL_FAILURE(AssertHash("command", e->command_hash)); + + e = log.LookupByOutput("out2"); + ASSERT_TRUE(e); + ASSERT_EQ(456, e->start_time); + ASSERT_EQ(789, e->end_time); + ASSERT_EQ(789, e->restat_mtime); + ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash)); +} + +TEST_F(BuildLogTest, VeryLongInputLine) { + // Ninja's build log buffer is currently 256kB. Lines longer than that are + // silently ignored, but don't affect parsing of other lines. + FILE* f = fopen(kTestFilename, "wb"); + fprintf(f, "# ninja log v4\n"); + fprintf(f, "123\t456\t456\tout\tcommand start"); + for (size_t i = 0; i < (512 << 10) / strlen(" more_command"); ++i) + fputs(" more_command", f); + fprintf(f, "\n"); + fprintf(f, "456\t789\t789\tout2\tcommand2\n"); + fclose(f); + + string err; + BuildLog log; + EXPECT_TRUE(log.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + + BuildLog::LogEntry* e = log.LookupByOutput("out"); + ASSERT_EQ(NULL, e); + + e = log.LookupByOutput("out2"); + ASSERT_TRUE(e); + ASSERT_EQ(456, e->start_time); + ASSERT_EQ(789, e->end_time); + ASSERT_EQ(789, e->restat_mtime); + ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash)); } diff --git a/src/build_test.cc b/src/build_test.cc index 5b35513..a3f345b 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -179,7 +179,7 @@ TEST_F(PlanTest, DependencyCycle) { struct BuildTest : public StateTestWithBuiltinRules, public CommandRunner { BuildTest() : config_(MakeConfig()), builder_(&state_, config_), now_(1), - last_command_(NULL) { + last_command_(NULL), status_(config_) { builder_.disk_interface_ = &fs_; builder_.command_runner_.reset(this); AssertParse(&state_, @@ -202,8 +202,8 @@ struct BuildTest : public StateTestWithBuiltinRules, virtual bool CanRunMore(); virtual bool StartCommand(Edge* edge); virtual Edge* WaitForCommand(ExitStatus* status, string* output); - virtual vector<Edge*> GetActiveEdges(); - virtual void Abort(); + virtual vector<Edge*> GetActiveEdges(); + virtual void Abort(); BuildConfig MakeConfig() { BuildConfig config; @@ -219,6 +219,7 @@ struct BuildTest : public StateTestWithBuiltinRules, vector<string> commands_ran_; Edge* last_command_; + BuildStatus status_; }; void BuildTest::Dirty(const string& path) { @@ -801,6 +802,60 @@ TEST_F(BuildWithLogTest, RestatMissingFile) { ASSERT_EQ(1u, commands_ran_.size()); } +// Test scenario, in which an input file is removed, but output isn't changed +// https://github.com/martine/ninja/issues/295 +TEST_F(BuildWithLogTest, RestatMissingInput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "rule true\n" + " command = true\n" + " depfile = $out.d\n" + " restat = 1\n" + "rule cc\n" + " command = cc\n" + "build out1: true in\n" + "build out2: cc out1\n")); + + // Create all necessary files + fs_.Create("in", now_, ""); + + // The implicit dependencies and the depfile itself + // are newer than the output + TimeStamp restat_mtime = ++now_; + fs_.Create("out1.d", now_, "out1: will.be.deleted restat.file\n"); + fs_.Create("will.be.deleted", now_, ""); + fs_.Create("restat.file", now_, ""); + + // Run the build, out1 and out2 get built + string err; + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ(2u, commands_ran_.size()); + + // See that an entry in the logfile is created, capturing + // the right mtime + BuildLog::LogEntry * log_entry = build_log_.LookupByOutput("out1"); + ASSERT_TRUE(NULL != log_entry); + ASSERT_EQ(restat_mtime, log_entry->restat_mtime); + + // Now remove a file, referenced from depfile, so that target becomes + // dirty, but the output does not change + fs_.RemoveFile("will.be.deleted"); + + // Trigger the build again - only out1 gets built + commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ(1u, commands_ran_.size()); + + // Check that the logfile entry remains correctly set + log_entry = build_log_.LookupByOutput("out1"); + ASSERT_TRUE(NULL != log_entry); + ASSERT_EQ(restat_mtime, log_entry->restat_mtime); +} + struct BuildDryRun : public BuildWithLogTest { BuildDryRun() { config_.dry_run = true; @@ -853,7 +908,7 @@ TEST_F(BuildTest, RspFileSuccess) fs_.Create("out1", now_, ""); fs_.Create("out2", now_, ""); fs_.Create("out3", now_, ""); - + now_++; fs_.Create("in", now_, ""); @@ -868,15 +923,15 @@ TEST_F(BuildTest, RspFileSuccess) size_t files_removed = fs_.files_removed_.size(); EXPECT_TRUE(builder_.Build(&err)); - ASSERT_EQ(2, commands_ran_.size()); // cat + cat_rsp - + ASSERT_EQ(2u, commands_ran_.size()); // cat + cat_rsp + // The RSP file was created ASSERT_EQ(files_created + 1, fs_.files_created_.size()); - ASSERT_EQ(1, fs_.files_created_.count("out2.rsp")); - + ASSERT_EQ(1u, fs_.files_created_.count("out2.rsp")); + // The RSP file was removed ASSERT_EQ(files_removed + 1, fs_.files_removed_.size()); - ASSERT_EQ(1, fs_.files_removed_.count("out2.rsp")); + ASSERT_EQ(1u, fs_.files_removed_.count("out2.rsp")); } // Test that RSP file is created but not removed for commands, which fail @@ -903,21 +958,21 @@ TEST_F(BuildTest, RspFileFailure) { EXPECT_FALSE(builder_.Build(&err)); ASSERT_EQ("subcommand failed", err); - ASSERT_EQ(1, commands_ran_.size()); + ASSERT_EQ(1u, commands_ran_.size()); // The RSP file was created ASSERT_EQ(files_created + 1, fs_.files_created_.size()); - ASSERT_EQ(1, fs_.files_created_.count("out.rsp")); + ASSERT_EQ(1u, fs_.files_created_.count("out.rsp")); // The RSP file was NOT removed ASSERT_EQ(files_removed, fs_.files_removed_.size()); - ASSERT_EQ(0, fs_.files_removed_.count("out.rsp")); + ASSERT_EQ(0u, fs_.files_removed_.count("out.rsp")); // The RSP file contains what it should ASSERT_EQ("Another very long command", fs_.files_["out.rsp"].contents); } -// Test that contens of the RSP file behaves like a regular part of +// Test that contens of the RSP file behaves like a regular part of // command line, i.e. triggers a rebuild if changed TEST_F(BuildWithLogTest, RspFileCmdLineChange) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, @@ -939,7 +994,7 @@ TEST_F(BuildWithLogTest, RspFileCmdLineChange) { // 1. Build for the 1st time (-> populate log) EXPECT_TRUE(builder_.Build(&err)); - ASSERT_EQ(1, commands_ran_.size()); + ASSERT_EQ(1u, commands_ran_.size()); // 2. Build again (no change) commands_ran_.clear(); @@ -948,19 +1003,21 @@ TEST_F(BuildWithLogTest, RspFileCmdLineChange) { EXPECT_EQ("", err); ASSERT_TRUE(builder_.AlreadyUpToDate()); - // 3. Alter the entry in the logfile + // 3. Alter the entry in the logfile // (to simulate a change in the command line between 2 builds) BuildLog::LogEntry * log_entry = build_log_.LookupByOutput("out"); ASSERT_TRUE(NULL != log_entry); - ASSERT_EQ("cat out.rsp > out;rspfile=Original very long command", log_entry->command); - log_entry->command = "cat out.rsp > out;rspfile=Altered very long command"; + ASSERT_NO_FATAL_FAILURE(AssertHash( + "cat out.rsp > out;rspfile=Original very long command", + log_entry->command_hash)); + log_entry->command_hash++; // Change the command hash to something else. // Now expect the target to be rebuilt commands_ran_.clear(); state_.Reset(); EXPECT_TRUE(builder_.AddTarget("out", &err)); EXPECT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); - EXPECT_EQ(1, commands_ran_.size()); + EXPECT_EQ(1u, commands_ran_.size()); } TEST_F(BuildTest, InterruptCleanup) { @@ -1022,3 +1079,8 @@ TEST_F(BuildTest, PhonyWithNoInputs) { EXPECT_EQ("", err); ASSERT_EQ(1u, commands_ran_.size()); } + +TEST_F(BuildTest, StatusFormatReplacePlaceholder) { + EXPECT_EQ("[%/s0/t0/r0/u0/f0]", + status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]")); +} diff --git a/src/canon_perftest.cc b/src/canon_perftest.cc new file mode 100644 index 0000000..d0ed397 --- /dev/null +++ b/src/canon_perftest.cc @@ -0,0 +1,56 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stdio.h> +#include <string.h> + +#include "util.h" +#include "metrics.h" + +const char kPath[] = + "../../third_party/WebKit/Source/WebCore/" + "platform/leveldb/LevelDBWriteBatch.cpp"; + +int main() { + vector<int> times; + string err; + + char buf[200]; + int len = strlen(kPath); + strcpy(buf, kPath); + + for (int j = 0; j < 5; ++j) { + const int kNumRepetitions = 2000000; + int64_t start = GetTimeMillis(); + for (int i = 0; i < kNumRepetitions; ++i) { + CanonicalizePath(buf, &len, &err); + } + int delta = (int)(GetTimeMillis() - start); + times.push_back(delta); + } + + int min = times[0]; + int max = times[0]; + float total = 0; + for (size_t i = 0; i < times.size(); ++i) { + total += times[i]; + if (times[i] < min) + min = times[i]; + else if (times[i] > max) + max = times[i]; + } + + printf("min %dms max %dms avg %.1fms\n", + min, max, total / times.size()); +} diff --git a/src/clean.cc b/src/clean.cc index e09ab4e..3fe23ec 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -102,7 +102,7 @@ int Cleaner::CleanAll(bool generator) { for (vector<Edge*>::iterator e = state_->edges_.begin(); e != state_->edges_.end(); ++e) { // Do not try to remove phony targets - if ((*e)->rule_ == &State::kPhonyRule) + if ((*e)->is_phony()) continue; // Do not remove generator's files unless generator specified. if (!generator && (*e)->rule().generator()) @@ -123,12 +123,16 @@ int Cleaner::CleanAll(bool generator) { } void Cleaner::DoCleanTarget(Node* target) { - if (target->in_edge()) { - Remove(target->path()); - if (target->in_edge()->HasRspFile()) - Remove(target->in_edge()->GetRspFile()); - for (vector<Node*>::iterator n = target->in_edge()->inputs_.begin(); - n != target->in_edge()->inputs_.end(); + if (Edge* e = target->in_edge()) { + // Do not try to remove phony targets + if (!e->is_phony()) { + Remove(target->path()); + if (!target->in_edge()->rule().depfile().empty()) + Remove(target->in_edge()->EvaluateDepFile()); + if (e->HasRspFile()) + Remove(e->GetRspFile()); + } + for (vector<Node*>::iterator n = e->inputs_.begin(); n != e->inputs_.end(); ++n) { DoCleanTarget(*n); } @@ -187,6 +191,8 @@ void Cleaner::DoCleanRule(const Rule* rule) { for (vector<Node*>::iterator out_node = (*e)->outputs_.begin(); out_node != (*e)->outputs_.end(); ++out_node) { Remove((*out_node)->path()); + if (!(*e)->rule().depfile().empty()) + Remove((*e)->EvaluateDepFile()); if ((*e)->HasRspFile()) Remove((*e)->GetRspFile()); } diff --git a/src/clean.h b/src/clean.h index 4d9b4e6..5938dff 100644 --- a/src/clean.h +++ b/src/clean.h @@ -14,7 +14,6 @@ #ifndef NINJA_CLEAN_H_ #define NINJA_CLEAN_H_ -#pragma once #include <set> #include <string> diff --git a/src/clean_test.cc b/src/clean_test.cc index fbbe6a1..5ed48da 100644 --- a/src/clean_test.cc +++ b/src/clean_test.cc @@ -249,6 +249,36 @@ TEST_F(CleanTest, CleanDepFile) { EXPECT_EQ(2u, fs_.files_removed_.size()); } +TEST_F(CleanTest, CleanDepFileOnCleanTarget) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc\n" +" command = cc $in > $out\n" +" depfile = $out.d\n" +"build out1: cc in1\n")); + fs_.Create("out1", 1, ""); + fs_.Create("out1.d", 1, ""); + + Cleaner cleaner(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner.CleanTarget("out1")); + EXPECT_EQ(2, cleaner.cleaned_files_count()); + EXPECT_EQ(2u, fs_.files_removed_.size()); +} + +TEST_F(CleanTest, CleanDepFileOnCleanRule) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc\n" +" command = cc $in > $out\n" +" depfile = $out.d\n" +"build out1: cc in1\n")); + fs_.Create("out1", 1, ""); + fs_.Create("out1.d", 1, ""); + + Cleaner cleaner(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner.CleanRule("cc")); + EXPECT_EQ(2, cleaner.cleaned_files_count()); + EXPECT_EQ(2u, fs_.files_removed_.size()); +} + TEST_F(CleanTest, CleanRspFile) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule cc\n" @@ -317,3 +347,28 @@ TEST_F(CleanTest, CleanFailure) { Cleaner cleaner(&state_, config_, &fs_); EXPECT_NE(0, cleaner.CleanAll()); } + +TEST_F(CleanTest, CleanPhony) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build phony: phony t1 t2\n" +"build t1: cat\n" +"build t2: cat\n")); + + fs_.Create("phony", 1, ""); + fs_.Create("t1", 1, ""); + fs_.Create("t2", 1, ""); + + // Check that CleanAll does not remove "phony". + Cleaner cleaner(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner.CleanAll()); + EXPECT_EQ(2, cleaner.cleaned_files_count()); + EXPECT_NE(0, fs_.Stat("phony")); + + fs_.Create("t1", 1, ""); + fs_.Create("t2", 1, ""); + + // Check that CleanTarget does not remove "phony". + EXPECT_EQ(0, cleaner.CleanTarget("phony")); + EXPECT_EQ(2, cleaner.cleaned_files_count()); + EXPECT_NE(0, fs_.Stat("phony")); +} diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 261893f..03dad92 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -54,10 +54,10 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 128, 128, 128, 128, 128, + 128, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 0, 0, 128, 0, 0, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 0, 0, 0, 0, 0, - 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 128, @@ -84,26 +84,38 @@ bool DepfileParser::Parse(string* content, string* err) { }; yych = *in; - if (yych <= '\\') { - if (yych <= ':') { + if (yych <= 'Z') { + if (yych <= '*') { if (yych <= 0x00) goto yy6; - if (yych <= '*') goto yy8; - goto yy4; + if (yych <= '\'') goto yy8; + if (yych <= ')') goto yy4; + goto yy8; } else { - if (yych <= '@') goto yy8; - if (yych <= 'Z') goto yy4; - if (yych <= '[') goto yy8; + if (yych <= '<') { + if (yych <= ':') goto yy4; + goto yy8; + } else { + if (yych <= '=') goto yy4; + if (yych <= '?') goto yy8; + goto yy4; + } } } else { - if (yych <= '`') { - if (yych == '_') goto yy4; - goto yy8; + if (yych <= '_') { + if (yych == '\\') goto yy2; + if (yych <= '^') goto yy8; + goto yy4; } else { - if (yych <= 'z') goto yy4; - if (yych == '~') goto yy4; - goto yy8; + if (yych <= 'z') { + if (yych <= '`') goto yy8; + goto yy4; + } else { + if (yych == '~') goto yy4; + goto yy8; + } } } +yy2: ++in; if ((yych = *in) <= '$') { if (yych <= '\n') { diff --git a/src/depfile_parser.h b/src/depfile_parser.h index c900956..1e6ebb5 100644 --- a/src/depfile_parser.h +++ b/src/depfile_parser.h @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifndef NINJA_DEPFILE_PARSER_H_ +#define NINJA_DEPFILE_PARSER_H_ + #include <string> #include <vector> using namespace std; @@ -28,3 +31,5 @@ struct DepfileParser { StringPiece out_; vector<StringPiece> ins_; }; + +#endif // NINJA_DEPFILE_PARSER_H_ diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 5e073df..68b6a95 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -68,7 +68,7 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = yych; continue; } - [a-zA-Z0-9+,/_:.~-]+ { + [a-zA-Z0-9+,/_:.~()@=-]+ { // Got a span of plain text. int len = in - start; // Need to shift it over if we're overwriting backslashes. diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index ce122cf..2bb9105 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -103,12 +103,28 @@ TEST_F(DepfileParserTest, Escapes) { ASSERT_EQ(0u, parser_.ins_.size()); } -TEST_F(DepfileParserTest, UnifyMultupleOutputs) { +TEST_F(DepfileParserTest, SpecialChars) { + string err; + EXPECT_TRUE(Parse( +"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n" +" en@quot.header~ t+t-x=1", + &err)); + ASSERT_EQ("", err); + EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", + parser_.out_.AsString()); + ASSERT_EQ(2u, parser_.ins_.size()); + EXPECT_EQ("en@quot.header~", + parser_.ins_[0].AsString()); + EXPECT_EQ("t+t-x=1", + parser_.ins_[1].AsString()); +} + +TEST_F(DepfileParserTest, UnifyMultipleOutputs) { // check that multiple duplicate targets are properly unified string err; EXPECT_TRUE(Parse("foo foo: x y z", &err)); ASSERT_EQ(parser_.out_.AsString(), "foo"); - ASSERT_EQ(parser_.ins_.size(), 3); + ASSERT_EQ(parser_.ins_.size(), 3u); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); EXPECT_EQ("z", parser_.ins_[2].AsString()); diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 5a36685..74f33c4 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -103,7 +103,7 @@ TimeStamp RealDiskInterface::Stat(const string& path) { #endif } -bool RealDiskInterface::WriteFile(const string & path, const string & contents) { +bool RealDiskInterface::WriteFile(const string& path, const string& contents) { FILE * fp = fopen(path.c_str(), "w"); if (fp == NULL) { Error("WriteFile(%s): Unable to create file. %s", path.c_str(), strerror(errno)); diff --git a/src/disk_interface.h b/src/disk_interface.h index c0f5ee5..55f8a21 100644 --- a/src/disk_interface.h +++ b/src/disk_interface.h @@ -36,7 +36,7 @@ struct DiskInterface { /// Create a file, with the specified name and contents /// Returns true on success, false on failure - virtual bool WriteFile(const string & path, const string & contents) = 0; + virtual bool WriteFile(const string& path, const string& contents) = 0; /// Read a file to a string. Fill in |err| on error. virtual string ReadFile(const string& path, string* err) = 0; @@ -58,7 +58,7 @@ struct RealDiskInterface : public DiskInterface { virtual ~RealDiskInterface() {} virtual TimeStamp Stat(const string& path); virtual bool MakeDir(const string& path); - virtual bool WriteFile(const string & path, const string & contents); + virtual bool WriteFile(const string& path, const string& contents); virtual string ReadFile(const string& path, string* err); virtual int RemoveFile(const string& path); }; diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 9e460c5..985e991 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -107,7 +107,7 @@ struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { // DiskInterface implementation. virtual TimeStamp Stat(const string& path); - virtual bool WriteFile(const string& path, const string & contents) { + virtual bool WriteFile(const string& path, const string& contents) { assert(false); return true; } diff --git a/src/explain.cc b/src/explain.cc new file mode 100644 index 0000000..4e14c25 --- /dev/null +++ b/src/explain.cc @@ -0,0 +1,15 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +bool g_explaining = false; diff --git a/src/explain.h b/src/explain.h new file mode 100644 index 0000000..d4f6a6c --- /dev/null +++ b/src/explain.h @@ -0,0 +1,27 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_EXPLAIN_H_ +#define NINJA_EXPLAIN_H_ + +#include <stdio.h> + +#define EXPLAIN(fmt, ...) { \ + if (g_explaining) \ + fprintf(stderr, "ninja explain: " fmt "\n", __VA_ARGS__); \ +} + +extern bool g_explaining; + +#endif // NINJA_EXPLAIN_H_ diff --git a/src/getopt.c b/src/getopt.c index 1e3c09d..75ef99c 100644 --- a/src/getopt.c +++ b/src/getopt.c @@ -91,10 +91,6 @@ gpietsch@comcast.net #include "getopt.h" #endif -#ifdef _WIN32 -#pragma warning(disable: 4701) -#endif - /* macros */ /* types */ @@ -159,7 +155,7 @@ getopt_internal (int argc, char **argv, char *shortopts, char *possible_arg = NULL; int longopt_match = -1; int has_arg = -1; - char *cp; + char *cp = NULL; int arg_next = 0; /* first, deal with silly parameters and easy stuff */ @@ -255,7 +251,7 @@ getopt_internal (int argc, char **argv, char *shortopts, longopts[optindex].name, match_chars) == 0) { /* do we have an exact match? */ - if (match_chars == (int) (strlen (longopts[optindex].name))) + if (match_chars == strlen (longopts[optindex].name)) { longopt_match = optindex; break; diff --git a/src/graph.cc b/src/graph.cc index 9d45ce1..18adeee 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -20,8 +20,9 @@ #include "build_log.h" #include "depfile_parser.h" #include "disk_interface.h" +#include "explain.h" +#include "manifest_parser.h" #include "metrics.h" -#include "parsers.h" #include "state.h" #include "util.h" @@ -43,6 +44,7 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, // Visit all inputs; we're dirty if any of the inputs are dirty. TimeStamp most_recent_input = 1; + Node* most_recent_node = NULL; for (vector<Node*>::iterator i = inputs_.begin(); i != inputs_.end(); ++i) { if ((*i)->StatIfNecessary(disk_interface)) { if (Edge* edge = (*i)->in_edge()) { @@ -50,6 +52,8 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, return false; } else { // This input has no in-edge; it is dirty if it is missing. + if (!(*i)->exists()) + EXPLAIN("%s has no in-edge and is missing", (*i)->path().c_str()); (*i)->set_dirty(!(*i)->exists()); } } @@ -64,10 +68,13 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, // If a regular input is dirty (or missing), we're dirty. // Otherwise consider mtime. if ((*i)->dirty()) { + EXPLAIN("%s is dirty", (*i)->path().c_str()); dirty = true; } else { - if ((*i)->mtime() > most_recent_input) + if ((*i)->mtime() > most_recent_input) { most_recent_input = (*i)->mtime(); + most_recent_node = *i; + } } } } @@ -81,7 +88,7 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, for (vector<Node*>::iterator i = outputs_.begin(); i != outputs_.end(); ++i) { (*i)->StatIfNecessary(disk_interface); - if (RecomputeOutputDirty(build_log, most_recent_input, command, *i)) { + if (RecomputeOutputDirty(build_log, most_recent_input, most_recent_node, command, *i)) { dirty = true; break; } @@ -107,7 +114,9 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface, bool Edge::RecomputeOutputDirty(BuildLog* build_log, TimeStamp most_recent_input, - const string& command, Node* output) { + Node* most_recent_node, + const string& command, + Node* output) { if (is_phony()) { // Phony edges don't write any output. Outputs are only dirty if // there are no inputs and we're missing the output. @@ -117,8 +126,10 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, BuildLog::LogEntry* entry = 0; // Dirty if we're missing the output. - if (!output->exists()) + if (!output->exists()) { + EXPLAIN("output %s doesn't exist", output->path().c_str()); return true; + } // Dirty if the output is older than the input. if (output->mtime() < most_recent_input) { @@ -128,9 +139,18 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, // considered dirty if an input was modified since the previous run. if (rule_->restat() && build_log && (entry = build_log->LookupByOutput(output->path()))) { - if (entry->restat_mtime < most_recent_input) + if (entry->restat_mtime < most_recent_input) { + EXPLAIN("restat of output %s older than most recent input %s (%d vs %d)", + output->path().c_str(), + most_recent_node ? most_recent_node->path().c_str() : "", + entry->restat_mtime, most_recent_input); return true; + } } else { + EXPLAIN("output %s older than most recent input %s (%d vs %d)", + output->path().c_str(), + most_recent_node ? most_recent_node->path().c_str() : "", + output->mtime(), most_recent_input); return true; } } @@ -140,8 +160,10 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, // dirty. if (!rule_->generator() && build_log && (entry || (entry = build_log->LookupByOutput(output->path())))) { - if (command != entry->command) + if (BuildLog::LogEntry::HashCommand(command) != entry->command_hash) { + EXPLAIN("command line changed for %s", output->path().c_str()); return true; + } } return false; @@ -158,40 +180,44 @@ bool Edge::AllInputsReady() const { /// An Env for an Edge, providing $in and $out. struct EdgeEnv : public Env { - EdgeEnv(Edge* edge) : edge_(edge) {} + explicit EdgeEnv(Edge* edge) : edge_(edge) {} virtual string LookupVariable(const string& var); /// Given a span of Nodes, construct a list of paths suitable for a command /// line. XXX here is where shell-escaping of e.g spaces should happen. string MakePathList(vector<Node*>::iterator begin, - vector<Node*>::iterator end); + vector<Node*>::iterator end, + char sep); Edge* edge_; }; string EdgeEnv::LookupVariable(const string& var) { - if (var == "in") { + if (var == "in" || var == "in_newline") { int explicit_deps_count = edge_->inputs_.size() - edge_->implicit_deps_ - edge_->order_only_deps_; return MakePathList(edge_->inputs_.begin(), - edge_->inputs_.begin() + explicit_deps_count); + edge_->inputs_.begin() + explicit_deps_count, + var == "in" ? ' ' : '\n'); } else if (var == "out") { return MakePathList(edge_->outputs_.begin(), - edge_->outputs_.end()); + edge_->outputs_.end(), + ' '); } else if (edge_->env_) { return edge_->env_->LookupVariable(var); } else { - // XXX shoudl we warn here? + // XXX should we warn here? return string(); } } string EdgeEnv::MakePathList(vector<Node*>::iterator begin, - vector<Node*>::iterator end) { + vector<Node*>::iterator end, + char sep) { string result; for (vector<Node*>::iterator i = begin; i != end; ++i) { if (!result.empty()) - result.push_back(' '); + result.push_back(sep); const string& path = (*i)->path(); if (path.find(" ") != string::npos) { result.append("\""); @@ -297,18 +323,37 @@ bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface, return true; } -void Edge::Dump() { - printf("[ "); - for (vector<Node*>::iterator i = inputs_.begin(); i != inputs_.end(); ++i) { +void Edge::Dump(const char* prefix) const { + printf("%s[ ", prefix); + for (vector<Node*>::const_iterator i = inputs_.begin(); + i != inputs_.end() && *i != NULL; ++i) { printf("%s ", (*i)->path().c_str()); } printf("--%s-> ", rule_->name().c_str()); - for (vector<Node*>::iterator i = outputs_.begin(); i != outputs_.end(); ++i) { + for (vector<Node*>::const_iterator i = outputs_.begin(); + i != outputs_.end() && *i != NULL; ++i) { printf("%s ", (*i)->path().c_str()); } - printf("]\n"); + printf("] 0x%p\n", this); } bool Edge::is_phony() const { return rule_ == &State::kPhonyRule; } + +void Node::Dump(const char* prefix) const { + printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", + prefix, path().c_str(), this, + mtime(), mtime()?"":" (:missing)", + dirty()?" dirty":" clean"); + if (in_edge()) { + in_edge()->Dump("in-edge: "); + }else{ + printf("no in-edge\n"); + } + printf(" out edges:\n"); + for (vector<Edge*>::const_iterator e = out_edges().begin(); + e != out_edges().end() && *e != NULL; ++e) { + (*e)->Dump(" +- "); + } +} diff --git a/src/graph.h b/src/graph.h index aa1bf49..0ef4f3f 100644 --- a/src/graph.h +++ b/src/graph.h @@ -28,8 +28,11 @@ struct Edge; /// Information about a node in the dependency graph: the file, whether /// it's dirty, mtime, etc. struct Node { - Node(const string& path) : path_(path), mtime_(-1), dirty_(false), - in_edge_(NULL) {} + explicit Node(const string& path) + : path_(path), + mtime_(-1), + dirty_(false), + in_edge_(NULL) {} /// Return true if the file exists (mtime_ got a value). bool Stat(DiskInterface* disk_interface); @@ -74,6 +77,8 @@ struct Node { const vector<Edge*>& out_edges() const { return out_edges_; } void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); } + void Dump(const char* prefix="") const; + private: string path_; /// Possible values of mtime_: @@ -97,7 +102,8 @@ private: /// An invokable build command and associated metadata (description, etc.). struct Rule { - Rule(const string& name) : name_(name), generator_(false), restat_(false) { } + explicit Rule(const string& name) + : name_(name), generator_(false), restat_(false) {} const string& name() const { return name_; } @@ -145,7 +151,8 @@ struct Edge { /// Recompute whether a given single output should be marked dirty. /// Returns true if so. bool RecomputeOutputDirty(BuildLog* build_log, TimeStamp most_recent_input, - const string& command, Node* output); + Node* most_recent_node, const string& command, + Node* output); /// Return true if all inputs' in-edges are ready. bool AllInputsReady() const; @@ -168,7 +175,7 @@ struct Edge { bool LoadDepFile(State* state, DiskInterface* disk_interface, string* err); - void Dump(); + void Dump(const char* prefix="") const; const Rule* rule_; vector<Node*> inputs_; diff --git a/src/hash_collision_bench.cc b/src/hash_collision_bench.cc new file mode 100644 index 0000000..6736109 --- /dev/null +++ b/src/hash_collision_bench.cc @@ -0,0 +1,57 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "build_log.h" + +int random(int low, int high) { + return int(low + (rand() / double(RAND_MAX)) * (high - low) + 0.5); +} + +void RandomCommand(char** s) { + int len = random(5, 100); + *s = new char[len]; + for (int i = 0; i < len; ++i) + (*s)[i] = random(32, 127); +} + +int main() { + const int N = 20 * 1000 * 1000; + + // Leak these, else 10% of the runtime is spent destroying strings. + char** commands = new char*[N]; + pair<uint64_t, int>* hashes = new pair<uint64_t, int>[N]; + + srand(time(NULL)); + + for (int i = 0; i < N; ++i) { + RandomCommand(&commands[i]); + hashes[i] = make_pair(BuildLog::LogEntry::HashCommand(commands[i]), i); + } + + sort(hashes, hashes + N); + + int num_collisions = 0; + for (int i = 1; i < N; ++i) { + if (hashes[i - 1].first == hashes[i].first) { + if (strcmp(commands[hashes[i - 1].second], + commands[hashes[i].second]) != 0) { + printf("collision!\n string 1: '%s'\n string 2: '%s'\n", + commands[hashes[i - 1].second], + commands[hashes[i].second]); + num_collisions++; + } + } + } + printf("\n\n%d collisions after %d runs\n", num_collisions, N); +} diff --git a/src/lexer.cc b/src/lexer.cc index 9e4392c..ca6f367 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -123,11 +123,11 @@ Lexer::Token Lexer::ReadToken() { start = p; { - char yych; + unsigned char yych; unsigned int yyaccept = 0; static const unsigned char yybm[] = { 0, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 0, 64, 64, 64, 64, 64, + 64, 64, 0, 64, 64, 0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 192, 64, 64, 64, 64, 64, 64, 64, @@ -216,7 +216,8 @@ yy3: yy4: yyaccept = 1; yych = *(q = ++p); - if (yych >= 0x01) goto yy60; + if (yych <= 0x00) goto yy5; + if (yych != '\r') goto yy60; yy5: { token = ERROR; break; } yy6: @@ -354,7 +355,9 @@ yy60: if (yybm[0+yych] & 64) { goto yy59; } - if (yych >= 0x01) goto yy62; + if (yych <= 0x00) goto yy61; + if (yych <= '\f') goto yy62; +yy61: p = q; if (yyaccept <= 0) { goto yy3; @@ -404,7 +407,7 @@ void Lexer::EatWhitespace() { ofs_ = p; { - char yych; + unsigned char yych; static const unsigned char yybm[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -485,7 +488,7 @@ bool Lexer::ReadIdent(string* out) { const char* start = p; { - char yych; + unsigned char yych; static const unsigned char yybm[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -573,10 +576,10 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { start = p; { - char yych; + unsigned char yych; static const unsigned char yybm[] = { 0, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 16, 128, 128, 128, 0, 128, 128, 128, @@ -609,24 +612,25 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { 128, 128, 128, 128, 128, 128, 128, 128, }; yych = *p; - if (yych <= '#') { + if (yych <= ' ') { if (yych <= '\n') { if (yych <= 0x00) goto yy96; if (yych >= '\n') goto yy92; } else { - if (yych == ' ') goto yy92; + if (yych == '\r') goto yy98; + if (yych >= ' ') goto yy92; } } else { - if (yych <= ':') { - if (yych <= '$') goto yy94; - if (yych >= ':') goto yy92; + if (yych <= '9') { + if (yych == '$') goto yy94; } else { + if (yych <= ':') goto yy92; if (yych == '|') goto yy92; } } ++p; yych = *p; - goto yy120; + goto yy121; yy91: { eval->AddText(StringPiece(start, p - start)); @@ -649,39 +653,40 @@ yy94: ++p; if ((yych = *p) <= '/') { if (yych <= ' ') { - if (yych == '\n') goto yy109; - if (yych <= 0x1F) goto yy98; - goto yy100; + if (yych == '\n') goto yy110; + if (yych <= 0x1F) goto yy99; + goto yy101; } else { if (yych <= '$') { - if (yych <= '#') goto yy98; - goto yy102; + if (yych <= '#') goto yy99; + goto yy103; } else { - if (yych == '-') goto yy104; - goto yy98; + if (yych == '-') goto yy105; + goto yy99; } } } else { if (yych <= '^') { if (yych <= ':') { - if (yych <= '9') goto yy104; - goto yy106; + if (yych <= '9') goto yy105; + goto yy107; } else { - if (yych <= '@') goto yy98; - if (yych <= 'Z') goto yy104; - goto yy98; + if (yych <= '@') goto yy99; + if (yych <= 'Z') goto yy105; + goto yy99; } } else { if (yych <= '`') { - if (yych <= '_') goto yy104; - goto yy98; + if (yych <= '_') goto yy105; + goto yy99; } else { - if (yych <= 'z') goto yy104; - if (yych <= '{') goto yy108; - goto yy98; + if (yych <= 'z') goto yy105; + if (yych <= '{') goto yy109; + goto yy99; } } } +yy95: { last_token_ = start; return Error("lexing error", err); @@ -693,83 +698,86 @@ yy96: return Error("unexpected EOF", err); } yy98: - ++p; + yych = *++p; + goto yy95; yy99: + ++p; +yy100: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy100: +yy101: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy102: +yy103: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy104: +yy105: ++p; yych = *p; - goto yy118; -yy105: + goto yy119; +yy106: { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy106: +yy107: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy108: +yy109: yych = *(q = ++p); if (yybm[0+yych] & 32) { - goto yy112; + goto yy113; } - goto yy99; -yy109: + goto yy100; +yy110: ++p; yych = *p; if (yybm[0+yych] & 16) { - goto yy109; + goto yy110; } { continue; } -yy112: +yy113: ++p; yych = *p; if (yybm[0+yych] & 32) { - goto yy112; + goto yy113; } - if (yych == '}') goto yy115; + if (yych == '}') goto yy116; p = q; - goto yy99; -yy115: + goto yy100; +yy116: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy117: +yy118: ++p; yych = *p; -yy118: +yy119: if (yybm[0+yych] & 64) { - goto yy117; + goto yy118; } - goto yy105; -yy119: + goto yy106; +yy120: ++p; yych = *p; -yy120: +yy121: if (yybm[0+yych] & 128) { - goto yy119; + goto yy120; } goto yy91; } diff --git a/src/lexer.h b/src/lexer.h index 75c1b2f..19008d7 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifndef NINJA_LEXER_H_ +#define NINJA_LEXER_H_ + #include "string_piece.h" // Windows may #define ERROR. @@ -95,3 +98,4 @@ private: const char* last_token_; }; +#endif // NINJA_LEXER_H_ diff --git a/src/lexer.in.cc b/src/lexer.in.cc index 28a5bdf..852d6e9 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -121,7 +121,7 @@ Lexer::Token Lexer::ReadToken() { for (;;) { start = p; /*!re2c - re2c:define:YYCTYPE = "char"; + re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYCURSOR = p; re2c:define:YYMARKER = q; re2c:yyfill:enable = 0; @@ -130,7 +130,7 @@ Lexer::Token Lexer::ReadToken() { simple_varname = [a-zA-Z0-9_-]+; varname = [a-zA-Z0-9_.-]+; - [ ]*"#"[^\000\n]*"\n" { continue; } + [ ]*"#"[^\000\r\n]*"\n" { continue; } [ ]*[\n] { token = NEWLINE; break; } [ ]+ { token = INDENT; break; } "build" { token = BUILD; break; } @@ -200,7 +200,7 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { for (;;) { start = p; /*!re2c - [^$ :\n|\000]+ { + [^$ :\r\n|\000]+ { eval->AddText(StringPiece(start, p - start)); continue; } diff --git a/src/parsers.cc b/src/manifest_parser.cc index c3844fb..057e12c 100644 --- a/src/parsers.cc +++ b/src/manifest_parser.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "parsers.h" +#include "manifest_parser.h" #include <assert.h> #include <errno.h> @@ -218,7 +218,7 @@ bool ManifestParser::ParseEdge(string* err) { ins.push_back(in); } - // Add all order-only deps, counting how many as we go. + // Add all implicit deps, counting how many as we go. int implicit = 0; if (lexer_.PeekToken(Lexer::PIPE)) { for (;;) { diff --git a/src/parsers.h b/src/manifest_parser.h index f889156..a2c6c93 100644 --- a/src/parsers.h +++ b/src/manifest_parser.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef NINJA_PARSERS_H_ -#define NINJA_PARSERS_H_ +#ifndef NINJA_MANIFEST_PARSER_H_ +#define NINJA_MANIFEST_PARSER_H_ #include <string> #include <vector> @@ -68,4 +68,4 @@ private: Lexer lexer_; }; -#endif // NINJA_PARSERS_H_ +#endif // NINJA_MANIFEST_PARSER_H_ diff --git a/src/parsers_test.cc b/src/manifest_parser_test.cc index a8bf179..3261d39 100644 --- a/src/parsers_test.cc +++ b/src/manifest_parser_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "parsers.h" +#include "manifest_parser.h" #include <gtest/gtest.h> @@ -115,6 +115,23 @@ TEST_F(ParserTest, ResponseFiles) { EXPECT_EQ("[$in]", rule->rspfile_content().Serialize()); } +TEST_F(ParserTest, InNewline) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat_rsp\n" +" command = cat $in_newline > $out\n" +"\n" +"build out: cat_rsp in in2\n" +" rspfile=out.rsp\n")); + + ASSERT_EQ(2u, state.rules_.size()); + const Rule* rule = state.rules_.begin()->second; + EXPECT_EQ("cat_rsp", rule->name()); + EXPECT_EQ("[cat ][$in_newline][ > ][$out]", rule->command().Serialize()); + + Edge* edge = state.edges_[0]; + EXPECT_EQ("cat in\nin2 > out", edge->EvaluateCommand()); +} + TEST_F(ParserTest, Variables) { ASSERT_NO_FATAL_FAILURE(AssertParse( "l = one-letter-test\n" @@ -658,3 +675,30 @@ TEST_F(ParserTest, DefaultStatements) { EXPECT_EQ("b", nodes[1]->path()); EXPECT_EQ("c", nodes[2]->path()); } + +TEST_F(ParserTest, UTF8) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule utf8\n" +" command = true\n" +" description = compilaci\xC3\xB3\n")); +} + +// We might want to eventually allow CRLF to be nice to Windows developers, +// but for now just verify we error out with a nice message. +TEST_F(ParserTest, CRLF) { + State state; + ManifestParser parser(&state, NULL); + string err; + + EXPECT_FALSE(parser.ParseTest("# comment with crlf\r\n", + &err)); + EXPECT_EQ("input:1: lexing error\n", + err); + + EXPECT_FALSE(parser.ParseTest("foo = foo\nbar = bar\r\n", + &err)); + EXPECT_EQ("input:2: lexing error\n" + "bar = bar\r\n" + " ^ near here", + err); +} diff --git a/src/metrics.cc b/src/metrics.cc index b2433aa..8407717 100644 --- a/src/metrics.cc +++ b/src/metrics.cc @@ -72,6 +72,7 @@ int64_t TimerToMicros(int64_t dt) { } // anonymous namespace + ScopedMetric::ScopedMetric(Metric* metric) { metric_ = metric; if (!metric_) @@ -102,14 +103,23 @@ void Metrics::Report() { width = max((int)(*i)->name.size(), width); } - printf("%-*s\t%-6s\t%9s\t%s\n", width, - "metric", "count", "total (ms)" , "avg (us)"); + printf("%-*s\t%-6s\t%-9s\t%s\n", width, + "metric", "count", "avg (us)", "total (ms)"); for (vector<Metric*>::iterator i = metrics_.begin(); i != metrics_.end(); ++i) { Metric* metric = *i; double total = metric->sum / (double)1000; double avg = metric->sum / (double)metric->count; printf("%-*s\t%-6d\t%-8.1f\t%.1f\n", width, metric->name.c_str(), - metric->count, total, avg); + metric->count, avg, total); } } + +uint64_t Stopwatch::Now() const { + return TimerToMicros(HighResTimer()); +} + +int64_t GetTimeMillis() { + return 1000 * TimerToMicros(HighResTimer()); +} + diff --git a/src/metrics.h b/src/metrics.h index 74f5f8f..dae6b9f 100644 --- a/src/metrics.h +++ b/src/metrics.h @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifndef NINJA_METRICS_H_ +#define NINJA_METRICS_H_ + #include <string> #include <vector> using namespace std; @@ -30,6 +33,7 @@ struct Metric { int64_t sum; }; + /// A scoped object for recording a metric across the body of a function. /// Used by the METRIC_RECORD macro. struct ScopedMetric { @@ -54,6 +58,30 @@ private: vector<Metric*> metrics_; }; +/// Get the current time as relative to some epoch. +/// Epoch varies between platforms; only useful for measuring elapsed +/// time. +int64_t GetTimeMillis(); + + +/// A simple stopwatch which retruns the time +// in seconds since Restart() was called +class Stopwatch +{ +public: + Stopwatch() : started_(0) {} + + /// Seconds since Restart() call + double Elapsed() const { return 1e-6 * (Now() - started_); } + + void Restart() { started_ = Now(); } + +private: + uint64_t started_; + uint64_t Now() const; +}; + + /// The primary interface to metrics. Use METRIC_RECORD("foobar") at the top /// of a function to get timing stats recorded for each call of the function. #define METRIC_RECORD(name) \ @@ -62,3 +90,5 @@ private: ScopedMetric metrics_h_scoped(metrics_h_metric); extern Metrics* g_metrics; + +#endif // NINJA_METRICS_H_ diff --git a/src/ninja.cc b/src/ninja.cc index 1a02fd6..393f717 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -31,6 +31,7 @@ #include <windows.h> #else #include <getopt.h> +#include <unistd.h> #endif #include "browse.h" @@ -38,15 +39,20 @@ #include "build_log.h" #include "clean.h" #include "edit_distance.h" +#include "explain.h" #include "graph.h" #include "graphviz.h" +#include "manifest_parser.h" #include "metrics.h" -#include "parsers.h" #include "state.h" #include "util.h" namespace { +/// The version number of the current Ninja release. This will always +/// be "git" on trunk. +const char* kVersion = "git"; + /// Global information passed into subtools. struct Globals { Globals() : state(new State()) {} @@ -81,8 +87,10 @@ void Usage(const BuildConfig& config) { "options:\n" " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" +" -V print ninja version (\"%s\")\n" "\n" " -j N run N jobs in parallel [default=%d]\n" +" -l N do not start new jobs if the load average is greater than N\n" " -k N keep going until N jobs fail [default=1]\n" " -n dry run (don't run commands but pretend they succeeded)\n" " -v show all command lines while building\n" @@ -91,7 +99,7 @@ void Usage(const BuildConfig& config) { " -t TOOL run a subtool\n" " use '-t list' to list subtools.\n" " terminates toplevel options; further flags are passed to the tool.\n", - config.parallelism); + kVersion, config.parallelism); } /// Choose a default value for the -j (parallelism) flag. @@ -197,6 +205,8 @@ bool CollectTargetsFromArgs(State* state, int argc, char* argv[], if (path == "clean") { *err += ", did you mean 'ninja -t clean'?"; + } else if (path == "help") { + *err += ", did you mean 'ninja -h'?"; } else { Node* suggestion = state->SpellcheckNode(path); if (suggestion) { @@ -566,12 +576,16 @@ int RunTool(const string& tool, Globals* globals, int argc, char** argv) { bool DebugEnable(const string& name, Globals* globals) { if (name == "list") { printf("debugging modes:\n" -" stats print operation counts/timing info\n"); -//"multiple modes can be enabled via -d FOO -d BAR\n"); +" stats print operation counts/timing info\n" +" explain explain what caused a command to execute\n" +"multiple modes can be enabled via -d FOO -d BAR\n"); return false; } else if (name == "stats") { g_metrics = new Metrics; return true; + } else if (name == "explain") { + g_explaining = true; + return true; } else { printf("ninja: unknown debug setting '%s'\n", name.c_str()); return false; @@ -667,7 +681,7 @@ int main(int argc, char** argv) { int opt; while (tool.empty() && - (opt = getopt_long(argc, argv, "d:f:hj:k:nt:vC:", kLongOptions, + (opt = getopt_long(argc, argv, "d:f:hj:k:l:nt:vC:V", kLongOptions, NULL)) != -1) { switch (opt) { case 'd': @@ -680,11 +694,19 @@ int main(int argc, char** argv) { case 'j': globals.config.parallelism = atoi(optarg); break; + case 'l': { + char* end; + double value = strtod(optarg, &end); + if (end == optarg) + Fatal("-l parameter not numeric: did you mean -l 0.0?"); + globals.config.max_load_average = value; + break; + } case 'k': { char* end; int value = strtol(optarg, &end, 10); if (*end != 0) - Fatal("-k parameter not numeric; did you mean -k0?"); + Fatal("-k parameter not numeric; did you mean -k 0?"); // We want to go until N jobs fail, which means we should allow // N failures and then stop. For N <= 0, INT_MAX is close enough @@ -704,6 +726,9 @@ int main(int argc, char** argv) { case 'C': working_dir = optarg; break; + case 'V': + printf("%s\n", kVersion); + return 0; case 'h': default: Usage(globals.config); @@ -717,7 +742,10 @@ int main(int argc, char** argv) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for // subsequent commands. - printf("ninja: Entering directory `%s'\n", working_dir); + // Don't print this if a tool is being used, so that tool output + // can be piped into a file without this string showing up. + if (tool == "") + printf("ninja: Entering directory `%s'\n", working_dir); #ifdef _WIN32 if (_chdir(working_dir) < 0) { #else @@ -757,13 +785,12 @@ reload: log_path = build_dir + "/" + kLogPath; } - if (!build_log.Load(log_path.c_str(), &err)) { - Error("loading build log %s: %s", - log_path.c_str(), err.c_str()); + if (!build_log.Load(log_path, &err)) { + Error("loading build log %s: %s", log_path.c_str(), err.c_str()); return 1; } - if (!build_log.OpenForWrite(log_path.c_str(), &err)) { + if (!build_log.OpenForWrite(log_path, &err)) { Error("opening build log: %s", err.c_str()); return 1; } diff --git a/src/parser_perftest.cc b/src/parser_perftest.cc index b1d5bf0..b215221 100644 --- a/src/parser_perftest.cc +++ b/src/parser_perftest.cc @@ -17,6 +17,7 @@ #include "depfile_parser.h" #include "util.h" +#include "metrics.h" int main(int argc, char* argv[]) { if (argc < 2) { diff --git a/src/state.h b/src/state.h index 845c947..9197ef8 100644 --- a/src/state.h +++ b/src/state.h @@ -14,7 +14,6 @@ #ifndef NINJA_STATE_H_ #define NINJA_STATE_H_ -#pragma once #include <map> #include <string> @@ -72,7 +71,7 @@ struct State { BindingEnv bindings_; vector<Node*> defaults_; - struct BuildLog* build_log_; + BuildLog* build_log_; }; #endif // NINJA_STATE_H_ diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index fef0bc8..38b6957 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -1,4 +1,4 @@ -// Copyright 2011 Google Inc. All Rights Reserved. +// Copyright 2012 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ Subprocess::~Subprocess() { HANDLE Subprocess::SetupPipe(HANDLE ioport) { char pipe_name[100]; snprintf(pipe_name, sizeof(pipe_name), - "\\\\.\\pipe\\ninja_pid%u_sp%p", GetCurrentProcessId(), this); + "\\\\.\\pipe\\ninja_pid%lu_sp%p", GetCurrentProcessId(), this); pipe_ = ::CreateNamedPipeA(pipe_name, PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED, @@ -80,15 +80,24 @@ HANDLE Subprocess::SetupPipe(HANDLE ioport) { bool Subprocess::Start(SubprocessSet* set, const string& command) { HANDLE child_pipe = SetupPipe(set->ioport_); + SECURITY_ATTRIBUTES security_attributes; + memset(&security_attributes, 0, sizeof(SECURITY_ATTRIBUTES)); + security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); + security_attributes.bInheritHandle = TRUE; + // Must be inheritable so subprocesses can dup to children. + HANDLE nul = CreateFile("NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &security_attributes, OPEN_EXISTING, 0, NULL); + if (nul == INVALID_HANDLE_VALUE) + Fatal("couldn't open nul"); + STARTUPINFOA startup_info; memset(&startup_info, 0, sizeof(startup_info)); startup_info.cb = sizeof(STARTUPINFO); startup_info.dwFlags = STARTF_USESTDHANDLES; + startup_info.hStdInput = nul; startup_info.hStdOutput = child_pipe; - // TODO: what does this hook up stdin to? - startup_info.hStdInput = NULL; - // TODO: is it ok to reuse pipe like this? - startup_info.hStdError = child_pipe; + startup_info.hStdError = child_pipe; PROCESS_INFORMATION process_info; memset(&process_info, 0, sizeof(process_info)); @@ -104,6 +113,7 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { if (child_pipe) CloseHandle(child_pipe); CloseHandle(pipe_); + CloseHandle(nul); pipe_ = NULL; // child_ is already NULL; buf_ = "CreateProcess failed: The system cannot find the file specified.\n"; @@ -116,6 +126,7 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { // Close pipe channel only used by the child. if (child_pipe) CloseHandle(child_pipe); + CloseHandle(nul); CloseHandle(process_info.hThread); child_ = process_info.hProcess; @@ -207,7 +218,7 @@ BOOL WINAPI SubprocessSet::NotifyInterrupted(DWORD dwCtrlType) { return FALSE; } -Subprocess *SubprocessSet::Add(const string &command) { +Subprocess *SubprocessSet::Add(const string& command) { Subprocess *subprocess = new Subprocess; if (!subprocess->Start(this, command)) { delete subprocess; @@ -260,9 +271,12 @@ Subprocess* SubprocessSet::NextFinished() { void SubprocessSet::Clear() { for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) { - if ((*i)->child_) - if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, GetProcessId((*i)->child_))) + if ((*i)->child_) { + if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, + GetProcessId((*i)->child_))) { Win32Fatal("GenerateConsoleCtrlEvent"); + } + } } for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) diff --git a/src/subprocess.cc b/src/subprocess.cc index 99de93f..1c47fd1 100644 --- a/src/subprocess.cc +++ b/src/subprocess.cc @@ -25,6 +25,13 @@ #include <string.h> #include <sys/wait.h> +// Older versions of glibc (like 2.4) won't find this in <poll.h>. glibc +// 2.4 keeps it in <asm-generic/poll.h>, though attempting to include that +// will redefine the pollfd structure. +#ifndef POLLRDHUP +#define POLLRDHUP 0x2000 +#endif + #include "util.h" Subprocess::Subprocess() : fd_(-1), pid_(-1) { @@ -45,7 +52,7 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { #if !defined(linux) // On linux we use ppoll in DoWork(); elsewhere we use pselect and so must // avoid overly-large FDs. - if (fd_ >= FD_SETSIZE) + if (fd_ >= static_cast<int>(FD_SETSIZE)) Fatal("pipe: %s", strerror(EMFILE)); #endif // !linux SetCloseOnExec(fd_); @@ -172,7 +179,7 @@ SubprocessSet::~SubprocessSet() { Fatal("sigprocmask: %s", strerror(errno)); } -Subprocess *SubprocessSet::Add(const string &command) { +Subprocess *SubprocessSet::Add(const string& command) { Subprocess *subprocess = new Subprocess; if (!subprocess->Start(this, command)) { delete subprocess; diff --git a/src/subprocess.h b/src/subprocess.h index 3294416..4c1629c 100644 --- a/src/subprocess.h +++ b/src/subprocess.h @@ -75,7 +75,7 @@ struct SubprocessSet { SubprocessSet(); ~SubprocessSet(); - Subprocess *Add(const string &command); + Subprocess* Add(const string& command); bool DoWork(); Subprocess* NextFinished(); void Clear(); diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index c155012..7dbaf97 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -16,6 +16,13 @@ #include "test.h" +#ifndef _WIN32 +// SetWithLots need setrlimit. +#include <sys/time.h> +#include <sys/resource.h> +#include <unistd.h> +#endif + namespace { #ifdef _WIN32 @@ -142,7 +149,7 @@ TEST_F(SubprocessTest, SetWithMulti) { } } -#ifdef linux +#ifndef _WIN32 TEST_F(SubprocessTest, SetWithLots) { // Arbitrary big number; needs to be over 1024 to confirm we're no longer // hostage to pselect. @@ -169,4 +176,4 @@ TEST_F(SubprocessTest, SetWithLots) { } ASSERT_EQ(kNumProcs, subprocs_.finished_.size()); } -#endif // linux +#endif // _WIN32 diff --git a/src/test.cc b/src/test.cc index 2fbb4df..0138b3a 100644 --- a/src/test.cc +++ b/src/test.cc @@ -18,7 +18,8 @@ #include <errno.h> -#include "parsers.h" +#include "build_log.h" +#include "manifest_parser.h" #include "util.h" #ifdef _WIN32 @@ -89,6 +90,10 @@ void AssertParse(State* state, const char* input) { ASSERT_EQ("", err); } +void AssertHash(const char* expected, uint64_t actual) { + ASSERT_EQ(BuildLog::LogEntry::HashCommand(expected), actual); +} + void VirtualFileSystem::Create(const string& path, int time, const string& contents) { files_[path].mtime = time; @@ -35,6 +35,7 @@ struct StateTestWithBuiltinRules : public testing::Test { }; void AssertParse(State* state, const char* input); +void AssertHash(const char* expected, uint64_t actual); /// An implementation of DiskInterface that uses an in-memory representation /// of disk state. It also logs file accesses and directory creations diff --git a/src/util.cc b/src/util.cc index e59ff74..be74c4d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -16,6 +16,7 @@ #ifdef _WIN32 #include <windows.h> +#include <io.h> #endif #include <errno.h> @@ -43,7 +44,7 @@ void Fatal(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: FATAL: "); + fprintf(stderr, "ninja: fatal: "); va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); @@ -61,7 +62,7 @@ void Fatal(const char* msg, ...) { void Warning(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: WARNING: "); + fprintf(stderr, "ninja: warning: "); va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); @@ -70,7 +71,7 @@ void Warning(const char* msg, ...) { void Error(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: ERROR: "); + fprintf(stderr, "ninja: error: "); va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); @@ -113,40 +114,39 @@ bool CanonicalizePath(char* path, int* len, string* err) { } while (src < end) { - const char* sep = (const char*)memchr(src, '/', end - src); - if (sep == NULL) - sep = end; - if (*src == '.') { - if (sep - src == 1) { + if (src + 1 == end || src[1] == '/') { // '.' component; eliminate. src += 2; continue; - } else if (sep - src == 2 && src[1] == '.') { + } else if (src[1] == '.' && (src + 2 == end || src[2] == '/')) { // '..' component. Back up if possible. if (component_count > 0) { dst = components[component_count - 1]; src += 3; --component_count; } else { - while (src <= sep) - *dst++ = *src++; + *dst++ = *src++; + *dst++ = *src++; + *dst++ = *src++; } continue; } } - if (sep > src) { - if (component_count == kMaxPathComponents) - Fatal("path has too many components"); - components[component_count] = dst; - ++component_count; - while (src <= sep) { - *dst++ = *src++; - } + if (*src == '/') { + src++; + continue; } - src = sep + 1; + if (component_count == kMaxPathComponents) + Fatal("path has too many components"); + components[component_count] = dst; + ++component_count; + + while (*src != '/' && src != end) + *dst++ = *src++; + *dst++ = *src++; // Copy '/' or final \0 character as well. } if (dst == start) { @@ -205,16 +205,6 @@ void SetCloseOnExec(int fd) { #endif // ! _WIN32 } -int64_t GetTimeMillis() { -#ifdef _WIN32 - // GetTickCount64 is only available on Vista or later. - return GetTickCount(); -#else - timeval now; - gettimeofday(&now, NULL); - return ((int64_t)now.tv_sec * 1000) + (now.tv_usec / 1000); -#endif -} const char* SpellcheckStringV(const string& text, const vector<const char*>& words) { @@ -294,6 +284,35 @@ string StripAnsiEscapeCodes(const string& in) { return stripped; } +#ifdef _WIN32 +static double GetLoadAverage_win32() +{ + // TODO(nicolas.despres@gmail.com): Find a way to implement it on Windows. + return -0.0f; +} +#else +static double GetLoadAverage_unix() +{ + double loadavg[3] = { 0.0f, 0.0f, 0.0f }; + if (getloadavg(loadavg, 3) < 0) + { + // Maybe we should return an error here or the availability of + // getloadavg(3) should be checked when ninja is configured. + return -0.0f; + } + return loadavg[0]; +} +#endif // _WIN32 + +double GetLoadAverage() +{ +#ifdef _WIN32 + return GetLoadAverage_win32(); +#else + return GetLoadAverage_unix(); +#endif // _WIN32 +} + #ifdef _MSC_VER
typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) (
IN HANDLE,
@@ -370,3 +389,4 @@ int exception_filter(unsigned int code, struct _EXCEPTION_POINTERS *ep) { //on Linux or MinGW, core dumps are created automatically, no code needed
#endif
+ @@ -14,7 +14,6 @@ #ifndef NINJA_UTIL_H_ #define NINJA_UTIL_H_ -#pragma once #ifdef _WIN32 #include "win32port.h" @@ -53,11 +52,6 @@ int ReadFile(const string& path, string* contents, string* err); /// Mark a file descriptor to not be inherited on exec()s. void SetCloseOnExec(int fd); -/// Get the current time as relative to some epoch. -/// Epoch varies between platforms; only useful for measuring elapsed -/// time. -int64_t GetTimeMillis(); - /// Given a misspelled string and a list of correct spellings, returns /// the closest match or NULL if there is no close enough match. const char* SpellcheckStringV(const string& text, const vector<const char*>& words); @@ -68,11 +62,16 @@ const char* SpellcheckString(const string& text, ...); /// Removes all Ansi escape codes (http://www.termsys.demon.co.uk/vtansi.htm). string StripAnsiEscapeCodes(const string& in); +/// @return the load average of the machine. A negative value is returned +/// on error. +double GetLoadAverage(); + #ifdef _MSC_VER #define snprintf _snprintf #define fileno _fileno #define unlink _unlink #define chdir _chdir +#define strtoull _strtoui64 #endif #ifdef _WIN32 diff --git a/src/util_test.cc b/src/util_test.cc index f7ccf00..23d4b83 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -58,6 +58,14 @@ TEST(CanonicalizePath, PathSamples) { path = "foo/./."; EXPECT_TRUE(CanonicalizePath(&path, &err)); EXPECT_EQ("foo", path); + + path = "foo/bar/.."; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo", path); + + path = "foo/.hidden_bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/.hidden_bar", path); } TEST(CanonicalizePath, EmptyResult) { @@ -94,6 +102,24 @@ TEST(CanonicalizePath, AbsolutePath) { EXPECT_EQ("/usr/include/stdio.h", path); } +TEST(CanonicalizePath, NotNullTerminated) { + string path; + string err; + int len; + + path = "foo/. bar/."; + len = strlen("foo/."); // Canonicalize only the part before the space. + EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); + EXPECT_EQ(strlen("foo"), static_cast<size_t>(len)); + EXPECT_EQ("foo/. bar/.", string(path)); + + path = "foo/../file bar/."; + len = strlen("foo/../file"); + EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); + EXPECT_EQ(strlen("file"), static_cast<size_t>(len)); + EXPECT_EQ("file ./file bar/.", string(path)); +} + TEST(StripAnsiEscapeCodes, EscapeAtEnd) { string stripped = StripAnsiEscapeCodes("foo\33"); EXPECT_EQ("foo", stripped); diff --git a/src/win32port.h b/src/win32port.h index 4abdb14..ce3c949 100644 --- a/src/win32port.h +++ b/src/win32port.h @@ -1,10 +1,31 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #ifndef NINJA_WIN32PORT_H_ #define NINJA_WIN32PORT_H_ -#pragma once +typedef signed short int16_t; +typedef unsigned short uint16_t; /// A 64-bit integer type typedef signed long long int64_t; typedef unsigned long long uint64_t; +// printf format specifier for uint64_t, from C99. +#ifndef PRIu64 +#define PRIu64 "I64u" +#define PRIx64 "I64x" +#endif + #endif // NINJA_WIN32PORT_H_ |