diff options
Diffstat (limited to 'src')
34 files changed, 1041 insertions, 464 deletions
diff --git a/src/browse.py b/src/browse.py index 17e67cf..7f15e50 100755 --- a/src/browse.py +++ b/src/browse.py @@ -20,7 +20,12 @@ This script is inlined into the final executable and spawned by it when needed. """ -import BaseHTTPServer +from __future__ import print_function + +try: + import http.server as httpserver +except ImportError: + import BaseHTTPServer as httpserver import subprocess import sys import webbrowser @@ -55,12 +60,12 @@ def parse(text): outputs = [] try: - target = lines.next()[:-1] # strip trailing colon + target = next(lines)[:-1] # strip trailing colon - line = lines.next() + line = next(lines) (match, rule) = match_strip(line, ' input: ') if match: - (match, line) = match_strip(lines.next(), ' ') + (match, line) = match_strip(next(lines), ' ') while match: type = None (match, line) = match_strip(line, '| ') @@ -70,21 +75,21 @@ def parse(text): if match: type = 'order-only' inputs.append((line, type)) - (match, line) = match_strip(lines.next(), ' ') + (match, line) = match_strip(next(lines), ' ') match, _ = match_strip(line, ' outputs:') if match: - (match, line) = match_strip(lines.next(), ' ') + (match, line) = match_strip(next(lines), ' ') while match: outputs.append(line) - (match, line) = match_strip(lines.next(), ' ') + (match, line) = match_strip(next(lines), ' ') except StopIteration: pass return Node(inputs, rule, target, outputs) -def generate_html(node): - print '''<!DOCTYPE html> +def create_page(body): + return '''<!DOCTYPE html> <style> body { font-family: sans; @@ -108,34 +113,42 @@ tt { .filelist { -webkit-columns: auto 2; } -</style>''' +</style> +''' + body - print '<h1><tt>%s</tt></h1>' % node.target +def generate_html(node): + document = ['<h1><tt>%s</tt></h1>' % node.target] if node.inputs: - print '<h2>target is built using rule <tt>%s</tt> of</h2>' % node.rule + document.append('<h2>target is built using rule <tt>%s</tt> of</h2>' % + node.rule) if len(node.inputs) > 0: - print '<div class=filelist>' + document.append('<div class=filelist>') for input, type in sorted(node.inputs): extra = '' if type: extra = ' (%s)' % type - print '<tt><a href="?%s">%s</a>%s</tt><br>' % (input, input, extra) - print '</div>' + document.append('<tt><a href="?%s">%s</a>%s</tt><br>' % + (input, input, extra)) + document.append('</div>') if node.outputs: - print '<h2>dependent edges build:</h2>' - print '<div class=filelist>' + document.append('<h2>dependent edges build:</h2>') + document.append('<div class=filelist>') for output in sorted(node.outputs): - print '<tt><a href="?%s">%s</a></tt><br>' % (output, output) - print '</div>' + document.append('<tt><a href="?%s">%s</a></tt><br>' % + (output, output)) + document.append('</div>') + + return '\n'.join(document) def ninja_dump(target): proc = subprocess.Popen([sys.argv[1], '-t', 'query', target], - stdout=subprocess.PIPE) - return proc.communicate()[0] + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + return proc.communicate() + (proc.returncode,) -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(httpserver.BaseHTTPRequestHandler): def do_GET(self): assert self.path[0] == '/' target = self.path[1:] @@ -152,28 +165,28 @@ class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): return target = target[1:] - input = ninja_dump(target) + ninja_output, ninja_error, exit_code = ninja_dump(target) + if exit_code == 0: + page_body = generate_html(parse(ninja_output.strip())) + else: + # Relay ninja's error message. + page_body = '<h1><tt>%s</tt></h1>' % ninja_error self.send_response(200) self.end_headers() - stdout = sys.stdout - sys.stdout = self.wfile - try: - generate_html(parse(input.strip())) - finally: - sys.stdout = stdout + self.wfile.write(create_page(page_body).encode('utf-8')) def log_message(self, format, *args): pass # Swallow console spam. port = 8000 -httpd = BaseHTTPServer.HTTPServer(('',port), RequestHandler) +httpd = httpserver.HTTPServer(('',port), RequestHandler) try: - print 'Web server running on port %d, ctl-C to abort...' % port + print('Web server running on port %d, ctl-C to abort...' % port) webbrowser.open_new('http://localhost:%s' % port) httpd.serve_forever() except KeyboardInterrupt: - print + print() pass # Swallow console spam. diff --git a/src/build.cc b/src/build.cc index e1aaad1..b4229c4 100644 --- a/src/build.cc +++ b/src/build.cc @@ -17,6 +17,7 @@ #include <assert.h> #include <stdio.h> #include <stdlib.h> +#include <functional> #ifdef _WIN32 #include <windows.h> @@ -37,13 +38,50 @@ #include "subprocess.h" #include "util.h" +namespace { + +/// A CommandRunner that doesn't actually run the commands. +struct DryRunCommandRunner : public CommandRunner { + virtual ~DryRunCommandRunner() {} + + // Overridden from CommandRunner: + virtual bool CanRunMore(); + virtual bool StartCommand(Edge* edge); + virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */); + + private: + queue<Edge*> finished_; +}; + +bool DryRunCommandRunner::CanRunMore() { + return true; +} + +bool DryRunCommandRunner::StartCommand(Edge* edge) { + finished_.push(edge); + return true; +} + +Edge* DryRunCommandRunner::WaitForCommand(ExitStatus* status, + string* /*output*/) { + if (finished_.empty()) { + *status = ExitFailure; + return NULL; + } + *status = ExitSuccess; + Edge* edge = finished_.front(); + finished_.pop(); + return edge; +} + +} // namespace + BuildStatus::BuildStatus(const BuildConfig& config) : config_(config), start_time_millis_(GetTimeMillis()), started_edges_(0), finished_edges_(0), total_edges_(0), have_blank_line_(true), progress_status_format_(NULL), - overall_rate_(), current_rate_(), - current_rate_average_count_(config.parallelism) { + overall_rate_(), current_rate_(config.parallelism) { #ifndef _WIN32 const char* term = getenv("TERM"); smart_terminal_ = isatty(1) && term && string(term) != "dumb"; @@ -136,9 +174,11 @@ void BuildStatus::BuildFinished() { printf("\n"); } -string BuildStatus::FormatProgressStatus(const char* progress_status_format) const { +string BuildStatus::FormatProgressStatus( + const char* progress_status_format) const { string out; char buf[32]; + int percent; for (const char* s = progress_status_format; *s != '\0'; ++s) { if (*s == '%') { ++s; @@ -177,30 +217,31 @@ string BuildStatus::FormatProgressStatus(const char* progress_status_format) con out += buf; break; - // Overall finished edges per second. + // Overall finished edges per second. case 'o': - overall_rate_.UpdateRate(finished_edges_, finished_edges_); - overall_rate_.snprinfRate(buf, "%.1f"); + overall_rate_.UpdateRate(finished_edges_); + snprinfRate(overall_rate_.rate(), buf, "%.1f"); out += buf; break; - // Current rate, average over the last '-j' jobs. + // Current rate, average over the last '-j' jobs. case 'c': - // TODO use sliding window? - if (finished_edges_ > current_rate_.last_update() && - finished_edges_ - current_rate_.last_update() == current_rate_average_count_) { - current_rate_.UpdateRate(current_rate_average_count_, finished_edges_); - current_rate_.Restart(); - } - current_rate_.snprinfRate(buf, "%.0f"); + current_rate_.UpdateRate(finished_edges_); + snprinfRate(current_rate_.rate(), buf, "%.1f"); + out += buf; + break; + + // Percentage + case 'p': + percent = (100 * started_edges_) / total_edges_; + snprintf(buf, sizeof(buf), "%3i%%", percent); out += buf; break; - default: { + default: Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s); return ""; } - } } else { out.push_back(*s); } @@ -325,7 +366,7 @@ bool Plan::AddSubTarget(Node* node, vector<Node*>* stack, string* err) { want = true; ++wanted_edges_; if (edge->AllInputsReady()) - ready_.insert(edge); + ScheduleWork(edge); if (!edge->is_phony()) ++command_edges_; } @@ -374,6 +415,22 @@ Edge* Plan::FindWork() { return edge; } +void Plan::ScheduleWork(Edge* edge) { + Pool* pool = edge->pool(); + if (pool->ShouldDelayEdge()) { + pool->DelayEdge(edge); + pool->RetrieveReadyEdges(&ready_); + } else { + pool->EdgeScheduled(*edge); + ready_.insert(edge); + } +} + +void Plan::ResumeDelayedJobs(Edge* edge) { + edge->pool()->EdgeFinished(*edge); + edge->pool()->RetrieveReadyEdges(&ready_); +} + void Plan::EdgeFinished(Edge* edge) { map<Edge*, bool>::iterator i = want_.find(edge); assert(i != want_.end()); @@ -382,6 +439,9 @@ void Plan::EdgeFinished(Edge* edge) { want_.erase(i); edge->outputs_ready_ = true; + // See if this job frees up any delayed jobs + ResumeDelayedJobs(edge); + // Check off any nodes we were waiting for with this edge. for (vector<Node*>::iterator i = edge->outputs_.begin(); i != edge->outputs_.end(); ++i) { @@ -400,7 +460,7 @@ void Plan::NodeFinished(Node* node) { // See if the edge is now ready. if ((*i)->AllInputsReady()) { if (want_i->second) { - ready_.insert(*i); + ScheduleWork(*i); } else { // We do not need to build this edge, but we might need to build one of // its dependents. @@ -422,8 +482,9 @@ void Plan::CleanNode(DependencyScan* scan, Node* node) { // If all non-order-only inputs for this edge are now clean, // we might have changed the dirty state of the outputs. - vector<Node*>::iterator begin = (*ei)->inputs_.begin(), - end = (*ei)->inputs_.end() - (*ei)->order_only_deps_; + vector<Node*>::iterator + begin = (*ei)->inputs_.begin(), + end = (*ei)->inputs_.end() - (*ei)->order_only_deps_; if (find_if(begin, end, mem_fun(&Node::dirty)) == end) { // Recompute most_recent_input and command. Node* most_recent_input = NULL; @@ -533,30 +594,6 @@ Edge* RealCommandRunner::WaitForCommand(ExitStatus* status, string* output) { return edge; } -/// A CommandRunner that doesn't actually run the commands. -struct DryRunCommandRunner : public CommandRunner { - virtual ~DryRunCommandRunner() {} - virtual bool CanRunMore() { - return true; - } - virtual bool StartCommand(Edge* edge) { - finished_.push(edge); - return true; - } - virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */) { - if (finished_.empty()) { - *status = ExitFailure; - return NULL; - } - *status = ExitSuccess; - Edge* edge = finished_.front(); - finished_.pop(); - return edge; - } - - queue<Edge*> finished_; -}; - Builder::Builder(State* state, const BuildConfig& config, BuildLog* log, DiskInterface* disk_interface) : state_(state), config_(config), disk_interface_(disk_interface), @@ -718,6 +755,7 @@ bool Builder::Build(string* err) { } bool Builder::StartEdge(Edge* edge, string* err) { + METRIC_RECORD("StartEdge"); if (edge->is_phony()) return true; @@ -734,8 +772,10 @@ bool Builder::StartEdge(Edge* edge, string* err) { // Create response file, if needed // XXX: this may also block; do we care? if (edge->HasRspFile()) { - if (!disk_interface_->WriteFile(edge->GetRspFile(), edge->GetRspFileContent())) + if (!disk_interface_->WriteFile(edge->GetRspFile(), + edge->GetRspFileContent())) { return false; + } } // start command computing and run it @@ -748,6 +788,7 @@ bool Builder::StartEdge(Edge* edge, string* err) { } void Builder::FinishEdge(Edge* edge, bool success, const string& output) { + METRIC_RECORD("FinishEdge"); TimeStamp restat_mtime = 0; if (success) { @@ -777,7 +818,8 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { } if (restat_mtime != 0 && !edge->rule().depfile().empty()) { - TimeStamp depfile_mtime = disk_interface_->Stat(edge->EvaluateDepFile()); + TimeStamp depfile_mtime = + disk_interface_->Stat(edge->EvaluateDepFile()); if (depfile_mtime > restat_mtime) restat_mtime = depfile_mtime; } diff --git a/src/build.h b/src/build.h index 3e7a144..23f653e 100644 --- a/src/build.h +++ b/src/build.h @@ -15,13 +15,13 @@ #ifndef NINJA_BUILD_H_ #define NINJA_BUILD_H_ +#include <cstdio> #include <map> +#include <memory> +#include <queue> #include <set> #include <string> -#include <queue> #include <vector> -#include <memory> -#include <cstdio> #include "graph.h" // XXX needed for DependencyScan; should rearrange. #include "exit_status.h" @@ -70,6 +70,16 @@ private: bool CheckDependencyCycle(Node* node, vector<Node*>* stack, string* err); void NodeFinished(Node* node); + /// Submits a ready edge as a candidate for execution. + /// The edge may be delayed from running, for example if it's a member of a + /// currently-full pool. + void ScheduleWork(Edge* edge); + + /// Allows jobs blocking on |edge| to potentially resume. + /// For example, if |edge| is a member of a pool, calling this may schedule + /// previously pending jobs in that pool. + void ResumeDelayedJobs(Edge* edge); + /// Keep track of which edges we want to build in this plan. If this map does /// not contain an entry for an edge, we do not want to build the entry or its /// dependents. If an entry maps to false, we do not want to build it, but we @@ -175,7 +185,7 @@ struct BuildStatus { /// Format the progress status string by replacing the placeholders. /// See the user manual for more information about the available /// placeholders. - /// @param progress_status_format_ The format of the progress status. + /// @param progress_status_format The format of the progress status. string FormatProgressStatus(const char* progress_status_format) const; private: @@ -200,38 +210,56 @@ struct BuildStatus { /// The custom progress status format to use. const char* progress_status_format_; + template<size_t S> + void snprinfRate(double rate, char(&buf)[S], const char* format) const { + if (rate == -1) snprintf(buf, S, "?"); + else snprintf(buf, S, format, rate); + } + struct RateInfo { - RateInfo() : last_update_(0), rate_(-1) {} - - double rate() const { return rate_; } - int last_update() const { return last_update_; } - void Restart() { return stopwatch_.Restart(); } - - double UpdateRate(int edges, int update_hint) { - if (update_hint != last_update_) { - rate_ = edges / stopwatch_.Elapsed() + 0.5; - last_update_ = update_hint; - } - return rate_; + RateInfo() : rate_(-1) {} + + void Restart() { stopwatch_.Restart(); } + double rate() { return rate_; } + + void UpdateRate(int edges) { + if (edges && stopwatch_.Elapsed()) + rate_ = edges / stopwatch_.Elapsed(); } - template<class T> - void snprinfRate(T buf, const char* format) { - if (rate_ == -1) - snprintf(buf, sizeof(buf), "?"); - else - snprintf(buf, sizeof(buf), format, rate_); + private: + double rate_; + Stopwatch stopwatch_; + }; + + struct SlidingRateInfo { + SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {} + + void Restart() { stopwatch_.Restart(); } + double rate() { return rate_; } + + void UpdateRate(int update_hint) { + if (update_hint == last_update_) + return; + last_update_ = update_hint; + + if (times_.size() == N) + times_.pop(); + times_.push(stopwatch_.Elapsed()); + if (times_.back() != times_.front()) + rate_ = times_.size() / (times_.back() - times_.front()); } private: + double rate_; Stopwatch stopwatch_; + const size_t N; + std::queue<double> times_; int last_update_; - double rate_; }; mutable RateInfo overall_rate_; - mutable RateInfo current_rate_; - const int current_rate_average_count_; + mutable SlidingRateInfo current_rate_; #ifdef _WIN32 void* console_; diff --git a/src/build_log.cc b/src/build_log.cc index a633892..6b73002 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -56,7 +56,7 @@ uint64_t MurmurHash64A(const void* key, size_t len) { uint64_t h = seed ^ (len * m); const uint64_t * data = (const uint64_t *)key; const uint64_t * end = data + (len/8); - while(data != end) { + while (data != end) { uint64_t k = *data++; k *= m; k ^= k >> r; @@ -65,7 +65,7 @@ uint64_t MurmurHash64A(const void* key, size_t len) { h *= m; } const unsigned char* data2 = (const unsigned char*)data; - switch(len & 7) + switch (len & 7) { case 7: h ^= uint64_t(data2[6]) << 48; case 6: h ^= uint64_t(data2[5]) << 40; @@ -91,6 +91,15 @@ uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { return MurmurHash64A(command.str_, command.len_); } +BuildLog::LogEntry::LogEntry(const string& output) + : output(output) {} + +BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash, + int start_time, int end_time, TimeStamp restat_mtime) + : output(output), command_hash(command_hash), + start_time(start_time), end_time(end_time), restat_mtime(restat_mtime) +{} + BuildLog::BuildLog() : log_file_(NULL), needs_recompaction_(false) {} @@ -130,6 +139,7 @@ bool BuildLog::OpenForWrite(const string& path, string* err) { void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, TimeStamp restat_mtime) { string command = edge->EvaluateCommand(true); + uint64_t command_hash = LogEntry::HashCommand(command); for (vector<Node*>::iterator out = edge->outputs_.begin(); out != edge->outputs_.end(); ++out) { const string& path = (*out)->path(); @@ -138,11 +148,10 @@ void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, if (i != entries_.end()) { log_entry = i->second; } else { - log_entry = new LogEntry; - log_entry->output = path; + log_entry = new LogEntry(path); entries_.insert(Entries::value_type(log_entry->output, log_entry)); } - log_entry->command_hash = LogEntry::HashCommand(command); + log_entry->command_hash = command_hash; log_entry->start_time = start_time; log_entry->end_time = end_time; log_entry->restat_mtime = restat_mtime; @@ -158,8 +167,7 @@ void BuildLog::Close() { log_file_ = NULL; } -class LineReader { - public: +struct LineReader { explicit LineReader(FILE* file) : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) { memset(buf_, 0, sizeof(buf_)); @@ -287,8 +295,7 @@ bool BuildLog::Load(const string& path, string* err) { if (i != entries_.end()) { entry = i->second; } else { - entry = new LogEntry; - entry->output = output; + entry = new LogEntry(output); entries_.insert(Entries::value_type(entry->output, entry)); ++unique_entry_count; } @@ -341,6 +348,7 @@ void BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { } bool BuildLog::Recompact(const string& path, string* err) { + METRIC_RECORD(".ninja_log recompact"); printf("Recompacting log...\n"); string temp_path = path + ".recompact"; diff --git a/src/build_log.h b/src/build_log.h index 4141ff3..231bfd9 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -60,6 +60,10 @@ struct BuildLog { start_time == o.start_time && end_time == o.end_time && restat_mtime == o.restat_mtime; } + + explicit LogEntry(const string& output); + LogEntry(const string& output, uint64_t command_hash, + int start_time, int end_time, TimeStamp restat_mtime); }; /// Lookup a previously-run command by its output path. diff --git a/src/build_log_test.cc b/src/build_log_test.cc index a6c2a86..2dd6500 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -26,6 +26,8 @@ #include <unistd.h> #endif +namespace { + const char kTestFilename[] = "BuildLogTest-tempfile"; struct BuildLogTest : public StateTestWithBuiltinRules { @@ -145,7 +147,8 @@ TEST_F(BuildLogTest, Truncate) { ASSERT_EQ(0, truncate(kTestFilename, size)); #else int fh; - fh = _sopen(kTestFilename, _O_RDWR | _O_CREAT, _SH_DENYNO, _S_IREAD | _S_IWRITE); + fh = _sopen(kTestFilename, _O_RDWR | _O_CREAT, _SH_DENYNO, + _S_IREAD | _S_IWRITE); ASSERT_EQ(0, _chsize(fh, size)); _close(fh); #endif @@ -245,3 +248,25 @@ TEST_F(BuildLogTest, VeryLongInputLine) { ASSERT_EQ(789, e->restat_mtime); ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash)); } + +TEST_F(BuildLogTest, MultiTargetEdge) { + AssertParse(&state_, +"build out out.d: cat\n"); + + BuildLog log; + log.RecordCommand(state_.edges_[0], 21, 22); + + ASSERT_EQ(2u, log.entries().size()); + BuildLog::LogEntry* e1 = log.LookupByOutput("out"); + ASSERT_TRUE(e1); + BuildLog::LogEntry* e2 = log.LookupByOutput("out.d"); + ASSERT_TRUE(e2); + ASSERT_EQ("out", e1->output); + ASSERT_EQ("out.d", e2->output); + ASSERT_EQ(21, e1->start_time); + ASSERT_EQ(21, e2->start_time); + ASSERT_EQ(22, e2->end_time); + ASSERT_EQ(22, e2->end_time); +} + +} // anonymous namespace diff --git a/src/build_test.cc b/src/build_test.cc index 859e758..59c4c53 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -176,6 +176,132 @@ TEST_F(PlanTest, DependencyCycle) { ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err); } +TEST_F(PlanTest, PoolWithDepthOne) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"pool foobar\n" +" depth = 1\n" +"rule poolcat\n" +" command = cat $in > $out\n" +" pool = foobar\n" +"build out1: poolcat in\n" +"build out2: poolcat in\n")); + GetNode("out1")->MarkDirty(); + GetNode("out2")->MarkDirty(); + string err; + EXPECT_TRUE(plan_.AddTarget(GetNode("out1"), &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(plan_.AddTarget(GetNode("out2"), &err)); + ASSERT_EQ("", err); + ASSERT_TRUE(plan_.more_to_do()); + + Edge* edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("in", edge->inputs_[0]->path()); + ASSERT_EQ("out1", edge->outputs_[0]->path()); + + // This will be false since poolcat is serialized + ASSERT_FALSE(plan_.FindWork()); + + plan_.EdgeFinished(edge); + + edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("in", edge->inputs_[0]->path()); + ASSERT_EQ("out2", edge->outputs_[0]->path()); + + ASSERT_FALSE(plan_.FindWork()); + + plan_.EdgeFinished(edge); + + ASSERT_FALSE(plan_.more_to_do()); + edge = plan_.FindWork(); + ASSERT_EQ(0, edge); +} + +TEST_F(PlanTest, PoolsWithDepthTwo) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"pool foobar\n" +" depth = 2\n" +"pool bazbin\n" +" depth = 2\n" +"rule foocat\n" +" command = cat $in > $out\n" +" pool = foobar\n" +"rule bazcat\n" +" command = cat $in > $out\n" +" pool = bazbin\n" +"build out1: foocat in\n" +"build out2: foocat in\n" +"build out3: foocat in\n" +"build outb1: bazcat in\n" +"build outb2: bazcat in\n" +"build outb3: bazcat in\n" +" pool =\n" +"build allTheThings: cat out1 out2 out3 outb1 outb2 outb3\n" +)); + // Mark all the out* nodes dirty + for (int i = 0; i < 3; ++i) { + GetNode("out" + string(1, '1' + i))->MarkDirty(); + GetNode("outb" + string(1, '1' + i))->MarkDirty(); + } + GetNode("allTheThings")->MarkDirty(); + + string err; + EXPECT_TRUE(plan_.AddTarget(GetNode("allTheThings"), &err)); + ASSERT_EQ("", err); + + // Grab the first 4 edges, out1 out2 outb1 outb2 + deque<Edge*> edges; + for (int i = 0; i < 4; ++i) { + ASSERT_TRUE(plan_.more_to_do()); + Edge* edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("in", edge->inputs_[0]->path()); + string base_name(i < 2 ? "out" : "outb"); + ASSERT_EQ(base_name + string(1, '1' + (i % 2)), edge->outputs_[0]->path()); + edges.push_back(edge); + } + + // outb3 is exempt because it has an empty pool + ASSERT_TRUE(plan_.more_to_do()); + Edge* edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("in", edge->inputs_[0]->path()); + ASSERT_EQ("outb3", edge->outputs_[0]->path()); + edges.push_back(edge); + + ASSERT_FALSE(plan_.FindWork()); + + // finish out1 + plan_.EdgeFinished(edges.front()); + edges.pop_front(); + + // out3 should be available + Edge* out3 = plan_.FindWork(); + ASSERT_TRUE(out3); + ASSERT_EQ("in", out3->inputs_[0]->path()); + ASSERT_EQ("out3", out3->outputs_[0]->path()); + + ASSERT_FALSE(plan_.FindWork()); + + plan_.EdgeFinished(out3); + + ASSERT_FALSE(plan_.FindWork()); + + for (deque<Edge*>::iterator it = edges.begin(); it != edges.end(); ++it) { + plan_.EdgeFinished(*it); + } + + Edge* final = plan_.FindWork(); + ASSERT_TRUE(final); + ASSERT_EQ("allTheThings", final->outputs_[0]->path()); + + plan_.EdgeFinished(final); + + ASSERT_FALSE(plan_.more_to_do()); + ASSERT_FALSE(plan_.FindWork()); +} + struct BuildTest : public StateTestWithBuiltinRules, public CommandRunner { BuildTest() : config_(MakeConfig()), @@ -447,10 +573,12 @@ TEST_F(BuildTest, MakeDirs) { string err; #ifdef _WIN32 - ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir\\dir2\\file: cat in1\n")); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build subdir\\dir2\\file: cat in1\n")); EXPECT_TRUE(builder_.AddTarget("subdir\\dir2\\file", &err)); #else - ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir/dir2/file: cat in1\n")); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build subdir/dir2/file: cat in1\n")); EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err)); #endif @@ -869,7 +997,7 @@ TEST_F(BuildWithLogTest, RestatMissingInput) { // Create all necessary files fs_.Create("in", now_, ""); - // The implicit dependencies and the depfile itself + // The implicit dependencies and the depfile itself // are newer than the output TimeStamp restat_mtime = ++now_; fs_.Create("out1.d", now_, "out1: will.be.deleted restat.file\n"); @@ -889,10 +1017,10 @@ TEST_F(BuildWithLogTest, RestatMissingInput) { ASSERT_TRUE(NULL != log_entry); ASSERT_EQ(restat_mtime, log_entry->restat_mtime); - // Now remove a file, referenced from depfile, so that target becomes + // Now remove a file, referenced from depfile, so that target becomes // dirty, but the output does not change fs_.RemoveFile("will.be.deleted"); - + // Trigger the build again - only out1 gets built commands_ran_.clear(); state_.Reset(); @@ -943,7 +1071,7 @@ TEST_F(BuildDryRun, AllCommandsShown) { } // Test that RSP files are created when & where appropriate and deleted after -// succesful execution. +// successful execution. TEST_F(BuildTest, RspFileSuccess) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, @@ -1135,3 +1263,4 @@ TEST_F(BuildTest, StatusFormatReplacePlaceholder) { EXPECT_EQ("[%/s0/t0/r0/u0/f0]", status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]")); } + diff --git a/src/clean.cc b/src/clean.cc index 3fe23ec..0b8476b 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -29,6 +29,7 @@ Cleaner::Cleaner(State* state, const BuildConfig& config) : state_(state), config_(config), removed_(), + cleaned_(), cleaned_files_count_(0), disk_interface_(new RealDiskInterface), status_(0) { @@ -40,6 +41,7 @@ Cleaner::Cleaner(State* state, : state_(state), config_(config), removed_(), + cleaned_(), cleaned_files_count_(0), disk_interface_(disk_interface), status_(0) { @@ -80,6 +82,16 @@ bool Cleaner::IsAlreadyRemoved(const string& path) { return (i != removed_.end()); } +void Cleaner::RemoveEdgeFiles(Edge* edge) { + string depfile = edge->EvaluateDepFile(); + if (!depfile.empty()) + Remove(depfile); + + string rspfile = edge->GetRspFile(); + if (!rspfile.empty()) + Remove(rspfile); +} + void Cleaner::PrintHeader() { if (config_.verbosity == BuildConfig::QUIET) return; @@ -111,12 +123,8 @@ int Cleaner::CleanAll(bool generator) { out_node != (*e)->outputs_.end(); ++out_node) { Remove((*out_node)->path()); } - // Remove the depfile - if (!(*e)->rule().depfile().empty()) - Remove((*e)->EvaluateDepFile()); - // Remove the response file - if ((*e)->HasRspFile()) - Remove((*e)->GetRspFile()); + + RemoveEdgeFiles(*e); } PrintFooter(); return status_; @@ -127,16 +135,20 @@ void Cleaner::DoCleanTarget(Node* target) { // Do not try to remove phony targets if (!e->is_phony()) { Remove(target->path()); - if (!target->in_edge()->rule().depfile().empty()) - Remove(target->in_edge()->EvaluateDepFile()); - if (e->HasRspFile()) - Remove(e->GetRspFile()); + RemoveEdgeFiles(e); } for (vector<Node*>::iterator n = e->inputs_.begin(); n != e->inputs_.end(); ++n) { - DoCleanTarget(*n); + Node* next = *n; + // call DoCleanTarget recursively if this node has not been visited + if (cleaned_.count(next) == 0) { + DoCleanTarget(next); + } } } + + // mark this target to be cleaned already + cleaned_.insert(target); } int Cleaner::CleanTarget(Node* target) { @@ -191,10 +203,7 @@ void Cleaner::DoCleanRule(const Rule* rule) { for (vector<Node*>::iterator out_node = (*e)->outputs_.begin(); out_node != (*e)->outputs_.end(); ++out_node) { Remove((*out_node)->path()); - if (!(*e)->rule().depfile().empty()) - Remove((*e)->EvaluateDepFile()); - if ((*e)->HasRspFile()) - Remove((*e)->GetRspFile()); + RemoveEdgeFiles(*e); } } } @@ -249,4 +258,5 @@ void Cleaner::Reset() { status_ = 0; cleaned_files_count_ = 0; removed_.clear(); + cleaned_.clear(); } diff --git a/src/clean.h b/src/clean.h index 5938dff..19432ab 100644 --- a/src/clean.h +++ b/src/clean.h @@ -27,8 +27,7 @@ struct Node; struct Rule; struct DiskInterface; -class Cleaner { - public: +struct Cleaner { /// Build a cleaner object with a real disk interface. Cleaner(State* state, const BuildConfig& config); @@ -81,10 +80,14 @@ class Cleaner { /// @returns whether the file @a path exists. bool FileExists(const string& path); void Report(const string& path); + /// Remove the given @a path file only if it has not been already removed. void Remove(const string& path); /// @return whether the given @a path has already been removed. bool IsAlreadyRemoved(const string& path); + /// Remove the depfile and rspfile for an Edge. + void RemoveEdgeFiles(Edge* edge); + /// Helper recursive method for CleanTarget(). void DoCleanTarget(Node* target); void PrintHeader(); @@ -95,6 +98,7 @@ class Cleaner { State* state_; const BuildConfig& config_; set<string> removed_; + set<Node*> cleaned_; int cleaned_files_count_; DiskInterface* disk_interface_; int status_; diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 515ff59..7c557cd 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -80,7 +80,8 @@ TimeStamp RealDiskInterface::Stat(const string& path) { // MSDN: "Naming Files, Paths, and Namespaces" // http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx if (!path.empty() && path[0] != '\\' && path.size() > MAX_PATH) { - Error("Stat(%s): Filename longer than %i characters", path.c_str(), MAX_PATH); + Error("Stat(%s): Filename longer than %i characters", + path.c_str(), MAX_PATH); return -1; } WIN32_FILE_ATTRIBUTE_DATA attrs; @@ -116,18 +117,21 @@ TimeStamp RealDiskInterface::Stat(const string& path) { bool RealDiskInterface::WriteFile(const string& path, const string& contents) { FILE * fp = fopen(path.c_str(), "w"); if (fp == NULL) { - Error("WriteFile(%s): Unable to create file. %s", path.c_str(), strerror(errno)); + Error("WriteFile(%s): Unable to create file. %s", + path.c_str(), strerror(errno)); return false; } if (fwrite(contents.data(), 1, contents.length(), fp) < contents.length()) { - Error("WriteFile(%s): Unable to write to the file. %s", path.c_str(), strerror(errno)); + Error("WriteFile(%s): Unable to write to the file. %s", + path.c_str(), strerror(errno)); fclose(fp); return false; } if (fclose(fp) == EOF) { - Error("WriteFile(%s): Unable to close the file. %s", path.c_str(), strerror(errno)); + Error("WriteFile(%s): Unable to close the file. %s", + path.c_str(), strerror(errno)); return false; } diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 32fe9cb..c2315c7 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -25,8 +25,7 @@ namespace { -class DiskInterfaceTest : public testing::Test { - public: +struct DiskInterfaceTest : public testing::Test { virtual void SetUp() { // These tests do real disk accesses, so create a temp dir. temp_dir_.CreateAndEnter("Ninja-DiskInterfaceTest"); diff --git a/src/graph.cc b/src/graph.cc index 6ae324d..f9b9c6f 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -145,9 +145,10 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, if (edge->rule_->restat() && build_log() && (entry = build_log()->LookupByOutput(output->path()))) { if (entry->restat_mtime < most_recent_stamp) { - EXPLAIN("restat of output %s older than most recent input %s (%d vs %d)", - output->path().c_str(), most_recent_input->path().c_str(), - entry->restat_mtime, most_recent_stamp); + EXPLAIN("restat of output %s older than most recent input %s " + "(%d vs %d)", + output->path().c_str(), most_recent_input->path().c_str(), + entry->restat_mtime, most_recent_stamp); return true; } } else { @@ -192,7 +193,7 @@ struct EdgeEnv : public Env { virtual string LookupVariable(const string& var); /// Given a span of Nodes, construct a list of paths suitable for a command - /// line. XXX here is where shell-escaping of e.g spaces should happen. + /// line. string MakePathList(vector<Node*>::iterator begin, vector<Node*>::iterator end, char sep); @@ -214,7 +215,6 @@ string EdgeEnv::LookupVariable(const string& var) { } else if (edge_->env_) { return edge_->env_->LookupVariable(var); } else { - // XXX should we warn here? return string(); } } @@ -241,7 +241,7 @@ string EdgeEnv::MakePathList(vector<Node*>::iterator begin, string Edge::EvaluateCommand(bool incl_rsp_file) { EdgeEnv env(this); string command = rule_->command().Evaluate(&env); - if (incl_rsp_file && HasRspFile()) + if (incl_rsp_file && HasRspFile()) command += ";rspfile=" + GetRspFileContent(); return command; } @@ -317,7 +317,8 @@ bool DependencyScan::LoadDepFile(Edge* edge, string* err) { // create one; this makes us not abort if the input is missing, // but instead will rebuild in that circumstance. if (!node->in_edge()) { - Edge* phony_edge = state_->AddEdge(&State::kPhonyRule); + Edge* phony_edge = state_->AddEdge(&State::kPhonyRule, + &State::kDefaultPool); node->set_in_edge(phony_edge); phony_edge->outputs_.push_back(node); @@ -345,6 +346,13 @@ void Edge::Dump(const char* prefix) const { i != outputs_.end() && *i != NULL; ++i) { printf("%s ", (*i)->path().c_str()); } + if (pool_) { + if (!pool_->name().empty()) { + printf("(in pool '%s')", pool_->name().c_str()); + } + } else { + printf("(null pool?)"); + } printf("] 0x%p\n", this); } @@ -353,18 +361,18 @@ bool Edge::is_phony() const { } void Node::Dump(const char* prefix) const { - printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", - prefix, path().c_str(), this, - mtime(), mtime()?"":" (:missing)", - dirty()?" dirty":" clean"); - if (in_edge()) { - in_edge()->Dump("in-edge: "); - }else{ - printf("no in-edge\n"); - } - printf(" out edges:\n"); - for (vector<Edge*>::const_iterator e = out_edges().begin(); - e != out_edges().end() && *e != NULL; ++e) { - (*e)->Dump(" +- "); - } + printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", + prefix, path().c_str(), this, + mtime(), mtime() ? "" : " (:missing)", + dirty() ? " dirty" : " clean"); + if (in_edge()) { + in_edge()->Dump("in-edge: "); + } else { + printf("no in-edge\n"); + } + printf(" out edges:\n"); + for (vector<Edge*>::const_iterator e = out_edges().begin(); + e != out_edges().end() && *e != NULL; ++e) { + (*e)->Dump(" +- "); + } } diff --git a/src/graph.h b/src/graph.h index 272fcb9..3c31e19 100644 --- a/src/graph.h +++ b/src/graph.h @@ -131,6 +131,7 @@ struct Rule { EvalString command_; EvalString description_; EvalString depfile_; + EvalString pool_; EvalString rspfile_; EvalString rspfile_content_; }; @@ -138,6 +139,7 @@ struct Rule { struct BuildLog; struct Node; struct State; +struct Pool; /// An edge in the dependency graph; links between Nodes using Rules. struct Edge { @@ -150,7 +152,7 @@ struct Edge { /// Expand all variables in a command and return it as a string. /// If incl_rsp_file is enabled, the string will also contain the /// full contents of a response file (if applicable) - string EvaluateCommand(bool incl_rsp_file = false); // XXX move to env, take env ptr + string EvaluateCommand(bool incl_rsp_file = false); string EvaluateDepFile(); string GetDescription(); @@ -166,25 +168,25 @@ struct Edge { void Dump(const char* prefix="") const; const Rule* rule_; + Pool* pool_; vector<Node*> inputs_; vector<Node*> outputs_; Env* env_; bool outputs_ready_; const Rule& rule() const { return *rule_; } + Pool* pool() const { return pool_; } + int weight() const { return 1; } bool outputs_ready() const { return outputs_ready_; } - // XXX There are three types of inputs. + // There are three types of inputs. // 1) explicit deps, which show up as $in on the command line; // 2) implicit deps, which the target depends on implicitly (e.g. C headers), // and changes in them cause the target to rebuild; // 3) order-only deps, which are needed before the target builds but which // don't cause the target to rebuild. - // Currently we stuff all of these into inputs_ and keep counts of #2 and #3 - // when we need to compute subsets. This is suboptimal; should think of a - // better representation. (Could make each pointer into a pair of a pointer - // and a type of input, or if memory matters could use the low bits of the - // pointer...) + // These are stored in inputs_ in that order, and we keep counts of + // #2 and #3 when we need to access the various subsets. int implicit_deps_; int order_only_deps_; bool is_implicit(size_t index) { diff --git a/src/hash_map.h b/src/hash_map.h index 9904fb8..076f6c0 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -25,7 +25,7 @@ unsigned int MurmurHash2(const void* key, size_t len) { const int r = 24; unsigned int h = seed ^ len; const unsigned char * data = (const unsigned char *)key; - while(len >= 4) { + while (len >= 4) { unsigned int k = *(unsigned int *)data; k *= m; k ^= k >> r; @@ -35,7 +35,7 @@ unsigned int MurmurHash2(const void* key, size_t len) { data += 4; len -= 4; } - switch(len) { + switch (len) { case 3: h ^= data[2] << 16; case 2: h ^= data[1] << 8; case 1: h ^= data[0]; diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc index 77b5b3b..29e6755 100644 --- a/src/includes_normalize_test.cc +++ b/src/includes_normalize_test.cc @@ -40,7 +40,8 @@ string GetCurDir() { TEST(IncludesNormalize, WithRelative) { string currentdir = IncludesNormalize::ToLower(GetCurDir()); EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b")); - EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), NULL)); + EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), + NULL)); EXPECT_EQ(string("..\\") + currentdir + string("\\a"), IncludesNormalize::Normalize("a", "../b")); EXPECT_EQ(string("..\\") + currentdir + string("\\a\\b"), @@ -69,16 +70,21 @@ TEST(IncludesNormalize, Join) { } TEST(IncludesNormalize, Split) { - EXPECT_EQ("", IncludesNormalize::Join(IncludesNormalize::Split("", '/'), ':')); - EXPECT_EQ("a", IncludesNormalize::Join(IncludesNormalize::Split("a", '/'), ':')); - EXPECT_EQ("a:b:c", IncludesNormalize::Join(IncludesNormalize::Split("a/b/c", '/'), ':')); + EXPECT_EQ("", IncludesNormalize::Join(IncludesNormalize::Split("", '/'), + ':')); + EXPECT_EQ("a", IncludesNormalize::Join(IncludesNormalize::Split("a", '/'), + ':')); + EXPECT_EQ("a:b:c", + IncludesNormalize::Join( + IncludesNormalize::Split("a/b/c", '/'), ':')); } TEST(IncludesNormalize, ToLower) { EXPECT_EQ("", IncludesNormalize::ToLower("")); EXPECT_EQ("stuff", IncludesNormalize::ToLower("Stuff")); EXPECT_EQ("stuff and things", IncludesNormalize::ToLower("Stuff AND thINGS")); - EXPECT_EQ("stuff 3and thin43gs", IncludesNormalize::ToLower("Stuff 3AND thIN43GS")); + EXPECT_EQ("stuff 3and thin43gs", + IncludesNormalize::ToLower("Stuff 3AND thIN43GS")); } TEST(IncludesNormalize, DifferentDrive) { diff --git a/src/lexer.cc b/src/lexer.cc index 5d7d185..685fe81 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -83,6 +83,7 @@ const char* Lexer::TokenName(Token t) { case NEWLINE: return "newline"; case PIPE2: return "'||'"; case PIPE: return "'|'"; + case POOL: return "'pool'"; case RULE: return "'rule'"; case SUBNINJA: return "'subninja'"; case TEOF: return "eof"; @@ -162,63 +163,71 @@ Lexer::Token Lexer::ReadToken() { }; yych = *p; - if (yych <= 'Z') { + if (yych <= '^') { if (yych <= ',') { if (yych <= 0x1F) { - if (yych <= 0x00) goto yy21; + if (yych <= 0x00) goto yy22; if (yych == '\n') goto yy6; - goto yy23; + goto yy24; } else { if (yych <= ' ') goto yy2; if (yych == '#') goto yy4; - goto yy23; + goto yy24; } } else { if (yych <= ':') { - if (yych == '/') goto yy23; - if (yych <= '9') goto yy20; - goto yy14; + if (yych == '/') goto yy24; + if (yych <= '9') goto yy21; + goto yy15; } else { - if (yych == '=') goto yy12; - if (yych <= '@') goto yy23; - goto yy20; + if (yych <= '=') { + if (yych <= '<') goto yy24; + goto yy13; + } else { + if (yych <= '@') goto yy24; + if (yych <= 'Z') goto yy21; + goto yy24; + } } } } else { - if (yych <= 'h') { - if (yych <= 'a') { - if (yych == '_') goto yy20; - if (yych <= '`') goto yy23; - goto yy20; + if (yych <= 'i') { + if (yych <= 'b') { + if (yych == '`') goto yy24; + if (yych <= 'a') goto yy21; + goto yy8; } else { - if (yych <= 'b') goto yy8; - if (yych == 'd') goto yy11; - goto yy20; + if (yych == 'd') goto yy12; + if (yych <= 'h') goto yy21; + goto yy19; } } else { - if (yych <= 's') { - if (yych <= 'i') goto yy18; - if (yych <= 'q') goto yy20; - if (yych <= 'r') goto yy10; - goto yy19; + if (yych <= 'r') { + if (yych == 'p') goto yy10; + if (yych <= 'q') goto yy21; + goto yy11; } else { - if (yych <= 'z') goto yy20; - if (yych == '|') goto yy16; - goto yy23; + if (yych <= 'z') { + if (yych <= 's') goto yy20; + goto yy21; + } else { + if (yych == '|') goto yy17; + goto yy24; + } } } } yy2: yyaccept = 0; yych = *(q = ++p); - goto yy65; + goto yy70; yy3: { token = INDENT; break; } yy4: yyaccept = 1; yych = *(q = ++p); if (yych <= 0x00) goto yy5; - if (yych != '\r') goto yy60; + if (yych != '\r') goto yy65; yy5: { token = ERROR; break; } yy6: @@ -227,159 +236,173 @@ yy7: { token = NEWLINE; break; } yy8: ++p; - if ((yych = *p) == 'u') goto yy54; - goto yy25; + if ((yych = *p) == 'u') goto yy59; + goto yy26; yy9: { token = IDENT; break; } yy10: yych = *++p; - if (yych == 'u') goto yy50; - goto yy25; + if (yych == 'o') goto yy55; + goto yy26; yy11: yych = *++p; - if (yych == 'e') goto yy43; - goto yy25; + if (yych == 'u') goto yy51; + goto yy26; yy12: + yych = *++p; + if (yych == 'e') goto yy44; + goto yy26; +yy13: ++p; { token = EQUALS; break; } -yy14: +yy15: ++p; { token = COLON; break; } -yy16: +yy17: ++p; - if ((yych = *p) == '|') goto yy41; + if ((yych = *p) == '|') goto yy42; { token = PIPE; break; } -yy18: - yych = *++p; - if (yych == 'n') goto yy34; - goto yy25; yy19: yych = *++p; - if (yych == 'u') goto yy26; - goto yy25; + if (yych == 'n') goto yy35; + goto yy26; yy20: yych = *++p; - goto yy25; + if (yych == 'u') goto yy27; + goto yy26; yy21: + yych = *++p; + goto yy26; +yy22: ++p; { token = TEOF; break; } -yy23: +yy24: yych = *++p; goto yy5; -yy24: +yy25: ++p; yych = *p; -yy25: +yy26: if (yybm[0+yych] & 32) { - goto yy24; + goto yy25; } goto yy9; -yy26: +yy27: yych = *++p; - if (yych != 'b') goto yy25; + if (yych != 'b') goto yy26; yych = *++p; - if (yych != 'n') goto yy25; + if (yych != 'n') goto yy26; yych = *++p; - if (yych != 'i') goto yy25; + if (yych != 'i') goto yy26; yych = *++p; - if (yych != 'n') goto yy25; + if (yych != 'n') goto yy26; yych = *++p; - if (yych != 'j') goto yy25; + if (yych != 'j') goto yy26; yych = *++p; - if (yych != 'a') goto yy25; + if (yych != 'a') goto yy26; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy24; + goto yy25; } { token = SUBNINJA; break; } -yy34: +yy35: yych = *++p; - if (yych != 'c') goto yy25; + if (yych != 'c') goto yy26; yych = *++p; - if (yych != 'l') goto yy25; + if (yych != 'l') goto yy26; yych = *++p; - if (yych != 'u') goto yy25; + if (yych != 'u') goto yy26; yych = *++p; - if (yych != 'd') goto yy25; + if (yych != 'd') goto yy26; yych = *++p; - if (yych != 'e') goto yy25; + if (yych != 'e') goto yy26; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy24; + goto yy25; } { token = INCLUDE; break; } -yy41: +yy42: ++p; { token = PIPE2; break; } -yy43: +yy44: yych = *++p; - if (yych != 'f') goto yy25; + if (yych != 'f') goto yy26; yych = *++p; - if (yych != 'a') goto yy25; + if (yych != 'a') goto yy26; yych = *++p; - if (yych != 'u') goto yy25; + if (yych != 'u') goto yy26; yych = *++p; - if (yych != 'l') goto yy25; + if (yych != 'l') goto yy26; yych = *++p; - if (yych != 't') goto yy25; + if (yych != 't') goto yy26; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy24; + goto yy25; } { token = DEFAULT; break; } -yy50: +yy51: yych = *++p; - if (yych != 'l') goto yy25; + if (yych != 'l') goto yy26; yych = *++p; - if (yych != 'e') goto yy25; + if (yych != 'e') goto yy26; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy24; + goto yy25; } { token = RULE; break; } -yy54: +yy55: yych = *++p; - if (yych != 'i') goto yy25; + if (yych != 'o') goto yy26; yych = *++p; - if (yych != 'l') goto yy25; + if (yych != 'l') goto yy26; + ++p; + if (yybm[0+(yych = *p)] & 32) { + goto yy25; + } + { token = POOL; break; } +yy59: + yych = *++p; + if (yych != 'i') goto yy26; yych = *++p; - if (yych != 'd') goto yy25; + if (yych != 'l') goto yy26; + yych = *++p; + if (yych != 'd') goto yy26; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy24; + goto yy25; } { token = BUILD; break; } -yy59: +yy64: ++p; yych = *p; -yy60: +yy65: if (yybm[0+yych] & 64) { - goto yy59; + goto yy64; } - if (yych <= 0x00) goto yy61; - if (yych <= '\f') goto yy62; -yy61: + if (yych <= 0x00) goto yy66; + if (yych <= '\f') goto yy67; +yy66: p = q; if (yyaccept <= 0) { goto yy3; } else { goto yy5; } -yy62: +yy67: ++p; { continue; } -yy64: +yy69: yyaccept = 0; q = ++p; yych = *p; -yy65: +yy70: if (yybm[0+yych] & 128) { - goto yy64; + goto yy69; } - if (yych == '\n') goto yy66; - if (yych == '#') goto yy59; + if (yych == '\n') goto yy71; + if (yych == '#') goto yy64; goto yy3; -yy66: +yy71: ++p; yych = *p; goto yy7; @@ -445,39 +468,39 @@ void Lexer::EatWhitespace() { }; yych = *p; if (yych <= ' ') { - if (yych <= 0x00) goto yy73; - if (yych <= 0x1F) goto yy75; + if (yych <= 0x00) goto yy78; + if (yych <= 0x1F) goto yy80; } else { - if (yych == '$') goto yy71; - goto yy75; + if (yych == '$') goto yy76; + goto yy80; } ++p; yych = *p; - goto yy79; -yy70: + goto yy84; +yy75: { continue; } -yy71: +yy76: ++p; - if ((yych = *p) == '\n') goto yy76; -yy72: + if ((yych = *p) == '\n') goto yy81; +yy77: { break; } -yy73: +yy78: ++p; { break; } -yy75: +yy80: yych = *++p; - goto yy72; -yy76: + goto yy77; +yy81: ++p; { continue; } -yy78: +yy83: ++p; yych = *p; -yy79: +yy84: if (yybm[0+yych] & 128) { - goto yy78; + goto yy83; } - goto yy70; + goto yy75; } } @@ -527,40 +550,40 @@ bool Lexer::ReadIdent(string* out) { yych = *p; if (yych <= '@') { if (yych <= '.') { - if (yych <= ',') goto yy84; + if (yych <= ',') goto yy89; } else { - if (yych <= '/') goto yy84; - if (yych >= ':') goto yy84; + if (yych <= '/') goto yy89; + if (yych >= ':') goto yy89; } } else { if (yych <= '_') { - if (yych <= 'Z') goto yy82; - if (yych <= '^') goto yy84; + if (yych <= 'Z') goto yy87; + if (yych <= '^') goto yy89; } else { - if (yych <= '`') goto yy84; - if (yych >= '{') goto yy84; + if (yych <= '`') goto yy89; + if (yych >= '{') goto yy89; } } -yy82: +yy87: ++p; yych = *p; - goto yy87; -yy83: + goto yy92; +yy88: { out->assign(start, p - start); break; } -yy84: +yy89: ++p; { return false; } -yy86: +yy91: ++p; yych = *p; -yy87: +yy92: if (yybm[0+yych] & 128) { - goto yy86; + goto yy91; } - goto yy83; + goto yy88; } } @@ -615,29 +638,29 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { yych = *p; if (yych <= ' ') { if (yych <= '\n') { - if (yych <= 0x00) goto yy96; - if (yych >= '\n') goto yy92; + if (yych <= 0x00) goto yy101; + if (yych >= '\n') goto yy97; } else { - if (yych == '\r') goto yy98; - if (yych >= ' ') goto yy92; + if (yych == '\r') goto yy103; + if (yych >= ' ') goto yy97; } } else { if (yych <= '9') { - if (yych == '$') goto yy94; + if (yych == '$') goto yy99; } else { - if (yych <= ':') goto yy92; - if (yych == '|') goto yy92; + if (yych <= ':') goto yy97; + if (yych == '|') goto yy97; } } ++p; yych = *p; - goto yy121; -yy91: + goto yy126; +yy96: { eval->AddText(StringPiece(start, p - start)); continue; } -yy92: +yy97: ++p; { if (path) { @@ -650,137 +673,137 @@ yy92: continue; } } -yy94: +yy99: ++p; if ((yych = *p) <= '/') { if (yych <= ' ') { - if (yych == '\n') goto yy110; - if (yych <= 0x1F) goto yy99; - goto yy101; + if (yych == '\n') goto yy115; + if (yych <= 0x1F) goto yy104; + goto yy106; } else { if (yych <= '$') { - if (yych <= '#') goto yy99; - goto yy103; + if (yych <= '#') goto yy104; + goto yy108; } else { - if (yych == '-') goto yy105; - goto yy99; + if (yych == '-') goto yy110; + goto yy104; } } } else { if (yych <= '^') { if (yych <= ':') { - if (yych <= '9') goto yy105; - goto yy107; + if (yych <= '9') goto yy110; + goto yy112; } else { - if (yych <= '@') goto yy99; - if (yych <= 'Z') goto yy105; - goto yy99; + if (yych <= '@') goto yy104; + if (yych <= 'Z') goto yy110; + goto yy104; } } else { if (yych <= '`') { - if (yych <= '_') goto yy105; - goto yy99; + if (yych <= '_') goto yy110; + goto yy104; } else { - if (yych <= 'z') goto yy105; - if (yych <= '{') goto yy109; - goto yy99; + if (yych <= 'z') goto yy110; + if (yych <= '{') goto yy114; + goto yy104; } } } -yy95: +yy100: { last_token_ = start; return Error(DescribeLastError(), err); } -yy96: +yy101: ++p; { last_token_ = start; return Error("unexpected EOF", err); } -yy98: +yy103: yych = *++p; - goto yy95; -yy99: + goto yy100; +yy104: ++p; -yy100: +yy105: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy101: +yy106: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy103: +yy108: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy105: +yy110: ++p; yych = *p; - goto yy119; -yy106: + goto yy124; +yy111: { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy107: +yy112: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy109: +yy114: yych = *(q = ++p); if (yybm[0+yych] & 32) { - goto yy113; + goto yy118; } - goto yy100; -yy110: + goto yy105; +yy115: ++p; yych = *p; if (yybm[0+yych] & 16) { - goto yy110; + goto yy115; } { continue; } -yy113: +yy118: ++p; yych = *p; if (yybm[0+yych] & 32) { - goto yy113; + goto yy118; } - if (yych == '}') goto yy116; + if (yych == '}') goto yy121; p = q; - goto yy100; -yy116: + goto yy105; +yy121: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy118: +yy123: ++p; yych = *p; -yy119: +yy124: if (yybm[0+yych] & 64) { - goto yy118; + goto yy123; } - goto yy106; -yy120: + goto yy111; +yy125: ++p; yych = *p; -yy121: +yy126: if (yybm[0+yych] & 128) { - goto yy120; + goto yy125; } - goto yy91; + goto yy96; } } diff --git a/src/lexer.h b/src/lexer.h index 03c59f2..f366556 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -41,6 +41,7 @@ struct Lexer { NEWLINE, PIPE, PIPE2, + POOL, RULE, SUBNINJA, TEOF, diff --git a/src/lexer.in.cc b/src/lexer.in.cc index 7ae9c61..93d5540 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -82,6 +82,7 @@ const char* Lexer::TokenName(Token t) { case NEWLINE: return "newline"; case PIPE2: return "'||'"; case PIPE: return "'|'"; + case POOL: return "'pool'"; case RULE: return "'rule'"; case SUBNINJA: return "'subninja'"; case TEOF: return "eof"; @@ -135,6 +136,7 @@ Lexer::Token Lexer::ReadToken() { [ ]*[\n] { token = NEWLINE; break; } [ ]+ { token = INDENT; break; } "build" { token = BUILD; break; } + "pool" { token = POOL; break; } "rule" { token = RULE; break; } "default" { token = DEFAULT; break; } "=" { token = EQUALS; break; } diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 405e244..2d052b5 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -47,6 +47,10 @@ bool ManifestParser::Parse(const string& filename, const string& input, for (;;) { Lexer::Token token = lexer_.ReadToken(); switch (token) { + case Lexer::POOL: + if (!ParsePool(err)) + return false; + break; case Lexer::BUILD: if (!ParseEdge(err)) return false; @@ -91,6 +95,44 @@ bool ManifestParser::Parse(const string& filename, const string& input, return false; // not reached } + +bool ManifestParser::ParsePool(string* err) { + string name; + if (!lexer_.ReadIdent(&name)) + return lexer_.Error("expected pool name", err); + + if (!ExpectToken(Lexer::NEWLINE, err)) + return false; + + if (state_->LookupPool(name) != NULL) + return lexer_.Error("duplicate pool '" + name + "'", err); + + int depth = -1; + + while (lexer_.PeekToken(Lexer::INDENT)) { + string key; + EvalString value; + if (!ParseLet(&key, &value, err)) + return false; + + if (key == "depth") { + string depth_string = value.Evaluate(env_); + depth = atol(depth_string.c_str()); + if (depth < 0) + return lexer_.Error("invalid pool depth", err); + } else { + return lexer_.Error("unexpected variable '" + key + "'", err); + } + } + + if (depth < 0) + return lexer_.Error("expected 'depth =' line", err); + + state_->AddPool(new Pool(name, depth)); + return true; +} + + bool ManifestParser::ParseRule(string* err) { string name; if (!lexer_.ReadIdent(&name)) @@ -126,6 +168,8 @@ bool ManifestParser::ParseRule(string* err) { rule->rspfile_ = value; } else if (key == "rspfile_content") { rule->rspfile_content_ = value; + } else if (key == "pool") { + rule->pool_ = value; } else { // Die on other keyvals for now; revisit if we want to add a // scope here. @@ -133,8 +177,10 @@ bool ManifestParser::ParseRule(string* err) { } } - if (rule->rspfile_.empty() != rule->rspfile_content_.empty()) - return lexer_.Error("rspfile and rspfile_content need to be both specified", err); + if (rule->rspfile_.empty() != rule->rspfile_content_.empty()) { + return lexer_.Error("rspfile and rspfile_content need to be both specified", + err); + } if (rule->command_.empty()) return lexer_.Error("expected 'command =' line", err); @@ -252,6 +298,7 @@ bool ManifestParser::ParseEdge(string* err) { // Default to using outer env. BindingEnv* env = env_; + Pool* pool = NULL; // But create and fill a nested env if there are variables in scope. if (lexer_.PeekToken(Lexer::INDENT)) { @@ -262,11 +309,28 @@ bool ManifestParser::ParseEdge(string* err) { EvalString val; if (!ParseLet(&key, &val, err)) return false; - env->AddBinding(key, val.Evaluate(env_)); + if (key == "pool") { + string pool_name = val.Evaluate(env_); + pool = state_->LookupPool(pool_name); + if (pool == NULL) + return lexer_.Error("undefined pool '" + pool_name + "'", err); + } else { + env->AddBinding(key, val.Evaluate(env_)); + } } while (lexer_.PeekToken(Lexer::INDENT)); } - Edge* edge = state_->AddEdge(rule); + if (pool == NULL) { + if (!rule->pool_.empty()) { + pool = state_->LookupPool(rule->pool_.Evaluate(env_)); + if (pool == NULL) + return lexer_.Error("cannot resolve pool for this edge.", err); + } else { + pool = &State::kDefaultPool; + } + } + + Edge* edge = state_->AddEdge(rule, pool); edge->env_ = env; for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) { string path = i->Evaluate(env); diff --git a/src/manifest_parser.h b/src/manifest_parser.h index a2c6c93..a08e5af 100644 --- a/src/manifest_parser.h +++ b/src/manifest_parser.h @@ -50,6 +50,7 @@ private: bool Parse(const string& filename, const string& input, string* err); /// Parse various statement types. + bool ParsePool(string* err); bool ParseRule(string* err); bool ParseLet(string* key, EvalString* val, string* err); bool ParseEdge(string* err); diff --git a/src/metrics.h b/src/metrics.h index f5ac0de..b6da859 100644 --- a/src/metrics.h +++ b/src/metrics.h @@ -59,29 +59,27 @@ private: }; /// Get the current time as relative to some epoch. -/// Epoch varies between platforms; only useful for measuring elapsed -/// time. +/// Epoch varies between platforms; only useful for measuring elapsed time. int64_t GetTimeMillis(); - -/// A simple stopwatch which retruns the time -// in seconds since Restart() was called -class Stopwatch -{ -public: +/// A simple stopwatch which returns the time +/// in seconds since Restart() was called. +struct Stopwatch { + public: Stopwatch() : started_(0) {} - /// Seconds since Restart() call - double Elapsed() const { return 1e-6 * static_cast<double>(Now() - started_); } + /// Seconds since Restart() call. + double Elapsed() const { + return 1e-6 * static_cast<double>(Now() - started_); + } void Restart() { started_ = Now(); } -private: + private: uint64_t started_; uint64_t Now() const; }; - /// The primary interface to metrics. Use METRIC_RECORD("foobar") at the top /// of a function to get timing stats recorded for each call of the function. #define METRIC_RECORD(name) \ diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index 8e440fe..fd9b671 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -14,6 +14,7 @@ #include "msvc_helper.h" +#include <stdio.h> #include <string.h> #include <windows.h> @@ -28,6 +29,21 @@ bool EndsWith(const string& input, const string& needle) { input.substr(input.size() - needle.size()) == needle); } +string Replace(const string& input, const string& find, const string& replace) { + string result = input; + size_t start_pos = 0; + while ((start_pos = result.find(find, start_pos)) != string::npos) { + result.replace(start_pos, find.length(), replace); + start_pos += replace.length(); + } + return result; +} + +string EscapeForDepfile(const string& path) { + // Depfiles don't escape single \. + return Replace(path, " ", "\\ "); +} + } // anonymous namespace // static @@ -125,7 +141,7 @@ int CLWrapper::Run(const string& command, string* extra_output) { if (!include.empty()) { include = IncludesNormalize::Normalize(include, NULL); if (!IsSystemInclude(include)) - includes_.push_back(include); + includes_.insert(include); } else if (FilterInputFilename(line)) { // Drop it. // TODO: if we support compiling multiple output files in a single @@ -162,3 +178,11 @@ int CLWrapper::Run(const string& command, string* extra_output) { return exit_code; } + +vector<string> CLWrapper::GetEscapedResult() { + vector<string> result; + for (set<string>::iterator i = includes_.begin(); i != includes_.end(); ++i) { + result.push_back(EscapeForDepfile(*i)); + } + return result; +} diff --git a/src/msvc_helper.h b/src/msvc_helper.h index f623520..102201b 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -13,6 +13,7 @@ // limitations under the License. #include <string> +#include <set> #include <vector> using namespace std; @@ -49,6 +50,10 @@ struct CLWrapper { /// Exposed for testing. static bool FilterInputFilename(const string& line); + /// Fill a vector with the unique'd headers, escaped for output as a .d + /// file. + vector<string> GetEscapedResult(); + void* env_block_; - vector<string> includes_; + set<string> includes_; }; diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index 0c8db37..152450e 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -14,6 +14,7 @@ #include "msvc_helper.h" +#include <stdio.h> #include <windows.h> #include "util.h" @@ -27,7 +28,6 @@ void Usage() { "usage: ninja -t msvc [options] -- cl.exe /showIncludes /otherArgs\n" "options:\n" " -e ENVFILE load environment block from ENVFILE as environment\n" -" -r BASE normalize paths and make relative to BASE before output\n" " -o FILE write output dependency information to FILE.d\n" ); } @@ -48,7 +48,6 @@ void PushPathIntoEnvironment(const string& env_block) { int MSVCHelperMain(int argc, char** argv) { const char* output_filename = NULL; - const char* relative_to = NULL; const char* envfile = NULL; const option kLongOptions[] = { @@ -56,7 +55,7 @@ int MSVCHelperMain(int argc, char** argv) { { NULL, 0, NULL, 0 } }; int opt; - while ((opt = getopt_long(argc, argv, "e:o:r:h", kLongOptions, NULL)) != -1) { + while ((opt = getopt_long(argc, argv, "e:o:h", kLongOptions, NULL)) != -1) { switch (opt) { case 'e': envfile = optarg; @@ -64,9 +63,6 @@ int MSVCHelperMain(int argc, char** argv) { case 'o': output_filename = optarg; break; - case 'r': - relative_to = optarg; - break; case 'h': default: Usage(); @@ -105,8 +101,8 @@ int MSVCHelperMain(int argc, char** argv) { Fatal("opening %s: %s", depfile.c_str(), GetLastErrorString().c_str()); } fprintf(output, "%s: ", output_filename); - for (vector<string>::iterator i = cl.includes_.begin(); - i != cl.includes_.end(); ++i) { + vector<string> headers = cl.GetEscapedResult(); + for (vector<string>::iterator i = headers.begin(); i != headers.end(); ++i) { fprintf(output, "%s\n", i->c_str()); } fclose(output); diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index 29fefd4..7730425 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -48,7 +48,7 @@ TEST(MSVCHelperTest, Run) { &output); ASSERT_EQ("foo\nbar\n", output); ASSERT_EQ(1u, cl.includes_.size()); - ASSERT_EQ("foo.h", cl.includes_[0]); + ASSERT_EQ("foo.h", *cl.includes_.begin()); } TEST(MSVCHelperTest, RunFilenameFilter) { @@ -70,7 +70,7 @@ TEST(MSVCHelperTest, RunSystemInclude) { // system headers. ASSERT_EQ("", output); ASSERT_EQ(1u, cl.includes_.size()); - ASSERT_EQ("path.h", cl.includes_[0]); + ASSERT_EQ("path.h", *cl.includes_.begin()); } TEST(MSVCHelperTest, EnvBlock) { @@ -81,3 +81,38 @@ TEST(MSVCHelperTest, EnvBlock) { cl.Run("cmd /c \"echo foo is %foo%", &output); ASSERT_EQ("foo is bar\n", output); } + +TEST(MSVCHelperTest, DuplicatedHeader) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo Note: including file: foo.h&&" + "echo Note: including file: bar.h&&" + "echo Note: including file: foo.h\"", + &output); + // We should have dropped one copy of foo.h. + ASSERT_EQ("", output); + ASSERT_EQ(2u, cl.includes_.size()); +} + +TEST(MSVCHelperTest, DuplicatedHeaderPathConverted) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo Note: including file: sub/foo.h&&" + "echo Note: including file: bar.h&&" + "echo Note: including file: sub\\foo.h\"", + &output); + // We should have dropped one copy of foo.h. + ASSERT_EQ("", output); + ASSERT_EQ(2u, cl.includes_.size()); +} + +TEST(MSVCHelperTest, SpacesInFilename) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo Note: including file: sub\\some sdk\\foo.h", + &output); + ASSERT_EQ("", output); + vector<string> headers = cl.GetEscapedResult(); + ASSERT_EQ(1u, headers.size()); + ASSERT_EQ("sub\\some\\ sdk\\foo.h", headers[0]); +} diff --git a/src/ninja.cc b/src/ninja.cc index 5a3c530..08d4b14 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -49,7 +49,7 @@ namespace { /// The version number of the current Ninja release. This will always /// be "git" on trunk. -const char* kVersion = "1.0.0"; +const char* kVersion = "1.1.0"; /// Global information passed into subtools. struct Globals { @@ -140,7 +140,7 @@ int GuessParallelism() { /// An implementation of ManifestParser::FileReader that actually reads /// the file. struct RealFileReader : public ManifestParser::FileReader { - bool ReadFile(const string& path, string* content, string* err) { + virtual bool ReadFile(const string& path, string* content, string* err) { return ::ReadFile(path, content, err) == 0; } }; @@ -168,6 +168,50 @@ bool RebuildManifest(Builder* builder, const char* input_file, string* err) { return node->dirty(); } +Node* CollectTarget(State* state, const char* cpath, string* err) { + string path = cpath; + if (!CanonicalizePath(&path, err)) + return NULL; + + // Special syntax: "foo.cc^" means "the first output of foo.cc". + bool first_dependent = false; + if (!path.empty() && path[path.size() - 1] == '^') { + path.resize(path.size() - 1); + first_dependent = true; + } + + Node* node = state->LookupNode(path); + if (node) { + if (first_dependent) { + if (node->out_edges().empty()) { + *err = "'" + path + "' has no out edge"; + return NULL; + } + Edge* edge = node->out_edges()[0]; + if (edge->outputs_.empty()) { + edge->Dump(); + Fatal("edge has no outputs"); + } + node = edge->outputs_[0]; + } + return node; + } else { + *err = "unknown target '" + path + "'"; + + if (path == "clean") { + *err += ", did you mean 'ninja -t clean'?"; + } else if (path == "help") { + *err += ", did you mean 'ninja -h'?"; + } else { + Node* suggestion = state->SpellcheckNode(path); + if (suggestion) { + *err += ", did you mean '" + suggestion->path() + "'?"; + } + } + return NULL; + } +} + bool CollectTargetsFromArgs(State* state, int argc, char* argv[], vector<Node*>* targets, string* err) { if (argc == 0) { @@ -176,47 +220,10 @@ bool CollectTargetsFromArgs(State* state, int argc, char* argv[], } for (int i = 0; i < argc; ++i) { - string path = argv[i]; - if (!CanonicalizePath(&path, err)) + Node* node = CollectTarget(state, argv[i], err); + if (node == NULL) return false; - - // Special syntax: "foo.cc^" means "the first output of foo.cc". - bool first_dependent = false; - if (!path.empty() && path[path.size() - 1] == '^') { - path.resize(path.size() - 1); - first_dependent = true; - } - - Node* node = state->LookupNode(path); - if (node) { - if (first_dependent) { - if (node->out_edges().empty()) { - *err = "'" + path + "' has no out edge"; - return false; - } - Edge* edge = node->out_edges()[0]; - if (edge->outputs_.empty()) { - edge->Dump(); - Fatal("edge has no outputs"); - } - node = edge->outputs_[0]; - } - targets->push_back(node); - } else { - *err = "unknown target '" + path + "'"; - - if (path == "clean") { - *err += ", did you mean 'ninja -t clean'?"; - } else if (path == "help") { - *err += ", did you mean 'ninja -h'?"; - } else { - Node* suggestion = state->SpellcheckNode(path); - if (suggestion) { - *err += ", did you mean '" + suggestion->path() + "'?"; - } - } - return false; - } + targets->push_back(node); } return true; } @@ -244,19 +251,14 @@ int ToolQuery(Globals* globals, int argc, char* argv[]) { return 1; } for (int i = 0; i < argc; ++i) { - Node* node = globals->state->LookupNode(argv[i]); + string err; + Node* node = CollectTarget(globals->state, argv[i], &err); if (!node) { - Node* suggestion = globals->state->SpellcheckNode(argv[i]); - if (suggestion) { - printf("%s unknown, did you mean %s?\n", - argv[i], suggestion->path().c_str()); - } else { - printf("%s unknown\n", argv[i]); - } + Error("%s", err.c_str()); return 1; } - printf("%s:\n", argv[i]); + printf("%s:\n", node->path().c_str()); if (Edge* edge = node->in_edge()) { printf(" input: %s\n", edge->rule_->name().c_str()); for (int in = 0; in < (int)edge->inputs_.size(); in++) { @@ -292,7 +294,7 @@ int ToolBrowse(Globals* globals, int argc, char* argv[]) { } #endif // _WIN32 -#if defined(WIN32) +#if defined(_WIN32) int ToolMSVC(Globals* globals, int argc, char* argv[]) { // Reset getopt: push one argument onto the front of argv, reset optind. argc++; @@ -537,7 +539,7 @@ int ChooseTool(const string& tool_name, const Tool** tool_out) { { "browse", "browse dependency graph in a web browser", Tool::RUN_AFTER_LOAD, ToolBrowse }, #endif -#if defined(WIN32) +#if defined(_WIN32) { "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)", Tool::RUN_AFTER_FLAGS, ToolMSVC }, #endif @@ -682,6 +684,9 @@ int RunBuild(Builder* builder, int argc, char** argv) { if (!builder->Build(&err)) { printf("ninja: build stopped: %s.\n", err.c_str()); + if (err.find("interrupted by user") != string::npos) { + return 2; + } return 1; } @@ -740,7 +745,7 @@ int NinjaMain(int argc, char** argv) { int opt; while (tool_name.empty() && - (opt = getopt_long(argc, argv, "d:f:hj:k:l:nt:vC:V", kLongOptions, + (opt = getopt_long(argc, argv, "d:f:j:k:l:nt:vC:h", kLongOptions, NULL)) != -1) { switch (opt) { case 'd': @@ -753,14 +758,6 @@ int NinjaMain(int argc, char** argv) { case 'j': config.parallelism = atoi(optarg); break; - case 'l': { - char* end; - double value = strtod(optarg, &end); - if (end == optarg) - Fatal("-l parameter not numeric: did you mean -l 0.0?"); - config.max_load_average = value; - break; - } case 'k': { char* end; int value = strtol(optarg, &end, 10); @@ -773,15 +770,23 @@ int NinjaMain(int argc, char** argv) { config.failures_allowed = value > 0 ? value : INT_MAX; break; } + case 'l': { + char* end; + double value = strtod(optarg, &end); + if (end == optarg) + Fatal("-l parameter not numeric: did you mean -l 0.0?"); + config.max_load_average = value; + break; + } case 'n': config.dry_run = true; break; - case 'v': - config.verbosity = BuildConfig::VERBOSE; - break; case 't': tool_name = optarg; break; + case 'v': + config.verbosity = BuildConfig::VERBOSE; + break; case 'C': working_dir = optarg; break; @@ -825,7 +830,6 @@ int NinjaMain(int argc, char** argv) { bool rebuilt_manifest = false; reload: - RealDiskInterface disk_interface; RealFileReader file_reader; ManifestParser parser(globals.state, &file_reader); string err; @@ -838,6 +842,7 @@ reload: return tool->func(&globals, argc, argv); BuildLog build_log; + RealDiskInterface disk_interface; if (!OpenLog(&build_log, &globals, &disk_interface)) return 1; diff --git a/src/state.cc b/src/state.cc index 4c7168b..bb0cc15 100644 --- a/src/state.cc +++ b/src/state.cc @@ -22,10 +22,49 @@ #include "metrics.h" #include "util.h" + +void Pool::EdgeScheduled(const Edge& edge) { + if (depth_ != 0) + current_use_ += edge.weight(); +} + +void Pool::EdgeFinished(const Edge& edge) { + if (depth_ != 0) + current_use_ -= edge.weight(); +} + +void Pool::DelayEdge(Edge* edge) { + assert(depth_ != 0); + delayed_.push_back(edge); +} + +void Pool::RetrieveReadyEdges(set<Edge*>* ready_queue) { + while (!delayed_.empty()) { + Edge* edge = delayed_.front(); + if (current_use_ + edge->weight() > depth_) + break; + delayed_.pop_front(); + ready_queue->insert(edge); + EdgeScheduled(*edge); + } +} + +void Pool::Dump() const { + printf("%s (%d/%d) ->\n", name_.c_str(), current_use_, depth_); + for (deque<Edge*>::const_iterator it = delayed_.begin(); + it != delayed_.end(); ++it) + { + printf("\t"); + (*it)->Dump(); + } +} + +Pool State::kDefaultPool("", 0); const Rule State::kPhonyRule("phony"); State::State() { AddRule(&kPhonyRule); + AddPool(&kDefaultPool); } void State::AddRule(const Rule* rule) { @@ -40,9 +79,22 @@ const Rule* State::LookupRule(const string& rule_name) { return i->second; } -Edge* State::AddEdge(const Rule* rule) { +void State::AddPool(Pool* pool) { + assert(LookupPool(pool->name()) == NULL); + pools_[pool->name()] = pool; +} + +Pool* State::LookupPool(const string& pool_name) { + map<string, Pool*>::iterator i = pools_.find(pool_name); + if (i == pools_.end()) + return NULL; + return i->second; +} + +Edge* State::AddEdge(const Rule* rule, Pool* pool) { Edge* edge = new Edge(); edge->rule_ = rule; + edge->pool_ = pool; edge->env_ = &bindings_; edges_.push_back(edge); return edge; @@ -146,4 +198,14 @@ void State::Dump() { node->status_known() ? (node->dirty() ? "dirty" : "clean") : "unknown"); } + if (!pools_.empty()) { + printf("resource_pools:\n"); + for (map<string, Pool*>::const_iterator it = pools_.begin(); + it != pools_.end(); ++it) + { + if (!it->second->name().empty()) { + it->second->Dump(); + } + } + } } diff --git a/src/state.h b/src/state.h index 026acf3..918fe09 100644 --- a/src/state.h +++ b/src/state.h @@ -16,6 +16,8 @@ #define NINJA_STATE_H_ #include <map> +#include <deque> +#include <set> #include <string> #include <vector> using namespace std; @@ -27,8 +29,59 @@ struct Edge; struct Node; struct Rule; +/// A pool for delayed edges. +/// Pools are scoped to a State. Edges within a State will share Pools. A Pool +/// will keep a count of the total 'weight' of the currently scheduled edges. If +/// a Plan attempts to schedule an Edge which would cause the total weight to +/// exceed the depth of the Pool, the Pool will enque the Edge instead of +/// allowing the Plan to schedule it. The Pool will relinquish queued Edges when +/// the total scheduled weight diminishes enough (i.e. when a scheduled edge +/// completes). +struct Pool { + explicit Pool(const string& name, int depth) + : name_(name), current_use_(0), depth_(depth) { } + + // A depth of 0 is infinite + bool is_valid() const { return depth_ >= 0; } + int depth() const { return depth_; } + const string& name() const { return name_; } + + /// true if the Pool might delay this edge + bool ShouldDelayEdge() const { return depth_ != 0; } + + /// informs this Pool that the given edge is committed to be run. + /// Pool will count this edge as using resources from this pool. + void EdgeScheduled(const Edge& edge); + + /// informs this Pool that the given edge is no longer runnable, and should + /// relinquish its resources back to the pool + void EdgeFinished(const Edge& edge); + + /// adds the given edge to this Pool to be delayed. + void DelayEdge(Edge* edge); + + /// Pool will add zero or more edges to the ready_queue + void RetrieveReadyEdges(set<Edge*>* ready_queue); + + /// Dump the Pool and its edges (useful for debugging). + void Dump() const; + +private: + int UnitsWaiting() { return delayed_.size(); } + + string name_; + + /// |current_use_| is the total of the weights of the edges which are + /// currently scheduled in the Plan (i.e. the edges in Plan::ready_). + int current_use_; + int depth_; + + deque<Edge*> delayed_; +}; + /// Global state (file status, loaded rules) for a single run. struct State { + static Pool kDefaultPool; static const Rule kPhonyRule; State(); @@ -36,7 +89,10 @@ struct State { void AddRule(const Rule* rule); const Rule* LookupRule(const string& rule_name); - Edge* AddEdge(const Rule* rule); + void AddPool(Pool* pool); + Pool* LookupPool(const string& pool_name); + + Edge* AddEdge(const Rule* rule, Pool* pool); Node* GetNode(StringPiece path); Node* LookupNode(StringPiece path); @@ -50,7 +106,7 @@ struct State { /// state where we haven't yet examined the disk for dirty state. void Reset(); - /// Dump the nodes (useful for debugging). + /// Dump the nodes and Pools (useful for debugging). void Dump(); /// @return the root node(s) of the graph. (Root nodes have no output edges). @@ -65,6 +121,9 @@ struct State { /// All the rules used in the graph. map<string, const Rule*> rules_; + /// All the pools used in the graph. + map<string, Pool*> pools_; + /// All the edges of the graph. vector<Edge*> edges_; diff --git a/src/state_test.cc b/src/state_test.cc index bc24edd..26177ff 100644 --- a/src/state_test.cc +++ b/src/state_test.cc @@ -32,7 +32,7 @@ TEST(State, Basic) { rule->set_command(command); state.AddRule(rule); - Edge* edge = state.AddEdge(rule); + Edge* edge = state.AddEdge(rule, &State::kDefaultPool); state.AddIn(edge, "in1"); state.AddIn(edge, "in2"); state.AddOut(edge, "out"); diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc index 1c47fd1..8f1a04e 100644 --- a/src/subprocess-posix.cc +++ b/src/subprocess-posix.cc @@ -76,7 +76,7 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { break; // Open /dev/null over stdin. - int devnull = open("/dev/null", O_WRONLY); + int devnull = open("/dev/null", O_RDONLY); if (devnull < 0) break; if (dup2(devnull, 0) < 0) diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 4b103a5..1b230b6 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -101,14 +101,17 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { NULL, NULL, &startup_info, &process_info)) { DWORD error = GetLastError(); - if (error == ERROR_FILE_NOT_FOUND) { // file (program) not found error is treated as a normal build action failure + if (error == ERROR_FILE_NOT_FOUND) { + // File (program) not found error is treated as a normal build + // action failure. if (child_pipe) CloseHandle(child_pipe); CloseHandle(pipe_); CloseHandle(nul); pipe_ = NULL; // child_ is already NULL; - buf_ = "CreateProcess failed: The system cannot find the file specified.\n"; + buf_ = "CreateProcess failed: The system cannot find the file " + "specified.\n"; return true; } else { Win32Fatal("CreateProcess"); // pass all other errors to Win32Fatal diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index d89525e..c3175da 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -64,7 +64,8 @@ TEST_F(SubprocessTest, NoSuchCommand) { EXPECT_EQ(ExitFailure, subproc->Finish()); EXPECT_NE("", subproc->GetOutput()); #ifdef _WIN32 - ASSERT_EQ("CreateProcess failed: The system cannot find the file specified.\n", subproc->GetOutput()); + ASSERT_EQ("CreateProcess failed: The system cannot find the file " + "specified.\n", subproc->GetOutput()); #endif } @@ -179,3 +180,18 @@ TEST_F(SubprocessTest, SetWithLots) { ASSERT_EQ(kNumProcs, subprocs_.finished_.size()); } #endif // linux + +// TODO: this test could work on Windows, just not sure how to simply +// read stdin. +#ifndef _WIN32 +// Verify that a command that attempts to read stdin correctly thinks +// that stdin is closed. +TEST_F(SubprocessTest, ReadStdin) { + Subprocess* subproc = subprocs_.Add("cat -"); + while (!subproc->Done()) { + subprocs_.DoWork(); + } + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_EQ(1u, subprocs_.finished_.size()); +} +#endif // _WIN32 diff --git a/src/util.cc b/src/util.cc index 0feb99d..4b2900f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -155,7 +155,7 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { } if (component_count == kMaxPathComponents) - Fatal("path has too many components"); + Fatal("path has too many components : %s", path); components[component_count] = dst; ++component_count; @@ -49,7 +49,8 @@ void SetCloseOnExec(int fd); /// Given a misspelled string and a list of correct spellings, returns /// the closest match or NULL if there is no close enough match. -const char* SpellcheckStringV(const string& text, const vector<const char*>& words); +const char* SpellcheckStringV(const string& text, + const vector<const char*>& words); /// Like SpellcheckStringV, but takes a NULL-terminated list. const char* SpellcheckString(const string& text, ...); |