diff options
author | Evan Martin <martine@danga.com> | 2013-04-09 15:57:02 (GMT) |
---|---|---|
committer | Evan Martin <martine@danga.com> | 2013-04-09 15:57:02 (GMT) |
commit | 8b9e3eaf7e7d2af4539eae167f92993f5ab84b86 (patch) | |
tree | 82582c0928cb33e12b4d0a6c68f7bd06ff5a9301 /src | |
parent | 76c8b11b0ad1c62ecdaa45fd553a6ac69c213663 (diff) | |
parent | 0a2fc151976277d8c0319cdc4ee3b1932b673d91 (diff) | |
download | Ninja-8b9e3eaf7e7d2af4539eae167f92993f5ab84b86.zip Ninja-8b9e3eaf7e7d2af4539eae167f92993f5ab84b86.tar.gz Ninja-8b9e3eaf7e7d2af4539eae167f92993f5ab84b86.tar.bz2 |
Merge branch 'dep-pipeless'
This merges a new mechanism for tracking "depfile" dependencies that
is faster on all platforms but dramatically so on Windows.
Diffstat (limited to 'src')
-rw-r--r-- | src/build.cc | 279 | ||||
-rw-r--r-- | src/build.h | 23 | ||||
-rw-r--r-- | src/build_test.cc | 245 | ||||
-rw-r--r-- | src/deps_log.cc | 263 | ||||
-rw-r--r-- | src/deps_log.h | 107 | ||||
-rw-r--r-- | src/deps_log_test.cc | 195 | ||||
-rw-r--r-- | src/disk_interface_test.cc | 2 | ||||
-rw-r--r-- | src/graph.cc | 192 | ||||
-rw-r--r-- | src/graph.h | 75 | ||||
-rw-r--r-- | src/graph_test.cc | 2 | ||||
-rw-r--r-- | src/manifest_parser.cc | 8 | ||||
-rw-r--r-- | src/manifest_parser_test.cc | 12 | ||||
-rw-r--r-- | src/msvc_helper-win32.cc | 94 | ||||
-rw-r--r-- | src/msvc_helper.h | 42 | ||||
-rw-r--r-- | src/msvc_helper_main-win32.cc | 26 | ||||
-rw-r--r-- | src/msvc_helper_test.cc | 123 | ||||
-rw-r--r-- | src/ninja.cc | 77 | ||||
-rw-r--r-- | src/state.cc | 5 | ||||
-rw-r--r-- | src/test.cc | 6 | ||||
-rw-r--r-- | src/test.h | 6 |
20 files changed, 1414 insertions, 368 deletions
diff --git a/src/build.cc b/src/build.cc index 340a3d9..ab3d781 100644 --- a/src/build.cc +++ b/src/build.cc @@ -15,6 +15,7 @@ #include "build.h" #include <assert.h> +#include <errno.h> #include <stdio.h> #include <stdlib.h> #include <functional> @@ -24,8 +25,11 @@ #endif #include "build_log.h" +#include "depfile_parser.h" +#include "deps_log.h" #include "disk_interface.h" #include "graph.h" +#include "msvc_helper.h" #include "state.h" #include "subprocess.h" #include "util.h" @@ -39,7 +43,7 @@ struct DryRunCommandRunner : public CommandRunner { // Overridden from CommandRunner: virtual bool CanRunMore(); virtual bool StartCommand(Edge* edge); - virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */); + virtual bool WaitForCommand(Result* result); private: queue<Edge*> finished_; @@ -54,16 +58,14 @@ bool DryRunCommandRunner::StartCommand(Edge* edge) { return true; } -Edge* DryRunCommandRunner::WaitForCommand(ExitStatus* status, - string* /*output*/) { - if (finished_.empty()) { - *status = ExitFailure; - return NULL; - } - *status = ExitSuccess; - Edge* edge = finished_.front(); +bool DryRunCommandRunner::WaitForCommand(Result* result) { + if (finished_.empty()) + return false; + + result->status = ExitSuccess; + result->edge = finished_.front(); finished_.pop(); - return edge; + return true; } } // namespace @@ -427,7 +429,7 @@ void Plan::CleanNode(DependencyScan* scan, Node* node) { if (!(*ni)->dirty()) continue; - if (scan->RecomputeOutputDirty(*ei, most_recent_input, + if (scan->RecomputeOutputDirty(*ei, most_recent_input, 0, command, *ni)) { (*ni)->MarkDirty(); all_outputs_clean = false; @@ -462,7 +464,7 @@ struct RealCommandRunner : public CommandRunner { virtual ~RealCommandRunner() {} virtual bool CanRunMore(); virtual bool StartCommand(Edge* edge); - virtual Edge* WaitForCommand(ExitStatus* status, string* output); + virtual bool WaitForCommand(Result* result); virtual vector<Edge*> GetActiveEdges(); virtual void Abort(); @@ -499,31 +501,30 @@ bool RealCommandRunner::StartCommand(Edge* edge) { return true; } -Edge* RealCommandRunner::WaitForCommand(ExitStatus* status, string* output) { +bool RealCommandRunner::WaitForCommand(Result* result) { Subprocess* subproc; while ((subproc = subprocs_.NextFinished()) == NULL) { bool interrupted = subprocs_.DoWork(); - if (interrupted) { - *status = ExitInterrupted; - return 0; - } + if (interrupted) + return false; } - *status = subproc->Finish(); - *output = subproc->GetOutput(); + result->status = subproc->Finish(); + result->output = subproc->GetOutput(); map<Subprocess*, Edge*>::iterator i = subproc_to_edge_.find(subproc); - Edge* edge = i->second; + result->edge = i->second; subproc_to_edge_.erase(i); delete subproc; - return edge; + return true; } Builder::Builder(State* state, const BuildConfig& config, - BuildLog* log, DiskInterface* disk_interface) + BuildLog* build_log, DepsLog* deps_log, + DiskInterface* disk_interface) : state_(state), config_(config), disk_interface_(disk_interface), - scan_(state, log, disk_interface) { + scan_(state, build_log, deps_log, disk_interface) { status_ = new BuildStatus(config); } @@ -609,8 +610,6 @@ bool Builder::Build(string* err) { // First, we attempt to start as many commands as allowed by the // command runner. // Second, we attempt to wait for / reap the next finished command. - // If we can do neither of those, the build is stuck, and we report - // an error. while (plan_.more_to_do()) { // See if we can start any more commands. if (failures_allowed && command_runner_->CanRunMore()) { @@ -620,10 +619,11 @@ bool Builder::Build(string* err) { return false; } - if (edge->is_phony()) - FinishEdge(edge, true, ""); - else + if (edge->is_phony()) { + plan_.EdgeFinished(edge); + } else { ++pending_commands; + } // We made some progress; go back to the main loop. continue; @@ -632,34 +632,24 @@ bool Builder::Build(string* err) { // See if we can reap any finished commands. if (pending_commands) { - ExitStatus status; - string output; - Edge* edge = command_runner_->WaitForCommand(&status, &output); - if (edge && status != ExitInterrupted) { - bool success = (status == ExitSuccess); - --pending_commands; - FinishEdge(edge, success, output); - if (!success) { - if (failures_allowed) - failures_allowed--; - } - - // We made some progress; start the main loop over. - continue; - } - - if (status == ExitInterrupted) { + CommandRunner::Result result; + if (!command_runner_->WaitForCommand(&result) || + result.status == ExitInterrupted) { status_->BuildFinished(); *err = "interrupted by user"; return false; } - } - // If we get here, we can neither enqueue new commands nor are any running. - if (pending_commands) { - status_->BuildFinished(); - *err = "stuck: pending commands but none to wait for? [this is a bug]"; - return false; + --pending_commands; + FinishCommand(&result); + + if (!result.success()) { + if (failures_allowed) + failures_allowed--; + } + + // We made some progress; start the main loop over. + continue; } // If we get here, we cannot make any more progress. @@ -714,63 +704,156 @@ bool Builder::StartEdge(Edge* edge, string* err) { return true; } -void Builder::FinishEdge(Edge* edge, bool success, const string& output) { - METRIC_RECORD("FinishEdge"); - TimeStamp restat_mtime = 0; +void Builder::FinishCommand(CommandRunner::Result* result) { + METRIC_RECORD("FinishCommand"); - if (success) { - if (edge->GetBindingBool("restat") && !config_.dry_run) { - bool node_cleaned = false; - - for (vector<Node*>::iterator i = edge->outputs_.begin(); - i != edge->outputs_.end(); ++i) { - TimeStamp new_mtime = disk_interface_->Stat((*i)->path()); - if ((*i)->mtime() == new_mtime) { - // The rule command did not change the output. Propagate the clean - // state through the build graph. - // Note that this also applies to nonexistent outputs (mtime == 0). - plan_.CleanNode(&scan_, *i); - node_cleaned = true; - } - } + Edge* edge = result->edge; - if (node_cleaned) { - // If any output was cleaned, find the most recent mtime of any - // (existing) non-order-only input or the depfile. - for (vector<Node*>::iterator i = edge->inputs_.begin(); - i != edge->inputs_.end() - edge->order_only_deps_; ++i) { - TimeStamp input_mtime = disk_interface_->Stat((*i)->path()); - if (input_mtime > restat_mtime) - restat_mtime = input_mtime; - } + // First try to extract dependencies from the result, if any. + // This must happen first as it filters the command output and + // can fail, which makes the command fail from a build perspective. - string depfile = edge->GetBinding("depfile"); - if (restat_mtime != 0 && !depfile.empty()) { - TimeStamp depfile_mtime = disk_interface_->Stat(depfile); - if (depfile_mtime > restat_mtime) - restat_mtime = depfile_mtime; - } + vector<Node*> deps_nodes; + TimeStamp deps_mtime = 0; + string deps_type = edge->GetBinding("deps"); + if (result->success() && !deps_type.empty()) { + string extract_err; + if (!ExtractDeps(result, deps_type, &deps_nodes, &deps_mtime, + &extract_err)) { + if (!result->output.empty()) + result->output.append("\n"); + result->output.append(extract_err); + result->status = ExitFailure; + } + } - // The total number of edges in the plan may have changed as a result - // of a restat. - status_->PlanHasTotalEdges(plan_.command_edge_count()); + int start_time, end_time; + status_->BuildEdgeFinished(edge, result->success(), result->output, + &start_time, &end_time); + + // The rest of this function only applies to successful commands. + if (!result->success()) + return; + + // Restat the edge outputs, if necessary. + TimeStamp restat_mtime = 0; + if (edge->GetBindingBool("restat") && !config_.dry_run) { + bool node_cleaned = false; + + for (vector<Node*>::iterator i = edge->outputs_.begin(); + i != edge->outputs_.end(); ++i) { + TimeStamp new_mtime = disk_interface_->Stat((*i)->path()); + if ((*i)->mtime() == new_mtime) { + // The rule command did not change the output. Propagate the clean + // state through the build graph. + // Note that this also applies to nonexistent outputs (mtime == 0). + plan_.CleanNode(&scan_, *i); + node_cleaned = true; } } - // Delete the response file on success (if exists) - string rspfile = edge->GetBinding("rspfile"); - if (!rspfile.empty()) - disk_interface_->RemoveFile(rspfile); + if (node_cleaned) { + // If any output was cleaned, find the most recent mtime of any + // (existing) non-order-only input or the depfile. + for (vector<Node*>::iterator i = edge->inputs_.begin(); + i != edge->inputs_.end() - edge->order_only_deps_; ++i) { + TimeStamp input_mtime = disk_interface_->Stat((*i)->path()); + if (input_mtime > restat_mtime) + restat_mtime = input_mtime; + } + + string depfile = edge->GetBinding("depfile"); + if (restat_mtime != 0 && !depfile.empty()) { + TimeStamp depfile_mtime = disk_interface_->Stat(depfile); + if (depfile_mtime > restat_mtime) + restat_mtime = depfile_mtime; + } - plan_.EdgeFinished(edge); + // The total number of edges in the plan may have changed as a result + // of a restat. + status_->PlanHasTotalEdges(plan_.command_edge_count()); + } } - if (edge->is_phony()) - return; + plan_.EdgeFinished(edge); + + // Delete any left over response file. + string rspfile = edge->GetBinding("rspfile"); + if (!rspfile.empty()) + disk_interface_->RemoveFile(rspfile); + + if (scan_.build_log()) { + scan_.build_log()->RecordCommand(edge, start_time, end_time, + restat_mtime); + } + + if (!deps_type.empty()) { + assert(edge->outputs_.size() == 1 && "should have been rejected by parser"); + Node* out = edge->outputs_[0]; + if (!deps_mtime) { + // On Windows there's no separate file to compare against; just reuse + // the output's mtime. + deps_mtime = disk_interface_->Stat(out->path()); + } + scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes); + } - int start_time, end_time; - status_->BuildEdgeFinished(edge, success, output, &start_time, &end_time); - if (success && scan_.build_log()) - scan_.build_log()->RecordCommand(edge, start_time, end_time, restat_mtime); } +bool Builder::ExtractDeps(CommandRunner::Result* result, + const string& deps_type, + vector<Node*>* deps_nodes, + TimeStamp* deps_mtime, + string* err) { +#ifdef _WIN32 + if (deps_type == "msvc") { + CLParser parser; + result->output = parser.Parse(result->output); + for (set<string>::iterator i = parser.includes_.begin(); + i != parser.includes_.end(); ++i) { + deps_nodes->push_back(state_->GetNode(*i)); + } + } else +#endif + if (deps_type == "gcc") { + string depfile = result->edge->GetBinding("depfile"); + if (depfile.empty()) { + *err = string("edge with deps=gcc but no depfile makes no sense"); + return false; + } + + *deps_mtime = disk_interface_->Stat(depfile); + if (*deps_mtime <= 0) { + *err = string("unable to read depfile"); + return false; + } + + string content = disk_interface_->ReadFile(depfile, err); + if (!err->empty()) + return false; + if (content.empty()) + return true; + + DepfileParser deps; + if (!deps.Parse(&content, err)) + return false; + + // XXX check depfile matches expected output. + deps_nodes->reserve(deps.ins_.size()); + for (vector<StringPiece>::iterator i = deps.ins_.begin(); + i != deps.ins_.end(); ++i) { + if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err)) + return false; + deps_nodes->push_back(state_->GetNode(*i)); + } + + if (disk_interface_->RemoveFile(depfile) < 0) { + *err = string("deleting depfile: ") + strerror(errno) + string("\n"); + return false; + } + } else { + Fatal("unknown deps type '%s'", deps_type.c_str()); + } + + return true; +} diff --git a/src/build.h b/src/build.h index 52c277a..9a16990 100644 --- a/src/build.h +++ b/src/build.h @@ -104,8 +104,18 @@ struct CommandRunner { virtual ~CommandRunner() {} virtual bool CanRunMore() = 0; virtual bool StartCommand(Edge* edge) = 0; - /// Wait for a command to complete. - virtual Edge* WaitForCommand(ExitStatus* status, string* output) = 0; + + /// The result of waiting for a command. + struct Result { + Result() : edge(NULL) {} + Edge* edge; + ExitStatus status; + string output; + bool success() const { return status == ExitSuccess; } + }; + /// Wait for a command to complete, or return false if interrupted. + virtual bool WaitForCommand(Result* result) = 0; + virtual vector<Edge*> GetActiveEdges() { return vector<Edge*>(); } virtual void Abort() {} }; @@ -132,7 +142,8 @@ struct BuildConfig { /// Builder wraps the build process: starting commands, updating status. struct Builder { Builder(State* state, const BuildConfig& config, - BuildLog* log, DiskInterface* disk_interface); + BuildLog* build_log, DepsLog* deps_log, + DiskInterface* disk_interface); ~Builder(); /// Clean up after interrupted commands by deleting output files. @@ -152,7 +163,7 @@ struct Builder { bool Build(string* err); bool StartEdge(Edge* edge, string* err); - void FinishEdge(Edge* edge, bool success, const string& output); + void FinishCommand(CommandRunner::Result* result); /// Used for tests. void SetBuildLog(BuildLog* log) { @@ -166,6 +177,10 @@ struct Builder { BuildStatus* status_; private: + bool ExtractDeps(CommandRunner::Result* result, const string& deps_type, + vector<Node*>* deps_nodes, TimeStamp* deps_mtime, + string* err); + DiskInterface* disk_interface_; DependencyScan scan_; diff --git a/src/build_test.cc b/src/build_test.cc index d4468a3..2a0fa0f 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -15,6 +15,7 @@ #include "build.h" #include "build_log.h" +#include "deps_log.h" #include "graph.h" #include "test.h" @@ -383,7 +384,7 @@ struct FakeCommandRunner : public CommandRunner { // CommandRunner impl virtual bool CanRunMore(); virtual bool StartCommand(Edge* edge); - virtual Edge* WaitForCommand(ExitStatus* status, string* output); + virtual bool WaitForCommand(Result* result); virtual vector<Edge*> GetActiveEdges(); virtual void Abort(); @@ -394,8 +395,13 @@ struct FakeCommandRunner : public CommandRunner { struct BuildTest : public StateTestWithBuiltinRules { BuildTest() : config_(MakeConfig()), command_runner_(&fs_), - builder_(&state_, config_, NULL, &fs_), + builder_(&state_, config_, NULL, NULL, &fs_), status_(config_) { + } + + virtual void SetUp() { + StateTestWithBuiltinRules::SetUp(); + builder_.command_runner_.reset(&command_runner_); AssertParse(&state_, "build cat1: cat in1\n" @@ -457,24 +463,25 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { return true; } -Edge* FakeCommandRunner::WaitForCommand(ExitStatus* status, - string* /* output */) { - if (Edge* edge = last_command_) { - if (edge->rule().name() == "interrupt" || - edge->rule().name() == "touch-interrupt") { - *status = ExitInterrupted; - return NULL; - } +bool FakeCommandRunner::WaitForCommand(Result* result) { + if (!last_command_) + return false; - if (edge->rule().name() == "fail") - *status = ExitFailure; - else - *status = ExitSuccess; - last_command_ = NULL; - return edge; + Edge* edge = last_command_; + result->edge = edge; + + if (edge->rule().name() == "interrupt" || + edge->rule().name() == "touch-interrupt") { + result->status = ExitInterrupted; + return true; } - *status = ExitFailure; - return NULL; + + if (edge->rule().name() == "fail") + result->status = ExitFailure; + else + result->status = ExitSuccess; + last_command_ = NULL; + return true; } vector<Edge*> FakeCommandRunner::GetActiveEdges() { @@ -761,6 +768,9 @@ TEST_F(BuildTest, OrderOnlyDeps) { fs_.Tick(); + // Recreate the depfile, as it should have been deleted by the build. + fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n"); + // implicit dep dirty, expect a rebuild. fs_.Create("blah.h", ""); fs_.Create("bar.h", ""); @@ -773,6 +783,9 @@ TEST_F(BuildTest, OrderOnlyDeps) { fs_.Tick(); + // Recreate the depfile, as it should have been deleted by the build. + fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n"); + // order only dep dirty, no rebuild. fs_.Create("otherfile", ""); command_runner_.commands_ran_.clear(); @@ -1340,8 +1353,204 @@ TEST_F(BuildTest, PhonyWithNoInputs) { ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } +TEST_F(BuildTest, DepsGccWithEmptyDeps) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc\n" +" command = cc\n" +" deps = gcc\n" +"build out: cc\n")); + Dirty("out"); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + EXPECT_FALSE(builder_.Build(&err)); + ASSERT_EQ("subcommand failed", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); +} + TEST_F(BuildTest, StatusFormatReplacePlaceholder) { EXPECT_EQ("[%/s0/t0/r0/u0/f0]", status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]")); } +TEST_F(BuildTest, FailedDepsParse) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build bad_deps.o: cat in1\n" +" deps = gcc\n" +" depfile = in1.d\n")); + + string err; + EXPECT_TRUE(builder_.AddTarget("bad_deps.o", &err)); + ASSERT_EQ("", err); + + // These deps will fail to parse, as they should only have one + // path to the left of the colon. + fs_.Create("in1.d", "AAA BBB"); + + EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ("subcommand failed", err); +} + +/// Tests of builds involving deps logs necessarily must span +/// multiple builds. We reuse methods on BuildTest but not the +/// builder_ it sets up, because we want pristine objects for +/// each build. +struct BuildWithDepsLogTest : public BuildTest { + BuildWithDepsLogTest() {} + + virtual void SetUp() { + BuildTest::SetUp(); + + temp_dir_.CreateAndEnter("BuildWithDepsLogTest"); + } + + virtual void TearDown() { + temp_dir_.Cleanup(); + } + + ScopedTempDir temp_dir_; + + /// Shadow parent class builder_ so we don't accidentally use it. + void* builder_; +}; + +/// Run a straightforwad build where the deps log is used. +TEST_F(BuildWithDepsLogTest, Straightforward) { + string err; + // Note: in1 was created by the superclass SetUp(). + const char* manifest = + "build out: cat in1\n" + " deps = gcc\n" + " depfile = in1.d\n"; + { + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + // Run the build once, everything should be ok. + DepsLog deps_log; + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + fs_.Create("in1.d", "out: in2"); + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + // The deps file should have been removed. + EXPECT_EQ(0, fs_.Stat("in1.d")); + // Recreate it for the next step. + fs_.Create("in1.d", "out: in2"); + deps_log.Close(); + builder.command_runner_.release(); + } + + { + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + // Touch the file only mentioned in the deps. + fs_.Tick(); + fs_.Create("in2", ""); + + // Run the build again. + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + + Builder builder(&state, config_, NULL, &deps_log, &fs_); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + // We should have rebuilt the output due to in2 being + // out of date. + EXPECT_EQ(1u, command_runner_.commands_ran_.size()); + + builder.command_runner_.release(); + } +} + +/// Verify that obsolete deps still cause a rebuild. +TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { + string err; + // Note: in1 was created by the superclass SetUp(). + const char* manifest = + "build out: cat in1\n" + " deps = gcc\n" + " depfile = in1.d\n"; + { + // Create the obsolete deps, then run a build to incorporate them. + // The idea is that the inputs/outputs are newer than the logged + // deps. + fs_.Create("in1.d", "out: "); + fs_.Tick(); + + fs_.Create("in1", ""); + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + // Run the build once, everything should be ok. + DepsLog deps_log; + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + fs_.Create("out", ""); + // The deps file should have been removed. + EXPECT_EQ(0, fs_.Stat("in1.d")); + deps_log.Close(); + builder.command_runner_.release(); + } + + // Now we should be in a situation where in1/out2 both have recent + // timestamps but the deps are old. Verify we rebuild. + fs_.Tick(); + + { + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + + Builder builder(&state, config_, NULL, &deps_log, &fs_); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + + // Recreate the deps here just to prove the old recorded deps are + // the problem. + fs_.Create("in1.d", "out: "); + + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + // We should have rebuilt the output due to the deps being + // out of date. + EXPECT_EQ(1u, command_runner_.commands_ran_.size()); + + builder.command_runner_.release(); + } +} diff --git a/src/deps_log.cc b/src/deps_log.cc new file mode 100644 index 0000000..8946e32 --- /dev/null +++ b/src/deps_log.cc @@ -0,0 +1,263 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "deps_log.h" + +#include <assert.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <unistd.h> + +#include "graph.h" +#include "metrics.h" +#include "state.h" +#include "util.h" + +// The version is stored as 4 bytes after the signature and also serves as a +// byte order mark. Signature and version combined are 16 bytes long. +const char kFileSignature[] = "# ninjadeps\n"; +const int kCurrentVersion = 1; + +DepsLog::~DepsLog() { + Close(); +} + +bool DepsLog::OpenForWrite(const string& path, string* err) { + file_ = fopen(path.c_str(), "ab"); + if (!file_) { + *err = strerror(errno); + return false; + } + SetCloseOnExec(fileno(file_)); + + // Opening a file in append mode doesn't set the file pointer to the file's + // end on Windows. Do that explicitly. + fseek(file_, 0, SEEK_END); + + if (ftell(file_) == 0) { + if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) { + *err = strerror(errno); + return false; + } + if (fwrite(&kCurrentVersion, 4, 1, file_) < 1) { + *err = strerror(errno); + return false; + } + } + + return true; +} + +bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, + const vector<Node*>& nodes) { + return RecordDeps(node, mtime, nodes.size(), + nodes.empty() ? NULL : (Node**)&nodes.front()); +} + +bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, + int node_count, Node** nodes) { + // Track whether there's any new data to be recorded. + bool made_change = false; + + // Assign ids to all nodes that are missing one. + if (node->id() < 0) { + RecordId(node); + made_change = true; + } + for (int i = 0; i < node_count; ++i) { + if (nodes[i]->id() < 0) { + RecordId(nodes[i]); + made_change = true; + } + } + + // See if the new data is different than the existing data, if any. + if (!made_change) { + Deps* deps = GetDeps(node); + if (!deps || + deps->mtime != mtime || + deps->node_count != node_count) { + made_change = true; + } else { + for (int i = 0; i < node_count; ++i) { + if (deps->nodes[i] != nodes[i]) { + made_change = true; + break; + } + } + } + } + + // Don't write anything if there's no new info. + if (!made_change) + return true; + + uint16_t size = 4 * (1 + 1 + (uint16_t)node_count); + size |= 0x8000; // Deps record: set high bit. + fwrite(&size, 2, 1, file_); + int id = node->id(); + fwrite(&id, 4, 1, file_); + int timestamp = mtime; + fwrite(×tamp, 4, 1, file_); + for (int i = 0; i < node_count; ++i) { + id = nodes[i]->id(); + fwrite(&id, 4, 1, file_); + } + + return true; +} + +void DepsLog::Close() { + if (file_) + fclose(file_); + file_ = NULL; +} + +bool DepsLog::Load(const string& path, State* state, string* err) { + METRIC_RECORD(".ninja_deps load"); + char buf[32 << 10]; + FILE* f = fopen(path.c_str(), "rb"); + if (!f) { + if (errno == ENOENT) + return true; + *err = strerror(errno); + return false; + } + + if (!fgets(buf, sizeof(buf), f)) { + *err = strerror(errno); + return false; + } + int version = 0; + if (fread(&version, 4, 1, f) < 1) { + *err = strerror(errno); + return false; + } + if (version != kCurrentVersion) { + *err = "bad deps log signature or version; starting over"; + fclose(f); + unlink(path.c_str()); + // Don't report this as a failure. An empty deps log will cause + // us to rebuild the outputs anyway. + return true; + } + + for (;;) { + uint16_t size; + if (fread(&size, 2, 1, f) < 1) + break; + bool is_deps = (size >> 15) != 0; + size = size & 0x7FFF; + + if (fread(buf, size, 1, f) < 1) + break; + + if (is_deps) { + assert(size % 4 == 0); + int* deps_data = reinterpret_cast<int*>(buf); + int out_id = deps_data[0]; + int mtime = deps_data[1]; + deps_data += 2; + int deps_count = (size / 4) - 2; + + Deps* deps = new Deps; + deps->mtime = mtime; + deps->node_count = deps_count; + deps->nodes = new Node*[deps_count]; + for (int i = 0; i < deps_count; ++i) { + assert(deps_data[i] < (int)nodes_.size()); + assert(nodes_[deps_data[i]]); + deps->nodes[i] = nodes_[deps_data[i]]; + } + + if (out_id >= (int)deps_.size()) + deps_.resize(out_id + 1); + if (deps_[out_id]) { + ++dead_record_count_; + delete deps_[out_id]; + } + deps_[out_id] = deps; + } else { + StringPiece path(buf, size); + Node* node = state->GetNode(path); + assert(node->id() < 0); + node->set_id(nodes_.size()); + nodes_.push_back(node); + } + } + if (ferror(f)) { + *err = strerror(ferror(f)); + return false; + } + fclose(f); + return true; +} + +DepsLog::Deps* DepsLog::GetDeps(Node* node) { + if (node->id() < 0) + return NULL; + return deps_[node->id()]; +} + +bool DepsLog::Recompact(const string& path, string* err) { + METRIC_RECORD(".ninja_deps recompact"); + printf("Recompacting deps...\n"); + + string temp_path = path + ".recompact"; + DepsLog new_log; + if (!new_log.OpenForWrite(temp_path, err)) + return false; + + // Clear all known ids so that new ones can be reassigned. + for (vector<Node*>::iterator i = nodes_.begin(); + i != nodes_.end(); ++i) { + (*i)->set_id(-1); + } + + // Write out all deps again. + for (int old_id = 0; old_id < (int)deps_.size(); ++old_id) { + Deps* deps = deps_[old_id]; + if (!new_log.RecordDeps(nodes_[old_id], deps->mtime, + deps->node_count, deps->nodes)) { + new_log.Close(); + return false; + } + } + + new_log.Close(); + + if (unlink(path.c_str()) < 0) { + *err = strerror(errno); + return false; + } + + if (rename(temp_path.c_str(), path.c_str()) < 0) { + *err = strerror(errno); + return false; + } + + return true; +} + +bool DepsLog::RecordId(Node* node) { + uint16_t size = (uint16_t)node->path().size(); + fwrite(&size, 2, 1, file_); + fwrite(node->path().data(), node->path().size(), 1, file_); + + node->set_id(nodes_.size()); + nodes_.push_back(node); + + return true; +} diff --git a/src/deps_log.h b/src/deps_log.h new file mode 100644 index 0000000..99b006e --- /dev/null +++ b/src/deps_log.h @@ -0,0 +1,107 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_DEPS_LOG_H_ +#define NINJA_DEPS_LOG_H_ + +#include <string> +#include <vector> +using namespace std; + +#include <stdio.h> + +#include "timestamp.h" + +struct Node; +struct State; + +/// As build commands run they can output extra dependency information +/// (e.g. header dependencies for C source) dynamically. DepsLog collects +/// that information at build time and uses it for subsequent builds. +/// +/// The on-disk format is based on two primary design constraints: +/// - it must be written to as a stream (during the build, which may be +/// interrupted); +/// - it can be read all at once on startup. (Alternative designs, where +/// it contains indexing information, were considered and discarded as +/// too complicated to implement; if the file is small than reading it +/// fully on startup is acceptable.) +/// Here are some stats from the Windows Chrome dependency files, to +/// help guide the design space. The total text in the files sums to +/// 90mb so some compression is warranted to keep load-time fast. +/// There's about 10k files worth of dependencies that reference about +/// 40k total paths totalling 2mb of unique strings. +/// +/// Based on these stats, here's the current design. +/// The file is structured as version header followed by a sequence of records. +/// Each record is either a path string or a dependency list. +/// Numbering the path strings in file order gives them dense integer ids. +/// A dependency list maps an output id to a list of input ids. +/// +/// Concretely, a record is: +/// two bytes record length, high bit indicates record type +/// (implies max record length 32k) +/// path records contain just the string name of the path +/// dependency records are an array of 4-byte integers +/// [output path id, output path mtime, input path id, input path id...] +/// (The mtime is compared against the on-disk output path mtime +/// to verify the stored data is up-to-date.) +/// If two records reference the same output the latter one in the file +/// wins, allowing updates to just be appended to the file. A separate +/// repacking step can run occasionally to remove dead records. +struct DepsLog { + DepsLog() : dead_record_count_(0), file_(NULL) {} + ~DepsLog(); + + // Writing (build-time) interface. + bool OpenForWrite(const string& path, string* err); + bool RecordDeps(Node* node, TimeStamp mtime, const vector<Node*>& nodes); + bool RecordDeps(Node* node, TimeStamp mtime, int node_count, Node** nodes); + void Close(); + + // Reading (startup-time) interface. + struct Deps { + Deps() : mtime(-1), node_count(0), nodes(NULL) {} + ~Deps() { delete [] nodes; } + int mtime; + int node_count; + Node** nodes; + }; + bool Load(const string& path, State* state, string* err); + Deps* GetDeps(Node* node); + + /// Rewrite the known log entries, throwing away old data. + bool Recompact(const string& path, string* err); + + /// Used for tests. + const vector<Node*>& nodes() const { return nodes_; } + + private: + // Write a node name record, assigning it an id. + bool RecordId(Node* node); + + /// Number of deps record read while loading the file that ended up + /// being unused (due to a latter entry superceding it). + int dead_record_count_; + + FILE* file_; + /// Maps id -> Node. + vector<Node*> nodes_; + /// Maps id -> deps of that id. + vector<Deps*> deps_; + + friend struct DepsLogTest; +}; + +#endif // NINJA_DEPS_LOG_H_ diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc new file mode 100644 index 0000000..2d91c0e --- /dev/null +++ b/src/deps_log_test.cc @@ -0,0 +1,195 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "deps_log.h" + +#include "graph.h" +#include "util.h" +#include "test.h" + +namespace { + +const char kTestFilename[] = "DepsLogTest-tempfile"; + +struct DepsLogTest : public testing::Test { + virtual void SetUp() { + // In case a crashing test left a stale file behind. + unlink(kTestFilename); + } + virtual void TearDown() { + //unlink(kTestFilename); + } +}; + +TEST_F(DepsLogTest, WriteRead) { + State state1; + DepsLog log1; + string err; + EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + { + vector<Node*> deps; + deps.push_back(state1.GetNode("foo.h")); + deps.push_back(state1.GetNode("bar.h")); + log1.RecordDeps(state1.GetNode("out.o"), 1, deps); + + deps.clear(); + deps.push_back(state1.GetNode("foo.h")); + deps.push_back(state1.GetNode("bar2.h")); + log1.RecordDeps(state1.GetNode("out2.o"), 2, deps); + } + + log1.Close(); + + State state2; + DepsLog log2; + EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err)); + ASSERT_EQ("", err); + + ASSERT_EQ(log1.nodes().size(), log2.nodes().size()); + for (int i = 0; i < (int)log1.nodes().size(); ++i) { + Node* node1 = log1.nodes()[i]; + Node* node2 = log2.nodes()[i]; + ASSERT_EQ(i, node1->id()); + ASSERT_EQ(node1->id(), node2->id()); + } + + // log1 has no deps entries, as it was only used for writing. + // Manually check the entries in log2. + DepsLog::Deps* deps = log2.GetDeps(state2.GetNode("out.o")); + ASSERT_TRUE(deps); + ASSERT_EQ(1, deps->mtime); + ASSERT_EQ(2, deps->node_count); + ASSERT_EQ("foo.h", deps->nodes[0]->path()); + ASSERT_EQ("bar.h", deps->nodes[1]->path()); +} + +// Verify that adding the same deps twice doesn't grow the file. +TEST_F(DepsLogTest, DoubleEntry) { + // Write some deps to the file and grab its size. + int file_size; + { + State state; + DepsLog log; + string err; + EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + vector<Node*> deps; + deps.push_back(state.GetNode("foo.h")); + deps.push_back(state.GetNode("bar.h")); + log.RecordDeps(state.GetNode("out.o"), 1, deps); + log.Close(); + + struct stat st; + ASSERT_EQ(0, stat(kTestFilename, &st)); + file_size = (int)st.st_size; + ASSERT_GT(file_size, 0); + } + + // Now reload the file, and readd the same deps. + { + State state; + DepsLog log; + string err; + EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); + + EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + vector<Node*> deps; + deps.push_back(state.GetNode("foo.h")); + deps.push_back(state.GetNode("bar.h")); + log.RecordDeps(state.GetNode("out.o"), 1, deps); + log.Close(); + + struct stat st; + ASSERT_EQ(0, stat(kTestFilename, &st)); + int file_size_2 = (int)st.st_size; + ASSERT_EQ(file_size, file_size_2); + } +} + +// Verify that adding the new deps works and can be compacted away. +TEST_F(DepsLogTest, Recompact) { + // Write some deps to the file and grab its size. + int file_size; + { + State state; + DepsLog log; + string err; + ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + vector<Node*> deps; + deps.push_back(state.GetNode("foo.h")); + deps.push_back(state.GetNode("bar.h")); + log.RecordDeps(state.GetNode("out.o"), 1, deps); + log.Close(); + + struct stat st; + ASSERT_EQ(0, stat(kTestFilename, &st)); + file_size = (int)st.st_size; + ASSERT_GT(file_size, 0); + } + + // Now reload the file, and add slighly different deps. + int file_size_2; + { + State state; + DepsLog log; + string err; + ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); + + ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + vector<Node*> deps; + deps.push_back(state.GetNode("foo.h")); + log.RecordDeps(state.GetNode("out.o"), 1, deps); + log.Close(); + + struct stat st; + ASSERT_EQ(0, stat(kTestFilename, &st)); + file_size_2 = (int)st.st_size; + // The file should grow to record the new deps. + ASSERT_GT(file_size_2, file_size); + } + + // Now reload the file, verify the new deps have replaced the old, then + // recompact. + { + State state; + DepsLog log; + string err; + ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); + + DepsLog::Deps* deps = log.GetDeps(state.GetNode("out.o")); + ASSERT_TRUE(deps); + ASSERT_EQ(1, deps->mtime); + ASSERT_EQ(1, deps->node_count); + ASSERT_EQ("foo.h", deps->nodes[0]->path()); + + ASSERT_TRUE(log.Recompact(kTestFilename, &err)); + + struct stat st; + ASSERT_EQ(0, stat(kTestFilename, &st)); + int file_size_3 = (int)st.st_size; + // The file should have shrunk a bit for the smaller deps. + ASSERT_LT(file_size_3, file_size_2); + } +} + +} // anonymous namespace diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index c2315c7..9b43d0f 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -104,7 +104,7 @@ TEST_F(DiskInterfaceTest, RemoveFile) { struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { - StatTest() : scan_(&state_, NULL, this) {} + StatTest() : scan_(&state_, NULL, NULL, this) {} // DiskInterface implementation. virtual TimeStamp Stat(const string& path); diff --git a/src/graph.cc b/src/graph.cc index 0f99698..2614882 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -19,6 +19,7 @@ #include "build_log.h" #include "depfile_parser.h" +#include "deps_log.h" #include "disk_interface.h" #include "explain.h" #include "manifest_parser.h" @@ -48,6 +49,7 @@ bool Rule::IsReservedBinding(const string& var) { return var == "command" || var == "depfile" || var == "description" || + var == "deps" || var == "generator" || var == "pool" || var == "restat" || @@ -59,15 +61,12 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { bool dirty = false; edge->outputs_ready_ = true; - string depfile = edge->GetBinding("depfile"); - if (!depfile.empty()) { - if (!LoadDepFile(edge, depfile, err)) { - if (!err->empty()) - return false; - EXPLAIN("Edge targets are dirty because depfile '%s' is missing", - depfile.c_str()); - dirty = true; - } + TimeStamp deps_mtime = 0; + if (!dep_loader_.LoadDeps(edge, &deps_mtime, err)) { + if (!err->empty()) + return false; + // Failed to load dependency info: rebuild to regenerate it. + dirty = true; } // Visit all inputs; we're dirty if any of the inputs are dirty. @@ -114,7 +113,8 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { for (vector<Node*>::iterator i = edge->outputs_.begin(); i != edge->outputs_.end(); ++i) { (*i)->StatIfNecessary(disk_interface_); - if (RecomputeOutputDirty(edge, most_recent_input, command, *i)) { + if (RecomputeOutputDirty(edge, most_recent_input, deps_mtime, + command, *i)) { dirty = true; break; } @@ -143,6 +143,7 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { bool DependencyScan::RecomputeOutputDirty(Edge* edge, Node* most_recent_input, + TimeStamp deps_mtime, const string& command, Node* output) { if (edge->is_phony()) { @@ -184,6 +185,12 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, } } + // Dirty if the output is newer than the deps. + if (deps_mtime && output->mtime() > deps_mtime) { + EXPLAIN("stored deps info out of date for for %s", output->path().c_str()); + return true; + } + // May also be dirty due to the command changing since the last build. // But if this is a generator rule, the command changing does not make us // dirty. @@ -282,7 +289,77 @@ bool Edge::GetBindingBool(const string& key) { return !GetBinding(key).empty(); } -bool DependencyScan::LoadDepFile(Edge* edge, const string& path, string* err) { +void Edge::Dump(const char* prefix) const { + printf("%s[ ", prefix); + for (vector<Node*>::const_iterator i = inputs_.begin(); + i != inputs_.end() && *i != NULL; ++i) { + printf("%s ", (*i)->path().c_str()); + } + printf("--%s-> ", rule_->name().c_str()); + for (vector<Node*>::const_iterator i = outputs_.begin(); + i != outputs_.end() && *i != NULL; ++i) { + printf("%s ", (*i)->path().c_str()); + } + if (pool_) { + if (!pool_->name().empty()) { + printf("(in pool '%s')", pool_->name().c_str()); + } + } else { + printf("(null pool?)"); + } + printf("] 0x%p\n", this); +} + +bool Edge::is_phony() const { + return rule_ == &State::kPhonyRule; +} + +void Node::Dump(const char* prefix) const { + printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", + prefix, path().c_str(), this, + mtime(), mtime() ? "" : " (:missing)", + dirty() ? " dirty" : " clean"); + if (in_edge()) { + in_edge()->Dump("in-edge: "); + } else { + printf("no in-edge\n"); + } + printf(" out edges:\n"); + for (vector<Edge*>::const_iterator e = out_edges().begin(); + e != out_edges().end() && *e != NULL; ++e) { + (*e)->Dump(" +- "); + } +} + +bool ImplicitDepLoader::LoadDeps(Edge* edge, TimeStamp* mtime, string* err) { + string deps_type = edge->GetBinding("deps"); + if (!deps_type.empty()) { + if (!LoadDepsFromLog(edge, mtime, err)) { + if (!err->empty()) + return false; + EXPLAIN("deps for %s are missing", edge->outputs_[0]->path().c_str()); + return false; + } + return true; + } + + string depfile = edge->GetBinding("depfile"); + if (!depfile.empty()) { + if (!LoadDepFile(edge, depfile, err)) { + if (!err->empty()) + return false; + EXPLAIN("depfile '%s' is missing", depfile.c_str()); + return false; + } + return true; + } + + // No deps to load. + return true; +} + +bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, + string* err) { METRIC_RECORD("depfile load"); string content = disk_interface_->ReadFile(path, err); if (!err->empty()) { @@ -310,11 +387,8 @@ bool DependencyScan::LoadDepFile(Edge* edge, const string& path, string* err) { } // Preallocate space in edge->inputs_ to be filled in below. - edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_, - depfile.ins_.size(), 0); - edge->implicit_deps_ += depfile.ins_.size(); vector<Node*>::iterator implicit_dep = - edge->inputs_.end() - edge->order_only_deps_ - depfile.ins_.size(); + PreallocateSpace(edge, depfile.ins_.size()); // Add all its in-edges. for (vector<StringPiece>::iterator i = depfile.ins_.begin(); @@ -325,66 +399,50 @@ bool DependencyScan::LoadDepFile(Edge* edge, const string& path, string* err) { Node* node = state_->GetNode(*i); *implicit_dep = node; node->AddOutEdge(edge); - - // If we don't have a edge that generates this input already, - // create one; this makes us not abort if the input is missing, - // but instead will rebuild in that circumstance. - if (!node->in_edge()) { - Edge* phony_edge = state_->AddEdge(&State::kPhonyRule); - node->set_in_edge(phony_edge); - phony_edge->outputs_.push_back(node); - - // RecomputeDirty might not be called for phony_edge if a previous call - // to RecomputeDirty had caused the file to be stat'ed. Because previous - // invocations of RecomputeDirty would have seen this node without an - // input edge (and therefore ready), we have to set outputs_ready_ to true - // to avoid a potential stuck build. If we do call RecomputeDirty for - // this node, it will simply set outputs_ready_ to the correct value. - phony_edge->outputs_ready_ = true; - } + CreatePhonyInEdge(node); } return true; } -void Edge::Dump(const char* prefix) const { - printf("%s[ ", prefix); - for (vector<Node*>::const_iterator i = inputs_.begin(); - i != inputs_.end() && *i != NULL; ++i) { - printf("%s ", (*i)->path().c_str()); - } - printf("--%s-> ", rule_->name().c_str()); - for (vector<Node*>::const_iterator i = outputs_.begin(); - i != outputs_.end() && *i != NULL; ++i) { - printf("%s ", (*i)->path().c_str()); - } - if (pool_) { - if (!pool_->name().empty()) { - printf("(in pool '%s')", pool_->name().c_str()); - } - } else { - printf("(null pool?)"); +bool ImplicitDepLoader::LoadDepsFromLog(Edge* edge, TimeStamp* deps_mtime, + string* err) { + DepsLog::Deps* deps = deps_log_->GetDeps(edge->outputs_[0]); + if (!deps) + return false; + + *deps_mtime = deps->mtime; + + vector<Node*>::iterator implicit_dep = + PreallocateSpace(edge, deps->node_count); + for (int i = 0; i < deps->node_count; ++i, ++implicit_dep) { + *implicit_dep = deps->nodes[i]; + CreatePhonyInEdge(*implicit_dep); } - printf("] 0x%p\n", this); + return true; } -bool Edge::is_phony() const { - return rule_ == &State::kPhonyRule; +vector<Node*>::iterator ImplicitDepLoader::PreallocateSpace(Edge* edge, + int count) { + edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_, + (size_t)count, 0); + edge->implicit_deps_ += count; + return edge->inputs_.end() - edge->order_only_deps_ - count; } -void Node::Dump(const char* prefix) const { - printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", - prefix, path().c_str(), this, - mtime(), mtime() ? "" : " (:missing)", - dirty() ? " dirty" : " clean"); - if (in_edge()) { - in_edge()->Dump("in-edge: "); - } else { - printf("no in-edge\n"); - } - printf(" out edges:\n"); - for (vector<Edge*>::const_iterator e = out_edges().begin(); - e != out_edges().end() && *e != NULL; ++e) { - (*e)->Dump(" +- "); - } +void ImplicitDepLoader::CreatePhonyInEdge(Node* node) { + if (node->in_edge()) + return; + + Edge* phony_edge = state_->AddEdge(&State::kPhonyRule); + node->set_in_edge(phony_edge); + phony_edge->outputs_.push_back(node); + + // RecomputeDirty might not be called for phony_edge if a previous call + // to RecomputeDirty had caused the file to be stat'ed. Because previous + // invocations of RecomputeDirty would have seen this node without an + // input edge (and therefore ready), we have to set outputs_ready_ to true + // to avoid a potential stuck build. If we do call RecomputeDirty for + // this node, it will simply set outputs_ready_ to the correct value. + phony_edge->outputs_ready_ = true; } diff --git a/src/graph.h b/src/graph.h index 8b93e29..428ba01 100644 --- a/src/graph.h +++ b/src/graph.h @@ -22,8 +22,13 @@ using namespace std; #include "eval_env.h" #include "timestamp.h" +struct BuildLog; struct DiskInterface; +struct DepsLog; struct Edge; +struct Node; +struct Pool; +struct State; /// Information about a node in the dependency graph: the file, whether /// it's dirty, mtime, etc. @@ -32,7 +37,8 @@ struct Node { : path_(path), mtime_(-1), dirty_(false), - in_edge_(NULL) {} + in_edge_(NULL), + id_(-1) {} /// Return true if the file exists (mtime_ got a value). bool Stat(DiskInterface* disk_interface); @@ -74,6 +80,9 @@ struct Node { Edge* in_edge() const { return in_edge_; } void set_in_edge(Edge* edge) { in_edge_ = edge; } + int id() const { return id_; } + void set_id(int id) { id_ = id; } + const vector<Edge*>& out_edges() const { return out_edges_; } void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); } @@ -98,6 +107,9 @@ private: /// All Edges that use this Node as an input. vector<Edge*> out_edges_; + + /// A dense integer id for the node, assigned and used by DepsLog. + int id_; }; /// An invokable build command and associated metadata (description, etc.). @@ -121,11 +133,6 @@ struct Rule { map<string, EvalString> bindings_; }; -struct BuildLog; -struct Node; -struct State; -struct Pool; - /// An edge in the dependency graph; links between Nodes using Rules. struct Edge { Edge() : rule_(NULL), env_(NULL), outputs_ready_(false), implicit_deps_(0), @@ -178,13 +185,54 @@ struct Edge { }; +/// ImplicitDepLoader loads implicit dependencies, as referenced via the +/// "depfile" attribute in build files. +struct ImplicitDepLoader { + ImplicitDepLoader(State* state, DepsLog* deps_log, + DiskInterface* disk_interface) + : state_(state), disk_interface_(disk_interface), deps_log_(deps_log) {} + + /// Load implicit dependencies for \a edge. May fill in \a mtime with + /// the timestamp of the loaded information. + /// @return false on error (without filling \a err if info is just missing). + bool LoadDeps(Edge* edge, TimeStamp* mtime, string* err); + + DepsLog* deps_log() const { + return deps_log_; + } + + private: + /// Load implicit dependencies for \a edge from a depfile attribute. + /// @return false on error (without filling \a err if info is just missing). + bool LoadDepFile(Edge* edge, const string& path, string* err); + + /// Load implicit dependencies for \a edge from the DepsLog. + /// @return false on error (without filling \a err if info is just missing). + bool LoadDepsFromLog(Edge* edge, TimeStamp* mtime, string* err); + + /// Preallocate \a count spaces in the input array on \a edge, returning + /// an iterator pointing at the first new space. + vector<Node*>::iterator PreallocateSpace(Edge* edge, int count); + + /// If we don't have a edge that generates this input already, + /// create one; this makes us not abort if the input is missing, + /// but instead will rebuild in that circumstance. + void CreatePhonyInEdge(Node* node); + + State* state_; + DiskInterface* disk_interface_; + DepsLog* deps_log_; +}; + + /// DependencyScan manages the process of scanning the files in a graph /// and updating the dirty/outputs_ready state of all the nodes and edges. struct DependencyScan { - DependencyScan(State* state, BuildLog* build_log, + DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log, DiskInterface* disk_interface) - : state_(state), build_log_(build_log), - disk_interface_(disk_interface) {} + : build_log_(build_log), + disk_interface_(disk_interface), + dep_loader_(state, deps_log, disk_interface) {} /// Examine inputs, outputs, and command lines to judge whether an edge /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| @@ -195,10 +243,9 @@ struct DependencyScan { /// Recompute whether a given single output should be marked dirty. /// Returns true if so. bool RecomputeOutputDirty(Edge* edge, Node* most_recent_input, + TimeStamp deps_mtime, const string& command, Node* output); - bool LoadDepFile(Edge* edge, const string& path, string* err); - BuildLog* build_log() const { return build_log_; } @@ -206,10 +253,14 @@ struct DependencyScan { build_log_ = log; } + DepsLog* deps_log() const { + return dep_loader_.deps_log(); + } + private: - State* state_; BuildLog* build_log_; DiskInterface* disk_interface_; + ImplicitDepLoader dep_loader_; }; #endif // NINJA_GRAPH_H_ diff --git a/src/graph_test.cc b/src/graph_test.cc index c105684..63d5757 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -17,7 +17,7 @@ #include "test.h" struct GraphTest : public StateTestWithBuiltinRules { - GraphTest() : scan_(&state_, NULL, &fs_) {} + GraphTest() : scan_(&state_, NULL, NULL, &fs_) {} VirtualFileSystem fs_; DependencyScan scan_; diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 14fca73..a581114 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -329,6 +329,14 @@ bool ManifestParser::ParseEdge(string* err) { edge->implicit_deps_ = implicit; edge->order_only_deps_ = order_only; + // Multiple outputs aren't (yet?) supported with depslog. + string deps_type = edge->GetBinding("deps"); + if (!deps_type.empty() && edge->outputs_.size() > 1) { + return lexer_.Error("multiple outputs aren't (yet?) supported by depslog; " + "bring this up on the mailing list if it affects you", + err); + } + return true; } diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 4ac093f..76d235d 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -71,6 +71,7 @@ TEST_F(ParserTest, RuleAttributes) { "rule cat\n" " command = a\n" " depfile = a\n" +" deps = a\n" " description = a\n" " generator = a\n" " restat = a\n" @@ -599,6 +600,17 @@ TEST_F(ParserTest, MultipleOutputs) { EXPECT_EQ("", err); } +TEST_F(ParserTest, MultipleOutputsWithDeps) { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n" + "build a.o b.o: cc c.cc\n", + &err)); + EXPECT_EQ("input:5: multiple outputs aren't (yet?) supported by depslog; " + "bring this up on the mailing list if it affects you\n", err); +} + TEST_F(ParserTest, SubNinja) { files_["test.ninja"] = "var = inner\n" diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index fd9b671..be2a5e0 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -39,15 +39,15 @@ string Replace(const string& input, const string& find, const string& replace) { return result; } +} // anonymous namespace + string EscapeForDepfile(const string& path) { // Depfiles don't escape single \. return Replace(path, " ", "\\ "); } -} // anonymous namespace - // static -string CLWrapper::FilterShowIncludes(const string& line) { +string CLParser::FilterShowIncludes(const string& line) { static const char kMagicPrefix[] = "Note: including file: "; const char* in = line.c_str(); const char* end = in + line.size(); @@ -63,14 +63,14 @@ string CLWrapper::FilterShowIncludes(const string& line) { } // static -bool CLWrapper::IsSystemInclude(const string& path) { +bool CLParser::IsSystemInclude(const string& path) { // TODO: this is a heuristic, perhaps there's a better way? return (path.find("program files") != string::npos || path.find("microsoft visual studio") != string::npos); } // static -bool CLWrapper::FilterInputFilename(const string& line) { +bool CLParser::FilterInputFilename(const string& line) { // TODO: other extensions, like .asm? return EndsWith(line, ".c") || EndsWith(line, ".cc") || @@ -78,7 +78,42 @@ bool CLWrapper::FilterInputFilename(const string& line) { EndsWith(line, ".cpp"); } -int CLWrapper::Run(const string& command, string* extra_output) { +string CLParser::Parse(const string& output) { + string filtered_output; + + // Loop over all lines in the output to process them. + size_t start = 0; + while (start < output.size()) { + size_t end = output.find_first_of("\r\n", start); + if (end == string::npos) + end = output.size(); + string line = output.substr(start, end - start); + + string include = FilterShowIncludes(line); + if (!include.empty()) { + include = IncludesNormalize::Normalize(include, NULL); + if (!IsSystemInclude(include)) + includes_.insert(include); + } else if (FilterInputFilename(line)) { + // Drop it. + // TODO: if we support compiling multiple output files in a single + // cl.exe invocation, we should stash the filename. + } else { + filtered_output.append(line); + filtered_output.append("\n"); + } + + if (end < output.size() && output[end] == '\r') + ++end; + if (end < output.size() && output[end] == '\n') + ++end; + start = end; + } + + return filtered_output; +} + +int CLWrapper::Run(const string& command, string* output) { SECURITY_ATTRIBUTES security_attributes = {}; security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); security_attributes.bInheritHandle = TRUE; @@ -118,8 +153,7 @@ int CLWrapper::Run(const string& command, string* extra_output) { Win32Fatal("CloseHandle"); } - // Read output of the subprocess and parse it. - string output; + // Read all output of the subprocess. DWORD read_len = 1; while (read_len) { char buf[64 << 10]; @@ -128,44 +162,12 @@ int CLWrapper::Run(const string& command, string* extra_output) { GetLastError() != ERROR_BROKEN_PIPE) { Win32Fatal("ReadFile"); } - output.append(buf, read_len); - - // Loop over all lines in the output and process them. - for (;;) { - size_t ofs = output.find_first_of("\r\n"); - if (ofs == string::npos) - break; - string line = output.substr(0, ofs); - - string include = FilterShowIncludes(line); - if (!include.empty()) { - include = IncludesNormalize::Normalize(include, NULL); - if (!IsSystemInclude(include)) - includes_.insert(include); - } else if (FilterInputFilename(line)) { - // Drop it. - // TODO: if we support compiling multiple output files in a single - // cl.exe invocation, we should stash the filename. - } else { - if (extra_output) { - extra_output->append(line); - extra_output->append("\n"); - } else { - printf("%s\n", line.c_str()); - } - } - - if (ofs < output.size() && output[ofs] == '\r') - ++ofs; - if (ofs < output.size() && output[ofs] == '\n') - ++ofs; - output = output.substr(ofs); - } + output->append(buf, read_len); } + // Wait for it to exit and grab its exit code. if (WaitForSingleObject(process_info.hProcess, INFINITE) == WAIT_FAILED) Win32Fatal("WaitForSingleObject"); - DWORD exit_code = 0; if (!GetExitCodeProcess(process_info.hProcess, &exit_code)) Win32Fatal("GetExitCodeProcess"); @@ -178,11 +180,3 @@ int CLWrapper::Run(const string& command, string* extra_output) { return exit_code; } - -vector<string> CLWrapper::GetEscapedResult() { - vector<string> result; - for (set<string>::iterator i = includes_.begin(); i != includes_.end(); ++i) { - result.push_back(EscapeForDepfile(*i)); - } - return result; -} diff --git a/src/msvc_helper.h b/src/msvc_helper.h index 102201b..32ab606 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -17,23 +17,13 @@ #include <vector> using namespace std; +string EscapeForDepfile(const string& path); + /// Visual Studio's cl.exe requires some massaging to work with Ninja; /// for example, it emits include information on stderr in a funny -/// format when building with /showIncludes. This class wraps a CL -/// process and parses that output to extract the file list. -struct CLWrapper { - CLWrapper() : env_block_(NULL) {} - - /// Set the environment block (as suitable for CreateProcess) to be used - /// by Run(). - void SetEnvBlock(void* env_block) { env_block_ = env_block; } - - /// Start a process and parse its output. Returns its exit code. - /// Any non-parsed output is buffered into \a extra_output if provided, - /// otherwise it is printed to stdout while the process runs. - /// Crashes (calls Fatal()) on error. - int Run(const string& command, string* extra_output=NULL); - +/// format when building with /showIncludes. This class parses this +/// output. +struct CLParser { /// Parse a line of cl.exe output and extract /showIncludes info. /// If a dependency is extracted, returns a nonempty string. /// Exposed for testing. @@ -50,10 +40,24 @@ struct CLWrapper { /// Exposed for testing. static bool FilterInputFilename(const string& line); - /// Fill a vector with the unique'd headers, escaped for output as a .d - /// file. - vector<string> GetEscapedResult(); + /// Parse the full output of cl, returning the output (if any) that + /// should printed. + string Parse(const string& output); - void* env_block_; set<string> includes_; }; + +/// Wraps a synchronous execution of a CL subprocess. +struct CLWrapper { + CLWrapper() : env_block_(NULL) {} + + /// Set the environment block (as suitable for CreateProcess) to be used + /// by Run(). + void SetEnvBlock(void* env_block) { env_block_ = env_block; } + + /// Start a process and gather its raw output. Returns its exit code. + /// Crashes (calls Fatal()) on error. + int Run(const string& command, string* output); + + void* env_block_; +}; diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index 0bbe98b..3192821 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -44,12 +44,13 @@ void PushPathIntoEnvironment(const string& env_block) { } } -void WriteDepFileOrDie(const char* object_path, CLWrapper* cl) { +void WriteDepFileOrDie(const char* object_path, const CLParser& parse) { string depfile_path = string(object_path) + ".d"; FILE* depfile = fopen(depfile_path.c_str(), "w"); if (!depfile) { unlink(object_path); - Fatal("opening %s: %s", depfile_path.c_str(), GetLastErrorString().c_str()); + Fatal("opening %s: %s", depfile_path.c_str(), + GetLastErrorString().c_str()); } if (fprintf(depfile, "%s: ", object_path) < 0) { unlink(object_path); @@ -57,9 +58,9 @@ void WriteDepFileOrDie(const char* object_path, CLWrapper* cl) { unlink(depfile_path.c_str()); Fatal("writing %s", depfile_path.c_str()); } - vector<string> headers = cl->GetEscapedResult(); - for (vector<string>::iterator i = headers.begin(); i != headers.end(); ++i) { - if (fprintf(depfile, "%s\n", i->c_str()) < 0) { + const set<string>& headers = parse.includes_; + for (set<string>::iterator i = headers.begin(); i != headers.end(); ++i) { + if (fprintf(depfile, "%s\n", EscapeForDepfile(*i).c_str()) < 0) { unlink(object_path); fclose(depfile); unlink(depfile_path.c_str()); @@ -95,11 +96,6 @@ int MSVCHelperMain(int argc, char** argv) { } } - if (!output_filename) { - Usage(); - Fatal("-o required"); - } - string env; if (envfile) { string err; @@ -118,9 +114,15 @@ int MSVCHelperMain(int argc, char** argv) { CLWrapper cl; if (!env.empty()) cl.SetEnvBlock((void*)env.data()); - int exit_code = cl.Run(command); + string output; + int exit_code = cl.Run(command, &output); - WriteDepFileOrDie(output_filename, &cl); + if (output_filename) { + CLParser parser; + output = parser.Parse(output); + WriteDepFileOrDie(output_filename, parser); + } + printf("%s\n", output.c_str()); return exit_code; } diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index 7730425..1e1cbde 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -19,100 +19,93 @@ #include "test.h" #include "util.h" -TEST(MSVCHelperTest, ShowIncludes) { - ASSERT_EQ("", CLWrapper::FilterShowIncludes("")); +TEST(CLParserTest, ShowIncludes) { + ASSERT_EQ("", CLParser::FilterShowIncludes("")); - ASSERT_EQ("", CLWrapper::FilterShowIncludes("Sample compiler output")); + ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output")); ASSERT_EQ("c:\\Some Files\\foobar.h", - CLWrapper::FilterShowIncludes("Note: including file: " - "c:\\Some Files\\foobar.h")); + CLParser::FilterShowIncludes("Note: including file: " + "c:\\Some Files\\foobar.h")); ASSERT_EQ("c:\\initspaces.h", - CLWrapper::FilterShowIncludes("Note: including file: " - "c:\\initspaces.h")); + CLParser::FilterShowIncludes("Note: including file: " + "c:\\initspaces.h")); } -TEST(MSVCHelperTest, FilterInputFilename) { - ASSERT_TRUE(CLWrapper::FilterInputFilename("foobar.cc")); - ASSERT_TRUE(CLWrapper::FilterInputFilename("foo bar.cc")); - ASSERT_TRUE(CLWrapper::FilterInputFilename("baz.c")); +TEST(CLParserTest, FilterInputFilename) { + ASSERT_TRUE(CLParser::FilterInputFilename("foobar.cc")); + ASSERT_TRUE(CLParser::FilterInputFilename("foo bar.cc")); + ASSERT_TRUE(CLParser::FilterInputFilename("baz.c")); - ASSERT_FALSE(CLWrapper::FilterInputFilename( + ASSERT_FALSE(CLParser::FilterInputFilename( "src\\cl_helper.cc(166) : fatal error C1075: end " "of file found ...")); } -TEST(MSVCHelperTest, Run) { - CLWrapper cl; - string output; - cl.Run("cmd /c \"echo foo&& echo Note: including file: foo.h&&echo bar\"", - &output); +TEST(CLParserTest, ParseSimple) { + CLParser parser; + string output = parser.Parse( + "foo\r\n" + "Note: including file: foo.h\r\n" + "bar\r\n"); + ASSERT_EQ("foo\nbar\n", output); - ASSERT_EQ(1u, cl.includes_.size()); - ASSERT_EQ("foo.h", *cl.includes_.begin()); + ASSERT_EQ(1u, parser.includes_.size()); + ASSERT_EQ("foo.h", *parser.includes_.begin()); } -TEST(MSVCHelperTest, RunFilenameFilter) { - CLWrapper cl; - string output; - cl.Run("cmd /c \"echo foo.cc&& echo cl: warning\"", - &output); +TEST(CLParserTest, ParseFilenameFilter) { + CLParser parser; + string output = parser.Parse( + "foo.cc\r\n" + "cl: warning\r\n"); ASSERT_EQ("cl: warning\n", output); } -TEST(MSVCHelperTest, RunSystemInclude) { - CLWrapper cl; - string output; - cl.Run("cmd /c \"echo Note: including file: c:\\Program Files\\foo.h&&" - "echo Note: including file: d:\\Microsoft Visual Studio\\bar.h&&" - "echo Note: including file: path.h\"", - &output); +TEST(CLParserTest, ParseSystemInclude) { + CLParser parser; + string output = parser.Parse( + "Note: including file: c:\\Program Files\\foo.h\r\n" + "Note: including file: d:\\Microsoft Visual Studio\\bar.h\r\n" + "Note: including file: path.h\r\n"); // We should have dropped the first two includes because they look like // system headers. ASSERT_EQ("", output); - ASSERT_EQ(1u, cl.includes_.size()); - ASSERT_EQ("path.h", *cl.includes_.begin()); -} - -TEST(MSVCHelperTest, EnvBlock) { - char env_block[] = "foo=bar\0"; - CLWrapper cl; - cl.SetEnvBlock(env_block); - string output; - cl.Run("cmd /c \"echo foo is %foo%", &output); - ASSERT_EQ("foo is bar\n", output); + ASSERT_EQ(1u, parser.includes_.size()); + ASSERT_EQ("path.h", *parser.includes_.begin()); } -TEST(MSVCHelperTest, DuplicatedHeader) { - CLWrapper cl; - string output; - cl.Run("cmd /c \"echo Note: including file: foo.h&&" - "echo Note: including file: bar.h&&" - "echo Note: including file: foo.h\"", - &output); +TEST(CLParserTest, DuplicatedHeader) { + CLParser parser; + string output = parser.Parse( + "Note: including file: foo.h\r\n" + "Note: including file: bar.h\r\n" + "Note: including file: foo.h\r\n"); // We should have dropped one copy of foo.h. ASSERT_EQ("", output); - ASSERT_EQ(2u, cl.includes_.size()); + ASSERT_EQ(2u, parser.includes_.size()); } -TEST(MSVCHelperTest, DuplicatedHeaderPathConverted) { - CLWrapper cl; - string output; - cl.Run("cmd /c \"echo Note: including file: sub/foo.h&&" - "echo Note: including file: bar.h&&" - "echo Note: including file: sub\\foo.h\"", - &output); +TEST(CLParserTest, DuplicatedHeaderPathConverted) { + CLParser parser; + string output = parser.Parse( + "Note: including file: sub/foo.h\r\n" + "Note: including file: bar.h\r\n" + "Note: including file: sub\\foo.h\r\n"); // We should have dropped one copy of foo.h. ASSERT_EQ("", output); - ASSERT_EQ(2u, cl.includes_.size()); + ASSERT_EQ(2u, parser.includes_.size()); } -TEST(MSVCHelperTest, SpacesInFilename) { +TEST(CLParserTest, SpacesInFilename) { + ASSERT_EQ("sub\\some\\ sdk\\foo.h", + EscapeForDepfile("sub\\some sdk\\foo.h")); +} + +TEST(MSVCHelperTest, EnvBlock) { + char env_block[] = "foo=bar\0"; CLWrapper cl; + cl.SetEnvBlock(env_block); string output; - cl.Run("cmd /c \"echo Note: including file: sub\\some sdk\\foo.h", - &output); - ASSERT_EQ("", output); - vector<string> headers = cl.GetEscapedResult(); - ASSERT_EQ(1u, headers.size()); - ASSERT_EQ("sub\\some\\ sdk\\foo.h", headers[0]); + cl.Run("cmd /c \"echo foo is %foo%", &output); + ASSERT_EQ("foo is bar\r\n", output); } diff --git a/src/ninja.cc b/src/ninja.cc index 69646e1..8ba1aa6 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -32,6 +32,7 @@ #include "browse.h" #include "build.h" #include "build_log.h" +#include "deps_log.h" #include "clean.h" #include "disk_interface.h" #include "edit_distance.h" @@ -641,20 +642,13 @@ bool DebugEnable(const string& name, Globals* globals) { } } -bool OpenLog(BuildLog* build_log, Globals* globals, - DiskInterface* disk_interface) { - const string build_dir = - globals->state->bindings_.LookupVariable("builddir"); - const char* kLogPath = ".ninja_log"; - string log_path = kLogPath; - if (!build_dir.empty()) { - log_path = build_dir + "/" + kLogPath; - if (!disk_interface->MakeDirs(log_path) && errno != EEXIST) { - Error("creating build directory %s: %s", - build_dir.c_str(), strerror(errno)); - return false; - } - } +/// Open the build log. +/// @return false on error. +bool OpenBuildLog(BuildLog* build_log, const string& build_dir, + Globals* globals, DiskInterface* disk_interface) { + string log_path = ".ninja_log"; + if (!build_dir.empty()) + log_path = build_dir + "/" + log_path; string err; if (!build_log->Load(log_path, &err)) { @@ -677,6 +671,36 @@ bool OpenLog(BuildLog* build_log, Globals* globals, return true; } +/// Open the deps log: load it, then open for writing. +/// @return false on error. +bool OpenDepsLog(DepsLog* deps_log, const string& build_dir, + Globals* globals, DiskInterface* disk_interface) { + string path = ".ninja_deps"; + if (!build_dir.empty()) + path = build_dir + "/" + path; + + string err; + if (!deps_log->Load(path, globals->state, &err)) { + Error("loading deps log %s: %s", path.c_str(), err.c_str()); + return false; + } + if (!err.empty()) { + // Hack: Load() can return a warning via err by returning true. + Warning("%s", err.c_str()); + err.clear(); + } + + if (!globals->config->dry_run) { + if (!deps_log->OpenForWrite(path, &err)) { + Error("opening deps log: %s", err.c_str()); + return false; + } + } + + return true; +} + + /// Dump the output requested by '-d stats'. void DumpMetrics(Globals* globals) { g_metrics->Report(); @@ -872,14 +896,30 @@ reload: if (tool && tool->when == Tool::RUN_AFTER_LOAD) return tool->func(&globals, argc, argv); - BuildLog build_log; RealDiskInterface disk_interface; - if (!OpenLog(&build_log, &globals, &disk_interface)) + + // Create the build dir if it doesn't exist. + const string build_dir = globals.state->bindings_.LookupVariable("builddir"); + if (!build_dir.empty() && !config.dry_run) { + if (!disk_interface.MakeDirs(build_dir + "/.") && + errno != EEXIST) { + Error("creating build directory %s: %s", + build_dir.c_str(), strerror(errno)); + return 1; + } + } + + BuildLog build_log; + if (!OpenBuildLog(&build_log, build_dir, &globals, &disk_interface)) + return 1; + + DepsLog deps_log; + if (!OpenDepsLog(&deps_log, build_dir, &globals, &disk_interface)) return 1; if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild // target that is never up to date. - Builder manifest_builder(globals.state, config, &build_log, + Builder manifest_builder(globals.state, config, &build_log, &deps_log, &disk_interface); if (RebuildManifest(&manifest_builder, input_file, &err)) { rebuilt_manifest = true; @@ -891,7 +931,8 @@ reload: } } - Builder builder(globals.state, config, &build_log, &disk_interface); + Builder builder(globals.state, config, &build_log, &deps_log, + &disk_interface); int result = RunBuild(&builder, argc, argv); if (g_metrics) DumpMetrics(&globals); diff --git a/src/state.cc b/src/state.cc index 9f46fee..d2d5ebe 100644 --- a/src/state.cc +++ b/src/state.cc @@ -202,10 +202,11 @@ void State::Reset() { void State::Dump() { for (Paths::iterator i = paths_.begin(); i != paths_.end(); ++i) { Node* node = i->second; - printf("%s %s\n", + printf("%s %s [id:%d]\n", node->path().c_str(), node->status_known() ? (node->dirty() ? "dirty" : "clean") - : "unknown"); + : "unknown", + node->id()); } if (!pools_.empty()) { printf("resource_pools:\n"); diff --git a/src/test.cc b/src/test.cc index c48ca82..45a9226 100644 --- a/src/test.cc +++ b/src/test.cc @@ -74,7 +74,11 @@ string GetSystemTempDir() { } // anonymous namespace StateTestWithBuiltinRules::StateTestWithBuiltinRules() { - AssertParse(&state_, + AddCatRule(&state_); +} + +void StateTestWithBuiltinRules::AddCatRule(State* state) { + AssertParse(state, "rule cat\n" " command = cat $in > $out\n"); } @@ -29,6 +29,12 @@ struct Node; /// builtin "cat" rule. struct StateTestWithBuiltinRules : public testing::Test { StateTestWithBuiltinRules(); + + /// Add a "cat" rule to \a state. Used by some tests; it's + /// otherwise done by the ctor to state_. + void AddCatRule(State* state); + + /// Short way to get a Node by its path from state_. Node* GetNode(const string& path); State state_; |