summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xbootstrap.py4
-rwxr-xr-xconfigure.py14
-rw-r--r--misc/ninja_syntax.py4
-rw-r--r--src/build.cc279
-rw-r--r--src/build.h23
-rw-r--r--src/build_test.cc245
-rw-r--r--src/deps_log.cc263
-rw-r--r--src/deps_log.h107
-rw-r--r--src/deps_log_test.cc195
-rw-r--r--src/disk_interface_test.cc2
-rw-r--r--src/graph.cc192
-rw-r--r--src/graph.h75
-rw-r--r--src/graph_test.cc2
-rw-r--r--src/manifest_parser.cc8
-rw-r--r--src/manifest_parser_test.cc12
-rw-r--r--src/msvc_helper-win32.cc94
-rw-r--r--src/msvc_helper.h42
-rw-r--r--src/msvc_helper_main-win32.cc26
-rw-r--r--src/msvc_helper_test.cc123
-rw-r--r--src/ninja.cc77
-rw-r--r--src/state.cc5
-rw-r--r--src/test.cc6
-rw-r--r--src/test.h6
23 files changed, 1423 insertions, 381 deletions
diff --git a/bootstrap.py b/bootstrap.py
index fcf1a20..4d9bc84 100755
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -143,9 +143,7 @@ Done!
Note: to work around Windows file locking, where you can't rebuild an
in-use binary, to run ninja after making any changes to build ninja itself
-you should run ninja.bootstrap instead. Your build is also configured to
-use ninja.bootstrap.exe as the MSVC helper; see the --with-ninja flag of
-the --help output of configure.py.""")
+you should run ninja.bootstrap instead.""")
else:
print('Building ninja using itself...')
run([sys.executable, 'configure.py'] + conf_args)
diff --git a/configure.py b/configure.py
index 6db181c..dd26906 100755
--- a/configure.py
+++ b/configure.py
@@ -47,9 +47,6 @@ parser.add_option('--with-gtest', metavar='PATH',
parser.add_option('--with-python', metavar='EXE',
help='use EXE as the Python interpreter',
default=os.path.basename(sys.executable))
-parser.add_option('--with-ninja', metavar='NAME',
- help="name for ninja binary for -t msvc (MSVC only)",
- default="ninja")
(options, args) = parser.parse_args()
if args:
print('ERROR: extra unparsed command-line arguments:', args)
@@ -190,14 +187,11 @@ n.variable('ldflags', ' '.join(shell_escape(flag) for flag in ldflags))
n.newline()
if platform == 'windows':
- compiler = '$cxx'
- if options.with_ninja:
- compiler = ('%s -t msvc -o $out -- $cxx /showIncludes' %
- options.with_ninja)
n.rule('cxx',
- command='%s $cflags -c $in /Fo$out' % compiler,
+ command='$cxx /showIncludes $cflags -c $in /Fo$out',
depfile='$out.d',
- description='CXX $out')
+ description='CXX $out',
+ deps='msvc')
else:
n.rule('cxx',
command='$cxx -MMD -MT $out -MF $out.d $cflags -c $in -o $out',
@@ -269,6 +263,7 @@ for name in ['build',
'build_log',
'clean',
'depfile_parser',
+ 'deps_log',
'disk_interface',
'edit_distance',
'eval_env',
@@ -346,6 +341,7 @@ for name in ['build_log_test',
'build_test',
'clean_test',
'depfile_parser_test',
+ 'deps_log_test',
'disk_interface_test',
'edit_distance_test',
'graph_test',
diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py
index ece7eb5..d69e3e4 100644
--- a/misc/ninja_syntax.py
+++ b/misc/ninja_syntax.py
@@ -38,7 +38,7 @@ class Writer(object):
def rule(self, name, command, description=None, depfile=None,
generator=False, pool=None, restat=False, rspfile=None,
- rspfile_content=None):
+ rspfile_content=None, deps=None):
self._line('rule %s' % name)
self.variable('command', command, indent=1)
if description:
@@ -55,6 +55,8 @@ class Writer(object):
self.variable('rspfile', rspfile, indent=1)
if rspfile_content:
self.variable('rspfile_content', rspfile_content, indent=1)
+ if deps:
+ self.variable('deps', deps, indent=1)
def build(self, outputs, rule, inputs=None, implicit=None, order_only=None,
variables=None):
diff --git a/src/build.cc b/src/build.cc
index 340a3d9..ab3d781 100644
--- a/src/build.cc
+++ b/src/build.cc
@@ -15,6 +15,7 @@
#include "build.h"
#include <assert.h>
+#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <functional>
@@ -24,8 +25,11 @@
#endif
#include "build_log.h"
+#include "depfile_parser.h"
+#include "deps_log.h"
#include "disk_interface.h"
#include "graph.h"
+#include "msvc_helper.h"
#include "state.h"
#include "subprocess.h"
#include "util.h"
@@ -39,7 +43,7 @@ struct DryRunCommandRunner : public CommandRunner {
// Overridden from CommandRunner:
virtual bool CanRunMore();
virtual bool StartCommand(Edge* edge);
- virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */);
+ virtual bool WaitForCommand(Result* result);
private:
queue<Edge*> finished_;
@@ -54,16 +58,14 @@ bool DryRunCommandRunner::StartCommand(Edge* edge) {
return true;
}
-Edge* DryRunCommandRunner::WaitForCommand(ExitStatus* status,
- string* /*output*/) {
- if (finished_.empty()) {
- *status = ExitFailure;
- return NULL;
- }
- *status = ExitSuccess;
- Edge* edge = finished_.front();
+bool DryRunCommandRunner::WaitForCommand(Result* result) {
+ if (finished_.empty())
+ return false;
+
+ result->status = ExitSuccess;
+ result->edge = finished_.front();
finished_.pop();
- return edge;
+ return true;
}
} // namespace
@@ -427,7 +429,7 @@ void Plan::CleanNode(DependencyScan* scan, Node* node) {
if (!(*ni)->dirty())
continue;
- if (scan->RecomputeOutputDirty(*ei, most_recent_input,
+ if (scan->RecomputeOutputDirty(*ei, most_recent_input, 0,
command, *ni)) {
(*ni)->MarkDirty();
all_outputs_clean = false;
@@ -462,7 +464,7 @@ struct RealCommandRunner : public CommandRunner {
virtual ~RealCommandRunner() {}
virtual bool CanRunMore();
virtual bool StartCommand(Edge* edge);
- virtual Edge* WaitForCommand(ExitStatus* status, string* output);
+ virtual bool WaitForCommand(Result* result);
virtual vector<Edge*> GetActiveEdges();
virtual void Abort();
@@ -499,31 +501,30 @@ bool RealCommandRunner::StartCommand(Edge* edge) {
return true;
}
-Edge* RealCommandRunner::WaitForCommand(ExitStatus* status, string* output) {
+bool RealCommandRunner::WaitForCommand(Result* result) {
Subprocess* subproc;
while ((subproc = subprocs_.NextFinished()) == NULL) {
bool interrupted = subprocs_.DoWork();
- if (interrupted) {
- *status = ExitInterrupted;
- return 0;
- }
+ if (interrupted)
+ return false;
}
- *status = subproc->Finish();
- *output = subproc->GetOutput();
+ result->status = subproc->Finish();
+ result->output = subproc->GetOutput();
map<Subprocess*, Edge*>::iterator i = subproc_to_edge_.find(subproc);
- Edge* edge = i->second;
+ result->edge = i->second;
subproc_to_edge_.erase(i);
delete subproc;
- return edge;
+ return true;
}
Builder::Builder(State* state, const BuildConfig& config,
- BuildLog* log, DiskInterface* disk_interface)
+ BuildLog* build_log, DepsLog* deps_log,
+ DiskInterface* disk_interface)
: state_(state), config_(config), disk_interface_(disk_interface),
- scan_(state, log, disk_interface) {
+ scan_(state, build_log, deps_log, disk_interface) {
status_ = new BuildStatus(config);
}
@@ -609,8 +610,6 @@ bool Builder::Build(string* err) {
// First, we attempt to start as many commands as allowed by the
// command runner.
// Second, we attempt to wait for / reap the next finished command.
- // If we can do neither of those, the build is stuck, and we report
- // an error.
while (plan_.more_to_do()) {
// See if we can start any more commands.
if (failures_allowed && command_runner_->CanRunMore()) {
@@ -620,10 +619,11 @@ bool Builder::Build(string* err) {
return false;
}
- if (edge->is_phony())
- FinishEdge(edge, true, "");
- else
+ if (edge->is_phony()) {
+ plan_.EdgeFinished(edge);
+ } else {
++pending_commands;
+ }
// We made some progress; go back to the main loop.
continue;
@@ -632,34 +632,24 @@ bool Builder::Build(string* err) {
// See if we can reap any finished commands.
if (pending_commands) {
- ExitStatus status;
- string output;
- Edge* edge = command_runner_->WaitForCommand(&status, &output);
- if (edge && status != ExitInterrupted) {
- bool success = (status == ExitSuccess);
- --pending_commands;
- FinishEdge(edge, success, output);
- if (!success) {
- if (failures_allowed)
- failures_allowed--;
- }
-
- // We made some progress; start the main loop over.
- continue;
- }
-
- if (status == ExitInterrupted) {
+ CommandRunner::Result result;
+ if (!command_runner_->WaitForCommand(&result) ||
+ result.status == ExitInterrupted) {
status_->BuildFinished();
*err = "interrupted by user";
return false;
}
- }
- // If we get here, we can neither enqueue new commands nor are any running.
- if (pending_commands) {
- status_->BuildFinished();
- *err = "stuck: pending commands but none to wait for? [this is a bug]";
- return false;
+ --pending_commands;
+ FinishCommand(&result);
+
+ if (!result.success()) {
+ if (failures_allowed)
+ failures_allowed--;
+ }
+
+ // We made some progress; start the main loop over.
+ continue;
}
// If we get here, we cannot make any more progress.
@@ -714,63 +704,156 @@ bool Builder::StartEdge(Edge* edge, string* err) {
return true;
}
-void Builder::FinishEdge(Edge* edge, bool success, const string& output) {
- METRIC_RECORD("FinishEdge");
- TimeStamp restat_mtime = 0;
+void Builder::FinishCommand(CommandRunner::Result* result) {
+ METRIC_RECORD("FinishCommand");
- if (success) {
- if (edge->GetBindingBool("restat") && !config_.dry_run) {
- bool node_cleaned = false;
-
- for (vector<Node*>::iterator i = edge->outputs_.begin();
- i != edge->outputs_.end(); ++i) {
- TimeStamp new_mtime = disk_interface_->Stat((*i)->path());
- if ((*i)->mtime() == new_mtime) {
- // The rule command did not change the output. Propagate the clean
- // state through the build graph.
- // Note that this also applies to nonexistent outputs (mtime == 0).
- plan_.CleanNode(&scan_, *i);
- node_cleaned = true;
- }
- }
+ Edge* edge = result->edge;
- if (node_cleaned) {
- // If any output was cleaned, find the most recent mtime of any
- // (existing) non-order-only input or the depfile.
- for (vector<Node*>::iterator i = edge->inputs_.begin();
- i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
- TimeStamp input_mtime = disk_interface_->Stat((*i)->path());
- if (input_mtime > restat_mtime)
- restat_mtime = input_mtime;
- }
+ // First try to extract dependencies from the result, if any.
+ // This must happen first as it filters the command output and
+ // can fail, which makes the command fail from a build perspective.
- string depfile = edge->GetBinding("depfile");
- if (restat_mtime != 0 && !depfile.empty()) {
- TimeStamp depfile_mtime = disk_interface_->Stat(depfile);
- if (depfile_mtime > restat_mtime)
- restat_mtime = depfile_mtime;
- }
+ vector<Node*> deps_nodes;
+ TimeStamp deps_mtime = 0;
+ string deps_type = edge->GetBinding("deps");
+ if (result->success() && !deps_type.empty()) {
+ string extract_err;
+ if (!ExtractDeps(result, deps_type, &deps_nodes, &deps_mtime,
+ &extract_err)) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append(extract_err);
+ result->status = ExitFailure;
+ }
+ }
- // The total number of edges in the plan may have changed as a result
- // of a restat.
- status_->PlanHasTotalEdges(plan_.command_edge_count());
+ int start_time, end_time;
+ status_->BuildEdgeFinished(edge, result->success(), result->output,
+ &start_time, &end_time);
+
+ // The rest of this function only applies to successful commands.
+ if (!result->success())
+ return;
+
+ // Restat the edge outputs, if necessary.
+ TimeStamp restat_mtime = 0;
+ if (edge->GetBindingBool("restat") && !config_.dry_run) {
+ bool node_cleaned = false;
+
+ for (vector<Node*>::iterator i = edge->outputs_.begin();
+ i != edge->outputs_.end(); ++i) {
+ TimeStamp new_mtime = disk_interface_->Stat((*i)->path());
+ if ((*i)->mtime() == new_mtime) {
+ // The rule command did not change the output. Propagate the clean
+ // state through the build graph.
+ // Note that this also applies to nonexistent outputs (mtime == 0).
+ plan_.CleanNode(&scan_, *i);
+ node_cleaned = true;
}
}
- // Delete the response file on success (if exists)
- string rspfile = edge->GetBinding("rspfile");
- if (!rspfile.empty())
- disk_interface_->RemoveFile(rspfile);
+ if (node_cleaned) {
+ // If any output was cleaned, find the most recent mtime of any
+ // (existing) non-order-only input or the depfile.
+ for (vector<Node*>::iterator i = edge->inputs_.begin();
+ i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
+ TimeStamp input_mtime = disk_interface_->Stat((*i)->path());
+ if (input_mtime > restat_mtime)
+ restat_mtime = input_mtime;
+ }
+
+ string depfile = edge->GetBinding("depfile");
+ if (restat_mtime != 0 && !depfile.empty()) {
+ TimeStamp depfile_mtime = disk_interface_->Stat(depfile);
+ if (depfile_mtime > restat_mtime)
+ restat_mtime = depfile_mtime;
+ }
- plan_.EdgeFinished(edge);
+ // The total number of edges in the plan may have changed as a result
+ // of a restat.
+ status_->PlanHasTotalEdges(plan_.command_edge_count());
+ }
}
- if (edge->is_phony())
- return;
+ plan_.EdgeFinished(edge);
+
+ // Delete any left over response file.
+ string rspfile = edge->GetBinding("rspfile");
+ if (!rspfile.empty())
+ disk_interface_->RemoveFile(rspfile);
+
+ if (scan_.build_log()) {
+ scan_.build_log()->RecordCommand(edge, start_time, end_time,
+ restat_mtime);
+ }
+
+ if (!deps_type.empty()) {
+ assert(edge->outputs_.size() == 1 && "should have been rejected by parser");
+ Node* out = edge->outputs_[0];
+ if (!deps_mtime) {
+ // On Windows there's no separate file to compare against; just reuse
+ // the output's mtime.
+ deps_mtime = disk_interface_->Stat(out->path());
+ }
+ scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes);
+ }
- int start_time, end_time;
- status_->BuildEdgeFinished(edge, success, output, &start_time, &end_time);
- if (success && scan_.build_log())
- scan_.build_log()->RecordCommand(edge, start_time, end_time, restat_mtime);
}
+bool Builder::ExtractDeps(CommandRunner::Result* result,
+ const string& deps_type,
+ vector<Node*>* deps_nodes,
+ TimeStamp* deps_mtime,
+ string* err) {
+#ifdef _WIN32
+ if (deps_type == "msvc") {
+ CLParser parser;
+ result->output = parser.Parse(result->output);
+ for (set<string>::iterator i = parser.includes_.begin();
+ i != parser.includes_.end(); ++i) {
+ deps_nodes->push_back(state_->GetNode(*i));
+ }
+ } else
+#endif
+ if (deps_type == "gcc") {
+ string depfile = result->edge->GetBinding("depfile");
+ if (depfile.empty()) {
+ *err = string("edge with deps=gcc but no depfile makes no sense");
+ return false;
+ }
+
+ *deps_mtime = disk_interface_->Stat(depfile);
+ if (*deps_mtime <= 0) {
+ *err = string("unable to read depfile");
+ return false;
+ }
+
+ string content = disk_interface_->ReadFile(depfile, err);
+ if (!err->empty())
+ return false;
+ if (content.empty())
+ return true;
+
+ DepfileParser deps;
+ if (!deps.Parse(&content, err))
+ return false;
+
+ // XXX check depfile matches expected output.
+ deps_nodes->reserve(deps.ins_.size());
+ for (vector<StringPiece>::iterator i = deps.ins_.begin();
+ i != deps.ins_.end(); ++i) {
+ if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err))
+ return false;
+ deps_nodes->push_back(state_->GetNode(*i));
+ }
+
+ if (disk_interface_->RemoveFile(depfile) < 0) {
+ *err = string("deleting depfile: ") + strerror(errno) + string("\n");
+ return false;
+ }
+ } else {
+ Fatal("unknown deps type '%s'", deps_type.c_str());
+ }
+
+ return true;
+}
diff --git a/src/build.h b/src/build.h
index 52c277a..9a16990 100644
--- a/src/build.h
+++ b/src/build.h
@@ -104,8 +104,18 @@ struct CommandRunner {
virtual ~CommandRunner() {}
virtual bool CanRunMore() = 0;
virtual bool StartCommand(Edge* edge) = 0;
- /// Wait for a command to complete.
- virtual Edge* WaitForCommand(ExitStatus* status, string* output) = 0;
+
+ /// The result of waiting for a command.
+ struct Result {
+ Result() : edge(NULL) {}
+ Edge* edge;
+ ExitStatus status;
+ string output;
+ bool success() const { return status == ExitSuccess; }
+ };
+ /// Wait for a command to complete, or return false if interrupted.
+ virtual bool WaitForCommand(Result* result) = 0;
+
virtual vector<Edge*> GetActiveEdges() { return vector<Edge*>(); }
virtual void Abort() {}
};
@@ -132,7 +142,8 @@ struct BuildConfig {
/// Builder wraps the build process: starting commands, updating status.
struct Builder {
Builder(State* state, const BuildConfig& config,
- BuildLog* log, DiskInterface* disk_interface);
+ BuildLog* build_log, DepsLog* deps_log,
+ DiskInterface* disk_interface);
~Builder();
/// Clean up after interrupted commands by deleting output files.
@@ -152,7 +163,7 @@ struct Builder {
bool Build(string* err);
bool StartEdge(Edge* edge, string* err);
- void FinishEdge(Edge* edge, bool success, const string& output);
+ void FinishCommand(CommandRunner::Result* result);
/// Used for tests.
void SetBuildLog(BuildLog* log) {
@@ -166,6 +177,10 @@ struct Builder {
BuildStatus* status_;
private:
+ bool ExtractDeps(CommandRunner::Result* result, const string& deps_type,
+ vector<Node*>* deps_nodes, TimeStamp* deps_mtime,
+ string* err);
+
DiskInterface* disk_interface_;
DependencyScan scan_;
diff --git a/src/build_test.cc b/src/build_test.cc
index d4468a3..2a0fa0f 100644
--- a/src/build_test.cc
+++ b/src/build_test.cc
@@ -15,6 +15,7 @@
#include "build.h"
#include "build_log.h"
+#include "deps_log.h"
#include "graph.h"
#include "test.h"
@@ -383,7 +384,7 @@ struct FakeCommandRunner : public CommandRunner {
// CommandRunner impl
virtual bool CanRunMore();
virtual bool StartCommand(Edge* edge);
- virtual Edge* WaitForCommand(ExitStatus* status, string* output);
+ virtual bool WaitForCommand(Result* result);
virtual vector<Edge*> GetActiveEdges();
virtual void Abort();
@@ -394,8 +395,13 @@ struct FakeCommandRunner : public CommandRunner {
struct BuildTest : public StateTestWithBuiltinRules {
BuildTest() : config_(MakeConfig()), command_runner_(&fs_),
- builder_(&state_, config_, NULL, &fs_),
+ builder_(&state_, config_, NULL, NULL, &fs_),
status_(config_) {
+ }
+
+ virtual void SetUp() {
+ StateTestWithBuiltinRules::SetUp();
+
builder_.command_runner_.reset(&command_runner_);
AssertParse(&state_,
"build cat1: cat in1\n"
@@ -457,24 +463,25 @@ bool FakeCommandRunner::StartCommand(Edge* edge) {
return true;
}
-Edge* FakeCommandRunner::WaitForCommand(ExitStatus* status,
- string* /* output */) {
- if (Edge* edge = last_command_) {
- if (edge->rule().name() == "interrupt" ||
- edge->rule().name() == "touch-interrupt") {
- *status = ExitInterrupted;
- return NULL;
- }
+bool FakeCommandRunner::WaitForCommand(Result* result) {
+ if (!last_command_)
+ return false;
- if (edge->rule().name() == "fail")
- *status = ExitFailure;
- else
- *status = ExitSuccess;
- last_command_ = NULL;
- return edge;
+ Edge* edge = last_command_;
+ result->edge = edge;
+
+ if (edge->rule().name() == "interrupt" ||
+ edge->rule().name() == "touch-interrupt") {
+ result->status = ExitInterrupted;
+ return true;
}
- *status = ExitFailure;
- return NULL;
+
+ if (edge->rule().name() == "fail")
+ result->status = ExitFailure;
+ else
+ result->status = ExitSuccess;
+ last_command_ = NULL;
+ return true;
}
vector<Edge*> FakeCommandRunner::GetActiveEdges() {
@@ -761,6 +768,9 @@ TEST_F(BuildTest, OrderOnlyDeps) {
fs_.Tick();
+ // Recreate the depfile, as it should have been deleted by the build.
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
+
// implicit dep dirty, expect a rebuild.
fs_.Create("blah.h", "");
fs_.Create("bar.h", "");
@@ -773,6 +783,9 @@ TEST_F(BuildTest, OrderOnlyDeps) {
fs_.Tick();
+ // Recreate the depfile, as it should have been deleted by the build.
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
+
// order only dep dirty, no rebuild.
fs_.Create("otherfile", "");
command_runner_.commands_ran_.clear();
@@ -1340,8 +1353,204 @@ TEST_F(BuildTest, PhonyWithNoInputs) {
ASSERT_EQ(1u, command_runner_.commands_ran_.size());
}
+TEST_F(BuildTest, DepsGccWithEmptyDeps) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n"
+" command = cc\n"
+" deps = gcc\n"
+"build out: cc\n"));
+ Dirty("out");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_FALSE(builder_.AlreadyUpToDate());
+
+ EXPECT_FALSE(builder_.Build(&err));
+ ASSERT_EQ("subcommand failed", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
TEST_F(BuildTest, StatusFormatReplacePlaceholder) {
EXPECT_EQ("[%/s0/t0/r0/u0/f0]",
status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]"));
}
+TEST_F(BuildTest, FailedDepsParse) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build bad_deps.o: cat in1\n"
+" deps = gcc\n"
+" depfile = in1.d\n"));
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("bad_deps.o", &err));
+ ASSERT_EQ("", err);
+
+ // These deps will fail to parse, as they should only have one
+ // path to the left of the colon.
+ fs_.Create("in1.d", "AAA BBB");
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+}
+
+/// Tests of builds involving deps logs necessarily must span
+/// multiple builds. We reuse methods on BuildTest but not the
+/// builder_ it sets up, because we want pristine objects for
+/// each build.
+struct BuildWithDepsLogTest : public BuildTest {
+ BuildWithDepsLogTest() {}
+
+ virtual void SetUp() {
+ BuildTest::SetUp();
+
+ temp_dir_.CreateAndEnter("BuildWithDepsLogTest");
+ }
+
+ virtual void TearDown() {
+ temp_dir_.Cleanup();
+ }
+
+ ScopedTempDir temp_dir_;
+
+ /// Shadow parent class builder_ so we don't accidentally use it.
+ void* builder_;
+};
+
+/// Run a straightforwad build where the deps log is used.
+TEST_F(BuildWithDepsLogTest, Straightforward) {
+ string err;
+ // Note: in1 was created by the superclass SetUp().
+ const char* manifest =
+ "build out: cat in1\n"
+ " deps = gcc\n"
+ " depfile = in1.d\n";
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Run the build once, everything should be ok.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("in1.d", "out: in2");
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // The deps file should have been removed.
+ EXPECT_EQ(0, fs_.Stat("in1.d"));
+ // Recreate it for the next step.
+ fs_.Create("in1.d", "out: in2");
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Touch the file only mentioned in the deps.
+ fs_.Tick();
+ fs_.Create("in2", "");
+
+ // Run the build again.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ command_runner_.commands_ran_.clear();
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // We should have rebuilt the output due to in2 being
+ // out of date.
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ builder.command_runner_.release();
+ }
+}
+
+/// Verify that obsolete deps still cause a rebuild.
+TEST_F(BuildWithDepsLogTest, ObsoleteDeps) {
+ string err;
+ // Note: in1 was created by the superclass SetUp().
+ const char* manifest =
+ "build out: cat in1\n"
+ " deps = gcc\n"
+ " depfile = in1.d\n";
+ {
+ // Create the obsolete deps, then run a build to incorporate them.
+ // The idea is that the inputs/outputs are newer than the logged
+ // deps.
+ fs_.Create("in1.d", "out: ");
+ fs_.Tick();
+
+ fs_.Create("in1", "");
+
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Run the build once, everything should be ok.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ fs_.Create("out", "");
+ // The deps file should have been removed.
+ EXPECT_EQ(0, fs_.Stat("in1.d"));
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ // Now we should be in a situation where in1/out2 both have recent
+ // timestamps but the deps are old. Verify we rebuild.
+ fs_.Tick();
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ command_runner_.commands_ran_.clear();
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ // Recreate the deps here just to prove the old recorded deps are
+ // the problem.
+ fs_.Create("in1.d", "out: ");
+
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // We should have rebuilt the output due to the deps being
+ // out of date.
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ builder.command_runner_.release();
+ }
+}
diff --git a/src/deps_log.cc b/src/deps_log.cc
new file mode 100644
index 0000000..8946e32
--- /dev/null
+++ b/src/deps_log.cc
@@ -0,0 +1,263 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "deps_log.h"
+
+#include <assert.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "graph.h"
+#include "metrics.h"
+#include "state.h"
+#include "util.h"
+
+// The version is stored as 4 bytes after the signature and also serves as a
+// byte order mark. Signature and version combined are 16 bytes long.
+const char kFileSignature[] = "# ninjadeps\n";
+const int kCurrentVersion = 1;
+
+DepsLog::~DepsLog() {
+ Close();
+}
+
+bool DepsLog::OpenForWrite(const string& path, string* err) {
+ file_ = fopen(path.c_str(), "ab");
+ if (!file_) {
+ *err = strerror(errno);
+ return false;
+ }
+ SetCloseOnExec(fileno(file_));
+
+ // Opening a file in append mode doesn't set the file pointer to the file's
+ // end on Windows. Do that explicitly.
+ fseek(file_, 0, SEEK_END);
+
+ if (ftell(file_) == 0) {
+ if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) {
+ *err = strerror(errno);
+ return false;
+ }
+ if (fwrite(&kCurrentVersion, 4, 1, file_) < 1) {
+ *err = strerror(errno);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool DepsLog::RecordDeps(Node* node, TimeStamp mtime,
+ const vector<Node*>& nodes) {
+ return RecordDeps(node, mtime, nodes.size(),
+ nodes.empty() ? NULL : (Node**)&nodes.front());
+}
+
+bool DepsLog::RecordDeps(Node* node, TimeStamp mtime,
+ int node_count, Node** nodes) {
+ // Track whether there's any new data to be recorded.
+ bool made_change = false;
+
+ // Assign ids to all nodes that are missing one.
+ if (node->id() < 0) {
+ RecordId(node);
+ made_change = true;
+ }
+ for (int i = 0; i < node_count; ++i) {
+ if (nodes[i]->id() < 0) {
+ RecordId(nodes[i]);
+ made_change = true;
+ }
+ }
+
+ // See if the new data is different than the existing data, if any.
+ if (!made_change) {
+ Deps* deps = GetDeps(node);
+ if (!deps ||
+ deps->mtime != mtime ||
+ deps->node_count != node_count) {
+ made_change = true;
+ } else {
+ for (int i = 0; i < node_count; ++i) {
+ if (deps->nodes[i] != nodes[i]) {
+ made_change = true;
+ break;
+ }
+ }
+ }
+ }
+
+ // Don't write anything if there's no new info.
+ if (!made_change)
+ return true;
+
+ uint16_t size = 4 * (1 + 1 + (uint16_t)node_count);
+ size |= 0x8000; // Deps record: set high bit.
+ fwrite(&size, 2, 1, file_);
+ int id = node->id();
+ fwrite(&id, 4, 1, file_);
+ int timestamp = mtime;
+ fwrite(&timestamp, 4, 1, file_);
+ for (int i = 0; i < node_count; ++i) {
+ id = nodes[i]->id();
+ fwrite(&id, 4, 1, file_);
+ }
+
+ return true;
+}
+
+void DepsLog::Close() {
+ if (file_)
+ fclose(file_);
+ file_ = NULL;
+}
+
+bool DepsLog::Load(const string& path, State* state, string* err) {
+ METRIC_RECORD(".ninja_deps load");
+ char buf[32 << 10];
+ FILE* f = fopen(path.c_str(), "rb");
+ if (!f) {
+ if (errno == ENOENT)
+ return true;
+ *err = strerror(errno);
+ return false;
+ }
+
+ if (!fgets(buf, sizeof(buf), f)) {
+ *err = strerror(errno);
+ return false;
+ }
+ int version = 0;
+ if (fread(&version, 4, 1, f) < 1) {
+ *err = strerror(errno);
+ return false;
+ }
+ if (version != kCurrentVersion) {
+ *err = "bad deps log signature or version; starting over";
+ fclose(f);
+ unlink(path.c_str());
+ // Don't report this as a failure. An empty deps log will cause
+ // us to rebuild the outputs anyway.
+ return true;
+ }
+
+ for (;;) {
+ uint16_t size;
+ if (fread(&size, 2, 1, f) < 1)
+ break;
+ bool is_deps = (size >> 15) != 0;
+ size = size & 0x7FFF;
+
+ if (fread(buf, size, 1, f) < 1)
+ break;
+
+ if (is_deps) {
+ assert(size % 4 == 0);
+ int* deps_data = reinterpret_cast<int*>(buf);
+ int out_id = deps_data[0];
+ int mtime = deps_data[1];
+ deps_data += 2;
+ int deps_count = (size / 4) - 2;
+
+ Deps* deps = new Deps;
+ deps->mtime = mtime;
+ deps->node_count = deps_count;
+ deps->nodes = new Node*[deps_count];
+ for (int i = 0; i < deps_count; ++i) {
+ assert(deps_data[i] < (int)nodes_.size());
+ assert(nodes_[deps_data[i]]);
+ deps->nodes[i] = nodes_[deps_data[i]];
+ }
+
+ if (out_id >= (int)deps_.size())
+ deps_.resize(out_id + 1);
+ if (deps_[out_id]) {
+ ++dead_record_count_;
+ delete deps_[out_id];
+ }
+ deps_[out_id] = deps;
+ } else {
+ StringPiece path(buf, size);
+ Node* node = state->GetNode(path);
+ assert(node->id() < 0);
+ node->set_id(nodes_.size());
+ nodes_.push_back(node);
+ }
+ }
+ if (ferror(f)) {
+ *err = strerror(ferror(f));
+ return false;
+ }
+ fclose(f);
+ return true;
+}
+
+DepsLog::Deps* DepsLog::GetDeps(Node* node) {
+ if (node->id() < 0)
+ return NULL;
+ return deps_[node->id()];
+}
+
+bool DepsLog::Recompact(const string& path, string* err) {
+ METRIC_RECORD(".ninja_deps recompact");
+ printf("Recompacting deps...\n");
+
+ string temp_path = path + ".recompact";
+ DepsLog new_log;
+ if (!new_log.OpenForWrite(temp_path, err))
+ return false;
+
+ // Clear all known ids so that new ones can be reassigned.
+ for (vector<Node*>::iterator i = nodes_.begin();
+ i != nodes_.end(); ++i) {
+ (*i)->set_id(-1);
+ }
+
+ // Write out all deps again.
+ for (int old_id = 0; old_id < (int)deps_.size(); ++old_id) {
+ Deps* deps = deps_[old_id];
+ if (!new_log.RecordDeps(nodes_[old_id], deps->mtime,
+ deps->node_count, deps->nodes)) {
+ new_log.Close();
+ return false;
+ }
+ }
+
+ new_log.Close();
+
+ if (unlink(path.c_str()) < 0) {
+ *err = strerror(errno);
+ return false;
+ }
+
+ if (rename(temp_path.c_str(), path.c_str()) < 0) {
+ *err = strerror(errno);
+ return false;
+ }
+
+ return true;
+}
+
+bool DepsLog::RecordId(Node* node) {
+ uint16_t size = (uint16_t)node->path().size();
+ fwrite(&size, 2, 1, file_);
+ fwrite(node->path().data(), node->path().size(), 1, file_);
+
+ node->set_id(nodes_.size());
+ nodes_.push_back(node);
+
+ return true;
+}
diff --git a/src/deps_log.h b/src/deps_log.h
new file mode 100644
index 0000000..99b006e
--- /dev/null
+++ b/src/deps_log.h
@@ -0,0 +1,107 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_DEPS_LOG_H_
+#define NINJA_DEPS_LOG_H_
+
+#include <string>
+#include <vector>
+using namespace std;
+
+#include <stdio.h>
+
+#include "timestamp.h"
+
+struct Node;
+struct State;
+
+/// As build commands run they can output extra dependency information
+/// (e.g. header dependencies for C source) dynamically. DepsLog collects
+/// that information at build time and uses it for subsequent builds.
+///
+/// The on-disk format is based on two primary design constraints:
+/// - it must be written to as a stream (during the build, which may be
+/// interrupted);
+/// - it can be read all at once on startup. (Alternative designs, where
+/// it contains indexing information, were considered and discarded as
+/// too complicated to implement; if the file is small than reading it
+/// fully on startup is acceptable.)
+/// Here are some stats from the Windows Chrome dependency files, to
+/// help guide the design space. The total text in the files sums to
+/// 90mb so some compression is warranted to keep load-time fast.
+/// There's about 10k files worth of dependencies that reference about
+/// 40k total paths totalling 2mb of unique strings.
+///
+/// Based on these stats, here's the current design.
+/// The file is structured as version header followed by a sequence of records.
+/// Each record is either a path string or a dependency list.
+/// Numbering the path strings in file order gives them dense integer ids.
+/// A dependency list maps an output id to a list of input ids.
+///
+/// Concretely, a record is:
+/// two bytes record length, high bit indicates record type
+/// (implies max record length 32k)
+/// path records contain just the string name of the path
+/// dependency records are an array of 4-byte integers
+/// [output path id, output path mtime, input path id, input path id...]
+/// (The mtime is compared against the on-disk output path mtime
+/// to verify the stored data is up-to-date.)
+/// If two records reference the same output the latter one in the file
+/// wins, allowing updates to just be appended to the file. A separate
+/// repacking step can run occasionally to remove dead records.
+struct DepsLog {
+ DepsLog() : dead_record_count_(0), file_(NULL) {}
+ ~DepsLog();
+
+ // Writing (build-time) interface.
+ bool OpenForWrite(const string& path, string* err);
+ bool RecordDeps(Node* node, TimeStamp mtime, const vector<Node*>& nodes);
+ bool RecordDeps(Node* node, TimeStamp mtime, int node_count, Node** nodes);
+ void Close();
+
+ // Reading (startup-time) interface.
+ struct Deps {
+ Deps() : mtime(-1), node_count(0), nodes(NULL) {}
+ ~Deps() { delete [] nodes; }
+ int mtime;
+ int node_count;
+ Node** nodes;
+ };
+ bool Load(const string& path, State* state, string* err);
+ Deps* GetDeps(Node* node);
+
+ /// Rewrite the known log entries, throwing away old data.
+ bool Recompact(const string& path, string* err);
+
+ /// Used for tests.
+ const vector<Node*>& nodes() const { return nodes_; }
+
+ private:
+ // Write a node name record, assigning it an id.
+ bool RecordId(Node* node);
+
+ /// Number of deps record read while loading the file that ended up
+ /// being unused (due to a latter entry superceding it).
+ int dead_record_count_;
+
+ FILE* file_;
+ /// Maps id -> Node.
+ vector<Node*> nodes_;
+ /// Maps id -> deps of that id.
+ vector<Deps*> deps_;
+
+ friend struct DepsLogTest;
+};
+
+#endif // NINJA_DEPS_LOG_H_
diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc
new file mode 100644
index 0000000..2d91c0e
--- /dev/null
+++ b/src/deps_log_test.cc
@@ -0,0 +1,195 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "deps_log.h"
+
+#include "graph.h"
+#include "util.h"
+#include "test.h"
+
+namespace {
+
+const char kTestFilename[] = "DepsLogTest-tempfile";
+
+struct DepsLogTest : public testing::Test {
+ virtual void SetUp() {
+ // In case a crashing test left a stale file behind.
+ unlink(kTestFilename);
+ }
+ virtual void TearDown() {
+ //unlink(kTestFilename);
+ }
+};
+
+TEST_F(DepsLogTest, WriteRead) {
+ State state1;
+ DepsLog log1;
+ string err;
+ EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ {
+ vector<Node*> deps;
+ deps.push_back(state1.GetNode("foo.h"));
+ deps.push_back(state1.GetNode("bar.h"));
+ log1.RecordDeps(state1.GetNode("out.o"), 1, deps);
+
+ deps.clear();
+ deps.push_back(state1.GetNode("foo.h"));
+ deps.push_back(state1.GetNode("bar2.h"));
+ log1.RecordDeps(state1.GetNode("out2.o"), 2, deps);
+ }
+
+ log1.Close();
+
+ State state2;
+ DepsLog log2;
+ EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
+ ASSERT_EQ("", err);
+
+ ASSERT_EQ(log1.nodes().size(), log2.nodes().size());
+ for (int i = 0; i < (int)log1.nodes().size(); ++i) {
+ Node* node1 = log1.nodes()[i];
+ Node* node2 = log2.nodes()[i];
+ ASSERT_EQ(i, node1->id());
+ ASSERT_EQ(node1->id(), node2->id());
+ }
+
+ // log1 has no deps entries, as it was only used for writing.
+ // Manually check the entries in log2.
+ DepsLog::Deps* deps = log2.GetDeps(state2.GetNode("out.o"));
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(2, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+ ASSERT_EQ("bar.h", deps->nodes[1]->path());
+}
+
+// Verify that adding the same deps twice doesn't grow the file.
+TEST_F(DepsLogTest, DoubleEntry) {
+ // Write some deps to the file and grab its size.
+ int file_size;
+ {
+ State state;
+ DepsLog log;
+ string err;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+ log.Close();
+
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ file_size = (int)st.st_size;
+ ASSERT_GT(file_size, 0);
+ }
+
+ // Now reload the file, and readd the same deps.
+ {
+ State state;
+ DepsLog log;
+ string err;
+ EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
+
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+ log.Close();
+
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ int file_size_2 = (int)st.st_size;
+ ASSERT_EQ(file_size, file_size_2);
+ }
+}
+
+// Verify that adding the new deps works and can be compacted away.
+TEST_F(DepsLogTest, Recompact) {
+ // Write some deps to the file and grab its size.
+ int file_size;
+ {
+ State state;
+ DepsLog log;
+ string err;
+ ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+ log.Close();
+
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ file_size = (int)st.st_size;
+ ASSERT_GT(file_size, 0);
+ }
+
+ // Now reload the file, and add slighly different deps.
+ int file_size_2;
+ {
+ State state;
+ DepsLog log;
+ string err;
+ ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
+
+ ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+ log.Close();
+
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ file_size_2 = (int)st.st_size;
+ // The file should grow to record the new deps.
+ ASSERT_GT(file_size_2, file_size);
+ }
+
+ // Now reload the file, verify the new deps have replaced the old, then
+ // recompact.
+ {
+ State state;
+ DepsLog log;
+ string err;
+ ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
+
+ DepsLog::Deps* deps = log.GetDeps(state.GetNode("out.o"));
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(1, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+
+ ASSERT_TRUE(log.Recompact(kTestFilename, &err));
+
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ int file_size_3 = (int)st.st_size;
+ // The file should have shrunk a bit for the smaller deps.
+ ASSERT_LT(file_size_3, file_size_2);
+ }
+}
+
+} // anonymous namespace
diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc
index c2315c7..9b43d0f 100644
--- a/src/disk_interface_test.cc
+++ b/src/disk_interface_test.cc
@@ -104,7 +104,7 @@ TEST_F(DiskInterfaceTest, RemoveFile) {
struct StatTest : public StateTestWithBuiltinRules,
public DiskInterface {
- StatTest() : scan_(&state_, NULL, this) {}
+ StatTest() : scan_(&state_, NULL, NULL, this) {}
// DiskInterface implementation.
virtual TimeStamp Stat(const string& path);
diff --git a/src/graph.cc b/src/graph.cc
index 0f99698..2614882 100644
--- a/src/graph.cc
+++ b/src/graph.cc
@@ -19,6 +19,7 @@
#include "build_log.h"
#include "depfile_parser.h"
+#include "deps_log.h"
#include "disk_interface.h"
#include "explain.h"
#include "manifest_parser.h"
@@ -48,6 +49,7 @@ bool Rule::IsReservedBinding(const string& var) {
return var == "command" ||
var == "depfile" ||
var == "description" ||
+ var == "deps" ||
var == "generator" ||
var == "pool" ||
var == "restat" ||
@@ -59,15 +61,12 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) {
bool dirty = false;
edge->outputs_ready_ = true;
- string depfile = edge->GetBinding("depfile");
- if (!depfile.empty()) {
- if (!LoadDepFile(edge, depfile, err)) {
- if (!err->empty())
- return false;
- EXPLAIN("Edge targets are dirty because depfile '%s' is missing",
- depfile.c_str());
- dirty = true;
- }
+ TimeStamp deps_mtime = 0;
+ if (!dep_loader_.LoadDeps(edge, &deps_mtime, err)) {
+ if (!err->empty())
+ return false;
+ // Failed to load dependency info: rebuild to regenerate it.
+ dirty = true;
}
// Visit all inputs; we're dirty if any of the inputs are dirty.
@@ -114,7 +113,8 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) {
for (vector<Node*>::iterator i = edge->outputs_.begin();
i != edge->outputs_.end(); ++i) {
(*i)->StatIfNecessary(disk_interface_);
- if (RecomputeOutputDirty(edge, most_recent_input, command, *i)) {
+ if (RecomputeOutputDirty(edge, most_recent_input, deps_mtime,
+ command, *i)) {
dirty = true;
break;
}
@@ -143,6 +143,7 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) {
bool DependencyScan::RecomputeOutputDirty(Edge* edge,
Node* most_recent_input,
+ TimeStamp deps_mtime,
const string& command,
Node* output) {
if (edge->is_phony()) {
@@ -184,6 +185,12 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge,
}
}
+ // Dirty if the output is newer than the deps.
+ if (deps_mtime && output->mtime() > deps_mtime) {
+ EXPLAIN("stored deps info out of date for for %s", output->path().c_str());
+ return true;
+ }
+
// May also be dirty due to the command changing since the last build.
// But if this is a generator rule, the command changing does not make us
// dirty.
@@ -282,7 +289,77 @@ bool Edge::GetBindingBool(const string& key) {
return !GetBinding(key).empty();
}
-bool DependencyScan::LoadDepFile(Edge* edge, const string& path, string* err) {
+void Edge::Dump(const char* prefix) const {
+ printf("%s[ ", prefix);
+ for (vector<Node*>::const_iterator i = inputs_.begin();
+ i != inputs_.end() && *i != NULL; ++i) {
+ printf("%s ", (*i)->path().c_str());
+ }
+ printf("--%s-> ", rule_->name().c_str());
+ for (vector<Node*>::const_iterator i = outputs_.begin();
+ i != outputs_.end() && *i != NULL; ++i) {
+ printf("%s ", (*i)->path().c_str());
+ }
+ if (pool_) {
+ if (!pool_->name().empty()) {
+ printf("(in pool '%s')", pool_->name().c_str());
+ }
+ } else {
+ printf("(null pool?)");
+ }
+ printf("] 0x%p\n", this);
+}
+
+bool Edge::is_phony() const {
+ return rule_ == &State::kPhonyRule;
+}
+
+void Node::Dump(const char* prefix) const {
+ printf("%s <%s 0x%p> mtime: %d%s, (:%s), ",
+ prefix, path().c_str(), this,
+ mtime(), mtime() ? "" : " (:missing)",
+ dirty() ? " dirty" : " clean");
+ if (in_edge()) {
+ in_edge()->Dump("in-edge: ");
+ } else {
+ printf("no in-edge\n");
+ }
+ printf(" out edges:\n");
+ for (vector<Edge*>::const_iterator e = out_edges().begin();
+ e != out_edges().end() && *e != NULL; ++e) {
+ (*e)->Dump(" +- ");
+ }
+}
+
+bool ImplicitDepLoader::LoadDeps(Edge* edge, TimeStamp* mtime, string* err) {
+ string deps_type = edge->GetBinding("deps");
+ if (!deps_type.empty()) {
+ if (!LoadDepsFromLog(edge, mtime, err)) {
+ if (!err->empty())
+ return false;
+ EXPLAIN("deps for %s are missing", edge->outputs_[0]->path().c_str());
+ return false;
+ }
+ return true;
+ }
+
+ string depfile = edge->GetBinding("depfile");
+ if (!depfile.empty()) {
+ if (!LoadDepFile(edge, depfile, err)) {
+ if (!err->empty())
+ return false;
+ EXPLAIN("depfile '%s' is missing", depfile.c_str());
+ return false;
+ }
+ return true;
+ }
+
+ // No deps to load.
+ return true;
+}
+
+bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path,
+ string* err) {
METRIC_RECORD("depfile load");
string content = disk_interface_->ReadFile(path, err);
if (!err->empty()) {
@@ -310,11 +387,8 @@ bool DependencyScan::LoadDepFile(Edge* edge, const string& path, string* err) {
}
// Preallocate space in edge->inputs_ to be filled in below.
- edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_,
- depfile.ins_.size(), 0);
- edge->implicit_deps_ += depfile.ins_.size();
vector<Node*>::iterator implicit_dep =
- edge->inputs_.end() - edge->order_only_deps_ - depfile.ins_.size();
+ PreallocateSpace(edge, depfile.ins_.size());
// Add all its in-edges.
for (vector<StringPiece>::iterator i = depfile.ins_.begin();
@@ -325,66 +399,50 @@ bool DependencyScan::LoadDepFile(Edge* edge, const string& path, string* err) {
Node* node = state_->GetNode(*i);
*implicit_dep = node;
node->AddOutEdge(edge);
-
- // If we don't have a edge that generates this input already,
- // create one; this makes us not abort if the input is missing,
- // but instead will rebuild in that circumstance.
- if (!node->in_edge()) {
- Edge* phony_edge = state_->AddEdge(&State::kPhonyRule);
- node->set_in_edge(phony_edge);
- phony_edge->outputs_.push_back(node);
-
- // RecomputeDirty might not be called for phony_edge if a previous call
- // to RecomputeDirty had caused the file to be stat'ed. Because previous
- // invocations of RecomputeDirty would have seen this node without an
- // input edge (and therefore ready), we have to set outputs_ready_ to true
- // to avoid a potential stuck build. If we do call RecomputeDirty for
- // this node, it will simply set outputs_ready_ to the correct value.
- phony_edge->outputs_ready_ = true;
- }
+ CreatePhonyInEdge(node);
}
return true;
}
-void Edge::Dump(const char* prefix) const {
- printf("%s[ ", prefix);
- for (vector<Node*>::const_iterator i = inputs_.begin();
- i != inputs_.end() && *i != NULL; ++i) {
- printf("%s ", (*i)->path().c_str());
- }
- printf("--%s-> ", rule_->name().c_str());
- for (vector<Node*>::const_iterator i = outputs_.begin();
- i != outputs_.end() && *i != NULL; ++i) {
- printf("%s ", (*i)->path().c_str());
- }
- if (pool_) {
- if (!pool_->name().empty()) {
- printf("(in pool '%s')", pool_->name().c_str());
- }
- } else {
- printf("(null pool?)");
+bool ImplicitDepLoader::LoadDepsFromLog(Edge* edge, TimeStamp* deps_mtime,
+ string* err) {
+ DepsLog::Deps* deps = deps_log_->GetDeps(edge->outputs_[0]);
+ if (!deps)
+ return false;
+
+ *deps_mtime = deps->mtime;
+
+ vector<Node*>::iterator implicit_dep =
+ PreallocateSpace(edge, deps->node_count);
+ for (int i = 0; i < deps->node_count; ++i, ++implicit_dep) {
+ *implicit_dep = deps->nodes[i];
+ CreatePhonyInEdge(*implicit_dep);
}
- printf("] 0x%p\n", this);
+ return true;
}
-bool Edge::is_phony() const {
- return rule_ == &State::kPhonyRule;
+vector<Node*>::iterator ImplicitDepLoader::PreallocateSpace(Edge* edge,
+ int count) {
+ edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_,
+ (size_t)count, 0);
+ edge->implicit_deps_ += count;
+ return edge->inputs_.end() - edge->order_only_deps_ - count;
}
-void Node::Dump(const char* prefix) const {
- printf("%s <%s 0x%p> mtime: %d%s, (:%s), ",
- prefix, path().c_str(), this,
- mtime(), mtime() ? "" : " (:missing)",
- dirty() ? " dirty" : " clean");
- if (in_edge()) {
- in_edge()->Dump("in-edge: ");
- } else {
- printf("no in-edge\n");
- }
- printf(" out edges:\n");
- for (vector<Edge*>::const_iterator e = out_edges().begin();
- e != out_edges().end() && *e != NULL; ++e) {
- (*e)->Dump(" +- ");
- }
+void ImplicitDepLoader::CreatePhonyInEdge(Node* node) {
+ if (node->in_edge())
+ return;
+
+ Edge* phony_edge = state_->AddEdge(&State::kPhonyRule);
+ node->set_in_edge(phony_edge);
+ phony_edge->outputs_.push_back(node);
+
+ // RecomputeDirty might not be called for phony_edge if a previous call
+ // to RecomputeDirty had caused the file to be stat'ed. Because previous
+ // invocations of RecomputeDirty would have seen this node without an
+ // input edge (and therefore ready), we have to set outputs_ready_ to true
+ // to avoid a potential stuck build. If we do call RecomputeDirty for
+ // this node, it will simply set outputs_ready_ to the correct value.
+ phony_edge->outputs_ready_ = true;
}
diff --git a/src/graph.h b/src/graph.h
index 8b93e29..428ba01 100644
--- a/src/graph.h
+++ b/src/graph.h
@@ -22,8 +22,13 @@ using namespace std;
#include "eval_env.h"
#include "timestamp.h"
+struct BuildLog;
struct DiskInterface;
+struct DepsLog;
struct Edge;
+struct Node;
+struct Pool;
+struct State;
/// Information about a node in the dependency graph: the file, whether
/// it's dirty, mtime, etc.
@@ -32,7 +37,8 @@ struct Node {
: path_(path),
mtime_(-1),
dirty_(false),
- in_edge_(NULL) {}
+ in_edge_(NULL),
+ id_(-1) {}
/// Return true if the file exists (mtime_ got a value).
bool Stat(DiskInterface* disk_interface);
@@ -74,6 +80,9 @@ struct Node {
Edge* in_edge() const { return in_edge_; }
void set_in_edge(Edge* edge) { in_edge_ = edge; }
+ int id() const { return id_; }
+ void set_id(int id) { id_ = id; }
+
const vector<Edge*>& out_edges() const { return out_edges_; }
void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); }
@@ -98,6 +107,9 @@ private:
/// All Edges that use this Node as an input.
vector<Edge*> out_edges_;
+
+ /// A dense integer id for the node, assigned and used by DepsLog.
+ int id_;
};
/// An invokable build command and associated metadata (description, etc.).
@@ -121,11 +133,6 @@ struct Rule {
map<string, EvalString> bindings_;
};
-struct BuildLog;
-struct Node;
-struct State;
-struct Pool;
-
/// An edge in the dependency graph; links between Nodes using Rules.
struct Edge {
Edge() : rule_(NULL), env_(NULL), outputs_ready_(false), implicit_deps_(0),
@@ -178,13 +185,54 @@ struct Edge {
};
+/// ImplicitDepLoader loads implicit dependencies, as referenced via the
+/// "depfile" attribute in build files.
+struct ImplicitDepLoader {
+ ImplicitDepLoader(State* state, DepsLog* deps_log,
+ DiskInterface* disk_interface)
+ : state_(state), disk_interface_(disk_interface), deps_log_(deps_log) {}
+
+ /// Load implicit dependencies for \a edge. May fill in \a mtime with
+ /// the timestamp of the loaded information.
+ /// @return false on error (without filling \a err if info is just missing).
+ bool LoadDeps(Edge* edge, TimeStamp* mtime, string* err);
+
+ DepsLog* deps_log() const {
+ return deps_log_;
+ }
+
+ private:
+ /// Load implicit dependencies for \a edge from a depfile attribute.
+ /// @return false on error (without filling \a err if info is just missing).
+ bool LoadDepFile(Edge* edge, const string& path, string* err);
+
+ /// Load implicit dependencies for \a edge from the DepsLog.
+ /// @return false on error (without filling \a err if info is just missing).
+ bool LoadDepsFromLog(Edge* edge, TimeStamp* mtime, string* err);
+
+ /// Preallocate \a count spaces in the input array on \a edge, returning
+ /// an iterator pointing at the first new space.
+ vector<Node*>::iterator PreallocateSpace(Edge* edge, int count);
+
+ /// If we don't have a edge that generates this input already,
+ /// create one; this makes us not abort if the input is missing,
+ /// but instead will rebuild in that circumstance.
+ void CreatePhonyInEdge(Node* node);
+
+ State* state_;
+ DiskInterface* disk_interface_;
+ DepsLog* deps_log_;
+};
+
+
/// DependencyScan manages the process of scanning the files in a graph
/// and updating the dirty/outputs_ready state of all the nodes and edges.
struct DependencyScan {
- DependencyScan(State* state, BuildLog* build_log,
+ DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log,
DiskInterface* disk_interface)
- : state_(state), build_log_(build_log),
- disk_interface_(disk_interface) {}
+ : build_log_(build_log),
+ disk_interface_(disk_interface),
+ dep_loader_(state, deps_log, disk_interface) {}
/// Examine inputs, outputs, and command lines to judge whether an edge
/// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_|
@@ -195,10 +243,9 @@ struct DependencyScan {
/// Recompute whether a given single output should be marked dirty.
/// Returns true if so.
bool RecomputeOutputDirty(Edge* edge, Node* most_recent_input,
+ TimeStamp deps_mtime,
const string& command, Node* output);
- bool LoadDepFile(Edge* edge, const string& path, string* err);
-
BuildLog* build_log() const {
return build_log_;
}
@@ -206,10 +253,14 @@ struct DependencyScan {
build_log_ = log;
}
+ DepsLog* deps_log() const {
+ return dep_loader_.deps_log();
+ }
+
private:
- State* state_;
BuildLog* build_log_;
DiskInterface* disk_interface_;
+ ImplicitDepLoader dep_loader_;
};
#endif // NINJA_GRAPH_H_
diff --git a/src/graph_test.cc b/src/graph_test.cc
index c105684..63d5757 100644
--- a/src/graph_test.cc
+++ b/src/graph_test.cc
@@ -17,7 +17,7 @@
#include "test.h"
struct GraphTest : public StateTestWithBuiltinRules {
- GraphTest() : scan_(&state_, NULL, &fs_) {}
+ GraphTest() : scan_(&state_, NULL, NULL, &fs_) {}
VirtualFileSystem fs_;
DependencyScan scan_;
diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index 14fca73..a581114 100644
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -329,6 +329,14 @@ bool ManifestParser::ParseEdge(string* err) {
edge->implicit_deps_ = implicit;
edge->order_only_deps_ = order_only;
+ // Multiple outputs aren't (yet?) supported with depslog.
+ string deps_type = edge->GetBinding("deps");
+ if (!deps_type.empty() && edge->outputs_.size() > 1) {
+ return lexer_.Error("multiple outputs aren't (yet?) supported by depslog; "
+ "bring this up on the mailing list if it affects you",
+ err);
+ }
+
return true;
}
diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc
index 4ac093f..76d235d 100644
--- a/src/manifest_parser_test.cc
+++ b/src/manifest_parser_test.cc
@@ -71,6 +71,7 @@ TEST_F(ParserTest, RuleAttributes) {
"rule cat\n"
" command = a\n"
" depfile = a\n"
+" deps = a\n"
" description = a\n"
" generator = a\n"
" restat = a\n"
@@ -599,6 +600,17 @@ TEST_F(ParserTest, MultipleOutputs) {
EXPECT_EQ("", err);
}
+TEST_F(ParserTest, MultipleOutputsWithDeps) {
+ State state;
+ ManifestParser parser(&state, NULL);
+ string err;
+ EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n"
+ "build a.o b.o: cc c.cc\n",
+ &err));
+ EXPECT_EQ("input:5: multiple outputs aren't (yet?) supported by depslog; "
+ "bring this up on the mailing list if it affects you\n", err);
+}
+
TEST_F(ParserTest, SubNinja) {
files_["test.ninja"] =
"var = inner\n"
diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc
index fd9b671..be2a5e0 100644
--- a/src/msvc_helper-win32.cc
+++ b/src/msvc_helper-win32.cc
@@ -39,15 +39,15 @@ string Replace(const string& input, const string& find, const string& replace) {
return result;
}
+} // anonymous namespace
+
string EscapeForDepfile(const string& path) {
// Depfiles don't escape single \.
return Replace(path, " ", "\\ ");
}
-} // anonymous namespace
-
// static
-string CLWrapper::FilterShowIncludes(const string& line) {
+string CLParser::FilterShowIncludes(const string& line) {
static const char kMagicPrefix[] = "Note: including file: ";
const char* in = line.c_str();
const char* end = in + line.size();
@@ -63,14 +63,14 @@ string CLWrapper::FilterShowIncludes(const string& line) {
}
// static
-bool CLWrapper::IsSystemInclude(const string& path) {
+bool CLParser::IsSystemInclude(const string& path) {
// TODO: this is a heuristic, perhaps there's a better way?
return (path.find("program files") != string::npos ||
path.find("microsoft visual studio") != string::npos);
}
// static
-bool CLWrapper::FilterInputFilename(const string& line) {
+bool CLParser::FilterInputFilename(const string& line) {
// TODO: other extensions, like .asm?
return EndsWith(line, ".c") ||
EndsWith(line, ".cc") ||
@@ -78,7 +78,42 @@ bool CLWrapper::FilterInputFilename(const string& line) {
EndsWith(line, ".cpp");
}
-int CLWrapper::Run(const string& command, string* extra_output) {
+string CLParser::Parse(const string& output) {
+ string filtered_output;
+
+ // Loop over all lines in the output to process them.
+ size_t start = 0;
+ while (start < output.size()) {
+ size_t end = output.find_first_of("\r\n", start);
+ if (end == string::npos)
+ end = output.size();
+ string line = output.substr(start, end - start);
+
+ string include = FilterShowIncludes(line);
+ if (!include.empty()) {
+ include = IncludesNormalize::Normalize(include, NULL);
+ if (!IsSystemInclude(include))
+ includes_.insert(include);
+ } else if (FilterInputFilename(line)) {
+ // Drop it.
+ // TODO: if we support compiling multiple output files in a single
+ // cl.exe invocation, we should stash the filename.
+ } else {
+ filtered_output.append(line);
+ filtered_output.append("\n");
+ }
+
+ if (end < output.size() && output[end] == '\r')
+ ++end;
+ if (end < output.size() && output[end] == '\n')
+ ++end;
+ start = end;
+ }
+
+ return filtered_output;
+}
+
+int CLWrapper::Run(const string& command, string* output) {
SECURITY_ATTRIBUTES security_attributes = {};
security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES);
security_attributes.bInheritHandle = TRUE;
@@ -118,8 +153,7 @@ int CLWrapper::Run(const string& command, string* extra_output) {
Win32Fatal("CloseHandle");
}
- // Read output of the subprocess and parse it.
- string output;
+ // Read all output of the subprocess.
DWORD read_len = 1;
while (read_len) {
char buf[64 << 10];
@@ -128,44 +162,12 @@ int CLWrapper::Run(const string& command, string* extra_output) {
GetLastError() != ERROR_BROKEN_PIPE) {
Win32Fatal("ReadFile");
}
- output.append(buf, read_len);
-
- // Loop over all lines in the output and process them.
- for (;;) {
- size_t ofs = output.find_first_of("\r\n");
- if (ofs == string::npos)
- break;
- string line = output.substr(0, ofs);
-
- string include = FilterShowIncludes(line);
- if (!include.empty()) {
- include = IncludesNormalize::Normalize(include, NULL);
- if (!IsSystemInclude(include))
- includes_.insert(include);
- } else if (FilterInputFilename(line)) {
- // Drop it.
- // TODO: if we support compiling multiple output files in a single
- // cl.exe invocation, we should stash the filename.
- } else {
- if (extra_output) {
- extra_output->append(line);
- extra_output->append("\n");
- } else {
- printf("%s\n", line.c_str());
- }
- }
-
- if (ofs < output.size() && output[ofs] == '\r')
- ++ofs;
- if (ofs < output.size() && output[ofs] == '\n')
- ++ofs;
- output = output.substr(ofs);
- }
+ output->append(buf, read_len);
}
+ // Wait for it to exit and grab its exit code.
if (WaitForSingleObject(process_info.hProcess, INFINITE) == WAIT_FAILED)
Win32Fatal("WaitForSingleObject");
-
DWORD exit_code = 0;
if (!GetExitCodeProcess(process_info.hProcess, &exit_code))
Win32Fatal("GetExitCodeProcess");
@@ -178,11 +180,3 @@ int CLWrapper::Run(const string& command, string* extra_output) {
return exit_code;
}
-
-vector<string> CLWrapper::GetEscapedResult() {
- vector<string> result;
- for (set<string>::iterator i = includes_.begin(); i != includes_.end(); ++i) {
- result.push_back(EscapeForDepfile(*i));
- }
- return result;
-}
diff --git a/src/msvc_helper.h b/src/msvc_helper.h
index 102201b..32ab606 100644
--- a/src/msvc_helper.h
+++ b/src/msvc_helper.h
@@ -17,23 +17,13 @@
#include <vector>
using namespace std;
+string EscapeForDepfile(const string& path);
+
/// Visual Studio's cl.exe requires some massaging to work with Ninja;
/// for example, it emits include information on stderr in a funny
-/// format when building with /showIncludes. This class wraps a CL
-/// process and parses that output to extract the file list.
-struct CLWrapper {
- CLWrapper() : env_block_(NULL) {}
-
- /// Set the environment block (as suitable for CreateProcess) to be used
- /// by Run().
- void SetEnvBlock(void* env_block) { env_block_ = env_block; }
-
- /// Start a process and parse its output. Returns its exit code.
- /// Any non-parsed output is buffered into \a extra_output if provided,
- /// otherwise it is printed to stdout while the process runs.
- /// Crashes (calls Fatal()) on error.
- int Run(const string& command, string* extra_output=NULL);
-
+/// format when building with /showIncludes. This class parses this
+/// output.
+struct CLParser {
/// Parse a line of cl.exe output and extract /showIncludes info.
/// If a dependency is extracted, returns a nonempty string.
/// Exposed for testing.
@@ -50,10 +40,24 @@ struct CLWrapper {
/// Exposed for testing.
static bool FilterInputFilename(const string& line);
- /// Fill a vector with the unique'd headers, escaped for output as a .d
- /// file.
- vector<string> GetEscapedResult();
+ /// Parse the full output of cl, returning the output (if any) that
+ /// should printed.
+ string Parse(const string& output);
- void* env_block_;
set<string> includes_;
};
+
+/// Wraps a synchronous execution of a CL subprocess.
+struct CLWrapper {
+ CLWrapper() : env_block_(NULL) {}
+
+ /// Set the environment block (as suitable for CreateProcess) to be used
+ /// by Run().
+ void SetEnvBlock(void* env_block) { env_block_ = env_block; }
+
+ /// Start a process and gather its raw output. Returns its exit code.
+ /// Crashes (calls Fatal()) on error.
+ int Run(const string& command, string* output);
+
+ void* env_block_;
+};
diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc
index 0bbe98b..3192821 100644
--- a/src/msvc_helper_main-win32.cc
+++ b/src/msvc_helper_main-win32.cc
@@ -44,12 +44,13 @@ void PushPathIntoEnvironment(const string& env_block) {
}
}
-void WriteDepFileOrDie(const char* object_path, CLWrapper* cl) {
+void WriteDepFileOrDie(const char* object_path, const CLParser& parse) {
string depfile_path = string(object_path) + ".d";
FILE* depfile = fopen(depfile_path.c_str(), "w");
if (!depfile) {
unlink(object_path);
- Fatal("opening %s: %s", depfile_path.c_str(), GetLastErrorString().c_str());
+ Fatal("opening %s: %s", depfile_path.c_str(),
+ GetLastErrorString().c_str());
}
if (fprintf(depfile, "%s: ", object_path) < 0) {
unlink(object_path);
@@ -57,9 +58,9 @@ void WriteDepFileOrDie(const char* object_path, CLWrapper* cl) {
unlink(depfile_path.c_str());
Fatal("writing %s", depfile_path.c_str());
}
- vector<string> headers = cl->GetEscapedResult();
- for (vector<string>::iterator i = headers.begin(); i != headers.end(); ++i) {
- if (fprintf(depfile, "%s\n", i->c_str()) < 0) {
+ const set<string>& headers = parse.includes_;
+ for (set<string>::iterator i = headers.begin(); i != headers.end(); ++i) {
+ if (fprintf(depfile, "%s\n", EscapeForDepfile(*i).c_str()) < 0) {
unlink(object_path);
fclose(depfile);
unlink(depfile_path.c_str());
@@ -95,11 +96,6 @@ int MSVCHelperMain(int argc, char** argv) {
}
}
- if (!output_filename) {
- Usage();
- Fatal("-o required");
- }
-
string env;
if (envfile) {
string err;
@@ -118,9 +114,15 @@ int MSVCHelperMain(int argc, char** argv) {
CLWrapper cl;
if (!env.empty())
cl.SetEnvBlock((void*)env.data());
- int exit_code = cl.Run(command);
+ string output;
+ int exit_code = cl.Run(command, &output);
- WriteDepFileOrDie(output_filename, &cl);
+ if (output_filename) {
+ CLParser parser;
+ output = parser.Parse(output);
+ WriteDepFileOrDie(output_filename, parser);
+ }
+ printf("%s\n", output.c_str());
return exit_code;
}
diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc
index 7730425..1e1cbde 100644
--- a/src/msvc_helper_test.cc
+++ b/src/msvc_helper_test.cc
@@ -19,100 +19,93 @@
#include "test.h"
#include "util.h"
-TEST(MSVCHelperTest, ShowIncludes) {
- ASSERT_EQ("", CLWrapper::FilterShowIncludes(""));
+TEST(CLParserTest, ShowIncludes) {
+ ASSERT_EQ("", CLParser::FilterShowIncludes(""));
- ASSERT_EQ("", CLWrapper::FilterShowIncludes("Sample compiler output"));
+ ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output"));
ASSERT_EQ("c:\\Some Files\\foobar.h",
- CLWrapper::FilterShowIncludes("Note: including file: "
- "c:\\Some Files\\foobar.h"));
+ CLParser::FilterShowIncludes("Note: including file: "
+ "c:\\Some Files\\foobar.h"));
ASSERT_EQ("c:\\initspaces.h",
- CLWrapper::FilterShowIncludes("Note: including file: "
- "c:\\initspaces.h"));
+ CLParser::FilterShowIncludes("Note: including file: "
+ "c:\\initspaces.h"));
}
-TEST(MSVCHelperTest, FilterInputFilename) {
- ASSERT_TRUE(CLWrapper::FilterInputFilename("foobar.cc"));
- ASSERT_TRUE(CLWrapper::FilterInputFilename("foo bar.cc"));
- ASSERT_TRUE(CLWrapper::FilterInputFilename("baz.c"));
+TEST(CLParserTest, FilterInputFilename) {
+ ASSERT_TRUE(CLParser::FilterInputFilename("foobar.cc"));
+ ASSERT_TRUE(CLParser::FilterInputFilename("foo bar.cc"));
+ ASSERT_TRUE(CLParser::FilterInputFilename("baz.c"));
- ASSERT_FALSE(CLWrapper::FilterInputFilename(
+ ASSERT_FALSE(CLParser::FilterInputFilename(
"src\\cl_helper.cc(166) : fatal error C1075: end "
"of file found ..."));
}
-TEST(MSVCHelperTest, Run) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo foo&& echo Note: including file: foo.h&&echo bar\"",
- &output);
+TEST(CLParserTest, ParseSimple) {
+ CLParser parser;
+ string output = parser.Parse(
+ "foo\r\n"
+ "Note: including file: foo.h\r\n"
+ "bar\r\n");
+
ASSERT_EQ("foo\nbar\n", output);
- ASSERT_EQ(1u, cl.includes_.size());
- ASSERT_EQ("foo.h", *cl.includes_.begin());
+ ASSERT_EQ(1u, parser.includes_.size());
+ ASSERT_EQ("foo.h", *parser.includes_.begin());
}
-TEST(MSVCHelperTest, RunFilenameFilter) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo foo.cc&& echo cl: warning\"",
- &output);
+TEST(CLParserTest, ParseFilenameFilter) {
+ CLParser parser;
+ string output = parser.Parse(
+ "foo.cc\r\n"
+ "cl: warning\r\n");
ASSERT_EQ("cl: warning\n", output);
}
-TEST(MSVCHelperTest, RunSystemInclude) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo Note: including file: c:\\Program Files\\foo.h&&"
- "echo Note: including file: d:\\Microsoft Visual Studio\\bar.h&&"
- "echo Note: including file: path.h\"",
- &output);
+TEST(CLParserTest, ParseSystemInclude) {
+ CLParser parser;
+ string output = parser.Parse(
+ "Note: including file: c:\\Program Files\\foo.h\r\n"
+ "Note: including file: d:\\Microsoft Visual Studio\\bar.h\r\n"
+ "Note: including file: path.h\r\n");
// We should have dropped the first two includes because they look like
// system headers.
ASSERT_EQ("", output);
- ASSERT_EQ(1u, cl.includes_.size());
- ASSERT_EQ("path.h", *cl.includes_.begin());
-}
-
-TEST(MSVCHelperTest, EnvBlock) {
- char env_block[] = "foo=bar\0";
- CLWrapper cl;
- cl.SetEnvBlock(env_block);
- string output;
- cl.Run("cmd /c \"echo foo is %foo%", &output);
- ASSERT_EQ("foo is bar\n", output);
+ ASSERT_EQ(1u, parser.includes_.size());
+ ASSERT_EQ("path.h", *parser.includes_.begin());
}
-TEST(MSVCHelperTest, DuplicatedHeader) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo Note: including file: foo.h&&"
- "echo Note: including file: bar.h&&"
- "echo Note: including file: foo.h\"",
- &output);
+TEST(CLParserTest, DuplicatedHeader) {
+ CLParser parser;
+ string output = parser.Parse(
+ "Note: including file: foo.h\r\n"
+ "Note: including file: bar.h\r\n"
+ "Note: including file: foo.h\r\n");
// We should have dropped one copy of foo.h.
ASSERT_EQ("", output);
- ASSERT_EQ(2u, cl.includes_.size());
+ ASSERT_EQ(2u, parser.includes_.size());
}
-TEST(MSVCHelperTest, DuplicatedHeaderPathConverted) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo Note: including file: sub/foo.h&&"
- "echo Note: including file: bar.h&&"
- "echo Note: including file: sub\\foo.h\"",
- &output);
+TEST(CLParserTest, DuplicatedHeaderPathConverted) {
+ CLParser parser;
+ string output = parser.Parse(
+ "Note: including file: sub/foo.h\r\n"
+ "Note: including file: bar.h\r\n"
+ "Note: including file: sub\\foo.h\r\n");
// We should have dropped one copy of foo.h.
ASSERT_EQ("", output);
- ASSERT_EQ(2u, cl.includes_.size());
+ ASSERT_EQ(2u, parser.includes_.size());
}
-TEST(MSVCHelperTest, SpacesInFilename) {
+TEST(CLParserTest, SpacesInFilename) {
+ ASSERT_EQ("sub\\some\\ sdk\\foo.h",
+ EscapeForDepfile("sub\\some sdk\\foo.h"));
+}
+
+TEST(MSVCHelperTest, EnvBlock) {
+ char env_block[] = "foo=bar\0";
CLWrapper cl;
+ cl.SetEnvBlock(env_block);
string output;
- cl.Run("cmd /c \"echo Note: including file: sub\\some sdk\\foo.h",
- &output);
- ASSERT_EQ("", output);
- vector<string> headers = cl.GetEscapedResult();
- ASSERT_EQ(1u, headers.size());
- ASSERT_EQ("sub\\some\\ sdk\\foo.h", headers[0]);
+ cl.Run("cmd /c \"echo foo is %foo%", &output);
+ ASSERT_EQ("foo is bar\r\n", output);
}
diff --git a/src/ninja.cc b/src/ninja.cc
index 69646e1..8ba1aa6 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -32,6 +32,7 @@
#include "browse.h"
#include "build.h"
#include "build_log.h"
+#include "deps_log.h"
#include "clean.h"
#include "disk_interface.h"
#include "edit_distance.h"
@@ -641,20 +642,13 @@ bool DebugEnable(const string& name, Globals* globals) {
}
}
-bool OpenLog(BuildLog* build_log, Globals* globals,
- DiskInterface* disk_interface) {
- const string build_dir =
- globals->state->bindings_.LookupVariable("builddir");
- const char* kLogPath = ".ninja_log";
- string log_path = kLogPath;
- if (!build_dir.empty()) {
- log_path = build_dir + "/" + kLogPath;
- if (!disk_interface->MakeDirs(log_path) && errno != EEXIST) {
- Error("creating build directory %s: %s",
- build_dir.c_str(), strerror(errno));
- return false;
- }
- }
+/// Open the build log.
+/// @return false on error.
+bool OpenBuildLog(BuildLog* build_log, const string& build_dir,
+ Globals* globals, DiskInterface* disk_interface) {
+ string log_path = ".ninja_log";
+ if (!build_dir.empty())
+ log_path = build_dir + "/" + log_path;
string err;
if (!build_log->Load(log_path, &err)) {
@@ -677,6 +671,36 @@ bool OpenLog(BuildLog* build_log, Globals* globals,
return true;
}
+/// Open the deps log: load it, then open for writing.
+/// @return false on error.
+bool OpenDepsLog(DepsLog* deps_log, const string& build_dir,
+ Globals* globals, DiskInterface* disk_interface) {
+ string path = ".ninja_deps";
+ if (!build_dir.empty())
+ path = build_dir + "/" + path;
+
+ string err;
+ if (!deps_log->Load(path, globals->state, &err)) {
+ Error("loading deps log %s: %s", path.c_str(), err.c_str());
+ return false;
+ }
+ if (!err.empty()) {
+ // Hack: Load() can return a warning via err by returning true.
+ Warning("%s", err.c_str());
+ err.clear();
+ }
+
+ if (!globals->config->dry_run) {
+ if (!deps_log->OpenForWrite(path, &err)) {
+ Error("opening deps log: %s", err.c_str());
+ return false;
+ }
+ }
+
+ return true;
+}
+
+
/// Dump the output requested by '-d stats'.
void DumpMetrics(Globals* globals) {
g_metrics->Report();
@@ -872,14 +896,30 @@ reload:
if (tool && tool->when == Tool::RUN_AFTER_LOAD)
return tool->func(&globals, argc, argv);
- BuildLog build_log;
RealDiskInterface disk_interface;
- if (!OpenLog(&build_log, &globals, &disk_interface))
+
+ // Create the build dir if it doesn't exist.
+ const string build_dir = globals.state->bindings_.LookupVariable("builddir");
+ if (!build_dir.empty() && !config.dry_run) {
+ if (!disk_interface.MakeDirs(build_dir + "/.") &&
+ errno != EEXIST) {
+ Error("creating build directory %s: %s",
+ build_dir.c_str(), strerror(errno));
+ return 1;
+ }
+ }
+
+ BuildLog build_log;
+ if (!OpenBuildLog(&build_log, build_dir, &globals, &disk_interface))
+ return 1;
+
+ DepsLog deps_log;
+ if (!OpenDepsLog(&deps_log, build_dir, &globals, &disk_interface))
return 1;
if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild
// target that is never up to date.
- Builder manifest_builder(globals.state, config, &build_log,
+ Builder manifest_builder(globals.state, config, &build_log, &deps_log,
&disk_interface);
if (RebuildManifest(&manifest_builder, input_file, &err)) {
rebuilt_manifest = true;
@@ -891,7 +931,8 @@ reload:
}
}
- Builder builder(globals.state, config, &build_log, &disk_interface);
+ Builder builder(globals.state, config, &build_log, &deps_log,
+ &disk_interface);
int result = RunBuild(&builder, argc, argv);
if (g_metrics)
DumpMetrics(&globals);
diff --git a/src/state.cc b/src/state.cc
index 9f46fee..d2d5ebe 100644
--- a/src/state.cc
+++ b/src/state.cc
@@ -202,10 +202,11 @@ void State::Reset() {
void State::Dump() {
for (Paths::iterator i = paths_.begin(); i != paths_.end(); ++i) {
Node* node = i->second;
- printf("%s %s\n",
+ printf("%s %s [id:%d]\n",
node->path().c_str(),
node->status_known() ? (node->dirty() ? "dirty" : "clean")
- : "unknown");
+ : "unknown",
+ node->id());
}
if (!pools_.empty()) {
printf("resource_pools:\n");
diff --git a/src/test.cc b/src/test.cc
index c48ca82..45a9226 100644
--- a/src/test.cc
+++ b/src/test.cc
@@ -74,7 +74,11 @@ string GetSystemTempDir() {
} // anonymous namespace
StateTestWithBuiltinRules::StateTestWithBuiltinRules() {
- AssertParse(&state_,
+ AddCatRule(&state_);
+}
+
+void StateTestWithBuiltinRules::AddCatRule(State* state) {
+ AssertParse(state,
"rule cat\n"
" command = cat $in > $out\n");
}
diff --git a/src/test.h b/src/test.h
index bff0e5a..9f29e07 100644
--- a/src/test.h
+++ b/src/test.h
@@ -29,6 +29,12 @@ struct Node;
/// builtin "cat" rule.
struct StateTestWithBuiltinRules : public testing::Test {
StateTestWithBuiltinRules();
+
+ /// Add a "cat" rule to \a state. Used by some tests; it's
+ /// otherwise done by the ctor to state_.
+ void AddCatRule(State* state);
+
+ /// Short way to get a Node by its path from state_.
Node* GetNode(const string& path);
State state_;
generated by cgit v0.12 at 2024-07-21 18:05:37 (GMT)