diff options
Diffstat (limited to 'src')
50 files changed, 1101 insertions, 225 deletions
diff --git a/src/build.cc b/src/build.cc index 9718f85..64bcea3 100644 --- a/src/build.cc +++ b/src/build.cc @@ -97,6 +97,9 @@ void BuildStatus::BuildEdgeStarted(Edge* edge) { ++started_edges_; PrintStatus(edge); + + if (edge->use_console()) + printer_.SetConsoleLocked(true); } void BuildStatus::BuildEdgeFinished(Edge* edge, @@ -112,10 +115,13 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, *end_time = (int)(now - start_time_millis_); running_edges_.erase(i); + if (edge->use_console()) + printer_.SetConsoleLocked(false); + if (config_.verbosity == BuildConfig::QUIET) return; - if (printer_.is_smart_terminal()) + if (!edge->use_console() && printer_.is_smart_terminal()) PrintStatus(edge); // Print the command that is spewing before printing its output. @@ -145,6 +151,7 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, } void BuildStatus::BuildFinished() { + printer_.SetConsoleLocked(false); printer_.PrintOnNewLine(""); } @@ -488,7 +495,7 @@ bool RealCommandRunner::CanRunMore() { bool RealCommandRunner::StartCommand(Edge* edge) { string command = edge->EvaluateCommand(); - Subprocess* subproc = subprocs_.Add(command); + Subprocess* subproc = subprocs_.Add(command, edge->use_console()); if (!subproc) return false; subproc_to_edge_.insert(make_pair(subproc, edge)); @@ -534,7 +541,7 @@ void Builder::Cleanup() { for (vector<Edge*>::iterator i = active_edges.begin(); i != active_edges.end(); ++i) { - string depfile = (*i)->GetBinding("depfile"); + string depfile = (*i)->GetUnescapedDepfile(); for (vector<Node*>::iterator ni = (*i)->outputs_.begin(); ni != (*i)->outputs_.end(); ++ni) { // Only delete this output if it was actually modified. This is @@ -610,6 +617,7 @@ bool Builder::Build(string* err) { if (failures_allowed && command_runner_->CanRunMore()) { if (Edge* edge = plan_.FindWork()) { if (!StartEdge(edge, err)) { + Cleanup(); status_->BuildFinished(); return false; } @@ -630,6 +638,7 @@ bool Builder::Build(string* err) { CommandRunner::Result result; if (!command_runner_->WaitForCommand(&result) || result.status == ExitInterrupted) { + Cleanup(); status_->BuildFinished(); *err = "interrupted by user"; return false; @@ -637,6 +646,7 @@ bool Builder::Build(string* err) { --pending_commands; if (!FinishCommand(&result, err)) { + Cleanup(); status_->BuildFinished(); return false; } @@ -686,7 +696,7 @@ bool Builder::StartEdge(Edge* edge, string* err) { // Create response file, if needed // XXX: this may also block; do we care? - string rspfile = edge->GetBinding("rspfile"); + string rspfile = edge->GetUnescapedRspfile(); if (!rspfile.empty()) { string content = edge->GetBinding("rspfile_content"); if (!disk_interface_->WriteFile(rspfile, content)) @@ -714,9 +724,11 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { // build perspective. vector<Node*> deps_nodes; string deps_type = edge->GetBinding("deps"); + const string deps_prefix = edge->GetBinding("msvc_deps_prefix"); if (!deps_type.empty()) { string extract_err; - if (!ExtractDeps(result, deps_type, &deps_nodes, &extract_err) && + if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes, + &extract_err) && result->success()) { if (!result->output.empty()) result->output.append("\n"); @@ -760,7 +772,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { restat_mtime = input_mtime; } - string depfile = edge->GetBinding("depfile"); + string depfile = edge->GetUnescapedDepfile(); if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) { TimeStamp depfile_mtime = disk_interface_->Stat(depfile); if (depfile_mtime > restat_mtime) @@ -776,7 +788,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { plan_.EdgeFinished(edge); // Delete any left over response file. - string rspfile = edge->GetBinding("rspfile"); + string rspfile = edge->GetUnescapedRspfile(); if (!rspfile.empty() && !g_keep_rsp) disk_interface_->RemoveFile(rspfile); @@ -802,12 +814,13 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { bool Builder::ExtractDeps(CommandRunner::Result* result, const string& deps_type, + const string& deps_prefix, vector<Node*>* deps_nodes, string* err) { #ifdef _WIN32 if (deps_type == "msvc") { CLParser parser; - result->output = parser.Parse(result->output); + result->output = parser.Parse(result->output, deps_prefix); for (set<string>::iterator i = parser.includes_.begin(); i != parser.includes_.end(); ++i) { deps_nodes->push_back(state_->GetNode(*i)); @@ -815,7 +828,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, } else #endif if (deps_type == "gcc") { - string depfile = result->edge->GetBinding("depfile"); + string depfile = result->edge->GetUnescapedDepfile(); if (depfile.empty()) { *err = string("edge with deps=gcc but no depfile makes no sense"); return false; diff --git a/src/build.h b/src/build.h index 5b6c83c..eb3636a 100644 --- a/src/build.h +++ b/src/build.h @@ -180,8 +180,9 @@ struct Builder { BuildStatus* status_; private: - bool ExtractDeps(CommandRunner::Result* result, const string& deps_type, - vector<Node*>* deps_nodes, string* err); + bool ExtractDeps(CommandRunner::Result* result, const string& deps_type, + const string& deps_prefix, vector<Node*>* deps_nodes, + string* err); DiskInterface* disk_interface_; DependencyScan scan_; diff --git a/src/build_log.cc b/src/build_log.cc index b92a06f..3f24c16 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -108,9 +108,10 @@ BuildLog::~BuildLog() { Close(); } -bool BuildLog::OpenForWrite(const string& path, string* err) { +bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user, + string* err) { if (needs_recompaction_) { - if (!Recompact(path, err)) + if (!Recompact(path, user, err)) return false; } @@ -350,7 +351,8 @@ bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { entry.output.c_str(), entry.command_hash) > 0; } -bool BuildLog::Recompact(const string& path, string* err) { +bool BuildLog::Recompact(const string& path, const BuildLogUser& user, + string* err) { METRIC_RECORD(".ninja_log recompact"); printf("Recompacting log...\n"); @@ -368,7 +370,13 @@ bool BuildLog::Recompact(const string& path, string* err) { return false; } + vector<StringPiece> dead_outputs; for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { + if (user.IsPathDead(i->first)) { + dead_outputs.push_back(i->first); + continue; + } + if (!WriteEntry(f, *i->second)) { *err = strerror(errno); fclose(f); @@ -376,6 +384,9 @@ bool BuildLog::Recompact(const string& path, string* err) { } } + for (size_t i = 0; i < dead_outputs.size(); ++i) + entries_.erase(dead_outputs[i]); + fclose(f); if (unlink(path.c_str()) < 0) { *err = strerror(errno); diff --git a/src/build_log.h b/src/build_log.h index eeac5b3..fe81a85 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -25,6 +25,13 @@ using namespace std; struct Edge; +/// Can answer questions about the manifest for the BuildLog. +struct BuildLogUser { + /// Return if a given output no longer part of the build manifest. + /// This is only called during recompaction and doesn't have to be fast. + virtual bool IsPathDead(StringPiece s) const = 0; +}; + /// Store a log of every command ran for every build. /// It has a few uses: /// @@ -36,7 +43,7 @@ struct BuildLog { BuildLog(); ~BuildLog(); - bool OpenForWrite(const string& path, string* err); + bool OpenForWrite(const string& path, const BuildLogUser& user, string* err); bool RecordCommand(Edge* edge, int start_time, int end_time, TimeStamp restat_mtime = 0); void Close(); @@ -72,7 +79,7 @@ struct BuildLog { bool WriteEntry(FILE* f, const LogEntry& entry); /// Rewrite the known log entries, throwing away old data. - bool Recompact(const string& path, string* err); + bool Recompact(const string& path, const BuildLogUser& user, string* err); typedef ExternalStringHashMap<LogEntry*>::Type Entries; const Entries& entries() const { return entries_; } diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc index a09beb8..810c065 100644 --- a/src/build_log_perftest.cc +++ b/src/build_log_perftest.cc @@ -28,10 +28,15 @@ const char kTestFilename[] = "BuildLogPerfTest-tempfile"; +struct NoDeadPaths : public BuildLogUser { + virtual bool IsPathDead(StringPiece) const { return false; } +}; + bool WriteTestData(string* err) { BuildLog log; - if (!log.OpenForWrite(kTestFilename, err)) + NoDeadPaths no_dead_paths; + if (!log.OpenForWrite(kTestFilename, no_dead_paths, err)) return false; /* diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 4639bc9..6738c7b 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -30,7 +30,7 @@ namespace { const char kTestFilename[] = "BuildLogTest-tempfile"; -struct BuildLogTest : public StateTestWithBuiltinRules { +struct BuildLogTest : public StateTestWithBuiltinRules, public BuildLogUser { virtual void SetUp() { // In case a crashing test left a stale file behind. unlink(kTestFilename); @@ -38,6 +38,7 @@ struct BuildLogTest : public StateTestWithBuiltinRules { virtual void TearDown() { unlink(kTestFilename); } + virtual bool IsPathDead(StringPiece s) const { return false; } }; TEST_F(BuildLogTest, WriteRead) { @@ -47,7 +48,7 @@ TEST_F(BuildLogTest, WriteRead) { BuildLog log1; string err; - EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err)); + EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); log1.RecordCommand(state_.edges_[0], 15, 18); log1.RecordCommand(state_.edges_[1], 20, 25); @@ -75,7 +76,7 @@ TEST_F(BuildLogTest, FirstWriteAddsSignature) { BuildLog log; string contents, err; - EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + EXPECT_TRUE(log.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); log.Close(); @@ -86,7 +87,7 @@ TEST_F(BuildLogTest, FirstWriteAddsSignature) { EXPECT_EQ(kExpectedVersion, contents); // Opening the file anew shouldn't add a second version string. - EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + EXPECT_TRUE(log.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); log.Close(); @@ -122,7 +123,7 @@ TEST_F(BuildLogTest, Truncate) { BuildLog log1; string err; - EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err)); + EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); log1.RecordCommand(state_.edges_[0], 15, 18); log1.RecordCommand(state_.edges_[1], 20, 25); @@ -137,7 +138,7 @@ TEST_F(BuildLogTest, Truncate) { for (off_t size = statbuf.st_size; size > 0; --size) { BuildLog log2; string err; - EXPECT_TRUE(log2.OpenForWrite(kTestFilename, &err)); + EXPECT_TRUE(log2.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); log2.RecordCommand(state_.edges_[0], 15, 18); log2.RecordCommand(state_.edges_[1], 20, 25); @@ -261,4 +262,44 @@ TEST_F(BuildLogTest, MultiTargetEdge) { ASSERT_EQ(22, e2->end_time); } +struct BuildLogRecompactTest : public BuildLogTest { + virtual bool IsPathDead(StringPiece s) const { return s == "out2"; } +}; + +TEST_F(BuildLogRecompactTest, Recompact) { + AssertParse(&state_, +"build out: cat in\n" +"build out2: cat in\n"); + + BuildLog log1; + string err; + EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); + ASSERT_EQ("", err); + // Record the same edge several times, to trigger recompaction + // the next time the log is opened. + for (int i = 0; i < 200; ++i) + log1.RecordCommand(state_.edges_[0], 15, 18 + i); + log1.RecordCommand(state_.edges_[1], 21, 22); + log1.Close(); + + // Load... + BuildLog log2; + EXPECT_TRUE(log2.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + ASSERT_EQ(2u, log2.entries().size()); + ASSERT_TRUE(log2.LookupByOutput("out")); + ASSERT_TRUE(log2.LookupByOutput("out2")); + // ...and force a recompaction. + EXPECT_TRUE(log2.OpenForWrite(kTestFilename, *this, &err)); + log2.Close(); + + // "out2" is dead, it should've been removed. + BuildLog log3; + EXPECT_TRUE(log2.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, log2.entries().size()); + ASSERT_TRUE(log2.LookupByOutput("out")); + ASSERT_FALSE(log2.LookupByOutput("out2")); +} + } // anonymous namespace diff --git a/src/build_test.cc b/src/build_test.cc index e206cd8..dad69dc 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -44,6 +44,8 @@ struct PlanTest : public StateTestWithBuiltinRules { ASSERT_FALSE(plan_.FindWork()); sort(ret->begin(), ret->end(), CompareEdgesByOutput::cmp); } + + void TestPoolWithDepthOne(const char *test_case); }; TEST_F(PlanTest, Basic) { @@ -197,15 +199,8 @@ TEST_F(PlanTest, DependencyCycle) { ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err); } -TEST_F(PlanTest, PoolWithDepthOne) { - ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, -"pool foobar\n" -" depth = 1\n" -"rule poolcat\n" -" command = cat $in > $out\n" -" pool = foobar\n" -"build out1: poolcat in\n" -"build out2: poolcat in\n")); +void PlanTest::TestPoolWithDepthOne(const char* test_case) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, test_case)); GetNode("out1")->MarkDirty(); GetNode("out2")->MarkDirty(); string err; @@ -239,6 +234,26 @@ TEST_F(PlanTest, PoolWithDepthOne) { ASSERT_EQ(0, edge); } +TEST_F(PlanTest, PoolWithDepthOne) { + TestPoolWithDepthOne( +"pool foobar\n" +" depth = 1\n" +"rule poolcat\n" +" command = cat $in > $out\n" +" pool = foobar\n" +"build out1: poolcat in\n" +"build out2: poolcat in\n"); +} + +TEST_F(PlanTest, ConsolePool) { + TestPoolWithDepthOne( +"rule poolcat\n" +" command = cat $in > $out\n" +" pool = console\n" +"build out1: poolcat in\n" +"build out2: poolcat in\n"); +} + TEST_F(PlanTest, PoolsWithDepthTwo) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "pool foobar\n" @@ -412,7 +427,7 @@ struct FakeCommandRunner : public CommandRunner { VirtualFileSystem* fs_; }; -struct BuildTest : public StateTestWithBuiltinRules { +struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { BuildTest() : config_(MakeConfig()), command_runner_(&fs_), builder_(&state_, config_, NULL, NULL, &fs_), status_(config_) { @@ -435,6 +450,8 @@ struct BuildTest : public StateTestWithBuiltinRules { builder_.command_runner_.release(); } + virtual bool IsPathDead(StringPiece s) const { return false; } + /// Rebuild target in the 'working tree' (fs_). /// State of command_runner_ and logs contents (if specified) ARE MODIFIED. /// Handy to check for NOOP builds, and higher-level rebuild tests. @@ -469,7 +486,7 @@ void BuildTest::RebuildTarget(const string& target, const char* manifest, BuildLog build_log, *pbuild_log = NULL; if (log_path) { ASSERT_TRUE(build_log.Load(log_path, &err)); - ASSERT_TRUE(build_log.OpenForWrite(log_path, &err)); + ASSERT_TRUE(build_log.OpenForWrite(log_path, *this, &err)); ASSERT_EQ("", err); pbuild_log = &build_log; } @@ -504,6 +521,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { commands_ran_.push_back(edge->EvaluateCommand()); if (edge->rule().name() == "cat" || edge->rule().name() == "cat_rsp" || + edge->rule().name() == "cat_rsp_out" || edge->rule().name() == "cc" || edge->rule().name() == "touch" || edge->rule().name() == "touch-interrupt") { @@ -513,7 +531,8 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { } } else if (edge->rule().name() == "true" || edge->rule().name() == "fail" || - edge->rule().name() == "interrupt") { + edge->rule().name() == "interrupt" || + edge->rule().name() == "console") { // Don't do anything. } else { printf("unknown command\n"); @@ -537,6 +556,15 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { return true; } + if (edge->rule().name() == "console") { + if (edge->use_console()) + result->status = ExitSuccess; + else + result->status = ExitFailure; + last_command_ = NULL; + return true; + } + if (edge->rule().name() == "fail") result->status = ExitFailure; else @@ -748,13 +776,13 @@ TEST_F(BuildTest, DepFileMissing) { string err; ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule cc\n command = cc $in\n depfile = $out.d\n" -"build foo.o: cc foo.c\n")); +"build fo$ o.o: cc foo.c\n")); fs_.Create("foo.c", ""); - EXPECT_TRUE(builder_.AddTarget("foo.o", &err)); + EXPECT_TRUE(builder_.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); ASSERT_EQ(1u, fs_.files_read_.size()); - EXPECT_EQ("foo.o.d", fs_.files_read_[0]); + EXPECT_EQ("fo o.o.d", fs_.files_read_[0]); } TEST_F(BuildTest, DepFileOK) { @@ -1275,14 +1303,20 @@ TEST_F(BuildTest, RspFileSuccess) " command = cat $rspfile > $out\n" " rspfile = $rspfile\n" " rspfile_content = $long_command\n" + "rule cat_rsp_out\n" + " command = cat $rspfile > $out\n" + " rspfile = $out.rsp\n" + " rspfile_content = $long_command\n" "build out1: cat in\n" "build out2: cat_rsp in\n" - " rspfile = out2.rsp\n" + " rspfile = out 2.rsp\n" + " long_command = Some very long command\n" + "build out$ 3: cat_rsp_out in\n" " long_command = Some very long command\n")); fs_.Create("out1", ""); fs_.Create("out2", ""); - fs_.Create("out3", ""); + fs_.Create("out 3", ""); fs_.Tick(); @@ -1293,20 +1327,24 @@ TEST_F(BuildTest, RspFileSuccess) ASSERT_EQ("", err); EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("out 3", &err)); + ASSERT_EQ("", err); size_t files_created = fs_.files_created_.size(); size_t files_removed = fs_.files_removed_.size(); EXPECT_TRUE(builder_.Build(&err)); - ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // cat + cat_rsp + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); - // The RSP file was created - ASSERT_EQ(files_created + 1, fs_.files_created_.size()); - ASSERT_EQ(1u, fs_.files_created_.count("out2.rsp")); + // The RSP files were created + ASSERT_EQ(files_created + 2, fs_.files_created_.size()); + ASSERT_EQ(1u, fs_.files_created_.count("out 2.rsp")); + ASSERT_EQ(1u, fs_.files_created_.count("out 3.rsp")); - // The RSP file was removed - ASSERT_EQ(files_removed + 1, fs_.files_removed_.size()); - ASSERT_EQ(1u, fs_.files_removed_.count("out2.rsp")); + // The RSP files were removed + ASSERT_EQ(files_removed + 2, fs_.files_removed_.size()); + ASSERT_EQ(1u, fs_.files_removed_.count("out 2.rsp")); + ASSERT_EQ(1u, fs_.files_removed_.count("out 3.rsp")); } // Test that RSP file is created but not removed for commands, which fail @@ -1777,7 +1815,7 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { string err; const char* manifest = "rule cc\n command = cc $in\n depfile = $out.d\n deps = gcc\n" - "build foo.o: cc foo.c\n"; + "build fo$ o.o: cc foo.c\n"; fs_.Create("foo.c", ""); @@ -1792,9 +1830,9 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { Builder builder(&state, config_, NULL, &deps_log, &fs_); builder.command_runner_.reset(&command_runner_); - EXPECT_TRUE(builder.AddTarget("foo.o", &err)); + EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); - fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n"); + fs_.Create("fo o.o.d", "fo\\ o.o: blah.h bar.h\n"); EXPECT_TRUE(builder.Build(&err)); EXPECT_EQ("", err); @@ -1817,10 +1855,10 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { Edge* edge = state.edges_.back(); state.GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing. - EXPECT_TRUE(builder.AddTarget("foo.o", &err)); + EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); - // Expect three new edges: one generating foo.o, and two more from + // Expect three new edges: one generating fo o.o, and two more from // loading the depfile. ASSERT_EQ(3u, state.edges_.size()); // Expect our edge to now have three inputs: foo.c and two headers. @@ -1909,3 +1947,20 @@ TEST_F(BuildWithDepsLogTest, RestatMissingDepfileDepslog) { RebuildTarget("out", manifest, "build_log", "ninja_deps2"); ASSERT_EQ(0u, command_runner_.commands_ran_.size()); } + +TEST_F(BuildTest, Console) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule console\n" +" command = console\n" +" pool = console\n" +"build cons: console in.txt\n")); + + fs_.Create("in.txt", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("cons", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); +} diff --git a/src/clean.cc b/src/clean.cc index 5d1974e..98c638c 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -80,11 +80,11 @@ bool Cleaner::IsAlreadyRemoved(const string& path) { } void Cleaner::RemoveEdgeFiles(Edge* edge) { - string depfile = edge->GetBinding("depfile"); + string depfile = edge->GetUnescapedDepfile(); if (!depfile.empty()) Remove(depfile); - string rspfile = edge->GetBinding("rspfile"); + string rspfile = edge->GetUnescapedRspfile(); if (!rspfile.empty()) Remove(rspfile); } diff --git a/src/clean_test.cc b/src/clean_test.cc index 04cff73..5869bbb 100644 --- a/src/clean_test.cc +++ b/src/clean_test.cc @@ -286,8 +286,7 @@ TEST_F(CleanTest, CleanRspFile) { " rspfile = $rspfile\n" " rspfile_content=$in\n" "build out1: cc in1\n" -" rspfile = cc1.rsp\n" -" rspfile_content=$in\n")); +" rspfile = cc1.rsp\n")); fs_.Create("out1", ""); fs_.Create("cc1.rsp", ""); @@ -307,10 +306,9 @@ TEST_F(CleanTest, CleanRsp) { "build out1: cat in1\n" "build in2: cat_rsp src2\n" " rspfile=in2.rsp\n" -" rspfile_content=$in\n" "build out2: cat_rsp in2\n" " rspfile=out2.rsp\n" -" rspfile_content=$in\n")); +)); fs_.Create("in1", ""); fs_.Create("out1", ""); fs_.Create("in2.rsp", ""); @@ -336,8 +334,6 @@ TEST_F(CleanTest, CleanRsp) { EXPECT_EQ(0, fs_.Stat("out2")); EXPECT_EQ(0, fs_.Stat("in2.rsp")); EXPECT_EQ(0, fs_.Stat("out2.rsp")); - - fs_.files_removed_.clear(); } TEST_F(CleanTest, CleanFailure) { @@ -372,3 +368,31 @@ TEST_F(CleanTest, CleanPhony) { EXPECT_EQ(2, cleaner.cleaned_files_count()); EXPECT_NE(0, fs_.Stat("phony")); } + +TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc_dep\n" +" command = cc $in > $out\n" +" depfile = $out.d\n" +"rule cc_rsp\n" +" command = cc $in > $out\n" +" rspfile = $out.rsp\n" +" rspfile_content = $in\n" +"build out$ 1: cc_dep in$ 1\n" +"build out$ 2: cc_rsp in$ 1\n" +)); + fs_.Create("out 1", ""); + fs_.Create("out 2", ""); + fs_.Create("out 1.d", ""); + fs_.Create("out 2.rsp", ""); + + Cleaner cleaner(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner.CleanAll()); + EXPECT_EQ(4, cleaner.cleaned_files_count()); + EXPECT_EQ(4u, fs_.files_removed_.size()); + + EXPECT_EQ(0, fs_.Stat("out 1")); + EXPECT_EQ(0, fs_.Stat("out 2")); + EXPECT_EQ(0, fs_.Stat("out 1.d")); + EXPECT_EQ(0, fs_.Stat("out 2.rsp")); +} diff --git a/src/debug_flags.cc b/src/debug_flags.cc index 75f1ea5..8065001 100644 --- a/src/debug_flags.cc +++ b/src/debug_flags.cc @@ -15,3 +15,5 @@ bool g_explaining = false; bool g_keep_rsp = false; + +bool g_experimental_statcache = true; diff --git a/src/debug_flags.h b/src/debug_flags.h index ba3ebf3..7965585 100644 --- a/src/debug_flags.h +++ b/src/debug_flags.h @@ -26,4 +26,6 @@ extern bool g_explaining; extern bool g_keep_rsp; +extern bool g_experimental_statcache; + #endif // NINJA_EXPLAIN_H_ diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 5a30c6b..4ca3943 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -53,10 +53,10 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, + 0, 128, 0, 0, 0, 0, 0, 0, + 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 0, 0, + 128, 128, 128, 0, 0, 128, 0, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, @@ -64,7 +64,7 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 0, 0, 0, 128, 0, + 128, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -84,42 +84,59 @@ bool DepfileParser::Parse(string* content, string* err) { }; yych = *in; - if (yych <= '[') { + if (yych <= '=') { if (yych <= '$') { - if (yych <= 0x00) goto yy7; - if (yych <= ' ') goto yy9; - if (yych <= '#') goto yy6; - goto yy4; + if (yych <= ' ') { + if (yych <= 0x00) goto yy7; + goto yy9; + } else { + if (yych <= '!') goto yy5; + if (yych <= '#') goto yy9; + goto yy4; + } } else { - if (yych <= '=') goto yy6; - if (yych <= '?') goto yy9; - if (yych <= 'Z') goto yy6; - goto yy9; + if (yych <= '*') { + if (yych <= '\'') goto yy9; + if (yych <= ')') goto yy5; + goto yy9; + } else { + if (yych <= ':') goto yy5; + if (yych <= '<') goto yy9; + goto yy5; + } } } else { - if (yych <= '`') { - if (yych <= '\\') goto yy2; - if (yych == '_') goto yy6; - goto yy9; + if (yych <= '^') { + if (yych <= 'Z') { + if (yych <= '?') goto yy9; + goto yy5; + } else { + if (yych != '\\') goto yy9; + } } else { - if (yych <= 'z') goto yy6; - if (yych == '~') goto yy6; - goto yy9; + if (yych <= '{') { + if (yych == '`') goto yy9; + goto yy5; + } else { + if (yych <= '|') goto yy9; + if (yych <= '~') goto yy5; + goto yy9; + } } } -yy2: ++in; - if ((yych = *in) <= '#') { - if (yych <= '\n') { + if ((yych = *in) <= '"') { + if (yych <= '\f') { if (yych <= 0x00) goto yy3; - if (yych <= '\t') goto yy14; + if (yych != '\n') goto yy14; } else { + if (yych <= '\r') goto yy3; if (yych == ' ') goto yy16; - if (yych <= '"') goto yy14; - goto yy16; + goto yy14; } } else { if (yych <= 'Z') { + if (yych <= '#') goto yy16; if (yych == '*') goto yy16; goto yy14; } else { @@ -135,10 +152,14 @@ yy3: break; } yy4: + yych = *++in; + if (yych == '$') goto yy12; + goto yy3; +yy5: ++in; - if ((yych = *in) == '$') goto yy12; + yych = *in; goto yy11; -yy5: +yy6: { // Got a span of plain text. int len = (int)(in - start); @@ -148,9 +169,6 @@ yy5: out += len; continue; } -yy6: - yych = *++in; - goto yy11; yy7: ++in; { @@ -166,12 +184,9 @@ yy11: if (yybm[0+yych] & 128) { goto yy10; } - goto yy5; + goto yy6; yy12: ++in; - if (yybm[0+(yych = *in)] & 128) { - goto yy10; - } { // De-escape dollar character. *out++ = '$'; @@ -211,7 +226,7 @@ yy16: } else if (!out_.str_) { out_ = StringPiece(filename, len); } else if (out_ != StringPiece(filename, len)) { - *err = "depfile has multiple output paths."; + *err = "depfile has multiple output paths"; return false; } } diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index cf24a09..b59baf0 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -67,13 +67,13 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = '$'; continue; } - '\\' [^\000\n] { + '\\' [^\000\r\n] { // Let backslash before other characters through verbatim. *out++ = '\\'; *out++ = yych; continue; } - [a-zA-Z0-9+,/_:.~()@=-!]+ { + [a-zA-Z0-9+,/_:.~()}{@=!-]+ { // Got a span of plain text. int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. @@ -108,7 +108,7 @@ bool DepfileParser::Parse(string* content, string* err) { } else if (!out_.str_) { out_ = StringPiece(filename, len); } else if (out_ != StringPiece(filename, len)) { - *err = "depfile has multiple output paths."; + *err = "depfile has multiple output paths"; return false; } } diff --git a/src/parser_perftest.cc b/src/depfile_parser_perftest.cc index b215221..b215221 100644 --- a/src/parser_perftest.cc +++ b/src/depfile_parser_perftest.cc diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index 0f6771a..a5f3321 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -58,6 +58,17 @@ TEST_F(DepfileParserTest, Continuation) { EXPECT_EQ(2u, parser_.ins_.size()); } +TEST_F(DepfileParserTest, CarriageReturnContinuation) { + string err; + EXPECT_TRUE(Parse( +"foo.o: \\\r\n" +" bar.h baz.h\r\n", + &err)); + ASSERT_EQ("", err); + EXPECT_EQ("foo.o", parser_.out_.AsString()); + EXPECT_EQ(2u, parser_.ins_.size()); +} + TEST_F(DepfileParserTest, BackSlashes) { string err; EXPECT_TRUE(Parse( @@ -109,16 +120,19 @@ TEST_F(DepfileParserTest, SpecialChars) { string err; EXPECT_TRUE(Parse( "C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n" -" en@quot.header~ t+t-x!=1", +" en@quot.header~ t+t-x!=1 \n" +" openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif", &err)); ASSERT_EQ("", err); EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", parser_.out_.AsString()); - ASSERT_EQ(2u, parser_.ins_.size()); + ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("en@quot.header~", parser_.ins_[0].AsString()); EXPECT_EQ("t+t-x!=1", parser_.ins_[1].AsString()); + EXPECT_EQ("openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif", + parser_.ins_[2].AsString()); } TEST_F(DepfileParserTest, UnifyMultipleOutputs) { @@ -136,4 +150,5 @@ TEST_F(DepfileParserTest, RejectMultipleDifferentOutputs) { // check that multiple different outputs are rejected by the parser string err; EXPECT_FALSE(Parse("foo bar: x y z", &err)); + ASSERT_EQ("depfile has multiple output paths", err); } diff --git a/src/deps_log.cc b/src/deps_log.cc index 4f1214a..61df387 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -325,6 +325,9 @@ bool DepsLog::Recompact(const string& path, string* err) { Deps* deps = deps_[old_id]; if (!deps) continue; // If nodes_[old_id] is a leaf, it has no deps. + if (!IsDepsEntryLiveFor(nodes_[old_id])) + continue; + if (!new_log.RecordDeps(nodes_[old_id], deps->mtime, deps->node_count, deps->nodes)) { new_log.Close(); @@ -351,6 +354,16 @@ bool DepsLog::Recompact(const string& path, string* err) { return true; } +bool DepsLog::IsDepsEntryLiveFor(Node* node) { + // Skip entries that don't have in-edges or whose edges don't have a + // "deps" attribute. They were in the deps log from previous builds, but + // the the files they were for were removed from the build and their deps + // entries are no longer needed. + // (Without the check for "deps", a chain of two or more nodes that each + // had deps wouldn't be collected in a single recompaction.) + return node->in_edge() && !node->in_edge()->GetBinding("deps").empty(); +} + bool DepsLog::UpdateDeps(int out_id, Deps* deps) { if (out_id >= (int)deps_.size()) deps_.resize(out_id + 1); diff --git a/src/deps_log.h b/src/deps_log.h index babf828..cec0257 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -88,6 +88,14 @@ struct DepsLog { /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, string* err); + /// Returns if the deps entry for a node is still reachable from the manifest. + /// + /// The deps log can contain deps entries for files that were built in the + /// past but are no longer part of the manifest. This function returns if + /// this is the case for a given node. This function is slow, don't call + /// it from code that runs on every build. + bool IsDepsEntryLiveFor(Node* node); + /// Used for tests. const vector<Node*>& nodes() const { return nodes_; } const vector<Deps*>& deps() const { return deps_; } diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index 4e6cbac..e8e5138 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -163,10 +163,18 @@ TEST_F(DepsLogTest, DoubleEntry) { // Verify that adding the new deps works and can be compacted away. TEST_F(DepsLogTest, Recompact) { + const char kManifest[] = +"rule cc\n" +" command = cc\n" +" deps = gcc\n" +"build out.o: cc\n" +"build other_out.o: cc\n"; + // Write some deps to the file and grab its size. int file_size; { State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest)); DepsLog log; string err; ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err)); @@ -194,6 +202,7 @@ TEST_F(DepsLogTest, Recompact) { int file_size_2; { State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest)); DepsLog log; string err; ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); @@ -215,8 +224,10 @@ TEST_F(DepsLogTest, Recompact) { // Now reload the file, verify the new deps have replaced the old, then // recompact. + int file_size_3; { State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest)); DepsLog log; string err; ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); @@ -257,9 +268,53 @@ TEST_F(DepsLogTest, Recompact) { // The file should have shrunk a bit for the smaller deps. struct stat st; ASSERT_EQ(0, stat(kTestFilename, &st)); - int file_size_3 = (int)st.st_size; + file_size_3 = (int)st.st_size; ASSERT_LT(file_size_3, file_size_2); } + + // Now reload the file and recompact with an empty manifest. The previous + // entries should be removed. + { + State state; + // Intentionally not parsing kManifest here. + DepsLog log; + string err; + ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); + + Node* out = state.GetNode("out.o"); + DepsLog::Deps* deps = log.GetDeps(out); + ASSERT_TRUE(deps); + ASSERT_EQ(1, deps->mtime); + ASSERT_EQ(1, deps->node_count); + ASSERT_EQ("foo.h", deps->nodes[0]->path()); + + Node* other_out = state.GetNode("other_out.o"); + deps = log.GetDeps(other_out); + ASSERT_TRUE(deps); + ASSERT_EQ(1, deps->mtime); + ASSERT_EQ(2, deps->node_count); + ASSERT_EQ("foo.h", deps->nodes[0]->path()); + ASSERT_EQ("baz.h", deps->nodes[1]->path()); + + ASSERT_TRUE(log.Recompact(kTestFilename, &err)); + + // The previous entries should have been removed. + deps = log.GetDeps(out); + ASSERT_FALSE(deps); + + deps = log.GetDeps(other_out); + ASSERT_FALSE(deps); + + // The .h files pulled in via deps should no longer have ids either. + ASSERT_EQ(-1, state.LookupNode("foo.h")->id()); + ASSERT_EQ(-1, state.LookupNode("baz.h")->id()); + + // The file should have shrunk more. + struct stat st; + ASSERT_EQ(0, stat(kTestFilename, &st)); + int file_size_4 = (int)st.st_size; + ASSERT_LT(file_size_4, file_size_3); + } } // Verify that invalid file headers cause a new build. diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 3233144..ae2146e 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -14,6 +14,8 @@ #include "disk_interface.h" +#include <algorithm> + #include <errno.h> #include <stdio.h> #include <string.h> @@ -31,15 +33,16 @@ namespace { string DirName(const string& path) { #ifdef _WIN32 - const char kPathSeparator = '\\'; + const char kPathSeparators[] = "\\/"; #else - const char kPathSeparator = '/'; + const char kPathSeparators[] = "/"; #endif - - string::size_type slash_pos = path.rfind(kPathSeparator); + string::size_type slash_pos = path.find_last_of(kPathSeparators); if (slash_pos == string::npos) return string(); // Nothing to do. - while (slash_pos > 0 && path[slash_pos - 1] == kPathSeparator) + const char* const kEnd = kPathSeparators + strlen(kPathSeparators); + while (slash_pos > 0 && + std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd) --slash_pos; return path.substr(0, slash_pos); } @@ -52,6 +55,80 @@ int MakeDir(const string& path) { #endif } +#ifdef _WIN32 +TimeStamp TimeStampFromFileTime(const FILETIME& filetime) { + // FILETIME is in 100-nanosecond increments since the Windows epoch. + // We don't much care about epoch correctness but we do want the + // resulting value to fit in an integer. + uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | + ((uint64_t)filetime.dwLowDateTime); + mtime /= 1000000000LL / 100; // 100ns -> s. + mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years). + return (TimeStamp)mtime; +} + +TimeStamp StatSingleFile(const string& path, bool quiet) { + WIN32_FILE_ATTRIBUTE_DATA attrs; + if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) { + DWORD err = GetLastError(); + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + return 0; + if (!quiet) { + Error("GetFileAttributesEx(%s): %s", path.c_str(), + GetLastErrorString().c_str()); + } + return -1; + } + return TimeStampFromFileTime(attrs.ftLastWriteTime); +} + +#pragma warning(push) +#pragma warning(disable: 4996) // GetVersionExA is deprecated post SDK 8.1. +bool IsWindows7OrLater() { + OSVERSIONINFO version_info = { sizeof(version_info) }; + if (!GetVersionEx(&version_info)) + Fatal("GetVersionEx: %s", GetLastErrorString().c_str()); + return version_info.dwMajorVersion > 6 || + version_info.dwMajorVersion == 6 && version_info.dwMinorVersion >= 1; +} +#pragma warning(pop) + +bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps, + bool quiet) { + // FindExInfoBasic is 30% faster than FindExInfoStandard. + static bool can_use_basic_info = IsWindows7OrLater(); + // This is not in earlier SDKs. + const FINDEX_INFO_LEVELS kFindExInfoBasic = + static_cast<FINDEX_INFO_LEVELS>(1); + FINDEX_INFO_LEVELS level = + can_use_basic_info ? kFindExInfoBasic : FindExInfoStandard; + WIN32_FIND_DATAA ffd; + HANDLE find_handle = FindFirstFileExA((dir + "\\*").c_str(), level, &ffd, + FindExSearchNameMatch, NULL, 0); + + if (find_handle == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + return true; + if (!quiet) { + Error("FindFirstFileExA(%s): %s", dir.c_str(), + GetLastErrorString().c_str()); + } + return false; + } + do { + if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + string lowername = ffd.cFileName; + transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower); + stamps->insert(make_pair(lowername, + TimeStampFromFileTime(ffd.ftLastWriteTime))); + } while (FindNextFileA(find_handle, &ffd)); + FindClose(find_handle); + return true; +} +#endif // _WIN32 + } // namespace // DiskInterface --------------------------------------------------------------- @@ -75,7 +152,7 @@ bool DiskInterface::MakeDirs(const string& path) { // RealDiskInterface ----------------------------------------------------------- -TimeStamp RealDiskInterface::Stat(const string& path) { +TimeStamp RealDiskInterface::Stat(const string& path) const { #ifdef _WIN32 // MSDN: "Naming Files, Paths, and Namespaces" // http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx @@ -86,26 +163,25 @@ TimeStamp RealDiskInterface::Stat(const string& path) { } return -1; } - WIN32_FILE_ATTRIBUTE_DATA attrs; - if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) { - DWORD err = GetLastError(); - if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) - return 0; - if (!quiet_) { - Error("GetFileAttributesEx(%s): %s", path.c_str(), - GetLastErrorString().c_str()); + if (!use_cache_) + return StatSingleFile(path, quiet_); + + string dir = DirName(path); + string base(path.substr(dir.size() ? dir.size() + 1 : 0)); + + transform(dir.begin(), dir.end(), dir.begin(), ::tolower); + transform(base.begin(), base.end(), base.begin(), ::tolower); + + Cache::iterator ci = cache_.find(dir); + if (ci == cache_.end()) { + ci = cache_.insert(make_pair(dir, DirCache())).first; + if (!StatAllFilesInDir(dir.empty() ? "." : dir, &ci->second, quiet_)) { + cache_.erase(ci); + return -1; } - return -1; } - const FILETIME& filetime = attrs.ftLastWriteTime; - // FILETIME is in 100-nanosecond increments since the Windows epoch. - // We don't much care about epoch correctness but we do want the - // resulting value to fit in an integer. - uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | - ((uint64_t)filetime.dwLowDateTime); - mtime /= 1000000000LL / 100; // 100ns -> s. - mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years). - return (TimeStamp)mtime; + DirCache::iterator di = ci->second.find(base); + return di != ci->second.end() ? di->second : 0; #else struct stat st; if (stat(path.c_str(), &st) < 0) { @@ -146,6 +222,9 @@ bool RealDiskInterface::WriteFile(const string& path, const string& contents) { bool RealDiskInterface::MakeDir(const string& path) { if (::MakeDir(path) < 0) { + if (errno == EEXIST) { + return true; + } Error("mkdir(%s): %s", path.c_str(), strerror(errno)); return false; } @@ -175,3 +254,11 @@ int RealDiskInterface::RemoveFile(const string& path) { return 0; } } + +void RealDiskInterface::AllowStatCache(bool allow) { +#ifdef _WIN32 + use_cache_ = allow; + if (!use_cache_) + cache_.clear(); +#endif +} diff --git a/src/disk_interface.h b/src/disk_interface.h index ff1e21c..a13bced 100644 --- a/src/disk_interface.h +++ b/src/disk_interface.h @@ -15,6 +15,7 @@ #ifndef NINJA_DISK_INTERFACE_H_ #define NINJA_DISK_INTERFACE_H_ +#include <map> #include <string> using namespace std; @@ -29,7 +30,7 @@ struct DiskInterface { /// stat() a file, returning the mtime, or 0 if missing and -1 on /// other errors. - virtual TimeStamp Stat(const string& path) = 0; + virtual TimeStamp Stat(const string& path) const = 0; /// Create a directory, returning false on failure. virtual bool MakeDir(const string& path) = 0; @@ -55,9 +56,13 @@ struct DiskInterface { /// Implementation of DiskInterface that actually hits the disk. struct RealDiskInterface : public DiskInterface { - RealDiskInterface() : quiet_(false) {} + RealDiskInterface() : quiet_(false) +#ifdef _WIN32 + , use_cache_(false) +#endif + {} virtual ~RealDiskInterface() {} - virtual TimeStamp Stat(const string& path); + virtual TimeStamp Stat(const string& path) const; virtual bool MakeDir(const string& path); virtual bool WriteFile(const string& path, const string& contents); virtual string ReadFile(const string& path, string* err); @@ -65,6 +70,21 @@ struct RealDiskInterface : public DiskInterface { /// Whether to print on errors. Used to make a test quieter. bool quiet_; + + /// Whether stat information can be cached. Only has an effect on Windows. + void AllowStatCache(bool allow); + + private: +#ifdef _WIN32 + /// Whether stat information can be cached. + bool use_cache_; + + typedef map<string, TimeStamp> DirCache; + // TODO: Neither a map nor a hashmap seems ideal here. If the statcache + // works out, come up with a better data structure. + typedef map<string, DirCache> Cache; + mutable Cache cache_; +#endif }; #endif // NINJA_DISK_INTERFACE_H_ diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 55822a6..f4e0bb0 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -76,6 +76,33 @@ TEST_F(DiskInterfaceTest, StatExistingFile) { EXPECT_GT(disk_.Stat("file"), 1); } +#ifdef _WIN32 +TEST_F(DiskInterfaceTest, StatCache) { + disk_.AllowStatCache(true); + + ASSERT_TRUE(Touch("file1")); + ASSERT_TRUE(Touch("fiLE2")); + ASSERT_TRUE(disk_.MakeDir("subdir")); + ASSERT_TRUE(Touch("subdir\\subfile1")); + ASSERT_TRUE(Touch("subdir\\SUBFILE2")); + ASSERT_TRUE(Touch("subdir\\SUBFILE3")); + + EXPECT_GT(disk_.Stat("FIle1"), 1); + EXPECT_GT(disk_.Stat("file1"), 1); + + EXPECT_GT(disk_.Stat("subdir/subfile2"), 1); + EXPECT_GT(disk_.Stat("sUbdir\\suBFile1"), 1); + + // Test error cases. + disk_.quiet_ = true; + string bad_path("cc:\\foo"); + EXPECT_EQ(-1, disk_.Stat(bad_path)); + EXPECT_EQ(-1, disk_.Stat(bad_path)); + EXPECT_EQ(0, disk_.Stat("nosuchfile")); + EXPECT_EQ(0, disk_.Stat("nosuchdir/nosuchfile")); +} +#endif + TEST_F(DiskInterfaceTest, ReadFile) { string err; EXPECT_EQ("", disk_.ReadFile("foobar", &err)); @@ -93,7 +120,18 @@ TEST_F(DiskInterfaceTest, ReadFile) { } TEST_F(DiskInterfaceTest, MakeDirs) { - EXPECT_TRUE(disk_.MakeDirs("path/with/double//slash/")); + string path = "path/with/double//slash/"; + EXPECT_TRUE(disk_.MakeDirs(path.c_str())); + FILE* f = fopen((path + "a_file").c_str(), "w"); + EXPECT_TRUE(f); + EXPECT_EQ(0, fclose(f)); +#ifdef _WIN32 + string path2 = "another\\with\\back\\\\slashes\\"; + EXPECT_TRUE(disk_.MakeDirs(path2.c_str())); + FILE* f2 = fopen((path2 + "a_file").c_str(), "w"); + EXPECT_TRUE(f2); + EXPECT_EQ(0, fclose(f2)); +#endif } TEST_F(DiskInterfaceTest, RemoveFile) { @@ -109,7 +147,7 @@ struct StatTest : public StateTestWithBuiltinRules, StatTest() : scan_(&state_, NULL, NULL, this) {} // DiskInterface implementation. - virtual TimeStamp Stat(const string& path); + virtual TimeStamp Stat(const string& path) const; virtual bool WriteFile(const string& path, const string& contents) { assert(false); return true; @@ -129,12 +167,12 @@ struct StatTest : public StateTestWithBuiltinRules, DependencyScan scan_; map<string, TimeStamp> mtimes_; - vector<string> stats_; + mutable vector<string> stats_; }; -TimeStamp StatTest::Stat(const string& path) { +TimeStamp StatTest::Stat(const string& path) const { stats_.push_back(path); - map<string, TimeStamp>::iterator i = mtimes_.find(path); + map<string, TimeStamp>::const_iterator i = mtimes_.find(path); if (i == mtimes_.end()) return 0; // File not found. return i->second; diff --git a/src/edit_distance.cc b/src/edit_distance.cc index cc4483f..9553c6e 100644 --- a/src/edit_distance.cc +++ b/src/edit_distance.cc @@ -14,6 +14,7 @@ #include "edit_distance.h" +#include <algorithm> #include <vector> int EditDistance(const StringPiece& s1, diff --git a/src/graph.cc b/src/graph.cc index 9801a7b..aa9c0e8 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -215,7 +215,10 @@ bool Edge::AllInputsReady() const { /// An Env for an Edge, providing $in and $out. struct EdgeEnv : public Env { - explicit EdgeEnv(Edge* edge) : edge_(edge) {} + enum EscapeKind { kShellEscape, kDoNotEscape }; + + explicit EdgeEnv(Edge* edge, EscapeKind escape) + : edge_(edge), escape_in_out_(escape) {} virtual string LookupVariable(const string& var); /// Given a span of Nodes, construct a list of paths suitable for a command @@ -225,6 +228,7 @@ struct EdgeEnv : public Env { char sep); Edge* edge_; + EscapeKind escape_in_out_; }; string EdgeEnv::LookupVariable(const string& var) { @@ -253,10 +257,12 @@ string EdgeEnv::MakePathList(vector<Node*>::iterator begin, if (!result.empty()) result.push_back(sep); const string& path = (*i)->path(); - if (path.find(" ") != string::npos) { - result.append("\""); - result.append(path); - result.append("\""); + if (escape_in_out_ == kShellEscape) { +#if _WIN32 + GetWin32EscapedString(path, &result); +#else + GetShellEscapedString(path, &result); +#endif } else { result.append(path); } @@ -275,7 +281,7 @@ string Edge::EvaluateCommand(bool incl_rsp_file) { } string Edge::GetBinding(const string& key) { - EdgeEnv env(this); + EdgeEnv env(this, EdgeEnv::kShellEscape); return env.LookupVariable(key); } @@ -283,6 +289,16 @@ bool Edge::GetBindingBool(const string& key) { return !GetBinding(key).empty(); } +string Edge::GetUnescapedDepfile() { + EdgeEnv env(this, EdgeEnv::kDoNotEscape); + return env.LookupVariable("depfile"); +} + +string Edge::GetUnescapedRspfile() { + EdgeEnv env(this, EdgeEnv::kDoNotEscape); + return env.LookupVariable("rspfile"); +} + void Edge::Dump(const char* prefix) const { printf("%s[ ", prefix); for (vector<Node*>::const_iterator i = inputs_.begin(); @@ -308,6 +324,10 @@ bool Edge::is_phony() const { return rule_ == &State::kPhonyRule; } +bool Edge::use_console() const { + return pool() == &State::kConsolePool; +} + void Node::Dump(const char* prefix) const { printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", prefix, path().c_str(), this, @@ -330,7 +350,7 @@ bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) { if (!deps_type.empty()) return LoadDepsFromLog(edge, err); - string depfile = edge->GetBinding("depfile"); + string depfile = edge->GetUnescapedDepfile(); if (!depfile.empty()) return LoadDepFile(edge, depfile, err); diff --git a/src/graph.h b/src/graph.h index 868413c..66e31b5 100644 --- a/src/graph.h +++ b/src/graph.h @@ -146,9 +146,15 @@ struct Edge { /// full contents of a response file (if applicable) string EvaluateCommand(bool incl_rsp_file = false); + /// Returns the shell-escaped value of |key|. string GetBinding(const string& key); bool GetBindingBool(const string& key); + /// Like GetBinding("depfile"), but without shell escaping. + string GetUnescapedDepfile(); + /// Like GetBinding("rspfile"), but without shell escaping. + string GetUnescapedRspfile(); + void Dump(const char* prefix="") const; const Rule* rule_; @@ -183,6 +189,7 @@ struct Edge { } bool is_phony() const; + bool use_console() const; }; diff --git a/src/graph_test.cc b/src/graph_test.cc index 8521216..14dc678 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -139,13 +139,18 @@ TEST_F(GraphTest, RootNodes) { } } -TEST_F(GraphTest, VarInOutQuoteSpaces) { +TEST_F(GraphTest, VarInOutPathEscaping) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, -"build a$ b: cat nospace with$ space nospace2\n")); +"build a$ b: cat no'space with$ space$$ no\"space2\n")); Edge* edge = GetNode("a b")->in_edge(); - EXPECT_EQ("cat nospace \"with space\" nospace2 > \"a b\"", +#if _WIN32 + EXPECT_EQ("cat no'space \"with space$\" \"no\\\"space2\" > \"a b\"", edge->EvaluateCommand()); +#else + EXPECT_EQ("cat 'no'\\''space' 'with space$' 'no\"space2' > 'a b'", + edge->EvaluateCommand()); +#endif } // Regression test for https://github.com/martine/ninja/issues/380 diff --git a/src/hash_map.h b/src/hash_map.h index c63aa88..77e7586 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -15,6 +15,7 @@ #ifndef NINJA_MAP_H_ #define NINJA_MAP_H_ +#include <algorithm> #include <string.h> #include "string_piece.h" diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc index 1713d5d..419996f 100644 --- a/src/includes_normalize_test.cc +++ b/src/includes_normalize_test.cc @@ -38,7 +38,7 @@ string GetCurDir() { } // namespace TEST(IncludesNormalize, WithRelative) { - string currentdir = IncludesNormalize::ToLower(GetCurDir()); + string currentdir = GetCurDir(); EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b")); EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), NULL)); diff --git a/src/line_printer.cc b/src/line_printer.cc index 3537e88..ef1609c 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -26,7 +26,7 @@ #include "util.h" -LinePrinter::LinePrinter() : have_blank_line_(true) { +LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) { #ifndef _WIN32 const char* term = getenv("TERM"); smart_terminal_ = isatty(1) && term && string(term) != "dumb"; @@ -43,6 +43,12 @@ LinePrinter::LinePrinter() : have_blank_line_(true) { } void LinePrinter::Print(string to_print, LineType type) { + if (console_locked_) { + line_buffer_ = to_print; + line_type_ = type; + return; + } + #ifdef _WIN32 CONSOLE_SCREEN_BUFFER_INFO csbi; GetConsoleScreenBufferInfo(console_, &csbi); @@ -101,13 +107,46 @@ void LinePrinter::Print(string to_print, LineType type) { } } -void LinePrinter::PrintOnNewLine(const string& to_print) { - if (!have_blank_line_) - printf("\n"); - if (!to_print.empty()) { +void LinePrinter::PrintOrBuffer(const char* data, size_t size) { + if (console_locked_) { + output_buffer_.append(data, size); + } else { // Avoid printf and C strings, since the actual output might contain null // bytes like UTF-16 does (yuck). - fwrite(&to_print[0], sizeof(char), to_print.size(), stdout); + fwrite(data, 1, size, stdout); + } +} + +void LinePrinter::PrintOnNewLine(const string& to_print) { + if (console_locked_ && !line_buffer_.empty()) { + output_buffer_.append(line_buffer_); + output_buffer_.append(1, '\n'); + line_buffer_.clear(); + } + if (!have_blank_line_) { + PrintOrBuffer("\n", 1); + } + if (!to_print.empty()) { + PrintOrBuffer(&to_print[0], to_print.size()); } have_blank_line_ = to_print.empty() || *to_print.rbegin() == '\n'; } + +void LinePrinter::SetConsoleLocked(bool locked) { + if (locked == console_locked_) + return; + + if (locked) + PrintOnNewLine(""); + + console_locked_ = locked; + + if (!locked) { + PrintOnNewLine(output_buffer_); + if (!line_buffer_.empty()) { + Print(line_buffer_, line_type_); + } + output_buffer_.clear(); + line_buffer_.clear(); + } +} diff --git a/src/line_printer.h b/src/line_printer.h index aea2817..55225e5 100644 --- a/src/line_printer.h +++ b/src/line_printer.h @@ -15,6 +15,7 @@ #ifndef NINJA_LINE_PRINTER_H_ #define NINJA_LINE_PRINTER_H_ +#include <stddef.h> #include <string> using namespace std; @@ -37,6 +38,10 @@ struct LinePrinter { /// Prints a string on a new line, not overprinting previous output. void PrintOnNewLine(const string& to_print); + /// Lock or unlock the console. Any output sent to the LinePrinter while the + /// console is locked will not be printed until it is unlocked. + void SetConsoleLocked(bool locked); + private: /// Whether we can do fancy terminal control codes. bool smart_terminal_; @@ -44,9 +49,24 @@ struct LinePrinter { /// Whether the caret is at the beginning of a blank line. bool have_blank_line_; + /// Whether console is locked. + bool console_locked_; + + /// Buffered current line while console is locked. + string line_buffer_; + + /// Buffered line type while console is locked. + LineType line_type_; + + /// Buffered console output while console is locked. + string output_buffer_; + #ifdef _WIN32 void* console_; #endif + + /// Print the given data to the console, or buffer it if it is locked. + void PrintOrBuffer(const char *data, size_t size); }; #endif // NINJA_LINE_PRINTER_H_ diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 20be7f3..6fa4f7c 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -296,16 +296,17 @@ bool ManifestParser::ParseEdge(string* err) { if (!ExpectToken(Lexer::NEWLINE, err)) return false; - // XXX scoped_ptr to handle error case. - BindingEnv* env = new BindingEnv(env_); - - while (lexer_.PeekToken(Lexer::INDENT)) { + // Bindings on edges are rare, so allocate per-edge envs only when needed. + bool hasIdent = lexer_.PeekToken(Lexer::INDENT); + BindingEnv* env = hasIdent ? new BindingEnv(env_) : env_; + while (hasIdent) { string key; EvalString val; if (!ParseLet(&key, &val, err)) return false; env->AddBinding(key, val.Evaluate(env_)); + hasIdent = lexer_.PeekToken(Lexer::INDENT); } Edge* edge = state_->AddEdge(rule); diff --git a/src/manifest_parser_perftest.cc b/src/manifest_parser_perftest.cc new file mode 100644 index 0000000..ca62fb2 --- /dev/null +++ b/src/manifest_parser_perftest.cc @@ -0,0 +1,118 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests manifest parser performance. Expects to be run in ninja's root +// directory. + +#include <numeric> + +#include <errno.h> +#include <stdio.h> +#include <string.h> + +#ifdef _WIN32 +#include "getopt.h" +#include <direct.h> +#else +#include <getopt.h> +#include <unistd.h> +#endif + +#include "disk_interface.h" +#include "graph.h" +#include "manifest_parser.h" +#include "metrics.h" +#include "state.h" +#include "util.h" + +struct RealFileReader : public ManifestParser::FileReader { + virtual bool ReadFile(const string& path, string* content, string* err) { + return ::ReadFile(path, content, err) == 0; + } +}; + +bool WriteFakeManifests(const string& dir) { + RealDiskInterface disk_interface; + if (disk_interface.Stat(dir + "/build.ninja") > 0) + return true; + + printf("Creating manifest data..."); fflush(stdout); + int err = system(("python misc/write_fake_manifests.py " + dir).c_str()); + printf("done.\n"); + return err == 0; +} + +int LoadManifests(bool measure_command_evaluation) { + string err; + RealFileReader file_reader; + State state; + ManifestParser parser(&state, &file_reader); + if (!parser.Load("build.ninja", &err)) { + fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); + exit(1); + } + // Doing an empty build involves reading the manifest and evaluating all + // commands required for the requested targets. So include command + // evaluation in the perftest by default. + int optimization_guard = 0; + if (measure_command_evaluation) + for (size_t i = 0; i < state.edges_.size(); ++i) + optimization_guard += state.edges_[i]->EvaluateCommand().size(); + return optimization_guard; +} + +int main(int argc, char* argv[]) { + bool measure_command_evaluation = true; + int opt; + while ((opt = getopt(argc, argv, const_cast<char*>("fh"))) != -1) { + switch (opt) { + case 'f': + measure_command_evaluation = false; + break; + case 'h': + default: + printf("usage: manifest_parser_perftest\n" +"\n" +"options:\n" +" -f only measure manifest load time, not command evaluation time\n" + ); + return 1; + } + } + + const char kManifestDir[] = "build/manifest_perftest"; + + if (!WriteFakeManifests(kManifestDir)) { + fprintf(stderr, "Failed to write test data\n"); + return 1; + } + + if (chdir(kManifestDir) < 0) + Fatal("chdir: %s", strerror(errno)); + + const int kNumRepetitions = 5; + vector<int> times; + for (int i = 0; i < kNumRepetitions; ++i) { + int64_t start = GetTimeMillis(); + int optimization_guard = LoadManifests(measure_command_evaluation); + int delta = (int)(GetTimeMillis() - start); + printf("%dms (hash: %x)\n", delta, optimization_guard); + times.push_back(delta); + } + + int min = *min_element(times.begin(), times.end()); + int max = *max_element(times.begin(), times.end()); + float total = accumulate(times.begin(), times.end(), 0.0f); + printf("min %dms max %dms avg %.1fms\n", min, max, total / times.size()); +} diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 5ed1584..152b965 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -236,7 +236,11 @@ TEST_F(ParserTest, Dollars) { "build $x: foo y\n" )); EXPECT_EQ("$dollar", state.bindings_.LookupVariable("x")); +#ifdef _WIN32 EXPECT_EQ("$dollarbar$baz$blah", state.edges_[0]->EvaluateCommand()); +#else + EXPECT_EQ("'$dollar'bar$baz$blah", state.edges_[0]->EvaluateCommand()); +#endif } TEST_F(ParserTest, EscapeSpaces) { @@ -762,6 +766,21 @@ TEST_F(ParserTest, MissingSubNinja) { , err); } +TEST_F(ParserTest, DuplicateRuleInDifferentSubninjas) { + // Test that rules live in a global namespace and aren't scoped to subninjas. + files_["test.ninja"] = "rule cat\n" + " command = cat\n"; + ManifestParser parser(&state, this); + string err; + EXPECT_FALSE(parser.ParseTest("rule cat\n" + " command = cat\n" + "subninja test.ninja\n", &err)); + EXPECT_EQ("test.ninja:1: duplicate rule 'cat'\n" + "rule cat\n" + " ^ near here" + , err); +} + TEST_F(ParserTest, Include) { files_["include.ninja"] = "var = inner\n"; ASSERT_NO_FATAL_FAILURE(AssertParse( diff --git a/src/metrics.cc b/src/metrics.cc index ca4f97a..a7d3c7a 100644 --- a/src/metrics.cc +++ b/src/metrics.cc @@ -24,6 +24,8 @@ #include <windows.h> #endif +#include <algorithm> + #include "util.h" Metrics* g_metrics = NULL; diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index 7c45029..e465279 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -48,14 +48,15 @@ string EscapeForDepfile(const string& path) { } // static -string CLParser::FilterShowIncludes(const string& line) { - static const char kMagicPrefix[] = "Note: including file: "; +string CLParser::FilterShowIncludes(const string& line, + const string& deps_prefix) { + const string kDepsPrefixEnglish = "Note: including file: "; const char* in = line.c_str(); const char* end = in + line.size(); - - if (end - in > (int)sizeof(kMagicPrefix) - 1 && - memcmp(in, kMagicPrefix, sizeof(kMagicPrefix) - 1) == 0) { - in += sizeof(kMagicPrefix) - 1; + const string& prefix = deps_prefix.empty() ? kDepsPrefixEnglish : deps_prefix; + if (end - in > (int)prefix.size() && + memcmp(in, prefix.c_str(), (int)prefix.size()) == 0) { + in += prefix.size(); while (*in == ' ') ++in; return line.substr(in - line.c_str()); @@ -81,7 +82,7 @@ bool CLParser::FilterInputFilename(string line) { EndsWith(line, ".cpp"); } -string CLParser::Parse(const string& output) { +string CLParser::Parse(const string& output, const string& deps_prefix) { string filtered_output; // Loop over all lines in the output to process them. @@ -92,7 +93,7 @@ string CLParser::Parse(const string& output) { end = output.size(); string line = output.substr(start, end - start); - string include = FilterShowIncludes(line); + string include = FilterShowIncludes(line, deps_prefix); if (!include.empty()) { include = IncludesNormalize::Normalize(include, NULL); if (!IsSystemInclude(include)) @@ -140,7 +141,7 @@ int CLWrapper::Run(const string& command, string* output) { STARTUPINFO startup_info = {}; startup_info.cb = sizeof(STARTUPINFO); startup_info.hStdInput = nul; - startup_info.hStdError = stdout_write; + startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); startup_info.hStdOutput = stdout_write; startup_info.dwFlags |= STARTF_USESTDHANDLES; diff --git a/src/msvc_helper.h b/src/msvc_helper.h index e207485..5d7dcb0 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -27,7 +27,8 @@ struct CLParser { /// Parse a line of cl.exe output and extract /showIncludes info. /// If a dependency is extracted, returns a nonempty string. /// Exposed for testing. - static string FilterShowIncludes(const string& line); + static string FilterShowIncludes(const string& line, + const string& deps_prefix); /// Return true if a mentioned include file is a system path. /// Filtering these out reduces dependency information considerably. @@ -41,7 +42,7 @@ struct CLParser { /// Parse the full output of cl, returning the output (if any) that /// should printed. - string Parse(const string& output); + string Parse(const string& output, const string& deps_prefix); set<string> includes_; }; diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index e3a7846..58bc797 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -31,6 +31,7 @@ void Usage() { "options:\n" " -e ENVFILE load environment block from ENVFILE as environment\n" " -o FILE write output dependency information to FILE.d\n" +" -p STRING localized prefix of msvc's /showIncludes output\n" ); } @@ -84,7 +85,8 @@ int MSVCHelperMain(int argc, char** argv) { { NULL, 0, NULL, 0 } }; int opt; - while ((opt = getopt_long(argc, argv, "e:o:h", kLongOptions, NULL)) != -1) { + string deps_prefix; + while ((opt = getopt_long(argc, argv, "e:o:p:h", kLongOptions, NULL)) != -1) { switch (opt) { case 'e': envfile = optarg; @@ -92,6 +94,9 @@ int MSVCHelperMain(int argc, char** argv) { case 'o': output_filename = optarg; break; + case 'p': + deps_prefix = optarg; + break; case 'h': default: Usage(); @@ -122,7 +127,7 @@ int MSVCHelperMain(int argc, char** argv) { if (output_filename) { CLParser parser; - output = parser.Parse(output); + output = parser.Parse(output, deps_prefix); WriteDepFileOrDie(output_filename, parser); } diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index 02f2863..391c045 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -20,15 +20,19 @@ #include "util.h" TEST(CLParserTest, ShowIncludes) { - ASSERT_EQ("", CLParser::FilterShowIncludes("")); + ASSERT_EQ("", CLParser::FilterShowIncludes("", "")); - ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output")); + ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output", "")); ASSERT_EQ("c:\\Some Files\\foobar.h", CLParser::FilterShowIncludes("Note: including file: " - "c:\\Some Files\\foobar.h")); + "c:\\Some Files\\foobar.h", "")); ASSERT_EQ("c:\\initspaces.h", CLParser::FilterShowIncludes("Note: including file: " - "c:\\initspaces.h")); + "c:\\initspaces.h", "")); + ASSERT_EQ("c:\\initspaces.h", + CLParser::FilterShowIncludes("Non-default prefix: inc file: " + "c:\\initspaces.h", + "Non-default prefix: inc file:")); } TEST(CLParserTest, FilterInputFilename) { @@ -46,8 +50,9 @@ TEST(CLParserTest, ParseSimple) { CLParser parser; string output = parser.Parse( "foo\r\n" - "Note: including file: foo.h\r\n" - "bar\r\n"); + "Note: inc file prefix: foo.h\r\n" + "bar\r\n", + "Note: inc file prefix:"); ASSERT_EQ("foo\nbar\n", output); ASSERT_EQ(1u, parser.includes_.size()); @@ -58,7 +63,8 @@ TEST(CLParserTest, ParseFilenameFilter) { CLParser parser; string output = parser.Parse( "foo.cc\r\n" - "cl: warning\r\n"); + "cl: warning\r\n", + ""); ASSERT_EQ("cl: warning\n", output); } @@ -67,7 +73,8 @@ TEST(CLParserTest, ParseSystemInclude) { string output = parser.Parse( "Note: including file: c:\\Program Files\\foo.h\r\n" "Note: including file: d:\\Microsoft Visual Studio\\bar.h\r\n" - "Note: including file: path.h\r\n"); + "Note: including file: path.h\r\n", + ""); // We should have dropped the first two includes because they look like // system headers. ASSERT_EQ("", output); @@ -80,7 +87,8 @@ TEST(CLParserTest, DuplicatedHeader) { string output = parser.Parse( "Note: including file: foo.h\r\n" "Note: including file: bar.h\r\n" - "Note: including file: foo.h\r\n"); + "Note: including file: foo.h\r\n", + ""); // We should have dropped one copy of foo.h. ASSERT_EQ("", output); ASSERT_EQ(2u, parser.includes_.size()); @@ -91,7 +99,8 @@ TEST(CLParserTest, DuplicatedHeaderPathConverted) { string output = parser.Parse( "Note: including file: sub/foo.h\r\n" "Note: including file: bar.h\r\n" - "Note: including file: sub\\foo.h\r\n"); + "Note: including file: sub\\foo.h\r\n", + ""); // We should have dropped one copy of foo.h. ASSERT_EQ("", output); ASSERT_EQ(2u, parser.includes_.size()); @@ -110,3 +119,10 @@ TEST(MSVCHelperTest, EnvBlock) { cl.Run("cmd /c \"echo foo is %foo%", &output); ASSERT_EQ("foo is bar\r\n", output); } + +TEST(MSVCHelperTest, NoReadOfStderr) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo to stdout&& echo to stderr 1>&2", &output); + ASSERT_EQ("to stdout\r\n", output); +} diff --git a/src/ninja.cc b/src/ninja.cc index a313ecb..a381e83 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -68,7 +68,7 @@ struct Options { /// The Ninja main() loads up a series of data structures; various tools need /// to poke into these, so store them as fields on an object. -struct NinjaMain { +struct NinjaMain : public BuildLogUser { NinjaMain(const char* ninja_command, const BuildConfig& config) : ninja_command_(ninja_command), config_(config) {} @@ -137,6 +137,20 @@ struct NinjaMain { /// Dump the output requested by '-d stats'. void DumpMetrics(); + + virtual bool IsPathDead(StringPiece s) const { + Node* n = state_.LookupNode(s); + // Just checking n isn't enough: If an old output is both in the build log + // and in the deps log, it will have a Node object in state_. (It will also + // have an in edge if one of its inputs is another output that's in the deps + // log, but having a deps edge product an output thats input to another deps + // edge is rare, and the first recompaction will delete all old outputs from + // the deps log, and then a second recompaction will clear the build log, + // which seems good enough for this corner case.) + // Do keep entries around for files which still exist on disk, for + // generators that want to use this information. + return (!n || !n->in_edge()) && disk_interface_.Stat(s.AsString()) == 0; + } }; /// Subtools, accessible via "-t foo". @@ -444,9 +458,7 @@ int NinjaMain::ToolDeps(int argc, char** argv) { if (argc == 0) { for (vector<Node*>::const_iterator ni = deps_log_.nodes().begin(); ni != deps_log_.nodes().end(); ++ni) { - // Only query for targets with an incoming edge and deps - Edge* e = (*ni)->in_edge(); - if (e && !e->GetBinding("deps").empty()) + if (deps_log_.IsDepsEntryLiveFor(*ni)) nodes.push_back(*ni); } } else { @@ -621,6 +633,8 @@ int NinjaMain::ToolCompilationDatabase(int argc, char* argv[]) { putchar('['); for (vector<Edge*>::iterator e = state_.edges_.begin(); e != state_.edges_.end(); ++e) { + if ((*e)->inputs_.empty()) + continue; for (int i = 0; i != argc; ++i) { if ((*e)->rule_->name() == argv[i]) { if (!first) @@ -748,6 +762,9 @@ bool DebugEnable(const string& name) { " stats print operation counts/timing info\n" " explain explain what caused a command to execute\n" " keeprsp don't delete @response files on success\n" +#ifdef _WIN32 +" nostatcache don't batch stat() calls per directory and cache them\n" +#endif "multiple modes can be enabled via -d FOO -d BAR\n"); return false; } else if (name == "stats") { @@ -759,9 +776,13 @@ bool DebugEnable(const string& name) { } else if (name == "keeprsp") { g_keep_rsp = true; return true; + } else if (name == "nostatcache") { + g_experimental_statcache = false; + return true; } else { const char* suggestion = - SpellcheckString(name.c_str(), "stats", "explain", NULL); + SpellcheckString(name.c_str(), "stats", "explain", "keeprsp", + "nostatcache", NULL); if (suggestion) { Error("unknown debug setting '%s', did you mean '%s'?", name.c_str(), suggestion); @@ -789,14 +810,14 @@ bool NinjaMain::OpenBuildLog(bool recompact_only) { } if (recompact_only) { - bool success = build_log_.Recompact(log_path, &err); + bool success = build_log_.Recompact(log_path, *this, &err); if (!success) Error("failed recompaction: %s", err.c_str()); return success; } if (!config_.dry_run) { - if (!build_log_.OpenForWrite(log_path, &err)) { + if (!build_log_.OpenForWrite(log_path, *this, &err)) { Error("opening build log: %s", err.c_str()); return false; } @@ -870,6 +891,8 @@ int NinjaMain::RunBuild(int argc, char** argv) { return 1; } + disk_interface_.AllowStatCache(g_experimental_statcache); + Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_); for (size_t i = 0; i < targets.size(); ++i) { if (!builder.AddTarget(targets[i], &err)) { @@ -883,6 +906,9 @@ int NinjaMain::RunBuild(int argc, char** argv) { } } + // Make sure restat rules do not see stale timestamps. + disk_interface_.AllowStatCache(false); + if (builder.AlreadyUpToDate()) { printf("ninja: no work to do.\n"); return 0; diff --git a/src/state.cc b/src/state.cc index 9b6160b..7258272 100644 --- a/src/state.cc +++ b/src/state.cc @@ -69,11 +69,13 @@ bool Pool::WeightedEdgeCmp(const Edge* a, const Edge* b) { } Pool State::kDefaultPool("", 0); +Pool State::kConsolePool("console", 1); const Rule State::kPhonyRule("phony"); State::State() { AddRule(&kPhonyRule); AddPool(&kDefaultPool); + AddPool(&kConsolePool); } void State::AddRule(const Rule* rule) { @@ -118,9 +120,9 @@ Node* State::GetNode(StringPiece path) { return node; } -Node* State::LookupNode(StringPiece path) { +Node* State::LookupNode(StringPiece path) const { METRIC_RECORD("lookup node"); - Paths::iterator i = paths_.find(path); + Paths::const_iterator i = paths_.find(path); if (i != paths_.end()) return i->second; return NULL; diff --git a/src/state.h b/src/state.h index bde75ff..c382dc0 100644 --- a/src/state.h +++ b/src/state.h @@ -82,6 +82,7 @@ struct Pool { /// Global state (file status, loaded rules) for a single run. struct State { static Pool kDefaultPool; + static Pool kConsolePool; static const Rule kPhonyRule; State(); @@ -95,7 +96,7 @@ struct State { Edge* AddEdge(const Rule* rule); Node* GetNode(StringPiece path); - Node* LookupNode(StringPiece path); + Node* LookupNode(StringPiece path) const; Node* SpellcheckNode(const string& path); void AddIn(Edge* edge, StringPiece path); diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc index a9af756..743e406 100644 --- a/src/subprocess-posix.cc +++ b/src/subprocess-posix.cc @@ -25,7 +25,8 @@ #include "util.h" -Subprocess::Subprocess() : fd_(-1), pid_(-1) { +Subprocess::Subprocess(bool use_console) : fd_(-1), pid_(-1), + use_console_(use_console) { } Subprocess::~Subprocess() { if (fd_ >= 0) @@ -58,29 +59,34 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { // Track which fd we use to report errors on. int error_pipe = output_pipe[1]; do { - if (setpgid(0, 0) < 0) - break; - if (sigaction(SIGINT, &set->old_act_, 0) < 0) break; if (sigprocmask(SIG_SETMASK, &set->old_mask_, 0) < 0) break; - // Open /dev/null over stdin. - int devnull = open("/dev/null", O_RDONLY); - if (devnull < 0) - break; - if (dup2(devnull, 0) < 0) - break; - close(devnull); - - if (dup2(output_pipe[1], 1) < 0 || - dup2(output_pipe[1], 2) < 0) - break; - - // Now can use stderr for errors. - error_pipe = 2; - close(output_pipe[1]); + if (!use_console_) { + // Put the child in its own process group, so ctrl-c won't reach it. + if (setpgid(0, 0) < 0) + break; + + // Open /dev/null over stdin. + int devnull = open("/dev/null", O_RDONLY); + if (devnull < 0) + break; + if (dup2(devnull, 0) < 0) + break; + close(devnull); + + if (dup2(output_pipe[1], 1) < 0 || + dup2(output_pipe[1], 2) < 0) + break; + + // Now can use stderr for errors. + error_pipe = 2; + close(output_pipe[1]); + } + // In the console case, output_pipe is still inherited by the child and + // closed when the subprocess finishes, which then notifies ninja. execl("/bin/sh", "/bin/sh", "-c", command.c_str(), (char *) NULL); } while (false); @@ -168,8 +174,8 @@ SubprocessSet::~SubprocessSet() { Fatal("sigprocmask: %s", strerror(errno)); } -Subprocess *SubprocessSet::Add(const string& command) { - Subprocess *subprocess = new Subprocess; +Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + Subprocess *subprocess = new Subprocess(use_console); if (!subprocess->Start(this, command)) { delete subprocess; return 0; @@ -279,7 +285,10 @@ Subprocess* SubprocessSet::NextFinished() { void SubprocessSet::Clear() { for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) - kill(-(*i)->pid_, SIGINT); + // Since the foreground process is in our process group, it will receive a + // SIGINT at the same time as us. + if (!(*i)->use_console_) + kill(-(*i)->pid_, SIGINT); for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) delete *i; diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 1b230b6..fad66e8 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -14,13 +14,16 @@ #include "subprocess.h" +#include <assert.h> #include <stdio.h> #include <algorithm> #include "util.h" -Subprocess::Subprocess() : child_(NULL) , overlapped_(), is_reading_(false) { +Subprocess::Subprocess(bool use_console) : child_(NULL) , overlapped_(), + is_reading_(false), + use_console_(use_console) { } Subprocess::~Subprocess() { @@ -86,18 +89,25 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { STARTUPINFOA startup_info; memset(&startup_info, 0, sizeof(startup_info)); startup_info.cb = sizeof(STARTUPINFO); - startup_info.dwFlags = STARTF_USESTDHANDLES; - startup_info.hStdInput = nul; - startup_info.hStdOutput = child_pipe; - startup_info.hStdError = child_pipe; + if (!use_console_) { + startup_info.dwFlags = STARTF_USESTDHANDLES; + startup_info.hStdInput = nul; + startup_info.hStdOutput = child_pipe; + startup_info.hStdError = child_pipe; + } + // In the console case, child_pipe is still inherited by the child and closed + // when the subprocess finishes, which then notifies ninja. PROCESS_INFORMATION process_info; memset(&process_info, 0, sizeof(process_info)); + // Ninja handles ctrl-c, except for subprocesses in console pools. + DWORD process_flags = use_console_ ? 0 : CREATE_NEW_PROCESS_GROUP; + // Do not prepend 'cmd /c' on Windows, this breaks command // lines greater than 8,191 chars. if (!CreateProcessA(NULL, (char*)command.c_str(), NULL, NULL, - /* inherit handles */ TRUE, CREATE_NEW_PROCESS_GROUP, + /* inherit handles */ TRUE, process_flags, NULL, NULL, &startup_info, &process_info)) { DWORD error = GetLastError(); @@ -213,8 +223,8 @@ BOOL WINAPI SubprocessSet::NotifyInterrupted(DWORD dwCtrlType) { return FALSE; } -Subprocess *SubprocessSet::Add(const string& command) { - Subprocess *subprocess = new Subprocess; +Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + Subprocess *subprocess = new Subprocess(use_console); if (!subprocess->Start(this, command)) { delete subprocess; return 0; @@ -266,7 +276,9 @@ Subprocess* SubprocessSet::NextFinished() { void SubprocessSet::Clear() { for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) { - if ((*i)->child_) { + // Since the foreground process is in our process group, it will receive a + // CTRL_C_EVENT or CTRL_BREAK_EVENT at the same time as us. + if ((*i)->child_ && !(*i)->use_console_) { if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, GetProcessId((*i)->child_))) { Win32Fatal("GenerateConsoleCtrlEvent"); diff --git a/src/subprocess.h b/src/subprocess.h index 4c1629c..b7a1a4c 100644 --- a/src/subprocess.h +++ b/src/subprocess.h @@ -44,7 +44,7 @@ struct Subprocess { const string& GetOutput() const; private: - Subprocess(); + Subprocess(bool use_console); bool Start(struct SubprocessSet* set, const string& command); void OnPipeReady(); @@ -64,6 +64,7 @@ struct Subprocess { int fd_; pid_t pid_; #endif + bool use_console_; friend struct SubprocessSet; }; @@ -75,7 +76,7 @@ struct SubprocessSet { SubprocessSet(); ~SubprocessSet(); - Subprocess* Add(const string& command); + Subprocess* Add(const string& command, bool use_console = false); bool DoWork(); Subprocess* NextFinished(); void Clear(); diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index 9f8dcea..775a13a 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -95,6 +95,21 @@ TEST_F(SubprocessTest, InterruptParent) { ADD_FAILURE() << "We should have been interrupted"; } +TEST_F(SubprocessTest, Console) { + // Skip test if we don't have the console ourselves. + if (isatty(0) && isatty(1) && isatty(2)) { + Subprocess* subproc = subprocs_.Add("test -t 0 -a -t 1 -a -t 2", + /*use_console=*/true); + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { + subprocs_.DoWork(); + } + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + } +} + #endif TEST_F(SubprocessTest, SetWithSingle) { diff --git a/src/test.cc b/src/test.cc index 45a9226..21015ed 100644 --- a/src/test.cc +++ b/src/test.cc @@ -105,8 +105,8 @@ void VirtualFileSystem::Create(const string& path, files_created_.insert(path); } -TimeStamp VirtualFileSystem::Stat(const string& path) { - FileMap::iterator i = files_.find(path); +TimeStamp VirtualFileSystem::Stat(const string& path) const { + FileMap::const_iterator i = files_.find(path); if (i != files_.end()) return i->second.mtime; return 0; @@ -59,7 +59,7 @@ struct VirtualFileSystem : public DiskInterface { } // DiskInterface - virtual TimeStamp Stat(const string& path); + virtual TimeStamp Stat(const string& path) const; virtual bool WriteFile(const string& path, const string& contents); virtual bool MakeDir(const string& path); virtual string ReadFile(const string& path, string* err); diff --git a/src/util.cc b/src/util.cc index 6ba3c6c..484b0c1 100644 --- a/src/util.cc +++ b/src/util.cc @@ -20,6 +20,7 @@ #include <share.h> #endif +#include <assert.h> #include <errno.h> #include <fcntl.h> #include <stdarg.h> @@ -175,6 +176,109 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { return true; } +static inline bool IsKnownShellSafeCharacter(char ch) { + if ('A' <= ch && ch <= 'Z') return true; + if ('a' <= ch && ch <= 'z') return true; + if ('0' <= ch && ch <= '9') return true; + + switch (ch) { + case '_': + case '+': + case '-': + case '.': + case '/': + return true; + default: + return false; + } +} + +static inline bool IsKnownWin32SafeCharacter(char ch) { + switch (ch) { + case ' ': + case '"': + return false; + default: + return true; + } +} + +static inline bool StringNeedsShellEscaping(const string& input) { + for (size_t i = 0; i < input.size(); ++i) { + if (!IsKnownShellSafeCharacter(input[i])) return true; + } + return false; +} + +static inline bool StringNeedsWin32Escaping(const string& input) { + for (size_t i = 0; i < input.size(); ++i) { + if (!IsKnownWin32SafeCharacter(input[i])) return true; + } + return false; +} + +void GetShellEscapedString(const string& input, string* result) { + assert(result); + + if (!StringNeedsShellEscaping(input)) { + result->append(input); + return; + } + + const char kQuote = '\''; + const char kEscapeSequence[] = "'\\'"; + + result->push_back(kQuote); + + string::const_iterator span_begin = input.begin(); + for (string::const_iterator it = input.begin(), end = input.end(); it != end; + ++it) { + if (*it == kQuote) { + result->append(span_begin, it); + result->append(kEscapeSequence); + span_begin = it; + } + } + result->append(span_begin, input.end()); + result->push_back(kQuote); +} + + +void GetWin32EscapedString(const string& input, string* result) { + assert(result); + if (!StringNeedsWin32Escaping(input)) { + result->append(input); + return; + } + + const char kQuote = '"'; + const char kBackslash = '\\'; + + result->push_back(kQuote); + size_t consecutive_backslash_count = 0; + string::const_iterator span_begin = input.begin(); + for (string::const_iterator it = input.begin(), end = input.end(); it != end; + ++it) { + switch (*it) { + case kBackslash: + ++consecutive_backslash_count; + break; + case kQuote: + result->append(span_begin, it); + result->append(consecutive_backslash_count + 1, kBackslash); + span_begin = it; + consecutive_backslash_count = 0; + break; + default: + consecutive_backslash_count = 0; + break; + } + } + result->append(span_begin, input.end()); + result->append(consecutive_backslash_count, kBackslash); + result->push_back(kQuote); +} + int ReadFile(const string& path, string* contents, string* err) { FILE* f = fopen(path.c_str(), "r"); if (!f) { @@ -45,6 +45,13 @@ bool CanonicalizePath(string* path, string* err); bool CanonicalizePath(char* path, size_t* len, string* err); +/// Appends |input| to |*result|, escaping according to the whims of either +/// Bash, or Win32's CommandLineToArgvW(). +/// Appends the string directly to |result| without modification if we can +/// determine that it contains no problematic characters. +void GetShellEscapedString(const string& input, string* result); +void GetWin32EscapedString(const string& input, string* result); + /// Read a file to a string (in text mode: with CRLF conversion /// on Windows). /// Returns -errno and fills in \a err on error. diff --git a/src/util_test.cc b/src/util_test.cc index 1e29053..b58d15e 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -136,6 +136,37 @@ TEST(CanonicalizePath, NotNullTerminated) { EXPECT_EQ("file ./file bar/.", string(path)); } +TEST(PathEscaping, TortureTest) { + string result; + + GetWin32EscapedString("foo bar\\\"'$@d!st!c'\\path'\\", &result); + EXPECT_EQ("\"foo bar\\\\\\\"'$@d!st!c'\\path'\\\\\"", result); + result.clear(); + + GetShellEscapedString("foo bar\"/'$@d!st!c'/path'", &result); + EXPECT_EQ("'foo bar\"/'\\''$@d!st!c'\\''/path'\\'''", result); +} + +TEST(PathEscaping, SensiblePathsAreNotNeedlesslyEscaped) { + const char* path = "some/sensible/path/without/crazy/characters.c++"; + string result; + + GetWin32EscapedString(path, &result); + EXPECT_EQ(path, result); + result.clear(); + + GetShellEscapedString(path, &result); + EXPECT_EQ(path, result); +} + +TEST(PathEscaping, SensibleWin32PathsAreNotNeedlesslyEscaped) { + const char* path = "some\\sensible\\path\\without\\crazy\\characters.c++"; + string result; + + GetWin32EscapedString(path, &result); + EXPECT_EQ(path, result); +} + TEST(StripAnsiEscapeCodes, EscapeAtEnd) { string stripped = StripAnsiEscapeCodes("foo\33"); EXPECT_EQ("foo", stripped); diff --git a/src/version.cc b/src/version.cc index 17c71aa..6d2d37c 100644 --- a/src/version.cc +++ b/src/version.cc @@ -18,7 +18,7 @@ #include "util.h" -const char* kNinjaVersion = "1.4.0"; +const char* kNinjaVersion = "1.5.0"; void ParseVersion(const string& version, int* major, int* minor) { size_t end = version.find('.'); |