diff options
Diffstat (limited to 'src')
55 files changed, 1987 insertions, 649 deletions
diff --git a/src/build.cc b/src/build.cc index f91ff2f..3e74131 100644 --- a/src/build.cc +++ b/src/build.cc @@ -97,6 +97,9 @@ void BuildStatus::BuildEdgeStarted(Edge* edge) { ++started_edges_; PrintStatus(edge); + + if (edge->use_console()) + printer_.SetConsoleLocked(true); } void BuildStatus::BuildEdgeFinished(Edge* edge, @@ -112,10 +115,13 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, *end_time = (int)(now - start_time_millis_); running_edges_.erase(i); + if (edge->use_console()) + printer_.SetConsoleLocked(false); + if (config_.verbosity == BuildConfig::QUIET) return; - if (printer_.is_smart_terminal()) + if (!edge->use_console() && printer_.is_smart_terminal()) PrintStatus(edge); // Print the command that is spewing before printing its output. @@ -145,6 +151,7 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, } void BuildStatus::BuildFinished() { + printer_.SetConsoleLocked(false); printer_.PrintOnNewLine(""); } @@ -481,14 +488,16 @@ void RealCommandRunner::Abort() { } bool RealCommandRunner::CanRunMore() { - return ((int)subprocs_.running_.size()) < config_.parallelism + size_t subproc_number = + subprocs_.running_.size() + subprocs_.finished_.size(); + return (int)subproc_number < config_.parallelism && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f) || GetLoadAverage() < config_.max_load_average); } bool RealCommandRunner::StartCommand(Edge* edge) { string command = edge->EvaluateCommand(); - Subprocess* subproc = subprocs_.Add(command); + Subprocess* subproc = subprocs_.Add(command, edge->use_console()); if (!subproc) return false; subproc_to_edge_.insert(make_pair(subproc, edge)); @@ -534,7 +543,7 @@ void Builder::Cleanup() { for (vector<Edge*>::iterator i = active_edges.begin(); i != active_edges.end(); ++i) { - string depfile = (*i)->GetBinding("depfile"); + string depfile = (*i)->GetUnescapedDepfile(); for (vector<Node*>::iterator ni = (*i)->outputs_.begin(); ni != (*i)->outputs_.end(); ++ni) { // Only delete this output if it was actually modified. This is @@ -610,6 +619,7 @@ bool Builder::Build(string* err) { if (failures_allowed && command_runner_->CanRunMore()) { if (Edge* edge = plan_.FindWork()) { if (!StartEdge(edge, err)) { + Cleanup(); status_->BuildFinished(); return false; } @@ -630,6 +640,7 @@ bool Builder::Build(string* err) { CommandRunner::Result result; if (!command_runner_->WaitForCommand(&result) || result.status == ExitInterrupted) { + Cleanup(); status_->BuildFinished(); *err = "interrupted by user"; return false; @@ -637,6 +648,7 @@ bool Builder::Build(string* err) { --pending_commands; if (!FinishCommand(&result, err)) { + Cleanup(); status_->BuildFinished(); return false; } @@ -686,7 +698,7 @@ bool Builder::StartEdge(Edge* edge, string* err) { // Create response file, if needed // XXX: this may also block; do we care? - string rspfile = edge->GetBinding("rspfile"); + string rspfile = edge->GetUnescapedRspfile(); if (!rspfile.empty()) { string content = edge->GetBinding("rspfile_content"); if (!disk_interface_->WriteFile(rspfile, content)) @@ -762,7 +774,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { restat_mtime = input_mtime; } - string depfile = edge->GetBinding("depfile"); + string depfile = edge->GetUnescapedDepfile(); if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) { TimeStamp depfile_mtime = disk_interface_->Stat(depfile); if (depfile_mtime > restat_mtime) @@ -778,7 +790,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { plan_.EdgeFinished(edge); // Delete any left over response file. - string rspfile = edge->GetBinding("rspfile"); + string rspfile = edge->GetUnescapedRspfile(); if (!rspfile.empty() && !g_keep_rsp) disk_interface_->RemoveFile(rspfile); @@ -813,12 +825,16 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, result->output = parser.Parse(result->output, deps_prefix); for (set<string>::iterator i = parser.includes_.begin(); i != parser.includes_.end(); ++i) { - deps_nodes->push_back(state_->GetNode(*i)); + // ~0 is assuming that with MSVC-parsed headers, it's ok to always make + // all backslashes (as some of the slashes will certainly be backslashes + // anyway). This could be fixed if necessary with some additional + // complexity in IncludesNormalize::Relativize. + deps_nodes->push_back(state_->GetNode(*i, ~0u)); } } else #endif if (deps_type == "gcc") { - string depfile = result->edge->GetBinding("depfile"); + string depfile = result->edge->GetUnescapedDepfile(); if (depfile.empty()) { *err = string("edge with deps=gcc but no depfile makes no sense"); return false; @@ -838,9 +854,11 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, deps_nodes->reserve(deps.ins_.size()); for (vector<StringPiece>::iterator i = deps.ins_.begin(); i != deps.ins_.end(); ++i) { - if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err)) + unsigned int slash_bits; + if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, &slash_bits, + err)) return false; - deps_nodes->push_back(state_->GetNode(*i)); + deps_nodes->push_back(state_->GetNode(*i, slash_bits)); } if (disk_interface_->RemoveFile(depfile) < 0) { diff --git a/src/build_log.cc b/src/build_log.cc index 3f24c16..b6f9874 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -102,7 +102,7 @@ BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash, {} BuildLog::BuildLog() - : log_file_(NULL), needs_recompaction_(false) {} + : log_file_(NULL), needs_recompaction_(false), quiet_(false) {} BuildLog::~BuildLog() { Close(); @@ -354,7 +354,8 @@ bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { bool BuildLog::Recompact(const string& path, const BuildLogUser& user, string* err) { METRIC_RECORD(".ninja_log recompact"); - printf("Recompacting log...\n"); + if (!quiet_) + printf("Recompacting log...\n"); Close(); string temp_path = path + ".recompact"; diff --git a/src/build_log.h b/src/build_log.h index fe81a85..db0cfe0 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -80,6 +80,7 @@ struct BuildLog { /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, const BuildLogUser& user, string* err); + void set_quiet(bool quiet) { quiet_ = quiet; } typedef ExternalStringHashMap<LogEntry*>::Type Entries; const Entries& entries() const { return entries_; } @@ -88,6 +89,7 @@ struct BuildLog { Entries entries_; FILE* log_file_; bool needs_recompaction_; + bool quiet_; }; #endif // NINJA_BUILD_LOG_H_ diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 6738c7b..7ea2117 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -17,12 +17,12 @@ #include "util.h" #include "test.h" +#include <sys/stat.h> #ifdef _WIN32 #include <fcntl.h> #include <share.h> #else #include <sys/types.h> -#include <sys/stat.h> #include <unistd.h> #endif @@ -290,6 +290,7 @@ TEST_F(BuildLogRecompactTest, Recompact) { ASSERT_TRUE(log2.LookupByOutput("out")); ASSERT_TRUE(log2.LookupByOutput("out2")); // ...and force a recompaction. + log2.set_quiet(true); EXPECT_TRUE(log2.OpenForWrite(kTestFilename, *this, &err)); log2.Close(); diff --git a/src/build_test.cc b/src/build_test.cc index 86a911b..bd1cd30 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -14,6 +14,8 @@ #include "build.h" +#include <assert.h> + #include "build_log.h" #include "deps_log.h" #include "graph.h" @@ -44,6 +46,8 @@ struct PlanTest : public StateTestWithBuiltinRules { ASSERT_FALSE(plan_.FindWork()); sort(ret->begin(), ret->end(), CompareEdgesByOutput::cmp); } + + void TestPoolWithDepthOne(const char *test_case); }; TEST_F(PlanTest, Basic) { @@ -197,15 +201,8 @@ TEST_F(PlanTest, DependencyCycle) { ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err); } -TEST_F(PlanTest, PoolWithDepthOne) { - ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, -"pool foobar\n" -" depth = 1\n" -"rule poolcat\n" -" command = cat $in > $out\n" -" pool = foobar\n" -"build out1: poolcat in\n" -"build out2: poolcat in\n")); +void PlanTest::TestPoolWithDepthOne(const char* test_case) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, test_case)); GetNode("out1")->MarkDirty(); GetNode("out2")->MarkDirty(); string err; @@ -239,6 +236,26 @@ TEST_F(PlanTest, PoolWithDepthOne) { ASSERT_EQ(0, edge); } +TEST_F(PlanTest, PoolWithDepthOne) { + TestPoolWithDepthOne( +"pool foobar\n" +" depth = 1\n" +"rule poolcat\n" +" command = cat $in > $out\n" +" pool = foobar\n" +"build out1: poolcat in\n" +"build out2: poolcat in\n"); +} + +TEST_F(PlanTest, ConsolePool) { + TestPoolWithDepthOne( +"rule poolcat\n" +" command = cat $in > $out\n" +" pool = console\n" +"build out1: poolcat in\n" +"build out2: poolcat in\n"); +} + TEST_F(PlanTest, PoolsWithDepthTwo) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "pool foobar\n" @@ -491,7 +508,7 @@ void BuildTest::RebuildTarget(const string& target, const char* manifest, builder.command_runner_.reset(&command_runner_); if (!builder.AlreadyUpToDate()) { bool build_res = builder.Build(&err); - EXPECT_TRUE(build_res) << "builder.Build(&err)"; + EXPECT_TRUE(build_res); } builder.command_runner_.release(); } @@ -506,6 +523,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { commands_ran_.push_back(edge->EvaluateCommand()); if (edge->rule().name() == "cat" || edge->rule().name() == "cat_rsp" || + edge->rule().name() == "cat_rsp_out" || edge->rule().name() == "cc" || edge->rule().name() == "touch" || edge->rule().name() == "touch-interrupt") { @@ -515,7 +533,8 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { } } else if (edge->rule().name() == "true" || edge->rule().name() == "fail" || - edge->rule().name() == "interrupt") { + edge->rule().name() == "interrupt" || + edge->rule().name() == "console") { // Don't do anything. } else { printf("unknown command\n"); @@ -539,6 +558,15 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { return true; } + if (edge->rule().name() == "console") { + if (edge->use_console()) + result->status = ExitSuccess; + else + result->status = ExitFailure; + last_command_ = NULL; + return true; + } + if (edge->rule().name() == "fail") result->status = ExitFailure; else @@ -727,36 +755,31 @@ TEST_F(BuildTest, MakeDirs) { #ifdef _WIN32 ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir\\dir2\\file: cat in1\n")); - EXPECT_TRUE(builder_.AddTarget("subdir\\dir2\\file", &err)); #else ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir/dir2/file: cat in1\n")); - EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err)); #endif + EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err)); EXPECT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); ASSERT_EQ("", err); ASSERT_EQ(2u, fs_.directories_made_.size()); EXPECT_EQ("subdir", fs_.directories_made_[0]); -#ifdef _WIN32 - EXPECT_EQ("subdir\\dir2", fs_.directories_made_[1]); -#else EXPECT_EQ("subdir/dir2", fs_.directories_made_[1]); -#endif } TEST_F(BuildTest, DepFileMissing) { string err; ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule cc\n command = cc $in\n depfile = $out.d\n" -"build foo.o: cc foo.c\n")); +"build fo$ o.o: cc foo.c\n")); fs_.Create("foo.c", ""); - EXPECT_TRUE(builder_.AddTarget("foo.o", &err)); + EXPECT_TRUE(builder_.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); ASSERT_EQ(1u, fs_.files_read_.size()); - EXPECT_EQ("foo.o.d", fs_.files_read_[0]); + EXPECT_EQ("fo o.o.d", fs_.files_read_[0]); } TEST_F(BuildTest, DepFileOK) { @@ -914,6 +937,38 @@ TEST_F(BuildTest, RebuildOrderOnlyDeps) { ASSERT_EQ("cc oo.h.in", command_runner_.commands_ran_[0]); } +#ifdef _WIN32 +TEST_F(BuildTest, DepFileCanonicalize) { + string err; + int orig_edges = state_.edges_.size(); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc\n command = cc $in\n depfile = $out.d\n" +"build gen/stuff\\things/foo.o: cc x\\y/z\\foo.c\n")); + Edge* edge = state_.edges_.back(); + + fs_.Create("x/y/z/foo.c", ""); + GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing. + // Note, different slashes from manifest. + fs_.Create("gen/stuff\\things/foo.o.d", + "gen\\stuff\\things\\foo.o: blah.h bar.h\n"); + EXPECT_TRUE(builder_.AddTarget("gen/stuff/things/foo.o", &err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, fs_.files_read_.size()); + // The depfile path does not get Canonicalize as it seems unnecessary. + EXPECT_EQ("gen/stuff\\things/foo.o.d", fs_.files_read_[0]); + + // Expect three new edges: one generating foo.o, and two more from + // loading the depfile. + ASSERT_EQ(orig_edges + 3, (int)state_.edges_.size()); + // Expect our edge to now have three inputs: foo.c and two headers. + ASSERT_EQ(3u, edge->inputs_.size()); + + // Expect the command line we generate to only use the original input, and + // using the slashes from the manifest. + ASSERT_EQ("cc x\\y/z\\foo.c", edge->EvaluateCommand()); +} +#endif + TEST_F(BuildTest, Phony) { string err; ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, @@ -1277,14 +1332,20 @@ TEST_F(BuildTest, RspFileSuccess) " command = cat $rspfile > $out\n" " rspfile = $rspfile\n" " rspfile_content = $long_command\n" + "rule cat_rsp_out\n" + " command = cat $rspfile > $out\n" + " rspfile = $out.rsp\n" + " rspfile_content = $long_command\n" "build out1: cat in\n" "build out2: cat_rsp in\n" - " rspfile = out2.rsp\n" + " rspfile = out 2.rsp\n" + " long_command = Some very long command\n" + "build out$ 3: cat_rsp_out in\n" " long_command = Some very long command\n")); fs_.Create("out1", ""); fs_.Create("out2", ""); - fs_.Create("out3", ""); + fs_.Create("out 3", ""); fs_.Tick(); @@ -1295,20 +1356,24 @@ TEST_F(BuildTest, RspFileSuccess) ASSERT_EQ("", err); EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("out 3", &err)); + ASSERT_EQ("", err); size_t files_created = fs_.files_created_.size(); size_t files_removed = fs_.files_removed_.size(); EXPECT_TRUE(builder_.Build(&err)); - ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // cat + cat_rsp + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); - // The RSP file was created - ASSERT_EQ(files_created + 1, fs_.files_created_.size()); - ASSERT_EQ(1u, fs_.files_created_.count("out2.rsp")); + // The RSP files were created + ASSERT_EQ(files_created + 2, fs_.files_created_.size()); + ASSERT_EQ(1u, fs_.files_created_.count("out 2.rsp")); + ASSERT_EQ(1u, fs_.files_created_.count("out 3.rsp")); - // The RSP file was removed - ASSERT_EQ(files_removed + 1, fs_.files_removed_.size()); - ASSERT_EQ(1u, fs_.files_removed_.count("out2.rsp")); + // The RSP files were removed + ASSERT_EQ(files_removed + 2, fs_.files_removed_.size()); + ASSERT_EQ(1u, fs_.files_removed_.count("out 2.rsp")); + ASSERT_EQ(1u, fs_.files_removed_.count("out 3.rsp")); } // Test that RSP file is created but not removed for commands, which fail @@ -1779,7 +1844,7 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { string err; const char* manifest = "rule cc\n command = cc $in\n depfile = $out.d\n deps = gcc\n" - "build foo.o: cc foo.c\n"; + "build fo$ o.o: cc foo.c\n"; fs_.Create("foo.c", ""); @@ -1794,9 +1859,9 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { Builder builder(&state, config_, NULL, &deps_log, &fs_); builder.command_runner_.reset(&command_runner_); - EXPECT_TRUE(builder.AddTarget("foo.o", &err)); + EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); - fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n"); + fs_.Create("fo o.o.d", "fo\\ o.o: blah.h bar.h\n"); EXPECT_TRUE(builder.Build(&err)); EXPECT_EQ("", err); @@ -1818,11 +1883,11 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { Edge* edge = state.edges_.back(); - state.GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing. - EXPECT_TRUE(builder.AddTarget("foo.o", &err)); + state.GetNode("bar.h", 0)->MarkDirty(); // Mark bar.h as missing. + EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); - // Expect three new edges: one generating foo.o, and two more from + // Expect three new edges: one generating fo o.o, and two more from // loading the depfile. ASSERT_EQ(3u, state.edges_.size()); // Expect our edge to now have three inputs: foo.c and two headers. @@ -1836,6 +1901,72 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { } } +#ifdef _WIN32 +TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { + string err; + const char* manifest = + "rule cc\n command = cc $in\n depfile = $out.d\n deps = gcc\n" + "build a/b\\c\\d/e/fo$ o.o: cc x\\y/z\\foo.c\n"; + + fs_.Create("x/y/z/foo.c", ""); + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + // Run the build once, everything should be ok. + DepsLog deps_log; + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err)); + ASSERT_EQ("", err); + // Note, different slashes from manifest. + fs_.Create("a/b\\c\\d/e/fo o.o.d", + "a\\b\\c\\d\\e\\fo\\ o.o: blah.h bar.h\n"); + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + deps_log.Close(); + builder.command_runner_.release(); + } + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_); + builder.command_runner_.reset(&command_runner_); + + Edge* edge = state.edges_.back(); + + state.GetNode("bar.h", 0)->MarkDirty(); // Mark bar.h as missing. + EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err)); + ASSERT_EQ("", err); + + // Expect three new edges: one generating fo o.o, and two more from + // loading the depfile. + ASSERT_EQ(3u, state.edges_.size()); + // Expect our edge to now have three inputs: foo.c and two headers. + ASSERT_EQ(3u, edge->inputs_.size()); + + // Expect the command line we generate to only use the original input. + // Note, slashes from manifest, not .d. + ASSERT_EQ("cc x\\y/z\\foo.c", edge->EvaluateCommand()); + + deps_log.Close(); + builder.command_runner_.release(); + } +} +#endif + /// Check that a restat rule doesn't clear an edge if the depfile is missing. /// Follows from: https://github.com/martine/ninja/issues/603 TEST_F(BuildTest, RestatMissingDepfile) { @@ -1911,3 +2042,20 @@ TEST_F(BuildWithDepsLogTest, RestatMissingDepfileDepslog) { RebuildTarget("out", manifest, "build_log", "ninja_deps2"); ASSERT_EQ(0u, command_runner_.commands_ran_.size()); } + +TEST_F(BuildTest, Console) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule console\n" +" command = console\n" +" pool = console\n" +"build cons: console in.txt\n")); + + fs_.Create("in.txt", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("cons", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); +} diff --git a/src/canon_perftest.cc b/src/canon_perftest.cc index 59bd18f..389ac24 100644 --- a/src/canon_perftest.cc +++ b/src/canon_perftest.cc @@ -33,8 +33,9 @@ int main() { for (int j = 0; j < 5; ++j) { const int kNumRepetitions = 2000000; int64_t start = GetTimeMillis(); + unsigned int slash_bits; for (int i = 0; i < kNumRepetitions; ++i) { - CanonicalizePath(buf, &len, &err); + CanonicalizePath(buf, &len, &slash_bits, &err); } int delta = (int)(GetTimeMillis() - start); times.push_back(delta); diff --git a/src/clean.cc b/src/clean.cc index 5d1974e..98c638c 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -80,11 +80,11 @@ bool Cleaner::IsAlreadyRemoved(const string& path) { } void Cleaner::RemoveEdgeFiles(Edge* edge) { - string depfile = edge->GetBinding("depfile"); + string depfile = edge->GetUnescapedDepfile(); if (!depfile.empty()) Remove(depfile); - string rspfile = edge->GetBinding("rspfile"); + string rspfile = edge->GetUnescapedRspfile(); if (!rspfile.empty()) Remove(rspfile); } diff --git a/src/clean_test.cc b/src/clean_test.cc index 04cff73..5869bbb 100644 --- a/src/clean_test.cc +++ b/src/clean_test.cc @@ -286,8 +286,7 @@ TEST_F(CleanTest, CleanRspFile) { " rspfile = $rspfile\n" " rspfile_content=$in\n" "build out1: cc in1\n" -" rspfile = cc1.rsp\n" -" rspfile_content=$in\n")); +" rspfile = cc1.rsp\n")); fs_.Create("out1", ""); fs_.Create("cc1.rsp", ""); @@ -307,10 +306,9 @@ TEST_F(CleanTest, CleanRsp) { "build out1: cat in1\n" "build in2: cat_rsp src2\n" " rspfile=in2.rsp\n" -" rspfile_content=$in\n" "build out2: cat_rsp in2\n" " rspfile=out2.rsp\n" -" rspfile_content=$in\n")); +)); fs_.Create("in1", ""); fs_.Create("out1", ""); fs_.Create("in2.rsp", ""); @@ -336,8 +334,6 @@ TEST_F(CleanTest, CleanRsp) { EXPECT_EQ(0, fs_.Stat("out2")); EXPECT_EQ(0, fs_.Stat("in2.rsp")); EXPECT_EQ(0, fs_.Stat("out2.rsp")); - - fs_.files_removed_.clear(); } TEST_F(CleanTest, CleanFailure) { @@ -372,3 +368,31 @@ TEST_F(CleanTest, CleanPhony) { EXPECT_EQ(2, cleaner.cleaned_files_count()); EXPECT_NE(0, fs_.Stat("phony")); } + +TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc_dep\n" +" command = cc $in > $out\n" +" depfile = $out.d\n" +"rule cc_rsp\n" +" command = cc $in > $out\n" +" rspfile = $out.rsp\n" +" rspfile_content = $in\n" +"build out$ 1: cc_dep in$ 1\n" +"build out$ 2: cc_rsp in$ 1\n" +)); + fs_.Create("out 1", ""); + fs_.Create("out 2", ""); + fs_.Create("out 1.d", ""); + fs_.Create("out 2.rsp", ""); + + Cleaner cleaner(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner.CleanAll()); + EXPECT_EQ(4, cleaner.cleaned_files_count()); + EXPECT_EQ(4u, fs_.files_removed_.size()); + + EXPECT_EQ(0, fs_.Stat("out 1")); + EXPECT_EQ(0, fs_.Stat("out 2")); + EXPECT_EQ(0, fs_.Stat("out 1.d")); + EXPECT_EQ(0, fs_.Stat("out 2.rsp")); +} diff --git a/src/debug_flags.cc b/src/debug_flags.cc index 75f1ea5..8065001 100644 --- a/src/debug_flags.cc +++ b/src/debug_flags.cc @@ -15,3 +15,5 @@ bool g_explaining = false; bool g_keep_rsp = false; + +bool g_experimental_statcache = true; diff --git a/src/debug_flags.h b/src/debug_flags.h index ba3ebf3..7965585 100644 --- a/src/debug_flags.h +++ b/src/debug_flags.h @@ -26,4 +26,6 @@ extern bool g_explaining; extern bool g_keep_rsp; +extern bool g_experimental_statcache; + #endif // NINJA_EXPLAIN_H_ diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 49c7d7b..4ca3943 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -64,7 +64,7 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 0, 0, 0, 128, 0, + 128, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -114,27 +114,29 @@ bool DepfileParser::Parse(string* content, string* err) { if (yych != '\\') goto yy9; } } else { - if (yych <= 'z') { + if (yych <= '{') { if (yych == '`') goto yy9; goto yy5; } else { - if (yych == '~') goto yy5; + if (yych <= '|') goto yy9; + if (yych <= '~') goto yy5; goto yy9; } } } ++in; - if ((yych = *in) <= '#') { - if (yych <= '\n') { + if ((yych = *in) <= '"') { + if (yych <= '\f') { if (yych <= 0x00) goto yy3; - if (yych <= '\t') goto yy14; + if (yych != '\n') goto yy14; } else { + if (yych <= '\r') goto yy3; if (yych == ' ') goto yy16; - if (yych <= '"') goto yy14; - goto yy16; + goto yy14; } } else { if (yych <= 'Z') { + if (yych <= '#') goto yy16; if (yych == '*') goto yy16; goto yy14; } else { @@ -224,7 +226,7 @@ yy16: } else if (!out_.str_) { out_ = StringPiece(filename, len); } else if (out_ != StringPiece(filename, len)) { - *err = "depfile has multiple output paths."; + *err = "depfile has multiple output paths"; return false; } } diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 8bb6d84..b59baf0 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -67,13 +67,13 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = '$'; continue; } - '\\' [^\000\n] { + '\\' [^\000\r\n] { // Let backslash before other characters through verbatim. *out++ = '\\'; *out++ = yych; continue; } - [a-zA-Z0-9+,/_:.~()@=!-]+ { + [a-zA-Z0-9+,/_:.~()}{@=!-]+ { // Got a span of plain text. int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. @@ -108,7 +108,7 @@ bool DepfileParser::Parse(string* content, string* err) { } else if (!out_.str_) { out_ = StringPiece(filename, len); } else if (out_ != StringPiece(filename, len)) { - *err = "depfile has multiple output paths."; + *err = "depfile has multiple output paths"; return false; } } diff --git a/src/parser_perftest.cc b/src/depfile_parser_perftest.cc index b215221..b215221 100644 --- a/src/parser_perftest.cc +++ b/src/depfile_parser_perftest.cc diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index 0f6771a..e67ef79 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -14,7 +14,7 @@ #include "depfile_parser.h" -#include <gtest/gtest.h> +#include "test.h" struct DepfileParserTest : public testing::Test { bool Parse(const char* input, string* err); @@ -58,6 +58,17 @@ TEST_F(DepfileParserTest, Continuation) { EXPECT_EQ(2u, parser_.ins_.size()); } +TEST_F(DepfileParserTest, CarriageReturnContinuation) { + string err; + EXPECT_TRUE(Parse( +"foo.o: \\\r\n" +" bar.h baz.h\r\n", + &err)); + ASSERT_EQ("", err); + EXPECT_EQ("foo.o", parser_.out_.AsString()); + EXPECT_EQ(2u, parser_.ins_.size()); +} + TEST_F(DepfileParserTest, BackSlashes) { string err; EXPECT_TRUE(Parse( @@ -109,16 +120,19 @@ TEST_F(DepfileParserTest, SpecialChars) { string err; EXPECT_TRUE(Parse( "C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n" -" en@quot.header~ t+t-x!=1", +" en@quot.header~ t+t-x!=1 \n" +" openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif", &err)); ASSERT_EQ("", err); EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", parser_.out_.AsString()); - ASSERT_EQ(2u, parser_.ins_.size()); + ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("en@quot.header~", parser_.ins_[0].AsString()); EXPECT_EQ("t+t-x!=1", parser_.ins_[1].AsString()); + EXPECT_EQ("openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif", + parser_.ins_[2].AsString()); } TEST_F(DepfileParserTest, UnifyMultipleOutputs) { @@ -136,4 +150,5 @@ TEST_F(DepfileParserTest, RejectMultipleDifferentOutputs) { // check that multiple different outputs are rejected by the parser string err; EXPECT_FALSE(Parse("foo bar: x y z", &err)); + ASSERT_EQ("depfile has multiple output paths", err); } diff --git a/src/deps_log.cc b/src/deps_log.cc index 61df387..fc46497 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -240,7 +240,12 @@ bool DepsLog::Load(const string& path, State* state, string* err) { if (buf[path_size - 1] == '\0') --path_size; if (buf[path_size - 1] == '\0') --path_size; StringPiece path(buf, path_size); - Node* node = state->GetNode(path); + // It is not necessary to pass in a correct slash_bits here. It will + // either be a Node that's in the manifest (in which case it will already + // have a correct slash_bits that GetNode will look up), or it is an + // implicit dependency from a .d which does not affect the build command + // (and so need not have its slashes maintained). + Node* node = state->GetNode(path, 0); // Check that the expected index matches the actual index. This can only // happen if two ninja processes write to the same deps log concurrently. @@ -302,7 +307,8 @@ DepsLog::Deps* DepsLog::GetDeps(Node* node) { bool DepsLog::Recompact(const string& path, string* err) { METRIC_RECORD(".ninja_deps recompact"); - printf("Recompacting deps...\n"); + if (!quiet_) + printf("Recompacting deps...\n"); Close(); string temp_path = path + ".recompact"; diff --git a/src/deps_log.h b/src/deps_log.h index cec0257..9b81bc1 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -64,7 +64,7 @@ struct State; /// wins, allowing updates to just be appended to the file. A separate /// repacking step can run occasionally to remove dead records. struct DepsLog { - DepsLog() : needs_recompaction_(false), file_(NULL) {} + DepsLog() : needs_recompaction_(false), quiet_(false), file_(NULL) {} ~DepsLog(); // Writing (build-time) interface. @@ -87,6 +87,7 @@ struct DepsLog { /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, string* err); + void set_quiet(bool quiet) { quiet_ = quiet; } /// Returns if the deps entry for a node is still reachable from the manifest. /// @@ -108,6 +109,7 @@ struct DepsLog { bool RecordId(Node* node); bool needs_recompaction_; + bool quiet_; FILE* file_; /// Maps id -> Node. diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index e8e5138..ac2b315 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -14,6 +14,11 @@ #include "deps_log.h" +#include <sys/stat.h> +#ifndef _WIN32 +#include <unistd.h> +#endif + #include "graph.h" #include "util.h" #include "test.h" @@ -41,16 +46,16 @@ TEST_F(DepsLogTest, WriteRead) { { vector<Node*> deps; - deps.push_back(state1.GetNode("foo.h")); - deps.push_back(state1.GetNode("bar.h")); - log1.RecordDeps(state1.GetNode("out.o"), 1, deps); + deps.push_back(state1.GetNode("foo.h", 0)); + deps.push_back(state1.GetNode("bar.h", 0)); + log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps); deps.clear(); - deps.push_back(state1.GetNode("foo.h")); - deps.push_back(state1.GetNode("bar2.h")); - log1.RecordDeps(state1.GetNode("out2.o"), 2, deps); + deps.push_back(state1.GetNode("foo.h", 0)); + deps.push_back(state1.GetNode("bar2.h", 0)); + log1.RecordDeps(state1.GetNode("out2.o", 0), 2, deps); - DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o")); + DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0)); ASSERT_TRUE(log_deps); ASSERT_EQ(1, log_deps->mtime); ASSERT_EQ(2, log_deps->node_count); @@ -74,7 +79,7 @@ TEST_F(DepsLogTest, WriteRead) { } // Spot-check the entries in log2. - DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o")); + DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o", 0)); ASSERT_TRUE(log_deps); ASSERT_EQ(2, log_deps->mtime); ASSERT_EQ(2, log_deps->node_count); @@ -96,11 +101,11 @@ TEST_F(DepsLogTest, LotsOfDeps) { for (int i = 0; i < kNumDeps; ++i) { char buf[32]; sprintf(buf, "file%d.h", i); - deps.push_back(state1.GetNode(buf)); + deps.push_back(state1.GetNode(buf, 0)); } - log1.RecordDeps(state1.GetNode("out.o"), 1, deps); + log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps); - DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o")); + DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0)); ASSERT_EQ(kNumDeps, log_deps->node_count); } @@ -111,7 +116,7 @@ TEST_F(DepsLogTest, LotsOfDeps) { EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err)); ASSERT_EQ("", err); - DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o")); + DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o", 0)); ASSERT_EQ(kNumDeps, log_deps->node_count); } @@ -127,9 +132,9 @@ TEST_F(DepsLogTest, DoubleEntry) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); log.Close(); struct stat st; @@ -149,9 +154,9 @@ TEST_F(DepsLogTest, DoubleEntry) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); log.Close(); struct stat st; @@ -181,14 +186,14 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); deps.clear(); - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("baz.h")); - log.RecordDeps(state.GetNode("other_out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("baz.h", 0)); + log.RecordDeps(state.GetNode("other_out.o", 0), 1, deps); log.Close(); @@ -211,8 +216,8 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); log.Close(); struct stat st; @@ -232,14 +237,14 @@ TEST_F(DepsLogTest, Recompact) { string err; ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); - Node* out = state.GetNode("out.o"); + Node* out = state.GetNode("out.o", 0); DepsLog::Deps* deps = log.GetDeps(out); ASSERT_TRUE(deps); ASSERT_EQ(1, deps->mtime); ASSERT_EQ(1, deps->node_count); ASSERT_EQ("foo.h", deps->nodes[0]->path()); - Node* other_out = state.GetNode("other_out.o"); + Node* other_out = state.GetNode("other_out.o", 0); deps = log.GetDeps(other_out); ASSERT_TRUE(deps); ASSERT_EQ(1, deps->mtime); @@ -247,6 +252,7 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_EQ("foo.h", deps->nodes[0]->path()); ASSERT_EQ("baz.h", deps->nodes[1]->path()); + log.set_quiet(true); ASSERT_TRUE(log.Recompact(kTestFilename, &err)); // The in-memory deps graph should still be valid after recompaction. @@ -281,14 +287,14 @@ TEST_F(DepsLogTest, Recompact) { string err; ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); - Node* out = state.GetNode("out.o"); + Node* out = state.GetNode("out.o", 0); DepsLog::Deps* deps = log.GetDeps(out); ASSERT_TRUE(deps); ASSERT_EQ(1, deps->mtime); ASSERT_EQ(1, deps->node_count); ASSERT_EQ("foo.h", deps->nodes[0]->path()); - Node* other_out = state.GetNode("other_out.o"); + Node* other_out = state.GetNode("other_out.o", 0); deps = log.GetDeps(other_out); ASSERT_TRUE(deps); ASSERT_EQ(1, deps->mtime); @@ -296,6 +302,7 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_EQ("foo.h", deps->nodes[0]->path()); ASSERT_EQ("baz.h", deps->nodes[1]->path()); + log.set_quiet(true); ASSERT_TRUE(log.Recompact(kTestFilename, &err)); // The previous entries should have been removed. @@ -354,14 +361,14 @@ TEST_F(DepsLogTest, Truncated) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); deps.clear(); - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar2.h")); - log.RecordDeps(state.GetNode("out2.o"), 2, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar2.h", 0)); + log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); log.Close(); } @@ -413,14 +420,14 @@ TEST_F(DepsLogTest, TruncatedRecovery) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); deps.clear(); - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar2.h")); - log.RecordDeps(state.GetNode("out2.o"), 2, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar2.h", 0)); + log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); log.Close(); } @@ -441,16 +448,16 @@ TEST_F(DepsLogTest, TruncatedRecovery) { err.clear(); // The truncated entry should've been discarded. - EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o"))); + EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0))); EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); ASSERT_EQ("", err); // Add a new entry. vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar2.h")); - log.RecordDeps(state.GetNode("out2.o"), 3, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar2.h", 0)); + log.RecordDeps(state.GetNode("out2.o", 0), 3, deps); log.Close(); } @@ -464,7 +471,7 @@ TEST_F(DepsLogTest, TruncatedRecovery) { EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); // The truncated entry should exist. - DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o")); + DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o", 0)); ASSERT_TRUE(deps); } } diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 3233144..9810708 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -14,6 +14,8 @@ #include "disk_interface.h" +#include <algorithm> + #include <errno.h> #include <stdio.h> #include <string.h> @@ -31,15 +33,16 @@ namespace { string DirName(const string& path) { #ifdef _WIN32 - const char kPathSeparator = '\\'; + const char kPathSeparators[] = "\\/"; #else - const char kPathSeparator = '/'; + const char kPathSeparators[] = "/"; #endif - - string::size_type slash_pos = path.rfind(kPathSeparator); + string::size_type slash_pos = path.find_last_of(kPathSeparators); if (slash_pos == string::npos) return string(); // Nothing to do. - while (slash_pos > 0 && path[slash_pos - 1] == kPathSeparator) + const char* const kEnd = kPathSeparators + strlen(kPathSeparators); + while (slash_pos > 0 && + std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd) --slash_pos; return path.substr(0, slash_pos); } @@ -52,6 +55,82 @@ int MakeDir(const string& path) { #endif } +#ifdef _WIN32 +TimeStamp TimeStampFromFileTime(const FILETIME& filetime) { + // FILETIME is in 100-nanosecond increments since the Windows epoch. + // We don't much care about epoch correctness but we do want the + // resulting value to fit in an integer. + uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | + ((uint64_t)filetime.dwLowDateTime); + mtime /= 1000000000LL / 100; // 100ns -> s. + mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years). + return (TimeStamp)mtime; +} + +TimeStamp StatSingleFile(const string& path, bool quiet) { + WIN32_FILE_ATTRIBUTE_DATA attrs; + if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) { + DWORD err = GetLastError(); + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + return 0; + if (!quiet) { + Error("GetFileAttributesEx(%s): %s", path.c_str(), + GetLastErrorString().c_str()); + } + return -1; + } + return TimeStampFromFileTime(attrs.ftLastWriteTime); +} + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4996) // GetVersionExA is deprecated post SDK 8.1. +#endif +bool IsWindows7OrLater() { + OSVERSIONINFO version_info = { sizeof(version_info) }; + if (!GetVersionEx(&version_info)) + Fatal("GetVersionEx: %s", GetLastErrorString().c_str()); + return version_info.dwMajorVersion > 6 || + (version_info.dwMajorVersion == 6 && version_info.dwMinorVersion >= 1); +} +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps, + bool quiet) { + // FindExInfoBasic is 30% faster than FindExInfoStandard. + static bool can_use_basic_info = IsWindows7OrLater(); + // This is not in earlier SDKs. + const FINDEX_INFO_LEVELS kFindExInfoBasic = + static_cast<FINDEX_INFO_LEVELS>(1); + FINDEX_INFO_LEVELS level = + can_use_basic_info ? kFindExInfoBasic : FindExInfoStandard; + WIN32_FIND_DATAA ffd; + HANDLE find_handle = FindFirstFileExA((dir + "\\*").c_str(), level, &ffd, + FindExSearchNameMatch, NULL, 0); + + if (find_handle == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + return true; + if (!quiet) { + Error("FindFirstFileExA(%s): %s", dir.c_str(), + GetLastErrorString().c_str()); + } + return false; + } + do { + string lowername = ffd.cFileName; + transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower); + stamps->insert(make_pair(lowername, + TimeStampFromFileTime(ffd.ftLastWriteTime))); + } while (FindNextFileA(find_handle, &ffd)); + FindClose(find_handle); + return true; +} +#endif // _WIN32 + } // namespace // DiskInterface --------------------------------------------------------------- @@ -75,7 +154,7 @@ bool DiskInterface::MakeDirs(const string& path) { // RealDiskInterface ----------------------------------------------------------- -TimeStamp RealDiskInterface::Stat(const string& path) { +TimeStamp RealDiskInterface::Stat(const string& path) const { #ifdef _WIN32 // MSDN: "Naming Files, Paths, and Namespaces" // http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx @@ -86,26 +165,25 @@ TimeStamp RealDiskInterface::Stat(const string& path) { } return -1; } - WIN32_FILE_ATTRIBUTE_DATA attrs; - if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) { - DWORD err = GetLastError(); - if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) - return 0; - if (!quiet_) { - Error("GetFileAttributesEx(%s): %s", path.c_str(), - GetLastErrorString().c_str()); + if (!use_cache_) + return StatSingleFile(path, quiet_); + + string dir = DirName(path); + string base(path.substr(dir.size() ? dir.size() + 1 : 0)); + + transform(dir.begin(), dir.end(), dir.begin(), ::tolower); + transform(base.begin(), base.end(), base.begin(), ::tolower); + + Cache::iterator ci = cache_.find(dir); + if (ci == cache_.end()) { + ci = cache_.insert(make_pair(dir, DirCache())).first; + if (!StatAllFilesInDir(dir.empty() ? "." : dir, &ci->second, quiet_)) { + cache_.erase(ci); + return -1; } - return -1; } - const FILETIME& filetime = attrs.ftLastWriteTime; - // FILETIME is in 100-nanosecond increments since the Windows epoch. - // We don't much care about epoch correctness but we do want the - // resulting value to fit in an integer. - uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | - ((uint64_t)filetime.dwLowDateTime); - mtime /= 1000000000LL / 100; // 100ns -> s. - mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years). - return (TimeStamp)mtime; + DirCache::iterator di = ci->second.find(base); + return di != ci->second.end() ? di->second : 0; #else struct stat st; if (stat(path.c_str(), &st) < 0) { @@ -146,6 +224,9 @@ bool RealDiskInterface::WriteFile(const string& path, const string& contents) { bool RealDiskInterface::MakeDir(const string& path) { if (::MakeDir(path) < 0) { + if (errno == EEXIST) { + return true; + } Error("mkdir(%s): %s", path.c_str(), strerror(errno)); return false; } @@ -175,3 +256,11 @@ int RealDiskInterface::RemoveFile(const string& path) { return 0; } } + +void RealDiskInterface::AllowStatCache(bool allow) { +#ifdef _WIN32 + use_cache_ = allow; + if (!use_cache_) + cache_.clear(); +#endif +} diff --git a/src/disk_interface.h b/src/disk_interface.h index ff1e21c..a13bced 100644 --- a/src/disk_interface.h +++ b/src/disk_interface.h @@ -15,6 +15,7 @@ #ifndef NINJA_DISK_INTERFACE_H_ #define NINJA_DISK_INTERFACE_H_ +#include <map> #include <string> using namespace std; @@ -29,7 +30,7 @@ struct DiskInterface { /// stat() a file, returning the mtime, or 0 if missing and -1 on /// other errors. - virtual TimeStamp Stat(const string& path) = 0; + virtual TimeStamp Stat(const string& path) const = 0; /// Create a directory, returning false on failure. virtual bool MakeDir(const string& path) = 0; @@ -55,9 +56,13 @@ struct DiskInterface { /// Implementation of DiskInterface that actually hits the disk. struct RealDiskInterface : public DiskInterface { - RealDiskInterface() : quiet_(false) {} + RealDiskInterface() : quiet_(false) +#ifdef _WIN32 + , use_cache_(false) +#endif + {} virtual ~RealDiskInterface() {} - virtual TimeStamp Stat(const string& path); + virtual TimeStamp Stat(const string& path) const; virtual bool MakeDir(const string& path); virtual bool WriteFile(const string& path, const string& contents); virtual string ReadFile(const string& path, string* err); @@ -65,6 +70,21 @@ struct RealDiskInterface : public DiskInterface { /// Whether to print on errors. Used to make a test quieter. bool quiet_; + + /// Whether stat information can be cached. Only has an effect on Windows. + void AllowStatCache(bool allow); + + private: +#ifdef _WIN32 + /// Whether stat information can be cached. + bool use_cache_; + + typedef map<string, TimeStamp> DirCache; + // TODO: Neither a map nor a hashmap seems ideal here. If the statcache + // works out, come up with a better data structure. + typedef map<string, DirCache> Cache; + mutable Cache cache_; +#endif }; #endif // NINJA_DISK_INTERFACE_H_ diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 55822a6..05d509c 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include <gtest/gtest.h> - +#include <assert.h> +#include <stdio.h> #ifdef _WIN32 #include <io.h> #include <windows.h> @@ -76,6 +76,54 @@ TEST_F(DiskInterfaceTest, StatExistingFile) { EXPECT_GT(disk_.Stat("file"), 1); } +TEST_F(DiskInterfaceTest, StatExistingDir) { + ASSERT_TRUE(disk_.MakeDir("subdir")); + ASSERT_TRUE(disk_.MakeDir("subdir/subsubdir")); + EXPECT_GT(disk_.Stat("."), 1); + EXPECT_GT(disk_.Stat("subdir"), 1); + EXPECT_GT(disk_.Stat("subdir/subsubdir"), 1); + + EXPECT_EQ(disk_.Stat("subdir"), disk_.Stat("subdir/.")); + EXPECT_EQ(disk_.Stat("subdir"), disk_.Stat("subdir/subsubdir/..")); + EXPECT_EQ(disk_.Stat("subdir/subsubdir"), disk_.Stat("subdir/subsubdir/.")); +} + +#ifdef _WIN32 +TEST_F(DiskInterfaceTest, StatCache) { + disk_.AllowStatCache(true); + + ASSERT_TRUE(Touch("file1")); + ASSERT_TRUE(Touch("fiLE2")); + ASSERT_TRUE(disk_.MakeDir("subdir")); + ASSERT_TRUE(disk_.MakeDir("subdir/subsubdir")); + ASSERT_TRUE(Touch("subdir\\subfile1")); + ASSERT_TRUE(Touch("subdir\\SUBFILE2")); + ASSERT_TRUE(Touch("subdir\\SUBFILE3")); + + EXPECT_GT(disk_.Stat("FIle1"), 1); + EXPECT_GT(disk_.Stat("file1"), 1); + + EXPECT_GT(disk_.Stat("subdir/subfile2"), 1); + EXPECT_GT(disk_.Stat("sUbdir\\suBFile1"), 1); + + EXPECT_GT(disk_.Stat("."), 1); + EXPECT_GT(disk_.Stat("subdir"), 1); + EXPECT_GT(disk_.Stat("subdir/subsubdir"), 1); + + EXPECT_EQ(disk_.Stat("subdir"), disk_.Stat("subdir/.")); + EXPECT_EQ(disk_.Stat("subdir"), disk_.Stat("subdir/subsubdir/..")); + EXPECT_EQ(disk_.Stat("subdir/subsubdir"), disk_.Stat("subdir/subsubdir/.")); + + // Test error cases. + disk_.quiet_ = true; + string bad_path("cc:\\foo"); + EXPECT_EQ(-1, disk_.Stat(bad_path)); + EXPECT_EQ(-1, disk_.Stat(bad_path)); + EXPECT_EQ(0, disk_.Stat("nosuchfile")); + EXPECT_EQ(0, disk_.Stat("nosuchdir/nosuchfile")); +} +#endif + TEST_F(DiskInterfaceTest, ReadFile) { string err; EXPECT_EQ("", disk_.ReadFile("foobar", &err)); @@ -93,7 +141,18 @@ TEST_F(DiskInterfaceTest, ReadFile) { } TEST_F(DiskInterfaceTest, MakeDirs) { - EXPECT_TRUE(disk_.MakeDirs("path/with/double//slash/")); + string path = "path/with/double//slash/"; + EXPECT_TRUE(disk_.MakeDirs(path.c_str())); + FILE* f = fopen((path + "a_file").c_str(), "w"); + EXPECT_TRUE(f); + EXPECT_EQ(0, fclose(f)); +#ifdef _WIN32 + string path2 = "another\\with\\back\\\\slashes\\"; + EXPECT_TRUE(disk_.MakeDirs(path2.c_str())); + FILE* f2 = fopen((path2 + "a_file").c_str(), "w"); + EXPECT_TRUE(f2); + EXPECT_EQ(0, fclose(f2)); +#endif } TEST_F(DiskInterfaceTest, RemoveFile) { @@ -109,7 +168,7 @@ struct StatTest : public StateTestWithBuiltinRules, StatTest() : scan_(&state_, NULL, NULL, this) {} // DiskInterface implementation. - virtual TimeStamp Stat(const string& path); + virtual TimeStamp Stat(const string& path) const; virtual bool WriteFile(const string& path, const string& contents) { assert(false); return true; @@ -129,12 +188,12 @@ struct StatTest : public StateTestWithBuiltinRules, DependencyScan scan_; map<string, TimeStamp> mtimes_; - vector<string> stats_; + mutable vector<string> stats_; }; -TimeStamp StatTest::Stat(const string& path) { +TimeStamp StatTest::Stat(const string& path) const { stats_.push_back(path); - map<string, TimeStamp>::iterator i = mtimes_.find(path); + map<string, TimeStamp>::const_iterator i = mtimes_.find(path); if (i == mtimes_.end()) return 0; // File not found. return i->second; diff --git a/src/getopt.c b/src/getopt.c index 75ef99c..3350fb9 100644 --- a/src/getopt.c +++ b/src/getopt.c @@ -299,7 +299,7 @@ getopt_internal (int argc, char **argv, char *shortopts, return (optopt = '?'); } has_arg = ((cp[1] == ':') - ? ((cp[2] == ':') ? OPTIONAL_ARG : REQUIRED_ARG) : no_argument); + ? ((cp[2] == ':') ? OPTIONAL_ARG : required_argument) : no_argument); possible_arg = argv[optind] + optwhere + 1; optopt = *cp; } @@ -318,7 +318,7 @@ getopt_internal (int argc, char **argv, char *shortopts, else optarg = NULL; break; - case REQUIRED_ARG: + case required_argument: if (*possible_arg == '=') possible_arg++; if (*possible_arg != '\0') diff --git a/src/getopt.h b/src/getopt.h index ead9878..b4247fb 100644 --- a/src/getopt.h +++ b/src/getopt.h @@ -4,9 +4,9 @@ /* include files needed by this include file */ /* macros defined by this include file */ -#define no_argument 0 -#define REQUIRED_ARG 1 -#define OPTIONAL_ARG 2 +#define no_argument 0 +#define required_argument 1 +#define OPTIONAL_ARG 2 /* types defined by this include file */ diff --git a/src/graph.cc b/src/graph.cc index 65f9244..2829669 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -131,7 +131,7 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { } bool DependencyScan::RecomputeOutputsDirty(Edge* edge, - Node* most_recent_input) { + Node* most_recent_input) { string command = edge->EvaluateCommand(true); for (vector<Node*>::iterator i = edge->outputs_.begin(); i != edge->outputs_.end(); ++i) { @@ -215,7 +215,10 @@ bool Edge::AllInputsReady() const { /// An Env for an Edge, providing $in and $out. struct EdgeEnv : public Env { - explicit EdgeEnv(Edge* edge) : edge_(edge) {} + enum EscapeKind { kShellEscape, kDoNotEscape }; + + explicit EdgeEnv(Edge* edge, EscapeKind escape) + : edge_(edge), escape_in_out_(escape) {} virtual string LookupVariable(const string& var); /// Given a span of Nodes, construct a list of paths suitable for a command @@ -225,6 +228,7 @@ struct EdgeEnv : public Env { char sep); Edge* edge_; + EscapeKind escape_in_out_; }; string EdgeEnv::LookupVariable(const string& var) { @@ -250,13 +254,18 @@ string EdgeEnv::MakePathList(vector<Node*>::iterator begin, char sep) { string result; for (vector<Node*>::iterator i = begin; i != end; ++i) { - if (!result.empty()) result.push_back(sep); - const string& path = (*i)->path(); + if (!result.empty()) + result.push_back(sep); + const string& path = (*i)->PathDecanonicalized(); + if (escape_in_out_ == kShellEscape) { #if _WIN32 - GetWin32EscapedString(path, &result); + GetWin32EscapedString(path, &result); #else - GetShellEscapedString(path, &result); + GetShellEscapedString(path, &result); #endif + } else { + result.append(path); + } } return result; } @@ -272,7 +281,7 @@ string Edge::EvaluateCommand(bool incl_rsp_file) { } string Edge::GetBinding(const string& key) { - EdgeEnv env(this); + EdgeEnv env(this, EdgeEnv::kShellEscape); return env.LookupVariable(key); } @@ -280,6 +289,16 @@ bool Edge::GetBindingBool(const string& key) { return !GetBinding(key).empty(); } +string Edge::GetUnescapedDepfile() { + EdgeEnv env(this, EdgeEnv::kDoNotEscape); + return env.LookupVariable("depfile"); +} + +string Edge::GetUnescapedRspfile() { + EdgeEnv env(this, EdgeEnv::kDoNotEscape); + return env.LookupVariable("rspfile"); +} + void Edge::Dump(const char* prefix) const { printf("%s[ ", prefix); for (vector<Node*>::const_iterator i = inputs_.begin(); @@ -305,6 +324,24 @@ bool Edge::is_phony() const { return rule_ == &State::kPhonyRule; } +bool Edge::use_console() const { + return pool() == &State::kConsolePool; +} + +string Node::PathDecanonicalized() const { + string result = path_; +#ifdef _WIN32 + unsigned int mask = 1; + for (char* c = &result[0]; (c = strchr(c, '/')) != NULL;) { + if (slash_bits_ & mask) + *c = '\\'; + c++; + mask <<= 1; + } +#endif + return result; +} + void Node::Dump(const char* prefix) const { printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", prefix, path().c_str(), this, @@ -327,7 +364,7 @@ bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) { if (!deps_type.empty()) return LoadDepsFromLog(edge, err); - string depfile = edge->GetBinding("depfile"); + string depfile = edge->GetUnescapedDepfile(); if (!depfile.empty()) return LoadDepFile(edge, depfile, err); @@ -356,6 +393,11 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, return false; } + unsigned int unused; + if (!CanonicalizePath(const_cast<char*>(depfile.out_.str_), + &depfile.out_.len_, &unused, err)) + return false; + // Check that this depfile matches the edge's output. Node* first_output = edge->outputs_[0]; StringPiece opath = StringPiece(first_output->path()); @@ -372,10 +414,12 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, // Add all its in-edges. for (vector<StringPiece>::iterator i = depfile.ins_.begin(); i != depfile.ins_.end(); ++i, ++implicit_dep) { - if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err)) + unsigned int slash_bits; + if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, &slash_bits, + err)) return false; - Node* node = state_->GetNode(*i); + Node* node = state_->GetNode(*i, slash_bits); *implicit_dep = node; node->AddOutEdge(edge); CreatePhonyInEdge(node); diff --git a/src/graph.h b/src/graph.h index 868413c..9aada38 100644 --- a/src/graph.h +++ b/src/graph.h @@ -33,8 +33,9 @@ struct State; /// Information about a node in the dependency graph: the file, whether /// it's dirty, mtime, etc. struct Node { - explicit Node(const string& path) + Node(const string& path, unsigned int slash_bits) : path_(path), + slash_bits_(slash_bits), mtime_(-1), dirty_(false), in_edge_(NULL), @@ -71,6 +72,9 @@ struct Node { } const string& path() const { return path_; } + /// Get |path()| but use slash_bits to convert back to original slash styles. + string PathDecanonicalized() const; + unsigned int slash_bits() const { return slash_bits_; } TimeStamp mtime() const { return mtime_; } bool dirty() const { return dirty_; } @@ -90,6 +94,11 @@ struct Node { private: string path_; + + /// Set bits starting from lowest for backslashes that were normalized to + /// forward slashes by CanonicalizePath. See |PathDecanonicalized|. + unsigned int slash_bits_; + /// Possible values of mtime_: /// -1: file hasn't been examined /// 0: we looked, and file doesn't exist @@ -146,9 +155,15 @@ struct Edge { /// full contents of a response file (if applicable) string EvaluateCommand(bool incl_rsp_file = false); + /// Returns the shell-escaped value of |key|. string GetBinding(const string& key); bool GetBindingBool(const string& key); + /// Like GetBinding("depfile"), but without shell escaping. + string GetUnescapedDepfile(); + /// Like GetBinding("rspfile"), but without shell escaping. + string GetUnescapedRspfile(); + void Dump(const char* prefix="") const; const Rule* rule_; @@ -183,6 +198,7 @@ struct Edge { } bool is_phony() const; + bool use_console() const; }; diff --git a/src/graph_test.cc b/src/graph_test.cc index 14dc678..382d352 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -251,3 +251,24 @@ TEST_F(GraphTest, NestedPhonyPrintsDone) { EXPECT_EQ(0, plan_.command_edge_count()); ASSERT_FALSE(plan_.more_to_do()); } + +#ifdef _WIN32 +TEST_F(GraphTest, Decanonicalize) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out\\out1: cat src\\in1\n" +"build out\\out2/out3\\out4: cat mid1\n" +"build out3 out4\\foo: cat mid1\n")); + + string err; + vector<Node*> root_nodes = state_.RootNodes(&err); + EXPECT_EQ(4u, root_nodes.size()); + EXPECT_EQ(root_nodes[0]->path(), "out/out1"); + EXPECT_EQ(root_nodes[1]->path(), "out/out2/out3/out4"); + EXPECT_EQ(root_nodes[2]->path(), "out3"); + EXPECT_EQ(root_nodes[3]->path(), "out4/foo"); + EXPECT_EQ(root_nodes[0]->PathDecanonicalized(), "out\\out1"); + EXPECT_EQ(root_nodes[1]->PathDecanonicalized(), "out\\out2/out3\\out4"); + EXPECT_EQ(root_nodes[2]->PathDecanonicalized(), "out3"); + EXPECT_EQ(root_nodes[3]->PathDecanonicalized(), "out4\\foo"); +} +#endif diff --git a/src/hash_collision_bench.cc b/src/hash_collision_bench.cc index d0eabde..5be0531 100644 --- a/src/hash_collision_bench.cc +++ b/src/hash_collision_bench.cc @@ -46,7 +46,7 @@ int main() { sort(hashes, hashes + N); - int num_collisions = 0; + int collision_count = 0; for (int i = 1; i < N; ++i) { if (hashes[i - 1].first == hashes[i].first) { if (strcmp(commands[hashes[i - 1].second], @@ -54,9 +54,9 @@ int main() { printf("collision!\n string 1: '%s'\n string 2: '%s'\n", commands[hashes[i - 1].second], commands[hashes[i].second]); - num_collisions++; + collision_count++; } } } - printf("\n\n%d collisions after %d runs\n", num_collisions, N); + printf("\n\n%d collisions after %d runs\n", collision_count, N); } diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 05ce75d..1e88a0a 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -68,12 +68,15 @@ string IncludesNormalize::ToLower(const string& s) { string IncludesNormalize::AbsPath(StringPiece s) { char result[_MAX_PATH]; GetFullPathName(s.AsString().c_str(), sizeof(result), result, NULL); + for (char* c = result; *c; ++c) + if (*c == '\\') + *c = '/'; return result; } string IncludesNormalize::Relativize(StringPiece path, const string& start) { - vector<string> start_list = Split(AbsPath(start), '\\'); - vector<string> path_list = Split(AbsPath(path), '\\'); + vector<string> start_list = Split(AbsPath(start), '/'); + vector<string> path_list = Split(AbsPath(path), '/'); int i; for (i = 0; i < static_cast<int>(min(start_list.size(), path_list.size())); ++i) { @@ -88,7 +91,7 @@ string IncludesNormalize::Relativize(StringPiece path, const string& start) { rel_list.push_back(path_list[j]); if (rel_list.size() == 0) return "."; - return Join(rel_list, '\\'); + return Join(rel_list, '/'); } string IncludesNormalize::Normalize(const string& input, @@ -96,19 +99,17 @@ string IncludesNormalize::Normalize(const string& input, char copy[_MAX_PATH]; size_t len = input.size(); strncpy(copy, input.c_str(), input.size() + 1); - for (size_t j = 0; j < len; ++j) - if (copy[j] == '/') - copy[j] = '\\'; string err; - if (!CanonicalizePath(copy, &len, &err)) { - Warning("couldn't canonicalize '%s: %s\n", input.c_str(), err.c_str()); - } + unsigned int slash_bits; + if (!CanonicalizePath(copy, &len, &slash_bits, &err)) + Warning("couldn't canonicalize '%s': %s\n", input.c_str(), err.c_str()); + StringPiece partially_fixed(copy, len); + string curdir; if (!relative_to) { curdir = AbsPath("."); relative_to = curdir.c_str(); } - StringPiece partially_fixed(copy, len); if (!SameDrive(partially_fixed, relative_to)) return partially_fixed.AsString(); return Relativize(partially_fixed, relative_to); diff --git a/src/includes_normalize.h b/src/includes_normalize.h index 43527af..634fef3 100644 --- a/src/includes_normalize.h +++ b/src/includes_normalize.h @@ -29,7 +29,6 @@ struct IncludesNormalize { static string Relativize(StringPiece path, const string& start); /// Normalize by fixing slashes style, fixing redundant .. and . and makes the - /// path relative to |relative_to|. Case is normalized to lowercase on - /// Windows too. + /// path relative to |relative_to|. static string Normalize(const string& input, const char* relative_to); }; diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc index 419996f..c4c2476 100644 --- a/src/includes_normalize_test.cc +++ b/src/includes_normalize_test.cc @@ -14,7 +14,7 @@ #include "includes_normalize.h" -#include <gtest/gtest.h> +#include <direct.h> #include "test.h" #include "util.h" @@ -22,8 +22,8 @@ TEST(IncludesNormalize, Simple) { EXPECT_EQ("b", IncludesNormalize::Normalize("a\\..\\b", NULL)); EXPECT_EQ("b", IncludesNormalize::Normalize("a\\../b", NULL)); - EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\.\\b", NULL)); - EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\./b", NULL)); + EXPECT_EQ("a/b", IncludesNormalize::Normalize("a\\.\\b", NULL)); + EXPECT_EQ("a/b", IncludesNormalize::Normalize("a\\./b", NULL)); } namespace { @@ -42,21 +42,21 @@ TEST(IncludesNormalize, WithRelative) { EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b")); EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), NULL)); - EXPECT_EQ(string("..\\") + currentdir + string("\\a"), + EXPECT_EQ(string("../") + currentdir + string("/a"), IncludesNormalize::Normalize("a", "../b")); - EXPECT_EQ(string("..\\") + currentdir + string("\\a\\b"), + EXPECT_EQ(string("../") + currentdir + string("/a/b"), IncludesNormalize::Normalize("a/b", "../c")); - EXPECT_EQ("..\\..\\a", IncludesNormalize::Normalize("a", "b/c")); + EXPECT_EQ("../../a", IncludesNormalize::Normalize("a", "b/c")); EXPECT_EQ(".", IncludesNormalize::Normalize("a", "a")); } TEST(IncludesNormalize, Case) { EXPECT_EQ("b", IncludesNormalize::Normalize("Abc\\..\\b", NULL)); EXPECT_EQ("BdEf", IncludesNormalize::Normalize("Abc\\..\\BdEf", NULL)); - EXPECT_EQ("A\\b", IncludesNormalize::Normalize("A\\.\\b", NULL)); - EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\./b", NULL)); - EXPECT_EQ("A\\B", IncludesNormalize::Normalize("A\\.\\B", NULL)); - EXPECT_EQ("A\\B", IncludesNormalize::Normalize("A\\./B", NULL)); + EXPECT_EQ("A/b", IncludesNormalize::Normalize("A\\.\\b", NULL)); + EXPECT_EQ("a/b", IncludesNormalize::Normalize("a\\./b", NULL)); + EXPECT_EQ("A/B", IncludesNormalize::Normalize("A\\.\\B", NULL)); + EXPECT_EQ("A/B", IncludesNormalize::Normalize("A\\./B", NULL)); } TEST(IncludesNormalize, Join) { @@ -92,13 +92,13 @@ TEST(IncludesNormalize, DifferentDrive) { IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "p:\\vs08")); EXPECT_EQ("stuff.h", IncludesNormalize::Normalize("P:\\Vs08\\stuff.h", "p:\\vs08")); - EXPECT_EQ("p:\\vs08\\stuff.h", + EXPECT_EQ("p:/vs08/stuff.h", IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "c:\\vs08")); - EXPECT_EQ("P:\\vs08\\stufF.h", + EXPECT_EQ("P:/vs08/stufF.h", IncludesNormalize::Normalize("P:\\vs08\\stufF.h", "D:\\stuff/things")); - EXPECT_EQ("P:\\vs08\\stuff.h", + EXPECT_EQ("P:/vs08/stuff.h", IncludesNormalize::Normalize("P:/vs08\\stuff.h", "D:\\stuff/things")); - // TODO: this fails; fix it. - //EXPECT_EQ("P:\\wee\\stuff.h", - // IncludesNormalize::Normalize("P:/vs08\\../wee\\stuff.h", "D:\\stuff/things")); + EXPECT_EQ("P:/wee/stuff.h", + IncludesNormalize::Normalize("P:/vs08\\../wee\\stuff.h", + "D:\\stuff/things")); } diff --git a/src/lexer.cc b/src/lexer.cc index 685fe81..37b8678 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -103,8 +103,6 @@ const char* Lexer::TokenErrorHint(Token expected) { string Lexer::DescribeLastError() { if (last_token_) { switch (last_token_[0]) { - case '\r': - return "carriage returns are not allowed, use newlines"; case '\t': return "tabs are not allowed, use spaces"; } @@ -129,7 +127,7 @@ Lexer::Token Lexer::ReadToken() { unsigned int yyaccept = 0; static const unsigned char yybm[] = { 0, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 0, 64, 64, 0, 64, 64, + 64, 64, 0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 192, 64, 64, 64, 64, 64, 64, 64, @@ -163,56 +161,66 @@ Lexer::Token Lexer::ReadToken() { }; yych = *p; - if (yych <= '^') { - if (yych <= ',') { - if (yych <= 0x1F) { - if (yych <= 0x00) goto yy22; - if (yych == '\n') goto yy6; - goto yy24; + if (yych <= 'Z') { + if (yych <= '#') { + if (yych <= '\f') { + if (yych <= 0x00) goto yy23; + if (yych == '\n') goto yy7; + goto yy25; } else { - if (yych <= ' ') goto yy2; - if (yych == '#') goto yy4; - goto yy24; + if (yych <= 0x1F) { + if (yych <= '\r') goto yy6; + goto yy25; + } else { + if (yych <= ' ') goto yy2; + if (yych <= '"') goto yy25; + goto yy4; + } } } else { - if (yych <= ':') { - if (yych == '/') goto yy24; - if (yych <= '9') goto yy21; - goto yy15; + if (yych <= '9') { + if (yych <= ',') goto yy25; + if (yych == '/') goto yy25; + goto yy22; } else { - if (yych <= '=') { - if (yych <= '<') goto yy24; - goto yy13; + if (yych <= '<') { + if (yych <= ':') goto yy16; + goto yy25; } else { - if (yych <= '@') goto yy24; - if (yych <= 'Z') goto yy21; - goto yy24; + if (yych <= '=') goto yy14; + if (yych <= '@') goto yy25; + goto yy22; } } } } else { if (yych <= 'i') { - if (yych <= 'b') { - if (yych == '`') goto yy24; - if (yych <= 'a') goto yy21; - goto yy8; + if (yych <= 'a') { + if (yych == '_') goto yy22; + if (yych <= '`') goto yy25; + goto yy22; } else { - if (yych == 'd') goto yy12; - if (yych <= 'h') goto yy21; - goto yy19; + if (yych <= 'c') { + if (yych <= 'b') goto yy9; + goto yy22; + } else { + if (yych <= 'd') goto yy13; + if (yych <= 'h') goto yy22; + goto yy20; + } } } else { if (yych <= 'r') { - if (yych == 'p') goto yy10; - if (yych <= 'q') goto yy21; - goto yy11; + if (yych == 'p') goto yy11; + if (yych <= 'q') goto yy22; + goto yy12; } else { if (yych <= 'z') { - if (yych <= 's') goto yy20; - goto yy21; + if (yych <= 's') goto yy21; + goto yy22; } else { - if (yych == '|') goto yy17; - goto yy24; + if (yych == '|') goto yy18; + goto yy25; } } } @@ -220,192 +228,203 @@ Lexer::Token Lexer::ReadToken() { yy2: yyaccept = 0; yych = *(q = ++p); - goto yy70; + goto yy73; yy3: { token = INDENT; break; } yy4: yyaccept = 1; yych = *(q = ++p); - if (yych <= 0x00) goto yy5; - if (yych != '\r') goto yy65; + if (yych >= 0x01) goto yy68; yy5: { token = ERROR; break; } yy6: - ++p; + yych = *++p; + if (yych == '\n') goto yy65; + goto yy5; yy7: - { token = NEWLINE; break; } -yy8: ++p; - if ((yych = *p) == 'u') goto yy59; - goto yy26; +yy8: + { token = NEWLINE; break; } yy9: - { token = IDENT; break; } + ++p; + if ((yych = *p) == 'u') goto yy60; + goto yy27; yy10: - yych = *++p; - if (yych == 'o') goto yy55; - goto yy26; + { token = IDENT; break; } yy11: yych = *++p; - if (yych == 'u') goto yy51; - goto yy26; + if (yych == 'o') goto yy56; + goto yy27; yy12: yych = *++p; - if (yych == 'e') goto yy44; - goto yy26; + if (yych == 'u') goto yy52; + goto yy27; yy13: + yych = *++p; + if (yych == 'e') goto yy45; + goto yy27; +yy14: ++p; { token = EQUALS; break; } -yy15: +yy16: ++p; { token = COLON; break; } -yy17: +yy18: ++p; - if ((yych = *p) == '|') goto yy42; + if ((yych = *p) == '|') goto yy43; { token = PIPE; break; } -yy19: - yych = *++p; - if (yych == 'n') goto yy35; - goto yy26; yy20: yych = *++p; - if (yych == 'u') goto yy27; - goto yy26; + if (yych == 'n') goto yy36; + goto yy27; yy21: yych = *++p; - goto yy26; + if (yych == 'u') goto yy28; + goto yy27; yy22: + yych = *++p; + goto yy27; +yy23: ++p; { token = TEOF; break; } -yy24: +yy25: yych = *++p; goto yy5; -yy25: +yy26: ++p; yych = *p; -yy26: +yy27: if (yybm[0+yych] & 32) { - goto yy25; + goto yy26; } - goto yy9; -yy27: + goto yy10; +yy28: yych = *++p; - if (yych != 'b') goto yy26; + if (yych != 'b') goto yy27; yych = *++p; - if (yych != 'n') goto yy26; + if (yych != 'n') goto yy27; yych = *++p; - if (yych != 'i') goto yy26; + if (yych != 'i') goto yy27; yych = *++p; - if (yych != 'n') goto yy26; + if (yych != 'n') goto yy27; yych = *++p; - if (yych != 'j') goto yy26; + if (yych != 'j') goto yy27; yych = *++p; - if (yych != 'a') goto yy26; + if (yych != 'a') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = SUBNINJA; break; } -yy35: +yy36: yych = *++p; - if (yych != 'c') goto yy26; + if (yych != 'c') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 'u') goto yy26; + if (yych != 'u') goto yy27; yych = *++p; - if (yych != 'd') goto yy26; + if (yych != 'd') goto yy27; yych = *++p; - if (yych != 'e') goto yy26; + if (yych != 'e') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = INCLUDE; break; } -yy42: +yy43: ++p; { token = PIPE2; break; } -yy44: +yy45: yych = *++p; - if (yych != 'f') goto yy26; + if (yych != 'f') goto yy27; yych = *++p; - if (yych != 'a') goto yy26; + if (yych != 'a') goto yy27; yych = *++p; - if (yych != 'u') goto yy26; + if (yych != 'u') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 't') goto yy26; + if (yych != 't') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = DEFAULT; break; } -yy51: +yy52: yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 'e') goto yy26; + if (yych != 'e') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = RULE; break; } -yy55: +yy56: yych = *++p; - if (yych != 'o') goto yy26; + if (yych != 'o') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = POOL; break; } -yy59: +yy60: yych = *++p; - if (yych != 'i') goto yy26; + if (yych != 'i') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 'd') goto yy26; + if (yych != 'd') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = BUILD; break; } -yy64: +yy65: + ++p; + { token = NEWLINE; break; } +yy67: ++p; yych = *p; -yy65: +yy68: if (yybm[0+yych] & 64) { - goto yy64; + goto yy67; } - if (yych <= 0x00) goto yy66; - if (yych <= '\f') goto yy67; -yy66: + if (yych >= 0x01) goto yy70; +yy69: p = q; if (yyaccept <= 0) { goto yy3; } else { goto yy5; } -yy67: +yy70: ++p; { continue; } -yy69: +yy72: yyaccept = 0; q = ++p; yych = *p; -yy70: +yy73: if (yybm[0+yych] & 128) { - goto yy69; + goto yy72; + } + if (yych <= '\f') { + if (yych != '\n') goto yy3; + } else { + if (yych <= '\r') goto yy75; + if (yych == '#') goto yy67; + goto yy3; } - if (yych == '\n') goto yy71; - if (yych == '#') goto yy64; - goto yy3; -yy71: + yych = *++p; + goto yy8; +yy75: ++p; - yych = *p; - goto yy7; + if ((yych = *p) == '\n') goto yy65; + goto yy69; } } @@ -427,6 +446,7 @@ bool Lexer::PeekToken(Token token) { void Lexer::EatWhitespace() { const char* p = ofs_; + const char* q; for (;;) { ofs_ = p; @@ -468,39 +488,48 @@ void Lexer::EatWhitespace() { }; yych = *p; if (yych <= ' ') { - if (yych <= 0x00) goto yy78; - if (yych <= 0x1F) goto yy80; + if (yych <= 0x00) goto yy82; + if (yych <= 0x1F) goto yy84; } else { - if (yych == '$') goto yy76; - goto yy80; + if (yych == '$') goto yy80; + goto yy84; } ++p; yych = *p; - goto yy84; -yy75: + goto yy92; +yy79: { continue; } -yy76: - ++p; - if ((yych = *p) == '\n') goto yy81; -yy77: +yy80: + yych = *(q = ++p); + if (yych == '\n') goto yy85; + if (yych == '\r') goto yy87; +yy81: { break; } -yy78: +yy82: ++p; { break; } -yy80: +yy84: yych = *++p; - goto yy77; -yy81: + goto yy81; +yy85: ++p; { continue; } -yy83: +yy87: + yych = *++p; + if (yych == '\n') goto yy89; + p = q; + goto yy81; +yy89: + ++p; + { continue; } +yy91: ++p; yych = *p; -yy84: +yy92: if (yybm[0+yych] & 128) { - goto yy83; + goto yy91; } - goto yy75; + goto yy79; } } @@ -550,40 +579,40 @@ bool Lexer::ReadIdent(string* out) { yych = *p; if (yych <= '@') { if (yych <= '.') { - if (yych <= ',') goto yy89; + if (yych <= ',') goto yy97; } else { - if (yych <= '/') goto yy89; - if (yych >= ':') goto yy89; + if (yych <= '/') goto yy97; + if (yych >= ':') goto yy97; } } else { if (yych <= '_') { - if (yych <= 'Z') goto yy87; - if (yych <= '^') goto yy89; + if (yych <= 'Z') goto yy95; + if (yych <= '^') goto yy97; } else { - if (yych <= '`') goto yy89; - if (yych >= '{') goto yy89; + if (yych <= '`') goto yy97; + if (yych >= '{') goto yy97; } } -yy87: +yy95: ++p; yych = *p; - goto yy92; -yy88: + goto yy100; +yy96: { out->assign(start, p - start); break; } -yy89: +yy97: ++p; { return false; } -yy91: +yy99: ++p; yych = *p; -yy92: +yy100: if (yybm[0+yych] & 128) { - goto yy91; + goto yy99; } - goto yy88; + goto yy96; } } @@ -638,29 +667,36 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { yych = *p; if (yych <= ' ') { if (yych <= '\n') { - if (yych <= 0x00) goto yy101; - if (yych >= '\n') goto yy97; + if (yych <= 0x00) goto yy110; + if (yych >= '\n') goto yy107; } else { - if (yych == '\r') goto yy103; - if (yych >= ' ') goto yy97; + if (yych == '\r') goto yy105; + if (yych >= ' ') goto yy107; } } else { if (yych <= '9') { - if (yych == '$') goto yy99; + if (yych == '$') goto yy109; } else { - if (yych <= ':') goto yy97; - if (yych == '|') goto yy97; + if (yych <= ':') goto yy107; + if (yych == '|') goto yy107; } } ++p; yych = *p; - goto yy126; -yy96: + goto yy140; +yy104: { eval->AddText(StringPiece(start, p - start)); continue; } -yy97: +yy105: + ++p; + if ((yych = *p) == '\n') goto yy137; + { + last_token_ = start; + return Error(DescribeLastError(), err); + } +yy107: ++p; { if (path) { @@ -673,137 +709,152 @@ yy97: continue; } } -yy99: - ++p; - if ((yych = *p) <= '/') { - if (yych <= ' ') { - if (yych == '\n') goto yy115; - if (yych <= 0x1F) goto yy104; - goto yy106; +yy109: + yych = *++p; + if (yych <= '-') { + if (yych <= 0x1F) { + if (yych <= '\n') { + if (yych <= '\t') goto yy112; + goto yy124; + } else { + if (yych == '\r') goto yy114; + goto yy112; + } } else { - if (yych <= '$') { - if (yych <= '#') goto yy104; - goto yy108; + if (yych <= '#') { + if (yych <= ' ') goto yy115; + goto yy112; } else { - if (yych == '-') goto yy110; - goto yy104; + if (yych <= '$') goto yy117; + if (yych <= ',') goto yy112; + goto yy119; } } } else { - if (yych <= '^') { - if (yych <= ':') { - if (yych <= '9') goto yy110; - goto yy112; + if (yych <= 'Z') { + if (yych <= '9') { + if (yych <= '/') goto yy112; + goto yy119; } else { - if (yych <= '@') goto yy104; - if (yych <= 'Z') goto yy110; - goto yy104; + if (yych <= ':') goto yy121; + if (yych <= '@') goto yy112; + goto yy119; } } else { if (yych <= '`') { - if (yych <= '_') goto yy110; - goto yy104; + if (yych == '_') goto yy119; + goto yy112; } else { - if (yych <= 'z') goto yy110; - if (yych <= '{') goto yy114; - goto yy104; + if (yych <= 'z') goto yy119; + if (yych <= '{') goto yy123; + goto yy112; } } } -yy100: - { - last_token_ = start; - return Error(DescribeLastError(), err); - } -yy101: +yy110: ++p; { last_token_ = start; return Error("unexpected EOF", err); } -yy103: - yych = *++p; - goto yy100; -yy104: +yy112: ++p; -yy105: +yy113: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy106: +yy114: + yych = *++p; + if (yych == '\n') goto yy134; + goto yy113; +yy115: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy108: +yy117: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy110: +yy119: ++p; yych = *p; - goto yy124; -yy111: + goto yy133; +yy120: { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy112: +yy121: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy114: +yy123: yych = *(q = ++p); if (yybm[0+yych] & 32) { - goto yy118; + goto yy127; } - goto yy105; -yy115: + goto yy113; +yy124: ++p; yych = *p; if (yybm[0+yych] & 16) { - goto yy115; + goto yy124; } { continue; } -yy118: +yy127: ++p; yych = *p; if (yybm[0+yych] & 32) { - goto yy118; + goto yy127; } - if (yych == '}') goto yy121; + if (yych == '}') goto yy130; p = q; - goto yy105; -yy121: + goto yy113; +yy130: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy123: +yy132: ++p; yych = *p; -yy124: +yy133: if (yybm[0+yych] & 64) { - goto yy123; + goto yy132; } - goto yy111; -yy125: + goto yy120; +yy134: ++p; yych = *p; -yy126: + if (yych == ' ') goto yy134; + { + continue; + } +yy137: + ++p; + { + if (path) + p = start; + break; + } +yy139: + ++p; + yych = *p; +yy140: if (yybm[0+yych] & 128) { - goto yy125; + goto yy139; } - goto yy96; + goto yy104; } } diff --git a/src/lexer.in.cc b/src/lexer.in.cc index 93d5540..f861239 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -102,8 +102,6 @@ const char* Lexer::TokenErrorHint(Token expected) { string Lexer::DescribeLastError() { if (last_token_) { switch (last_token_[0]) { - case '\r': - return "carriage returns are not allowed, use newlines"; case '\t': return "tabs are not allowed, use spaces"; } @@ -132,8 +130,9 @@ Lexer::Token Lexer::ReadToken() { simple_varname = [a-zA-Z0-9_-]+; varname = [a-zA-Z0-9_.-]+; - [ ]*"#"[^\000\r\n]*"\n" { continue; } - [ ]*[\n] { token = NEWLINE; break; } + [ ]*"#"[^\000\n]*"\n" { continue; } + [ ]*"\r\n" { token = NEWLINE; break; } + [ ]*"\n" { token = NEWLINE; break; } [ ]+ { token = INDENT; break; } "build" { token = BUILD; break; } "pool" { token = POOL; break; } @@ -168,13 +167,15 @@ bool Lexer::PeekToken(Token token) { void Lexer::EatWhitespace() { const char* p = ofs_; + const char* q; for (;;) { ofs_ = p; /*!re2c - [ ]+ { continue; } - "$\n" { continue; } - nul { break; } - [^] { break; } + [ ]+ { continue; } + "$\r\n" { continue; } + "$\n" { continue; } + nul { break; } + [^] { break; } */ } } @@ -207,6 +208,11 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { eval->AddText(StringPiece(start, p - start)); continue; } + "\r\n" { + if (path) + p = start; + break; + } [ :|\n] { if (path) { p = start; @@ -226,6 +232,9 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { eval->AddText(StringPiece(" ", 1)); continue; } + "$\r\n"[ ]* { + continue; + } "$\n"[ ]* { continue; } diff --git a/src/lexer_test.cc b/src/lexer_test.cc index e8a1642..331d8e1 100644 --- a/src/lexer_test.cc +++ b/src/lexer_test.cc @@ -14,9 +14,8 @@ #include "lexer.h" -#include <gtest/gtest.h> - #include "eval_env.h" +#include "test.h" TEST(Lexer, ReadVarValue) { Lexer lexer("plain text $var $VaR ${x}\n"); diff --git a/src/line_printer.cc b/src/line_printer.cc index 3537e88..2cd3e17 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -21,12 +21,13 @@ #else #include <unistd.h> #include <sys/ioctl.h> +#include <termios.h> #include <sys/time.h> #endif #include "util.h" -LinePrinter::LinePrinter() : have_blank_line_(true) { +LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) { #ifndef _WIN32 const char* term = getenv("TERM"); smart_terminal_ = isatty(1) && term && string(term) != "dumb"; @@ -43,29 +44,27 @@ LinePrinter::LinePrinter() : have_blank_line_(true) { } void LinePrinter::Print(string to_print, LineType type) { -#ifdef _WIN32 - CONSOLE_SCREEN_BUFFER_INFO csbi; - GetConsoleScreenBufferInfo(console_, &csbi); -#endif + if (console_locked_) { + line_buffer_ = to_print; + line_type_ = type; + return; + } if (smart_terminal_) { -#ifndef _WIN32 printf("\r"); // Print over previous line, if any. -#else - csbi.dwCursorPosition.X = 0; - SetConsoleCursorPosition(console_, csbi.dwCursorPosition); -#endif + // On Windows, calling a C library function writing to stdout also handles + // pausing the executable when the "Pause" key or Ctrl-S is pressed. } if (smart_terminal_ && type == ELIDE) { #ifdef _WIN32 - // Don't use the full width or console will move to next line. - size_t width = static_cast<size_t>(csbi.dwSize.X) - 1; - to_print = ElideMiddle(to_print, width); - // We don't want to have the cursor spamming back and forth, so - // use WriteConsoleOutput instead which updates the contents of - // the buffer, but doesn't move the cursor position. + CONSOLE_SCREEN_BUFFER_INFO csbi; GetConsoleScreenBufferInfo(console_, &csbi); + + to_print = ElideMiddle(to_print, static_cast<size_t>(csbi.dwSize.X)); + // We don't want to have the cursor spamming back and forth, so instead of + // printf use WriteConsoleOutput which updates the contents of the buffer, + // but doesn't move the cursor position. COORD buf_size = { csbi.dwSize.X, 1 }; COORD zero_zero = { 0, 0 }; SMALL_RECT target = { @@ -73,16 +72,12 @@ void LinePrinter::Print(string to_print, LineType type) { static_cast<SHORT>(csbi.dwCursorPosition.X + csbi.dwSize.X - 1), csbi.dwCursorPosition.Y }; - CHAR_INFO* char_data = new CHAR_INFO[csbi.dwSize.X]; - memset(char_data, 0, sizeof(CHAR_INFO) * csbi.dwSize.X); - for (int i = 0; i < csbi.dwSize.X; ++i) { - char_data[i].Char.AsciiChar = ' '; + vector<CHAR_INFO> char_data(csbi.dwSize.X); + for (size_t i = 0; i < static_cast<size_t>(csbi.dwSize.X); ++i) { + char_data[i].Char.AsciiChar = i < to_print.size() ? to_print[i] : ' '; char_data[i].Attributes = csbi.wAttributes; } - for (size_t i = 0; i < to_print.size(); ++i) - char_data[i].Char.AsciiChar = to_print[i]; - WriteConsoleOutput(console_, char_data, buf_size, zero_zero, &target); - delete[] char_data; + WriteConsoleOutput(console_, &char_data[0], buf_size, zero_zero, &target); #else // Limit output to width of the terminal if provided so we don't cause // line-wrapping. @@ -101,13 +96,46 @@ void LinePrinter::Print(string to_print, LineType type) { } } -void LinePrinter::PrintOnNewLine(const string& to_print) { - if (!have_blank_line_) - printf("\n"); - if (!to_print.empty()) { +void LinePrinter::PrintOrBuffer(const char* data, size_t size) { + if (console_locked_) { + output_buffer_.append(data, size); + } else { // Avoid printf and C strings, since the actual output might contain null // bytes like UTF-16 does (yuck). - fwrite(&to_print[0], sizeof(char), to_print.size(), stdout); + fwrite(data, 1, size, stdout); + } +} + +void LinePrinter::PrintOnNewLine(const string& to_print) { + if (console_locked_ && !line_buffer_.empty()) { + output_buffer_.append(line_buffer_); + output_buffer_.append(1, '\n'); + line_buffer_.clear(); + } + if (!have_blank_line_) { + PrintOrBuffer("\n", 1); + } + if (!to_print.empty()) { + PrintOrBuffer(&to_print[0], to_print.size()); } have_blank_line_ = to_print.empty() || *to_print.rbegin() == '\n'; } + +void LinePrinter::SetConsoleLocked(bool locked) { + if (locked == console_locked_) + return; + + if (locked) + PrintOnNewLine(""); + + console_locked_ = locked; + + if (!locked) { + PrintOnNewLine(output_buffer_); + if (!line_buffer_.empty()) { + Print(line_buffer_, line_type_); + } + output_buffer_.clear(); + line_buffer_.clear(); + } +} diff --git a/src/line_printer.h b/src/line_printer.h index aea2817..55225e5 100644 --- a/src/line_printer.h +++ b/src/line_printer.h @@ -15,6 +15,7 @@ #ifndef NINJA_LINE_PRINTER_H_ #define NINJA_LINE_PRINTER_H_ +#include <stddef.h> #include <string> using namespace std; @@ -37,6 +38,10 @@ struct LinePrinter { /// Prints a string on a new line, not overprinting previous output. void PrintOnNewLine(const string& to_print); + /// Lock or unlock the console. Any output sent to the LinePrinter while the + /// console is locked will not be printed until it is unlocked. + void SetConsoleLocked(bool locked); + private: /// Whether we can do fancy terminal control codes. bool smart_terminal_; @@ -44,9 +49,24 @@ struct LinePrinter { /// Whether the caret is at the beginning of a blank line. bool have_blank_line_; + /// Whether console is locked. + bool console_locked_; + + /// Buffered current line while console is locked. + string line_buffer_; + + /// Buffered line type while console is locked. + LineType line_type_; + + /// Buffered console output while console is locked. + string output_buffer_; + #ifdef _WIN32 void* console_; #endif + + /// Print the given data to the console, or buffer it if it is locked. + void PrintOrBuffer(const char *data, size_t size); }; #endif // NINJA_LINE_PRINTER_H_ diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 20be7f3..388b5bc 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -191,7 +191,7 @@ bool ManifestParser::ParseRule(string* err) { bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) { if (!lexer_.ReadIdent(key)) - return false; + return lexer_.Error("expected variable name", err); if (!ExpectToken(Lexer::EQUALS, err)) return false; if (!lexer_.ReadVarValue(value, err)) @@ -209,7 +209,8 @@ bool ManifestParser::ParseDefault(string* err) { do { string path = eval.Evaluate(env_); string path_err; - if (!CanonicalizePath(&path, &path_err)) + unsigned int slash_bits; // Unused because this only does lookup. + if (!CanonicalizePath(&path, &slash_bits, &path_err)) return lexer_.Error(path_err, err); if (!state_->AddDefault(path, &path_err)) return lexer_.Error(path_err, err); @@ -296,16 +297,17 @@ bool ManifestParser::ParseEdge(string* err) { if (!ExpectToken(Lexer::NEWLINE, err)) return false; - // XXX scoped_ptr to handle error case. - BindingEnv* env = new BindingEnv(env_); - - while (lexer_.PeekToken(Lexer::INDENT)) { + // Bindings on edges are rare, so allocate per-edge envs only when needed. + bool hasIdent = lexer_.PeekToken(Lexer::INDENT); + BindingEnv* env = hasIdent ? new BindingEnv(env_) : env_; + while (hasIdent) { string key; EvalString val; if (!ParseLet(&key, &val, err)) return false; env->AddBinding(key, val.Evaluate(env_)); + hasIdent = lexer_.PeekToken(Lexer::INDENT); } Edge* edge = state_->AddEdge(rule); @@ -322,16 +324,18 @@ bool ManifestParser::ParseEdge(string* err) { for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) { string path = i->Evaluate(env); string path_err; - if (!CanonicalizePath(&path, &path_err)) + unsigned int slash_bits; + if (!CanonicalizePath(&path, &slash_bits, &path_err)) return lexer_.Error(path_err, err); - state_->AddIn(edge, path); + state_->AddIn(edge, path, slash_bits); } for (vector<EvalString>::iterator i = outs.begin(); i != outs.end(); ++i) { string path = i->Evaluate(env); string path_err; - if (!CanonicalizePath(&path, &path_err)) + unsigned int slash_bits; + if (!CanonicalizePath(&path, &slash_bits, &path_err)) return lexer_.Error(path_err, err); - state_->AddOut(edge, path); + state_->AddOut(edge, path, slash_bits); } edge->implicit_deps_ = implicit; edge->order_only_deps_ = order_only; diff --git a/src/manifest_parser_perftest.cc b/src/manifest_parser_perftest.cc new file mode 100644 index 0000000..ca62fb2 --- /dev/null +++ b/src/manifest_parser_perftest.cc @@ -0,0 +1,118 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests manifest parser performance. Expects to be run in ninja's root +// directory. + +#include <numeric> + +#include <errno.h> +#include <stdio.h> +#include <string.h> + +#ifdef _WIN32 +#include "getopt.h" +#include <direct.h> +#else +#include <getopt.h> +#include <unistd.h> +#endif + +#include "disk_interface.h" +#include "graph.h" +#include "manifest_parser.h" +#include "metrics.h" +#include "state.h" +#include "util.h" + +struct RealFileReader : public ManifestParser::FileReader { + virtual bool ReadFile(const string& path, string* content, string* err) { + return ::ReadFile(path, content, err) == 0; + } +}; + +bool WriteFakeManifests(const string& dir) { + RealDiskInterface disk_interface; + if (disk_interface.Stat(dir + "/build.ninja") > 0) + return true; + + printf("Creating manifest data..."); fflush(stdout); + int err = system(("python misc/write_fake_manifests.py " + dir).c_str()); + printf("done.\n"); + return err == 0; +} + +int LoadManifests(bool measure_command_evaluation) { + string err; + RealFileReader file_reader; + State state; + ManifestParser parser(&state, &file_reader); + if (!parser.Load("build.ninja", &err)) { + fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); + exit(1); + } + // Doing an empty build involves reading the manifest and evaluating all + // commands required for the requested targets. So include command + // evaluation in the perftest by default. + int optimization_guard = 0; + if (measure_command_evaluation) + for (size_t i = 0; i < state.edges_.size(); ++i) + optimization_guard += state.edges_[i]->EvaluateCommand().size(); + return optimization_guard; +} + +int main(int argc, char* argv[]) { + bool measure_command_evaluation = true; + int opt; + while ((opt = getopt(argc, argv, const_cast<char*>("fh"))) != -1) { + switch (opt) { + case 'f': + measure_command_evaluation = false; + break; + case 'h': + default: + printf("usage: manifest_parser_perftest\n" +"\n" +"options:\n" +" -f only measure manifest load time, not command evaluation time\n" + ); + return 1; + } + } + + const char kManifestDir[] = "build/manifest_perftest"; + + if (!WriteFakeManifests(kManifestDir)) { + fprintf(stderr, "Failed to write test data\n"); + return 1; + } + + if (chdir(kManifestDir) < 0) + Fatal("chdir: %s", strerror(errno)); + + const int kNumRepetitions = 5; + vector<int> times; + for (int i = 0; i < kNumRepetitions; ++i) { + int64_t start = GetTimeMillis(); + int optimization_guard = LoadManifests(measure_command_evaluation); + int delta = (int)(GetTimeMillis() - start); + printf("%dms (hash: %x)\n", delta, optimization_guard); + times.push_back(delta); + } + + int min = *min_element(times.begin(), times.end()); + int max = *max_element(times.begin(), times.end()); + float total = accumulate(times.begin(), times.end(), 0.0f); + printf("min %dms max %dms avg %.1fms\n", min, max, total / times.size()); +} diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 152b965..6909ea9 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -17,17 +17,16 @@ #include <map> #include <vector> -#include <gtest/gtest.h> - #include "graph.h" #include "state.h" +#include "test.h" struct ParserTest : public testing::Test, public ManifestParser::FileReader { void AssertParse(const char* input) { ManifestParser parser(&state, this); string err; - ASSERT_TRUE(parser.ParseTest(input, &err)) << err; + EXPECT_TRUE(parser.ParseTest(input, &err)); ASSERT_EQ("", err); } @@ -97,7 +96,7 @@ TEST_F(ParserTest, IgnoreIndentedComments) { ASSERT_EQ(2u, state.rules_.size()); const Rule* rule = state.rules_.begin()->second; EXPECT_EQ("cat", rule->name()); - Edge* edge = state.GetNode("result")->in_edge(); + Edge* edge = state.GetNode("result", 0)->in_edge(); EXPECT_TRUE(edge->GetBindingBool("restat")); EXPECT_FALSE(edge->GetBindingBool("generator")); } @@ -270,6 +269,26 @@ TEST_F(ParserTest, CanonicalizeFile) { EXPECT_FALSE(state.LookupNode("in//2")); } +#ifdef _WIN32 +TEST_F(ParserTest, CanonicalizeFileBackslashes) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build out: cat in\\1 in\\\\2\n" +"build in\\1: cat\n" +"build in\\2: cat\n")); + + Node* node = state.LookupNode("in/1");; + EXPECT_TRUE(node); + EXPECT_EQ(1, node->slash_bits()); + node = state.LookupNode("in/2"); + EXPECT_TRUE(node); + EXPECT_EQ(1, node->slash_bits()); + EXPECT_FALSE(state.LookupNode("in//1")); + EXPECT_FALSE(state.LookupNode("in//2")); +} +#endif + TEST_F(ParserTest, PathVariables) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule cat\n" @@ -293,6 +312,34 @@ TEST_F(ParserTest, CanonicalizePaths) { EXPECT_TRUE(state.LookupNode("bar/foo.cc")); } +#ifdef _WIN32 +TEST_F(ParserTest, CanonicalizePathsBackslashes) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build ./out.o: cat ./bar/baz/../foo.cc\n" +"build .\\out2.o: cat .\\bar/baz\\..\\foo.cc\n" +"build .\\out3.o: cat .\\bar\\baz\\..\\foo3.cc\n" +)); + + EXPECT_FALSE(state.LookupNode("./out.o")); + EXPECT_FALSE(state.LookupNode(".\\out2.o")); + EXPECT_FALSE(state.LookupNode(".\\out3.o")); + EXPECT_TRUE(state.LookupNode("out.o")); + EXPECT_TRUE(state.LookupNode("out2.o")); + EXPECT_TRUE(state.LookupNode("out3.o")); + EXPECT_FALSE(state.LookupNode("./bar/baz/../foo.cc")); + EXPECT_FALSE(state.LookupNode(".\\bar/baz\\..\\foo.cc")); + EXPECT_FALSE(state.LookupNode(".\\bar/baz\\..\\foo3.cc")); + Node* node = state.LookupNode("bar/foo.cc"); + EXPECT_TRUE(node); + EXPECT_EQ(0, node->slash_bits()); + node = state.LookupNode("bar/foo3.cc"); + EXPECT_TRUE(node); + EXPECT_EQ(1, node->slash_bits()); +} +#endif + TEST_F(ParserTest, ReservedWords) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule build\n" @@ -553,6 +600,15 @@ TEST_F(ParserTest, Errors) { State state; ManifestParser parser(&state, NULL); string err; + EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n && bar", + &err)); + EXPECT_EQ("input:3: expected variable name\n", err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n" "build $: cc bar.cc\n", &err)); @@ -862,22 +918,18 @@ TEST_F(ParserTest, UTF8) { " description = compilaci\xC3\xB3\n")); } -// We might want to eventually allow CRLF to be nice to Windows developers, -// but for now just verify we error out with a nice message. TEST_F(ParserTest, CRLF) { State state; ManifestParser parser(&state, NULL); string err; - EXPECT_FALSE(parser.ParseTest("# comment with crlf\r\n", - &err)); - EXPECT_EQ("input:1: lexing error\n", - err); - - EXPECT_FALSE(parser.ParseTest("foo = foo\nbar = bar\r\n", - &err)); - EXPECT_EQ("input:2: carriage returns are not allowed, use newlines\n" - "bar = bar\r\n" - " ^ near here", - err); + EXPECT_TRUE(parser.ParseTest("# comment with crlf\r\n", &err)); + EXPECT_TRUE(parser.ParseTest("foo = foo\nbar = bar\r\n", &err)); + EXPECT_TRUE(parser.ParseTest( + "pool link_pool\r\n" + " depth = 15\r\n\r\n" + "rule xyz\r\n" + " command = something$expand \r\n" + " description = YAY!\r\n", + &err)); } diff --git a/src/minidump-win32.cc b/src/minidump-win32.cc index c79ec0e..c611919 100644 --- a/src/minidump-win32.cc +++ b/src/minidump-win32.cc @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef NINJA_BOOTSTRAP +#ifdef _MSC_VER #include <windows.h> #include <DbgHelp.h> - #include "util.h" typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( @@ -85,4 +84,4 @@ void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep) { Warning("minidump created: %s", temp_file); } -#endif // NINJA_BOOTSTRAP +#endif // _MSC_VER diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index d2e2eb5..e465279 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -141,7 +141,7 @@ int CLWrapper::Run(const string& command, string* output) { STARTUPINFO startup_info = {}; startup_info.cb = sizeof(STARTUPINFO); startup_info.hStdInput = nul; - startup_info.hStdError = stdout_write; + startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); startup_info.hStdOutput = stdout_write; startup_info.dwFlags |= STARTF_USESTDHANDLES; diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index 48fbe21..29db650 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -14,8 +14,6 @@ #include "msvc_helper.h" -#include <gtest/gtest.h> - #include "test.h" #include "util.h" @@ -119,3 +117,10 @@ TEST(MSVCHelperTest, EnvBlock) { cl.Run("cmd /c \"echo foo is %foo%", &output); ASSERT_EQ("foo is bar\r\n", output); } + +TEST(MSVCHelperTest, NoReadOfStderr) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo to stdout&& echo to stderr 1>&2", &output); + ASSERT_EQ("to stdout\r\n", output); +} diff --git a/src/ninja.cc b/src/ninja.cc index 03ca83b..3e99782 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -147,7 +147,9 @@ struct NinjaMain : public BuildLogUser { // edge is rare, and the first recompaction will delete all old outputs from // the deps log, and then a second recompaction will clear the build log, // which seems good enough for this corner case.) - return !n || !n->in_edge(); + // Do keep entries around for files which still exist on disk, for + // generators that want to use this information. + return (!n || !n->in_edge()) && disk_interface_.Stat(s.AsString()) == 0; } }; @@ -161,7 +163,8 @@ struct Tool { /// When to run the tool. enum { - /// Run after parsing the command-line flags (as early as possible). + /// Run after parsing the command-line flags and potentially changing + /// the current working directory (as early as possible). RUN_AFTER_FLAGS, /// Run after loading build.ninja. @@ -190,9 +193,6 @@ void Usage(const BuildConfig& config) { "\n" " -j N run N jobs in parallel [default=%d, derived from CPUs available]\n" " -l N do not start new jobs if the load average is greater than N\n" -#ifdef _WIN32 -" (not yet implemented on Windows)\n" -#endif " -k N keep going until N jobs fail [default=1]\n" " -n dry run (don't run commands but act like they succeeded)\n" " -v show all command lines while building\n" @@ -228,7 +228,8 @@ struct RealFileReader : public ManifestParser::FileReader { /// Returns true if the manifest was rebuilt. bool NinjaMain::RebuildManifest(const char* input_file, string* err) { string path = input_file; - if (!CanonicalizePath(&path, err)) + unsigned int slash_bits; // Unused because this path is only used for lookup. + if (!CanonicalizePath(&path, &slash_bits, err)) return false; Node* node = state_.LookupNode(path); if (!node) @@ -240,17 +241,17 @@ bool NinjaMain::RebuildManifest(const char* input_file, string* err) { if (builder.AlreadyUpToDate()) return false; // Not an error, but we didn't rebuild. - if (!builder.Build(err)) - return false; - // The manifest was only rebuilt if it is now dirty (it may have been cleaned - // by a restat). - return node->dirty(); + // Even if the manifest was cleaned by a restat rule, claim that it was + // rebuilt. Not doing so can lead to crashes, see + // https://github.com/martine/ninja/issues/874 + return builder.Build(err); } Node* NinjaMain::CollectTarget(const char* cpath, string* err) { string path = cpath; - if (!CanonicalizePath(&path, err)) + unsigned int slash_bits; // Unused because this path is only used for lookup. + if (!CanonicalizePath(&path, &slash_bits, err)) return NULL; // Special syntax: "foo.cc^" means "the first output of foo.cc". @@ -363,7 +364,7 @@ int NinjaMain::ToolQuery(int argc, char* argv[]) { return 0; } -#if !defined(_WIN32) && !defined(NINJA_BOOTSTRAP) +#if defined(NINJA_HAVE_BROWSE) int NinjaMain::ToolBrowse(int argc, char* argv[]) { if (argc < 1) { Error("expected a target to browse"); @@ -631,6 +632,8 @@ int NinjaMain::ToolCompilationDatabase(int argc, char* argv[]) { putchar('['); for (vector<Edge*>::iterator e = state_.edges_.begin(); e != state_.edges_.end(); ++e) { + if ((*e)->inputs_.empty()) + continue; for (int i = 0; i != argc; ++i) { if ((*e)->rule_->name() == argv[i]) { if (!first) @@ -694,7 +697,7 @@ int NinjaMain::ToolUrtle(int argc, char** argv) { /// Returns a Tool, or NULL if Ninja should exit. const Tool* ChooseTool(const string& tool_name) { static const Tool kTools[] = { -#if !defined(_WIN32) && !defined(NINJA_BOOTSTRAP) +#if defined(NINJA_HAVE_BROWSE) { "browse", "browse dependency graph in a web browser", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolBrowse }, #endif @@ -758,6 +761,9 @@ bool DebugEnable(const string& name) { " stats print operation counts/timing info\n" " explain explain what caused a command to execute\n" " keeprsp don't delete @response files on success\n" +#ifdef _WIN32 +" nostatcache don't batch stat() calls per directory and cache them\n" +#endif "multiple modes can be enabled via -d FOO -d BAR\n"); return false; } else if (name == "stats") { @@ -769,9 +775,13 @@ bool DebugEnable(const string& name) { } else if (name == "keeprsp") { g_keep_rsp = true; return true; + } else if (name == "nostatcache") { + g_experimental_statcache = false; + return true; } else { const char* suggestion = - SpellcheckString(name.c_str(), "stats", "explain", NULL); + SpellcheckString(name.c_str(), "stats", "explain", "keeprsp", + "nostatcache", NULL); if (suggestion) { Error("unknown debug setting '%s', did you mean '%s'?", name.c_str(), suggestion); @@ -880,6 +890,8 @@ int NinjaMain::RunBuild(int argc, char** argv) { return 1; } + disk_interface_.AllowStatCache(g_experimental_statcache); + Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_); for (size_t i = 0; i < targets.size(); ++i) { if (!builder.AddTarget(targets[i], &err)) { @@ -893,6 +905,9 @@ int NinjaMain::RunBuild(int argc, char** argv) { } } + // Make sure restat rules do not see stale timestamps. + disk_interface_.AllowStatCache(false); + if (builder.AlreadyUpToDate()) { printf("ninja: no work to do.\n"); return 0; @@ -1026,13 +1041,6 @@ int real_main(int argc, char** argv) { if (exit_code >= 0) return exit_code; - if (options.tool && options.tool->when == Tool::RUN_AFTER_FLAGS) { - // None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed - // by other tools. - NinjaMain ninja(ninja_command, config); - return (ninja.*options.tool->func)(argc, argv); - } - if (options.working_dir) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for @@ -1046,6 +1054,13 @@ int real_main(int argc, char** argv) { } } + if (options.tool && options.tool->when == Tool::RUN_AFTER_FLAGS) { + // None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed + // by other tools. + NinjaMain ninja(ninja_command, config); + return (ninja.*options.tool->func)(argc, argv); + } + // The build can take up to 2 passes: one to rebuild the manifest, then // another to build the desired target. for (int cycle = 0; cycle < 2; ++cycle) { @@ -1095,7 +1110,7 @@ int real_main(int argc, char** argv) { } // anonymous namespace int main(int argc, char** argv) { -#if !defined(NINJA_BOOTSTRAP) && defined(_MSC_VER) +#if defined(_MSC_VER) // Set a handler to catch crashes not caught by the __try..__except // block (e.g. an exception in a stack-unwind-block). set_terminate(TerminateHandler); diff --git a/src/ninja_test.cc b/src/ninja_test.cc index 989ea5c..54d8784 100644 --- a/src/ninja_test.cc +++ b/src/ninja_test.cc @@ -15,9 +15,35 @@ #include <stdarg.h> #include <stdio.h> -#include "gtest/gtest.h" +#ifdef _WIN32 +#include "getopt.h" +#else +#include <getopt.h> +#endif + +#include "test.h" #include "line_printer.h" +struct RegisteredTest { + testing::Test* (*factory)(); + const char *name; + bool should_run; +}; +// This can't be a vector because tests call RegisterTest from static +// initializers and the order static initializers run it isn't specified. So +// the vector constructor isn't guaranteed to run before all of the +// RegisterTest() calls. +static RegisteredTest tests[10000]; +testing::Test* g_current_test; +static int ntests; +static LinePrinter printer; + +void RegisterTest(testing::Test* (*factory)(), const char* name) { + tests[ntests].factory = factory; + tests[ntests++].name = name; +} + +namespace { string StringPrintf(const char* format, ...) { const int N = 1024; char buf[N]; @@ -30,59 +56,101 @@ string StringPrintf(const char* format, ...) { return buf; } -/// A test result printer that's less wordy than gtest's default. -struct LaconicPrinter : public testing::EmptyTestEventListener { - LaconicPrinter() : tests_started_(0), test_count_(0), iteration_(0) {} - virtual void OnTestProgramStart(const testing::UnitTest& unit_test) { - test_count_ = unit_test.test_to_run_count(); - } +void Usage() { + fprintf(stderr, +"usage: ninja_tests [options]\n" +"\n" +"options:\n" +" --gtest_filter=POSTIVE_PATTERN[-NEGATIVE_PATTERN]\n" +" Run tests whose names match the positive but not the negative pattern.\n" +" '*' matches any substring. (gtest's ':', '?' are not implemented).\n"); +} - virtual void OnTestIterationStart(const testing::UnitTest& test_info, - int iteration) { - tests_started_ = 0; - iteration_ = iteration; +bool PatternMatchesString(const char* pattern, const char* str) { + switch (*pattern) { + case '\0': + case '-': return *str == '\0'; + case '*': return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || + PatternMatchesString(pattern + 1, str); + default: return *pattern == *str && + PatternMatchesString(pattern + 1, str + 1); } +} - virtual void OnTestStart(const testing::TestInfo& test_info) { - ++tests_started_; - printer_.Print( - StringPrintf("[%d/%d%s] %s.%s", - tests_started_, - test_count_, - iteration_ ? StringPrintf(" iter %d", iteration_).c_str() - : "", - test_info.test_case_name(), - test_info.name()), - LinePrinter::ELIDE); - } +bool TestMatchesFilter(const char* test, const char* filter) { + // Split --gtest_filter at '-' into positive and negative filters. + const char* const dash = strchr(filter, '-'); + const char* pos = dash == filter ? "*" : filter; //Treat '-test1' as '*-test1' + const char* neg = dash ? dash + 1 : ""; + return PatternMatchesString(pos, test) && !PatternMatchesString(neg, test); +} - virtual void OnTestPartResult( - const testing::TestPartResult& test_part_result) { - if (!test_part_result.failed()) - return; - printer_.PrintOnNewLine(StringPrintf( - "*** Failure in %s:%d\n%s\n", test_part_result.file_name(), - test_part_result.line_number(), test_part_result.summary())); - } +bool ReadFlags(int* argc, char*** argv, const char** test_filter) { + enum { OPT_GTEST_FILTER = 1 }; + const option kLongOptions[] = { + { "gtest_filter", required_argument, NULL, OPT_GTEST_FILTER }, + { NULL, 0, NULL, 0 } + }; - virtual void OnTestProgramEnd(const testing::UnitTest& unit_test) { - printer_.PrintOnNewLine(unit_test.Passed() ? "passed\n" : "failed\n"); + int opt; + while ((opt = getopt_long(*argc, *argv, "h", kLongOptions, NULL)) != -1) { + switch (opt) { + case OPT_GTEST_FILTER: + if (strchr(optarg, '?') == NULL && strchr(optarg, ':') == NULL) { + *test_filter = optarg; + break; + } // else fall through. + default: + Usage(); + return false; + } } + *argv += optind; + *argc -= optind; + return true; +} - private: - LinePrinter printer_; - int tests_started_; - int test_count_; - int iteration_; -}; +} // namespace + +bool testing::Test::Check(bool condition, const char* file, int line, + const char* error) { + if (!condition) { + printer.PrintOnNewLine( + StringPrintf("*** Failure in %s:%d\n%s\n", file, line, error)); + failed_ = true; + } + return condition; +} int main(int argc, char **argv) { - testing::InitGoogleTest(&argc, argv); + int tests_started = 0; - testing::TestEventListeners& listeners = - testing::UnitTest::GetInstance()->listeners(); - delete listeners.Release(listeners.default_result_printer()); - listeners.Append(new LaconicPrinter); + const char* test_filter = "*"; + if (!ReadFlags(&argc, &argv, &test_filter)) + return 1; + + int nactivetests = 0; + for (int i = 0; i < ntests; i++) + if ((tests[i].should_run = TestMatchesFilter(tests[i].name, test_filter))) + ++nactivetests; + + bool passed = true; + for (int i = 0; i < ntests; i++) { + if (!tests[i].should_run) continue; + + ++tests_started; + testing::Test* test = tests[i].factory(); + printer.Print( + StringPrintf("[%d/%d] %s", tests_started, nactivetests, tests[i].name), + LinePrinter::ELIDE); + test->SetUp(); + test->Run(); + test->TearDown(); + if (test->Failed()) + passed = false; + delete test; + } - return RUN_ALL_TESTS(); + printer.PrintOnNewLine(passed ? "passed\n" : "failed\n"); + return passed ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/src/state.cc b/src/state.cc index 33f8423..6e3e10d 100644 --- a/src/state.cc +++ b/src/state.cc @@ -69,11 +69,13 @@ bool Pool::WeightedEdgeCmp(const Edge* a, const Edge* b) { } Pool State::kDefaultPool("", 0); +Pool State::kConsolePool("console", 1); const Rule State::kPhonyRule("phony"); State::State() { AddRule(&kPhonyRule); AddPool(&kDefaultPool); + AddPool(&kConsolePool); } void State::AddRule(const Rule* rule) { @@ -109,11 +111,11 @@ Edge* State::AddEdge(const Rule* rule) { return edge; } -Node* State::GetNode(StringPiece path) { +Node* State::GetNode(StringPiece path, unsigned int slash_bits) { Node* node = LookupNode(path); if (node) return node; - node = new Node(path.AsString()); + node = new Node(path.AsString(), slash_bits); paths_[node->path()] = node; return node; } @@ -143,14 +145,14 @@ Node* State::SpellcheckNode(const string& path) { return result; } -void State::AddIn(Edge* edge, StringPiece path) { - Node* node = GetNode(path); +void State::AddIn(Edge* edge, StringPiece path, unsigned int slash_bits) { + Node* node = GetNode(path, slash_bits); edge->inputs_.push_back(node); node->AddOutEdge(edge); } -void State::AddOut(Edge* edge, StringPiece path) { - Node* node = GetNode(path); +void State::AddOut(Edge* edge, StringPiece path, unsigned int slash_bits) { + Node* node = GetNode(path, slash_bits); edge->outputs_.push_back(node); if (node->in_edge()) { Warning("multiple rules generate %s. " diff --git a/src/state.h b/src/state.h index bcb0eff..8804532 100644 --- a/src/state.h +++ b/src/state.h @@ -82,6 +82,7 @@ struct Pool { /// Global state (file status, loaded rules) for a single run. struct State { static Pool kDefaultPool; + static Pool kConsolePool; static const Rule kPhonyRule; State(); @@ -94,12 +95,12 @@ struct State { Edge* AddEdge(const Rule* rule); - Node* GetNode(StringPiece path); + Node* GetNode(StringPiece path, unsigned int slash_bits); Node* LookupNode(StringPiece path) const; Node* SpellcheckNode(const string& path); - void AddIn(Edge* edge, StringPiece path); - void AddOut(Edge* edge, StringPiece path); + void AddIn(Edge* edge, StringPiece path, unsigned int slash_bits); + void AddOut(Edge* edge, StringPiece path, unsigned int slash_bits); bool AddDefault(StringPiece path, string* error); /// Reset state. Keeps all nodes and edges, but restores them to the diff --git a/src/state_test.cc b/src/state_test.cc index af2bff1..bd133b6 100644 --- a/src/state_test.cc +++ b/src/state_test.cc @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include <gtest/gtest.h> - #include "graph.h" #include "state.h" +#include "test.h" namespace { @@ -33,15 +32,15 @@ TEST(State, Basic) { state.AddRule(rule); Edge* edge = state.AddEdge(rule); - state.AddIn(edge, "in1"); - state.AddIn(edge, "in2"); - state.AddOut(edge, "out"); + state.AddIn(edge, "in1", 0); + state.AddIn(edge, "in2", 0); + state.AddOut(edge, "out", 0); EXPECT_EQ("cat in1 in2 > out", edge->EvaluateCommand()); - EXPECT_FALSE(state.GetNode("in1")->dirty()); - EXPECT_FALSE(state.GetNode("in2")->dirty()); - EXPECT_FALSE(state.GetNode("out")->dirty()); + EXPECT_FALSE(state.GetNode("in1", 0)->dirty()); + EXPECT_FALSE(state.GetNode("in2", 0)->dirty()); + EXPECT_FALSE(state.GetNode("out", 0)->dirty()); } } // namespace diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc index a9af756..743e406 100644 --- a/src/subprocess-posix.cc +++ b/src/subprocess-posix.cc @@ -25,7 +25,8 @@ #include "util.h" -Subprocess::Subprocess() : fd_(-1), pid_(-1) { +Subprocess::Subprocess(bool use_console) : fd_(-1), pid_(-1), + use_console_(use_console) { } Subprocess::~Subprocess() { if (fd_ >= 0) @@ -58,29 +59,34 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { // Track which fd we use to report errors on. int error_pipe = output_pipe[1]; do { - if (setpgid(0, 0) < 0) - break; - if (sigaction(SIGINT, &set->old_act_, 0) < 0) break; if (sigprocmask(SIG_SETMASK, &set->old_mask_, 0) < 0) break; - // Open /dev/null over stdin. - int devnull = open("/dev/null", O_RDONLY); - if (devnull < 0) - break; - if (dup2(devnull, 0) < 0) - break; - close(devnull); - - if (dup2(output_pipe[1], 1) < 0 || - dup2(output_pipe[1], 2) < 0) - break; - - // Now can use stderr for errors. - error_pipe = 2; - close(output_pipe[1]); + if (!use_console_) { + // Put the child in its own process group, so ctrl-c won't reach it. + if (setpgid(0, 0) < 0) + break; + + // Open /dev/null over stdin. + int devnull = open("/dev/null", O_RDONLY); + if (devnull < 0) + break; + if (dup2(devnull, 0) < 0) + break; + close(devnull); + + if (dup2(output_pipe[1], 1) < 0 || + dup2(output_pipe[1], 2) < 0) + break; + + // Now can use stderr for errors. + error_pipe = 2; + close(output_pipe[1]); + } + // In the console case, output_pipe is still inherited by the child and + // closed when the subprocess finishes, which then notifies ninja. execl("/bin/sh", "/bin/sh", "-c", command.c_str(), (char *) NULL); } while (false); @@ -168,8 +174,8 @@ SubprocessSet::~SubprocessSet() { Fatal("sigprocmask: %s", strerror(errno)); } -Subprocess *SubprocessSet::Add(const string& command) { - Subprocess *subprocess = new Subprocess; +Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + Subprocess *subprocess = new Subprocess(use_console); if (!subprocess->Start(this, command)) { delete subprocess; return 0; @@ -279,7 +285,10 @@ Subprocess* SubprocessSet::NextFinished() { void SubprocessSet::Clear() { for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) - kill(-(*i)->pid_, SIGINT); + // Since the foreground process is in our process group, it will receive a + // SIGINT at the same time as us. + if (!(*i)->use_console_) + kill(-(*i)->pid_, SIGINT); for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) delete *i; diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 1b230b6..fad66e8 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -14,13 +14,16 @@ #include "subprocess.h" +#include <assert.h> #include <stdio.h> #include <algorithm> #include "util.h" -Subprocess::Subprocess() : child_(NULL) , overlapped_(), is_reading_(false) { +Subprocess::Subprocess(bool use_console) : child_(NULL) , overlapped_(), + is_reading_(false), + use_console_(use_console) { } Subprocess::~Subprocess() { @@ -86,18 +89,25 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { STARTUPINFOA startup_info; memset(&startup_info, 0, sizeof(startup_info)); startup_info.cb = sizeof(STARTUPINFO); - startup_info.dwFlags = STARTF_USESTDHANDLES; - startup_info.hStdInput = nul; - startup_info.hStdOutput = child_pipe; - startup_info.hStdError = child_pipe; + if (!use_console_) { + startup_info.dwFlags = STARTF_USESTDHANDLES; + startup_info.hStdInput = nul; + startup_info.hStdOutput = child_pipe; + startup_info.hStdError = child_pipe; + } + // In the console case, child_pipe is still inherited by the child and closed + // when the subprocess finishes, which then notifies ninja. PROCESS_INFORMATION process_info; memset(&process_info, 0, sizeof(process_info)); + // Ninja handles ctrl-c, except for subprocesses in console pools. + DWORD process_flags = use_console_ ? 0 : CREATE_NEW_PROCESS_GROUP; + // Do not prepend 'cmd /c' on Windows, this breaks command // lines greater than 8,191 chars. if (!CreateProcessA(NULL, (char*)command.c_str(), NULL, NULL, - /* inherit handles */ TRUE, CREATE_NEW_PROCESS_GROUP, + /* inherit handles */ TRUE, process_flags, NULL, NULL, &startup_info, &process_info)) { DWORD error = GetLastError(); @@ -213,8 +223,8 @@ BOOL WINAPI SubprocessSet::NotifyInterrupted(DWORD dwCtrlType) { return FALSE; } -Subprocess *SubprocessSet::Add(const string& command) { - Subprocess *subprocess = new Subprocess; +Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + Subprocess *subprocess = new Subprocess(use_console); if (!subprocess->Start(this, command)) { delete subprocess; return 0; @@ -266,7 +276,9 @@ Subprocess* SubprocessSet::NextFinished() { void SubprocessSet::Clear() { for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) { - if ((*i)->child_) { + // Since the foreground process is in our process group, it will receive a + // CTRL_C_EVENT or CTRL_BREAK_EVENT at the same time as us. + if ((*i)->child_ && !(*i)->use_console_) { if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, GetProcessId((*i)->child_))) { Win32Fatal("GenerateConsoleCtrlEvent"); diff --git a/src/subprocess.h b/src/subprocess.h index 4c1629c..b7a1a4c 100644 --- a/src/subprocess.h +++ b/src/subprocess.h @@ -44,7 +44,7 @@ struct Subprocess { const string& GetOutput() const; private: - Subprocess(); + Subprocess(bool use_console); bool Start(struct SubprocessSet* set, const string& command); void OnPipeReady(); @@ -64,6 +64,7 @@ struct Subprocess { int fd_; pid_t pid_; #endif + bool use_console_; friend struct SubprocessSet; }; @@ -75,7 +76,7 @@ struct SubprocessSet { SubprocessSet(); ~SubprocessSet(); - Subprocess* Add(const string& command); + Subprocess* Add(const string& command, bool use_console = false); bool DoWork(); Subprocess* NextFinished(); void Clear(); diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index 9f8dcea..8a0787c 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -18,6 +18,7 @@ #ifndef _WIN32 // SetWithLots need setrlimit. +#include <stdio.h> #include <sys/time.h> #include <sys/resource.h> #include <unistd.h> @@ -92,7 +93,22 @@ TEST_F(SubprocessTest, InterruptParent) { return; } - ADD_FAILURE() << "We should have been interrupted"; + ASSERT_FALSE("We should have been interrupted"); +} + +TEST_F(SubprocessTest, Console) { + // Skip test if we don't have the console ourselves. + if (isatty(0) && isatty(1) && isatty(2)) { + Subprocess* subproc = subprocs_.Add("test -t 0 -a -t 1 -a -t 2", + /*use_console=*/true); + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { + subprocs_.DoWork(); + } + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + } } #endif @@ -156,14 +172,15 @@ TEST_F(SubprocessTest, SetWithMulti) { TEST_F(SubprocessTest, SetWithLots) { // Arbitrary big number; needs to be over 1024 to confirm we're no longer // hostage to pselect. - const size_t kNumProcs = 1025; + const unsigned kNumProcs = 1025; // Make sure [ulimit -n] isn't going to stop us from working. rlimit rlim; ASSERT_EQ(0, getrlimit(RLIMIT_NOFILE, &rlim)); - ASSERT_GT(rlim.rlim_cur, kNumProcs) - << "Raise [ulimit -n] well above " << kNumProcs - << " to make this test go"; + if (!EXPECT_GT(rlim.rlim_cur, kNumProcs)) { + printf("Raise [ulimit -n] well above %u to make this test go\n", kNumProcs); + return; + } vector<Subprocess*> procs; for (size_t i = 0; i < kNumProcs; ++i) { diff --git a/src/test.cc b/src/test.cc index 45a9226..f667fef 100644 --- a/src/test.cc +++ b/src/test.cc @@ -12,20 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifdef _WIN32 +#include <direct.h> // Has to be before util.h is included. +#endif + #include "test.h" #include <algorithm> #include <errno.h> +#ifdef _WIN32 +#include <windows.h> +#else +#include <unistd.h> +#endif #include "build_log.h" #include "manifest_parser.h" #include "util.h" -#ifdef _WIN32 -#include <windows.h> -#endif - namespace { #ifdef _WIN32 @@ -84,13 +89,14 @@ void StateTestWithBuiltinRules::AddCatRule(State* state) { } Node* StateTestWithBuiltinRules::GetNode(const string& path) { - return state_.GetNode(path); + EXPECT_FALSE(strpbrk(path.c_str(), "/\\")); + return state_.GetNode(path, 0); } void AssertParse(State* state, const char* input) { ManifestParser parser(state, NULL); string err; - ASSERT_TRUE(parser.ParseTest(input, &err)) << err; + EXPECT_TRUE(parser.ParseTest(input, &err)); ASSERT_EQ("", err); } @@ -105,8 +111,8 @@ void VirtualFileSystem::Create(const string& path, files_created_.insert(path); } -TimeStamp VirtualFileSystem::Stat(const string& path) { - FileMap::iterator i = files_.find(path); +TimeStamp VirtualFileSystem::Stat(const string& path) const { + FileMap::const_iterator i = files_.find(path); if (i != files_.end()) return i->second.mtime; return 0; @@ -15,12 +15,94 @@ #ifndef NINJA_TEST_H_ #define NINJA_TEST_H_ -#include <gtest/gtest.h> - #include "disk_interface.h" #include "state.h" #include "util.h" +// A tiny testing framework inspired by googletest, but much simpler and +// faster to compile. It supports most things commonly used from googltest. The +// most noticeable things missing: EXPECT_* and ASSERT_* don't support +// streaming notes to them with operator<<, and for failing tests the lhs and +// rhs are not printed. That's so that this header does not have to include +// sstream, which slows down building ninja_test almost 20%. +namespace testing { +class Test { + bool failed_; + int assertion_failures_; + public: + Test() : failed_(false), assertion_failures_(0) {} + virtual ~Test() {} + virtual void SetUp() {} + virtual void TearDown() {} + virtual void Run() = 0; + + bool Failed() const { return failed_; } + int AssertionFailures() const { return assertion_failures_; } + void AddAssertionFailure() { assertion_failures_++; } + bool Check(bool condition, const char* file, int line, const char* error); +}; +} + +void RegisterTest(testing::Test* (*)(), const char*); + +extern testing::Test* g_current_test; +#define TEST_F_(x, y, name) \ + struct y : public x { \ + static testing::Test* Create() { return g_current_test = new y; } \ + virtual void Run(); \ + }; \ + struct Register##y { \ + Register##y() { RegisterTest(y::Create, name); } \ + }; \ + Register##y g_register_##y; \ + void y::Run() + +#define TEST_F(x, y) TEST_F_(x, x##y, #x "." #y) +#define TEST(x, y) TEST_F_(testing::Test, x##y, #x "." #y) + +#define EXPECT_EQ(a, b) \ + g_current_test->Check(a == b, __FILE__, __LINE__, #a " == " #b) +#define EXPECT_NE(a, b) \ + g_current_test->Check(a != b, __FILE__, __LINE__, #a " != " #b) +#define EXPECT_GT(a, b) \ + g_current_test->Check(a > b, __FILE__, __LINE__, #a " > " #b) +#define EXPECT_LT(a, b) \ + g_current_test->Check(a < b, __FILE__, __LINE__, #a " < " #b) +#define EXPECT_GE(a, b) \ + g_current_test->Check(a >= b, __FILE__, __LINE__, #a " >= " #b) +#define EXPECT_LE(a, b) \ + g_current_test->Check(a <= b, __FILE__, __LINE__, #a " <= " #b) +#define EXPECT_TRUE(a) \ + g_current_test->Check(static_cast<bool>(a), __FILE__, __LINE__, #a) +#define EXPECT_FALSE(a) \ + g_current_test->Check(!static_cast<bool>(a), __FILE__, __LINE__, #a) + +#define ASSERT_EQ(a, b) \ + if (!EXPECT_EQ(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_NE(a, b) \ + if (!EXPECT_NE(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_GT(a, b) \ + if (!EXPECT_GT(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_LT(a, b) \ + if (!EXPECT_LT(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_GE(a, b) \ + if (!EXPECT_GE(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_LE(a, b) \ + if (!EXPECT_LE(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_TRUE(a) \ + if (!EXPECT_TRUE(a)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_FALSE(a) \ + if (!EXPECT_FALSE(a)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_NO_FATAL_FAILURE(a) \ + { \ + int f = g_current_test->AssertionFailures(); \ + a; \ + if (f != g_current_test->AssertionFailures()) { \ + g_current_test->AddAssertionFailure(); \ + return; \ + } \ + } + // Support utilites for tests. struct Node; @@ -59,7 +141,7 @@ struct VirtualFileSystem : public DiskInterface { } // DiskInterface - virtual TimeStamp Stat(const string& path); + virtual TimeStamp Stat(const string& path) const; virtual bool WriteFile(const string& path, const string& contents); virtual bool MakeDir(const string& path); virtual string ReadFile(const string& path, string* err); diff --git a/src/util.cc b/src/util.cc index 24d231f..720ccbd 100644 --- a/src/util.cc +++ b/src/util.cc @@ -14,7 +14,10 @@ #include "util.h" -#ifdef _WIN32 +#ifdef __CYGWIN__ +#include <windows.h> +#include <io.h> +#elif defined( _WIN32) #include <windows.h> #include <io.h> #include <share.h> @@ -85,19 +88,33 @@ void Error(const char* msg, ...) { fprintf(stderr, "\n"); } -bool CanonicalizePath(string* path, string* err) { +bool CanonicalizePath(string* path, unsigned int* slash_bits, string* err) { METRIC_RECORD("canonicalize str"); size_t len = path->size(); char* str = 0; if (len > 0) str = &(*path)[0]; - if (!CanonicalizePath(str, &len, err)) + if (!CanonicalizePath(str, &len, slash_bits, err)) return false; path->resize(len); return true; } -bool CanonicalizePath(char* path, size_t* len, string* err) { +#ifdef _WIN32 +static unsigned int ShiftOverBit(int offset, unsigned int bits) { + // e.g. for |offset| == 2: + // | ... 9 8 7 6 5 4 3 2 1 0 | + // \_________________/ \_/ + // above below + // So we drop the bit at offset and move above "down" into its place. + unsigned int above = bits & ~((1 << (offset + 1)) - 1); + unsigned int below = bits & ((1 << offset) - 1); + return (above >> 1) | below; +} +#endif + +bool CanonicalizePath(char* path, size_t* len, unsigned int* slash_bits, + string* err) { // WARNING: this function is performance-critical; please benchmark // any changes you make to it. METRIC_RECORD("canonicalize path"); @@ -115,12 +132,37 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { const char* src = start; const char* end = start + *len; +#ifdef _WIN32 + unsigned int bits = 0; + unsigned int bits_mask = 1; + int bits_offset = 0; + // Convert \ to /, setting a bit in |bits| for each \ encountered. + for (char* c = path; c < end; ++c) { + switch (*c) { + case '\\': + bits |= bits_mask; + *c = '/'; + // Intentional fallthrough. + case '/': + bits_mask <<= 1; + bits_offset++; + } + } + if (bits_offset > 32) { + *err = "too many path components"; + return false; + } + bits_offset = 0; +#endif + if (*src == '/') { #ifdef _WIN32 + bits_offset++; // network path starts with // if (*len > 1 && *(src + 1) == '/') { src += 2; dst += 2; + bits_offset++; } else { ++src; ++dst; @@ -136,6 +178,9 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { if (src + 1 == end || src[1] == '/') { // '.' component; eliminate. src += 2; +#ifdef _WIN32 + bits = ShiftOverBit(bits_offset, bits); +#endif continue; } else if (src[1] == '.' && (src + 2 == end || src[2] == '/')) { // '..' component. Back up if possible. @@ -143,6 +188,11 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { dst = components[component_count - 1]; src += 3; --component_count; +#ifdef _WIN32 + bits = ShiftOverBit(bits_offset, bits); + bits_offset--; + bits = ShiftOverBit(bits_offset, bits); +#endif } else { *dst++ = *src++; *dst++ = *src++; @@ -154,6 +204,9 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { if (*src == '/') { src++; +#ifdef _WIN32 + bits = ShiftOverBit(bits_offset, bits); +#endif continue; } @@ -164,6 +217,9 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { while (*src != '/' && src != end) *dst++ = *src++; +#ifdef _WIN32 + bits_offset++; +#endif *dst++ = *src++; // Copy '/' or final \0 character as well. } @@ -173,6 +229,11 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { } *len = dst - start - 1; +#ifdef _WIN32 + *slash_bits = bits; +#else + *slash_bits = 0; +#endif return true; } @@ -183,6 +244,7 @@ static inline bool IsKnownShellSafeCharacter(char ch) { switch (ch) { case '_': + case '+': case '-': case '.': case '/': @@ -279,7 +341,38 @@ void GetWin32EscapedString(const string& input, string* result) { } int ReadFile(const string& path, string* contents, string* err) { - FILE* f = fopen(path.c_str(), "r"); +#ifdef _WIN32 + // This makes a ninja run on a set of 1500 manifest files about 4% faster + // than using the generic fopen code below. + err->clear(); + HANDLE f = ::CreateFile(path.c_str(), + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_FLAG_SEQUENTIAL_SCAN, + NULL); + if (f == INVALID_HANDLE_VALUE) { + err->assign(GetLastErrorString()); + return -ENOENT; + } + + for (;;) { + DWORD len; + char buf[64 << 10]; + if (!::ReadFile(f, buf, sizeof(buf), &len, NULL)) { + err->assign(GetLastErrorString()); + contents->clear(); + return -1; + } + if (len == 0) + break; + contents->append(buf, len); + } + ::CloseHandle(f); + return 0; +#else + FILE* f = fopen(path.c_str(), "rb"); if (!f) { err->assign(strerror(errno)); return -errno; @@ -298,6 +391,7 @@ int ReadFile(const string& path, string* contents, string* err) { } fclose(f); return 0; +#endif } void SetCloseOnExec(int fd) { @@ -414,10 +508,70 @@ int GetProcessorCount() { } #if defined(_WIN32) || defined(__CYGWIN__) +static double CalculateProcessorLoad(uint64_t idle_ticks, uint64_t total_ticks) +{ + static uint64_t previous_idle_ticks = 0; + static uint64_t previous_total_ticks = 0; + static double previous_load = -0.0; + + uint64_t idle_ticks_since_last_time = idle_ticks - previous_idle_ticks; + uint64_t total_ticks_since_last_time = total_ticks - previous_total_ticks; + + bool first_call = (previous_total_ticks == 0); + bool ticks_not_updated_since_last_call = (total_ticks_since_last_time == 0); + + double load; + if (first_call || ticks_not_updated_since_last_call) { + load = previous_load; + } else { + // Calculate load. + double idle_to_total_ratio = + ((double)idle_ticks_since_last_time) / total_ticks_since_last_time; + double load_since_last_call = 1.0 - idle_to_total_ratio; + + // Filter/smooth result when possible. + if(previous_load > 0) { + load = 0.9 * previous_load + 0.1 * load_since_last_call; + } else { + load = load_since_last_call; + } + } + + previous_load = load; + previous_total_ticks = total_ticks; + previous_idle_ticks = idle_ticks; + + return load; +} + +static uint64_t FileTimeToTickCount(const FILETIME & ft) +{ + uint64_t high = (((uint64_t)(ft.dwHighDateTime)) << 32); + uint64_t low = ft.dwLowDateTime; + return (high | low); +} + double GetLoadAverage() { - // TODO(nicolas.despres@gmail.com): Find a way to implement it on Windows. - // Remember to also update Usage() when this is fixed. - return -0.0f; + FILETIME idle_time, kernel_time, user_time; + BOOL get_system_time_succeeded = + GetSystemTimes(&idle_time, &kernel_time, &user_time); + + double posix_compatible_load; + if (get_system_time_succeeded) { + uint64_t idle_ticks = FileTimeToTickCount(idle_time); + + // kernel_time from GetSystemTimes already includes idle_time. + uint64_t total_ticks = + FileTimeToTickCount(kernel_time) + FileTimeToTickCount(user_time); + + double processor_load = CalculateProcessorLoad(idle_ticks, total_ticks); + posix_compatible_load = processor_load * GetProcessorCount(); + + } else { + posix_compatible_load = -0.0; + } + + return posix_compatible_load; } #else double GetLoadAverage() { @@ -41,9 +41,11 @@ void Warning(const char* msg, ...); void Error(const char* msg, ...); /// Canonicalize a path like "foo/../bar.h" into just "bar.h". -bool CanonicalizePath(string* path, string* err); - -bool CanonicalizePath(char* path, size_t* len, string* err); +/// |slash_bits| has bits set starting from lowest for a backslash that was +/// normalized to a forward slash. (only used on Windows) +bool CanonicalizePath(string* path, unsigned int* slash_bits, string* err); +bool CanonicalizePath(char* path, size_t* len, unsigned int* slash_bits, + string* err); /// Appends |input| to |*result|, escaping according to the whims of either /// Bash, or Win32's CommandLineToArgvW(). diff --git a/src/util_test.cc b/src/util_test.cc index f827e5a..8ca7f56 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -16,6 +16,15 @@ #include "test.h" +namespace { + +bool CanonicalizePath(string* path, string* err) { + unsigned int unused; + return ::CanonicalizePath(path, &unused, err); +} + +} // namespace + TEST(CanonicalizePath, PathSamples) { string path; string err; @@ -84,6 +93,201 @@ TEST(CanonicalizePath, PathSamples) { EXPECT_EQ("", path); } +#ifdef _WIN32 +TEST(CanonicalizePath, PathSamplesWindows) { + string path; + string err; + + EXPECT_FALSE(CanonicalizePath(&path, &err)); + EXPECT_EQ("empty path", err); + + path = "foo.h"; err = ""; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo.h", path); + + path = ".\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo.h", path); + + path = ".\\foo\\.\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/bar.h", path); + + path = ".\\x\\foo\\..\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("x/bar.h", path); + + path = ".\\x\\foo\\..\\..\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("bar.h", path); + + path = "foo\\\\bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/bar", path); + + path = "foo\\\\.\\\\..\\\\\\bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("bar", path); + + path = ".\\x\\..\\foo\\..\\..\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("../bar.h", path); + + path = "foo\\.\\."; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo", path); + + path = "foo\\bar\\.."; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo", path); + + path = "foo\\.hidden_bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/.hidden_bar", path); + + path = "\\foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("/foo", path); + + path = "\\\\foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("//foo", path); + + path = "\\"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("", path); +} + +TEST(CanonicalizePath, SlashTracking) { + string path; + string err; + unsigned int slash_bits; + + path = "foo.h"; err = ""; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("foo.h", path); + EXPECT_EQ(0, slash_bits); + + path = "a\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a/bcd/efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/bcd/efh/foo.h", path); + EXPECT_EQ(4, slash_bits); + + path = "a\\bcd/efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/bcd/efh/foo.h", path); + EXPECT_EQ(5, slash_bits); + + path = "a\\bcd\\efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/bcd/efh/foo.h", path); + EXPECT_EQ(7, slash_bits); + + path = "a/bcd/efh/foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/bcd/efh/foo.h", path); + EXPECT_EQ(0, slash_bits); + + path = "a\\./efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/efh/foo.h", path); + EXPECT_EQ(3, slash_bits); + + path = "a\\../efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("efh/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a\\b\\c\\d\\e\\f\\g\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/b/c/d/e/f/g/foo.h", path); + EXPECT_EQ(127, slash_bits); + + path = "a\\b\\c\\..\\..\\..\\g\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("g/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a\\b/c\\../../..\\g\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("g/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a\\b/c\\./../..\\g\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/g/foo.h", path); + EXPECT_EQ(3, slash_bits); + + path = "a\\b/c\\./../..\\g/foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/g/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a\\\\\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a/\\\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/foo.h", path); + EXPECT_EQ(0, slash_bits); + + path = "a\\//foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/foo.h", path); + EXPECT_EQ(1, slash_bits); +} + +TEST(CanonicalizePath, CanonicalizeNotExceedingLen) { + // Make sure searching \/ doesn't go past supplied len. + char buf[] = "foo/bar\\baz.h\\"; // Last \ past end. + unsigned int slash_bits; + string err; + size_t size = 13; + EXPECT_TRUE(::CanonicalizePath(buf, &size, &slash_bits, &err)); + EXPECT_EQ(0, strncmp("foo/bar/baz.h", buf, size)); + EXPECT_EQ(2, slash_bits); // Not including the trailing one. +} + +TEST(CanonicalizePath, TooManyComponents) { + string path; + string err; + unsigned int slash_bits; + + // 32 is OK. + path = "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./x.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + + // Backslashes version. + path = + "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\." + "\\a\\.\\a\\.\\a\\.\\a\\.\\x.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(slash_bits, 0xffff); + + // 33 is not. + err = ""; + path = + "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/x.h"; + EXPECT_FALSE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(err, "too many path components"); + + // Backslashes version. + err = ""; + path = + "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\." + "\\a\\.\\a\\.\\a\\.\\a\\.\\a\\x.h"; + EXPECT_FALSE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(err, "too many path components"); +} +#endif + TEST(CanonicalizePath, EmptyResult) { string path; string err; @@ -122,33 +326,34 @@ TEST(CanonicalizePath, NotNullTerminated) { string path; string err; size_t len; + unsigned int unused; path = "foo/. bar/."; len = strlen("foo/."); // Canonicalize only the part before the space. - EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); + EXPECT_TRUE(CanonicalizePath(&path[0], &len, &unused, &err)); EXPECT_EQ(strlen("foo"), len); EXPECT_EQ("foo/. bar/.", string(path)); path = "foo/../file bar/."; len = strlen("foo/../file"); - EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); + EXPECT_TRUE(CanonicalizePath(&path[0], &len, &unused, &err)); EXPECT_EQ(strlen("file"), len); EXPECT_EQ("file ./file bar/.", string(path)); } TEST(PathEscaping, TortureTest) { string result; - + GetWin32EscapedString("foo bar\\\"'$@d!st!c'\\path'\\", &result); EXPECT_EQ("\"foo bar\\\\\\\"'$@d!st!c'\\path'\\\\\"", result); - result.clear(); + result.clear(); GetShellEscapedString("foo bar\"/'$@d!st!c'/path'", &result); EXPECT_EQ("'foo bar\"/'\\''$@d!st!c'\\''/path'\\'''", result); } TEST(PathEscaping, SensiblePathsAreNotNeedlesslyEscaped) { - const char* path = "some/sensible/path/without/crazy/characters.cc"; + const char* path = "some/sensible/path/without/crazy/characters.c++"; string result; GetWin32EscapedString(path, &result); @@ -160,7 +365,7 @@ TEST(PathEscaping, SensiblePathsAreNotNeedlesslyEscaped) { } TEST(PathEscaping, SensibleWin32PathsAreNotNeedlesslyEscaped) { - const char* path = "some\\sensible\\path\\without\\crazy\\characters.cc"; + const char* path = "some\\sensible\\path\\without\\crazy\\characters.c++"; string result; GetWin32EscapedString(path, &result); diff --git a/src/version.cc b/src/version.cc index 1406d91..f2d6711 100644 --- a/src/version.cc +++ b/src/version.cc @@ -18,7 +18,7 @@ #include "util.h" -const char* kNinjaVersion = "1.4.0.git"; +const char* kNinjaVersion = "1.5.3.git"; void ParseVersion(const string& version, int* major, int* minor) { size_t end = version.find('.'); |