diff options
author | Bruce Dawson <randomascii@users.noreply.github.com> | 2022-02-17 00:06:15 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-02-17 00:06:15 (GMT) |
commit | 34d03bd3db07f7cae57016eee0b60b4ebe5baecc (patch) | |
tree | 12e7703f8c2562ca25686a93e887f44e38c62c7d | |
parent | 87b8965080693f084e5784efde234b127711d8c1 (diff) | |
parent | f404f0059d71c8c86da7b56c48794266b5befd10 (diff) | |
download | Ninja-34d03bd3db07f7cae57016eee0b60b4ebe5baecc.zip Ninja-34d03bd3db07f7cae57016eee0b60b4ebe5baecc.tar.gz Ninja-34d03bd3db07f7cae57016eee0b60b4ebe5baecc.tar.bz2 |
Merge branch 'ninja-build:master' into compiler_detect
35 files changed, 1141 insertions, 222 deletions
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 80c88c6..3c93e00 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -13,6 +13,9 @@ jobs: image: centos:7 steps: - uses: actions/checkout@v2 + - uses: codespell-project/actions-codespell@master + with: + ignore_words_list: fo,wee - name: Install dependencies run: | curl -L -O https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-Linux-x86_64.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index b49c5b0..57ae548 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ if(RE2C) # the depfile parser and ninja lexers are generated using re2c. function(re2c IN OUT) add_custom_command(DEPENDS ${IN} OUTPUT ${OUT} - COMMAND ${RE2C} -b -i --no-generation-date -o ${OUT} ${IN} + COMMAND ${RE2C} -b -i --no-generation-date --no-version -o ${OUT} ${IN} ) endfunction() re2c(${PROJECT_SOURCE_DIR}/src/depfile_parser.in.cc ${PROJECT_BINARY_DIR}/depfile_parser.cc) @@ -122,10 +122,8 @@ if(WIN32) src/msvc_helper-win32.cc src/msvc_helper_main-win32.cc src/getopt.c + src/minidump-win32.cc ) - if(MSVC) - target_sources(libninja PRIVATE src/minidump-win32.cc) - endif() else() target_sources(libninja PRIVATE src/subprocess-posix.cc) if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") @@ -37,7 +37,7 @@ to build Ninja with itself. ### CMake ``` -cmake -Bbuild-cmake -H. +cmake -Bbuild-cmake cmake --build build-cmake ``` diff --git a/configure.py b/configure.py index c0d6712..fcc987c 100755 --- a/configure.py +++ b/configure.py @@ -84,7 +84,7 @@ class Platform(object): return self._platform == 'msvc' def msvc_needs_fs(self): - popen = subprocess.Popen(['cl', '/nologo', '/?'], + popen = subprocess.Popen(['cl', '/nologo', '/help'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = popen.communicate() @@ -488,7 +488,7 @@ def has_re2c(): return False if has_re2c(): n.rule('re2c', - command='re2c -b -i --no-generation-date -o $out $in', + command='re2c -b -i --no-generation-date --no-version -o $out $in', description='RE2C $out') # Generate the .cc files in the source directory so we can check them in. n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index a5012b4..9e2bec5 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -257,6 +257,10 @@ than the _depth_ mode. executed in order, may be used to rebuild those targets, assuming that all output files are out of date. +`inputs`:: given a list of targets, print a list of all inputs which are used +to rebuild those targets. +_Available since Ninja 1.11._ + `clean`:: remove built files. By default it removes all built files except for those created by the generator. Adding the `-g` flag also removes built files created by the generator (see <<ref_rule,the rule @@ -294,7 +298,7 @@ of a generator-target) implicitly, but does not have an explicit or order-only dependency path to the generator-target, is considered broken. + The tool's findings can be verified by trying to build the listed targets in -a clean outdir without buidling any other targets. The build should fail for +a clean outdir without building any other targets. The build should fail for each of them with a missing include error or equivalent pointing to the generated file. _Available since Ninja 1.11._ @@ -304,9 +308,9 @@ _Available since Ninja 1.11._ `restat`:: updates all recorded file modification timestamps in the `.ninja_log` file. _Available since Ninja 1.10._ -`rules`:: output the list of all rules (eventually with their description -if they have one). It can be used to know which rule name to pass to -+ninja -t targets rule _name_+ or +ninja -t compdb+. +`rules`:: output the list of all rules. It can be used to know which rule name +to pass to +ninja -t targets rule _name_+ or +ninja -t compdb+. Adding the `-d` +flag also prints the description of the rules. `wincodepage`:: Available on Windows hosts (_since Ninja 1.11_). Prints the Windows code page whose encoding is expected in the build file. @@ -483,6 +487,11 @@ nothing, but phony rules are handled specially in that they aren't printed when run, logged (see below), nor do they contribute to the command count printed as part of the build process. +When a `phony` target is used as an input to another build rule, the +other build rule will, semantically, consider the inputs of the +`phony` rule as its own. Therefore, `phony` rules can be used to group +inputs, e.g. header files. + `phony` can also be used to create dummy targets for files which may not exist at build time. If a phony build statement is written without any dependencies, the target will be considered out of date if @@ -743,6 +752,8 @@ A file is a series of declarations. A declaration can be one of: Order-only dependencies may be tacked on the end with +|| _dependency1_ _dependency2_+. (See <<ref_dependencies,the reference on dependency types>>.) + Validations may be taked on the end with +|@ _validation1_ _validation2_+. + (See <<validations,the reference on validations>>.) + Implicit outputs _(available since Ninja 1.7)_ may be added before the `:` with +| _output1_ _output2_+ and do not appear in `$out`. @@ -1001,6 +1012,31 @@ express the implicit dependency.) File paths are compared as is, which means that an absolute path and a relative path, pointing to the same file, are considered different by Ninja. +[[validations]] +Validations +~~~~~~~~~~~ +Validations listed on the build line cause the specified files to be +added to the top level of the build graph (as if they were specified +on the Ninja command line) whenever the build line is a transitive +dependency of one of the targets specified on the command line or a +default target. + +Validations are added to the build graph regardless of whether the output +files of the build statement are dirty are not, and the dirty state of +the build statement that outputs the file being used as a validation +has no effect on the dirty state of the build statement that requested it. + +A build edge can list another build edge as a validation even if the second +edge depends on the first. + +Validations are designed to handle rules that perform error checking but +don't produce any artifacts needed by the build, for example static +analysis tools. Marking the static analysis rule as an implicit input +of the main build rule of the source files or of the rules that depend +on the main build rule would slow down the critical path of the build, +but using a validation would allow the build to proceed in parallel with +the static analysis rule once the main build rule is complete. + Variable expansion ~~~~~~~~~~~~~~~~~~ diff --git a/misc/write_fake_manifests.py b/misc/write_fake_manifests.py index b3594de..abcb677 100644 --- a/misc/write_fake_manifests.py +++ b/misc/write_fake_manifests.py @@ -65,7 +65,7 @@ class GenRandom(object): def _n_unique_strings(self, n): seen = set([None]) return [self._unique_string(seen, avg_options=3, p_suffix=0.4) - for _ in xrange(n)] + for _ in range(n)] def target_name(self): return self._unique_string(p_suffix=0, seen=self.seen_names) @@ -73,7 +73,7 @@ class GenRandom(object): def path(self): return os.path.sep.join([ self._unique_string(self.seen_names, avg_options=1, p_suffix=0) - for _ in xrange(1 + paretoint(0.6, alpha=4))]) + for _ in range(1 + paretoint(0.6, alpha=4))]) def src_obj_pairs(self, path, name): num_sources = paretoint(55, alpha=2) + 1 @@ -84,7 +84,7 @@ class GenRandom(object): def defines(self): return [ '-DENABLE_' + self._unique_string(self.seen_defines).upper() - for _ in xrange(paretoint(20, alpha=3))] + for _ in range(paretoint(20, alpha=3))] LIB, EXE = 0, 1 @@ -227,7 +227,7 @@ def random_targets(num_targets, src_dir): gen = GenRandom(src_dir) # N-1 static libraries, and 1 executable depending on all of them. - targets = [Target(gen, LIB) for i in xrange(num_targets - 1)] + targets = [Target(gen, LIB) for i in range(num_targets - 1)] for i in range(len(targets)): targets[i].deps = [t for t in targets[0:i] if random.random() < 0.05] diff --git a/src/build.cc b/src/build.cc index cf07846..6f11ed7 100644 --- a/src/build.cc +++ b/src/build.cc @@ -384,8 +384,21 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node, Node* n = *i; // Check if this dependent node is now dirty. Also checks for new cycles. - if (!scan->RecomputeDirty(n, err)) + std::vector<Node*> validation_nodes; + if (!scan->RecomputeDirty(n, &validation_nodes, err)) return false; + + // Add any validation nodes found during RecomputeDirty as new top level + // targets. + for (std::vector<Node*>::iterator v = validation_nodes.begin(); + v != validation_nodes.end(); ++v) { + if (Edge* in_edge = (*v)->in_edge()) { + if (!in_edge->outputs_ready() && + !AddTarget(*v, err)) { + return false; + } + } + } if (!n->dirty()) continue; @@ -553,16 +566,28 @@ Node* Builder::AddTarget(const string& name, string* err) { } bool Builder::AddTarget(Node* target, string* err) { - if (!scan_.RecomputeDirty(target, err)) + std::vector<Node*> validation_nodes; + if (!scan_.RecomputeDirty(target, &validation_nodes, err)) return false; - if (Edge* in_edge = target->in_edge()) { - if (in_edge->outputs_ready()) - return true; // Nothing to do. + Edge* in_edge = target->in_edge(); + if (!in_edge || !in_edge->outputs_ready()) { + if (!plan_.AddTarget(target, err)) { + return false; + } } - if (!plan_.AddTarget(target, err)) - return false; + // Also add any validation nodes found during RecomputeDirty as top level + // targets. + for (std::vector<Node*>::iterator n = validation_nodes.begin(); + n != validation_nodes.end(); ++n) { + if (Edge* validation_in_edge = (*n)->in_edge()) { + if (!validation_in_edge->outputs_ready() && + !plan_.AddTarget(*n, err)) { + return false; + } + } + } return true; } diff --git a/src/build_test.cc b/src/build_test.cc index e0c43b1..4ef62b2 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -897,6 +897,14 @@ TEST_F(BuildTest, MissingTarget) { EXPECT_EQ("unknown target: 'meow'", err); } +TEST_F(BuildTest, MissingInputTarget) { + // Target is a missing input file + string err; + Dirty("in1"); + EXPECT_FALSE(builder_.AddTarget("in1", &err)); + EXPECT_EQ("'in1' missing and no known rule to make it", err); +} + TEST_F(BuildTest, MakeDirs) { string err; @@ -1182,6 +1190,152 @@ TEST_F(BuildTest, PhonySelfReference) { EXPECT_TRUE(builder_.AlreadyUpToDate()); } +// There are 6 different cases for phony rules: +// +// 1. output edge does not exist, inputs are not real +// 2. output edge does not exist, no inputs +// 3. output edge does not exist, inputs are real, newest mtime is M +// 4. output edge is real, inputs are not real +// 5. output edge is real, no inputs +// 6. output edge is real, inputs are real, newest mtime is M +// +// Expected results : +// 1. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +// 2. Edge is marked as dirty, causing dependent edges to always rebuild +// 3. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +// 4. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +// 5. Edge is marked as dirty, causing dependent edges to always rebuild +// 6. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +void TestPhonyUseCase(BuildTest* t, int i) { + State& state_ = t->state_; + Builder& builder_ = t->builder_; + FakeCommandRunner& command_runner_ = t->command_runner_; + VirtualFileSystem& fs_ = t->fs_; + + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"build notreal: phony blank\n" +"build phony1: phony notreal\n" +"build phony2: phony\n" +"build phony3: phony blank\n" +"build phony4: phony notreal\n" +"build phony5: phony\n" +"build phony6: phony blank\n" +"\n" +"build test1: touch phony1\n" +"build test2: touch phony2\n" +"build test3: touch phony3\n" +"build test4: touch phony4\n" +"build test5: touch phony5\n" +"build test6: touch phony6\n" +)); + + // Set up test. + builder_.command_runner_.release(); // BuildTest owns the CommandRunner + builder_.command_runner_.reset(&command_runner_); + + fs_.Create("blank", ""); // a "real" file + EXPECT_TRUE(builder_.AddTarget("test1", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test3", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test4", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test5", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test6", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + + string ci; + ci += static_cast<char>('0' + i); + + // Tests 1, 3, 4, and 6 should rebuild when the input is updated. + if (i != 2 && i != 5) { + Node* testNode = t->GetNode("test" + ci); + Node* phonyNode = t->GetNode("phony" + ci); + Node* inputNode = t->GetNode("blank"); + + state_.Reset(); + TimeStamp startTime = fs_.now_; + + // Build number 1 + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + if (!builder_.AlreadyUpToDate()) + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + + // Touch the input file + state_.Reset(); + command_runner_.commands_ran_.clear(); + fs_.Tick(); + fs_.Create("blank", ""); // a "real" file + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + + // Second build, expect testN edge to be rebuilt + // and phonyN node's mtime to be updated. + EXPECT_FALSE(builder_.AlreadyUpToDate()); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ(string("touch test") + ci, command_runner_.commands_ran_[0]); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + + TimeStamp inputTime = inputNode->mtime(); + + EXPECT_FALSE(phonyNode->exists()); + EXPECT_FALSE(phonyNode->dirty()); + + EXPECT_GT(phonyNode->mtime(), startTime); + EXPECT_EQ(phonyNode->mtime(), inputTime); + ASSERT_TRUE(testNode->Stat(&fs_, &err)); + EXPECT_TRUE(testNode->exists()); + EXPECT_GT(testNode->mtime(), startTime); + } else { + // Tests 2 and 5: Expect dependents to always rebuild. + + state_.Reset(); + command_runner_.commands_ran_.clear(); + fs_.Tick(); + command_runner_.commands_ran_.clear(); + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("touch test" + ci, command_runner_.commands_ran_[0]); + + state_.Reset(); + command_runner_.commands_ran_.clear(); + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("touch test" + ci, command_runner_.commands_ran_[0]); + } +} + +TEST_F(BuildTest, PhonyUseCase1) { TestPhonyUseCase(this, 1); } +TEST_F(BuildTest, PhonyUseCase2) { TestPhonyUseCase(this, 2); } +TEST_F(BuildTest, PhonyUseCase3) { TestPhonyUseCase(this, 3); } +TEST_F(BuildTest, PhonyUseCase4) { TestPhonyUseCase(this, 4); } +TEST_F(BuildTest, PhonyUseCase5) { TestPhonyUseCase(this, 5); } +TEST_F(BuildTest, PhonyUseCase6) { TestPhonyUseCase(this, 6); } + TEST_F(BuildTest, Fail) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule fail\n" @@ -2201,7 +2355,7 @@ struct BuildWithDepsLogTest : public BuildTest { void* builder_; }; -/// Run a straightforwad build where the deps log is used. +/// Run a straightforward build where the deps log is used. TEST_F(BuildWithDepsLogTest, Straightforward) { string err; // Note: in1 was created by the superclass SetUp(). @@ -3083,6 +3237,67 @@ TEST_F(BuildTest, DyndepBuildDiscoverNewInput) { EXPECT_EQ("touch out", command_runner_.commands_ran_[2]); } +TEST_F(BuildTest, DyndepBuildDiscoverNewInputWithValidation) { + // Verify that a dyndep file cannot contain the |@ validation + // syntax. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out: touch || dd\n" +" dyndep = dd\n" +)); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep |@ validation\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_FALSE(builder_.Build(&err)); + + string err_first_line = err.substr(0, err.find("\n")); + EXPECT_EQ("dd:2: expected newline, got '|@'", err_first_line); +} + +TEST_F(BuildTest, DyndepBuildDiscoverNewInputWithTransitiveValidation) { + // Verify that a dyndep file can be built and loaded to discover + // a new input to an edge that has a validation edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build in: touch |@ validation\n" +"build validation: touch in out\n" +"build out: touch || dd\n" +" dyndep = dd\n" + )); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep | in\n" +); + fs_.Tick(); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(4u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch in", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out", command_runner_.commands_ran_[2]); + EXPECT_EQ("touch validation", command_runner_.commands_ran_[3]); +} + TEST_F(BuildTest, DyndepBuildDiscoverImplicitConnection) { // Verify that a dyndep file can be built and loaded to discover // that one edge has an implicit output that is also an implicit @@ -3526,3 +3741,247 @@ TEST_F(BuildTest, DyndepTwoLevelDiscoveredDirty) { EXPECT_EQ("touch tmp", command_runner_.commands_ran_[3]); EXPECT_EQ("touch out", command_runner_.commands_ran_[4]); } + +TEST_F(BuildTest, Validation) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ validate\n" + "build validate: cat in2\n")); + + fs_.Create("in", ""); + fs_.Create("in2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in" only rebuilds "out" ("validate" doesn't depend on + // "out"). + fs_.Tick(); + fs_.Create("in", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in > out", command_runner_.commands_ran_[0]); + + // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on + // "validate"). + fs_.Tick(); + fs_.Create("in2", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in2 > validate", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildTest, ValidationDependsOnOutput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ validate\n" + "build validate: cat in2 | out\n")); + + fs_.Create("in", ""); + fs_.Create("in2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in" rebuilds "out" and "validate". + fs_.Tick(); + fs_.Create("in", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on + // "validate"). + fs_.Tick(); + fs_.Create("in2", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in2 > validate", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildWithDepsLogTest, ValidationThroughDepfile) { + const char* manifest = + "build out: cat in |@ validate\n" + "build validate: cat in2 | out\n" + "build out2: cat in3\n" + " deps = gcc\n" + " depfile = out2.d\n"; + + string err; + + { + fs_.Create("in", ""); + fs_.Create("in2", ""); + fs_.Create("in3", ""); + fs_.Create("out2.d", "out: out"); + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + + EXPECT_TRUE(builder.AddTarget("out2", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + // On the first build, only the out2 command is run. + ASSERT_EQ(command_runner_.commands_ran_.size(), 1); + EXPECT_EQ("cat in3 > out2", command_runner_.commands_ran_[0]); + + // The deps file should have been removed. + EXPECT_EQ(0, fs_.Stat("out2.d", &err)); + + deps_log.Close(); + builder.command_runner_.release(); + } + + fs_.Tick(); + command_runner_.commands_ran_.clear(); + + { + fs_.Create("in2", ""); + fs_.Create("in3", ""); + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + + EXPECT_TRUE(builder.AddTarget("out2", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + // The out and validate actions should have been run as well as out2. + ASSERT_EQ(command_runner_.commands_ran_.size(), 3); + // out has to run first, as both out2 and validate depend on it. + EXPECT_EQ("cat in > out", command_runner_.commands_ran_[0]); + + deps_log.Close(); + builder.command_runner_.release(); + } +} + +TEST_F(BuildTest, ValidationCircular) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ out2\n" + "build out2: cat in2 |@ out\n")); + + fs_.Create("in", ""); + fs_.Create("in2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in" rebuilds "out". + fs_.Tick(); + fs_.Create("in", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in > out", command_runner_.commands_ran_[0]); + + // Test touching "in2" rebuilds "out2". + fs_.Tick(); + fs_.Create("in2", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in2 > out2", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildTest, ValidationWithCircularDependency) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ validate\n" + "build validate: cat validate_in | out\n" + "build validate_in: cat validate\n")); + + fs_.Create("in", ""); + + string err; + EXPECT_FALSE(builder_.AddTarget("out", &err)); + EXPECT_EQ("dependency cycle: validate -> validate_in -> validate", err); +} diff --git a/src/clean.cc b/src/clean.cc index 72dee1f..575bf6b 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -129,7 +129,16 @@ int Cleaner::CleanDead(const BuildLog::Entries& entries) { PrintHeader(); for (BuildLog::Entries::const_iterator i = entries.begin(); i != entries.end(); ++i) { Node* n = state_->LookupNode(i->first); - if (!n || !n->in_edge()) { + // Detecting stale outputs works as follows: + // + // - If it has no Node, it is not in the build graph, or the deps log + // anymore, hence is stale. + // + // - If it isn't an output or input for any edge, it comes from a stale + // entry in the deps log, but no longer referenced from the build + // graph. + // + if (!n || (!n->in_edge() && n->out_edges().empty())) { Remove(i->first.AsString()); } } diff --git a/src/clean_test.cc b/src/clean_test.cc index 1b843a2..e99909c 100644 --- a/src/clean_test.cc +++ b/src/clean_test.cc @@ -537,4 +537,65 @@ TEST_F(CleanDeadTest, CleanDead) { EXPECT_NE(0, fs_.Stat("out2", &err)); log2.Close(); } + +TEST_F(CleanDeadTest, CleanDeadPreservesInputs) { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, +"rule cat\n" +" command = cat $in > $out\n" +"build out1: cat in\n" +"build out2: cat in\n" +)); + // This manifest does not build out1 anymore, but makes + // it an implicit input. CleanDead should detect this + // and preserve it. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out2: cat in | out1\n" +)); + fs_.Create("in", ""); + fs_.Create("out1", ""); + fs_.Create("out2", ""); + + BuildLog log1; + string err; + EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); + ASSERT_EQ("", err); + log1.RecordCommand(state.edges_[0], 15, 18); + log1.RecordCommand(state.edges_[1], 20, 25); + log1.Close(); + + BuildLog log2; + EXPECT_TRUE(log2.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + ASSERT_EQ(2u, log2.entries().size()); + ASSERT_TRUE(log2.LookupByOutput("out1")); + ASSERT_TRUE(log2.LookupByOutput("out2")); + + // First use the manifest that describe how to build out1. + Cleaner cleaner1(&state, config_, &fs_); + EXPECT_EQ(0, cleaner1.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner1.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Then use the manifest that does not build out1 anymore. + Cleaner cleaner2(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner2.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Nothing to do now. + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner2.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + log2.Close(); +} } // anonymous namespace diff --git a/src/clparser.cc b/src/clparser.cc index 070bcfd..3d3e7de 100644 --- a/src/clparser.cc +++ b/src/clparser.cc @@ -72,7 +72,8 @@ bool CLParser::FilterInputFilename(string line) { return EndsWith(line, ".c") || EndsWith(line, ".cc") || EndsWith(line, ".cxx") || - EndsWith(line, ".cpp"); + EndsWith(line, ".cpp") || + EndsWith(line, ".c++"); } // static diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index bffeb76..98fba2e 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 1.3 */ +/* Generated by re2c */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/deps_log.cc b/src/deps_log.cc index 191f300..7e48b38 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -295,6 +295,19 @@ DepsLog::Deps* DepsLog::GetDeps(Node* node) { return deps_[node->id()]; } +Node* DepsLog::GetFirstReverseDepsNode(Node* node) { + for (size_t id = 0; id < deps_.size(); ++id) { + Deps* deps = deps_[id]; + if (!deps) + continue; + for (int i = 0; i < deps->node_count; ++i) { + if (deps->nodes[i] == node) + return nodes_[id]; + } + } + return NULL; +} + bool DepsLog::Recompact(const string& path, string* err) { METRIC_RECORD(".ninja_deps recompact"); diff --git a/src/deps_log.h b/src/deps_log.h index cc44b41..09cc41c 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -86,6 +86,7 @@ struct DepsLog { }; LoadStatus Load(const std::string& path, State* state, std::string* err); Deps* GetDeps(Node* node); + Node* GetFirstReverseDepsNode(Node* node); /// Rewrite the known log entries, throwing away old data. bool Recompact(const std::string& path, std::string* err); diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index 4055941..13fcc78 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -390,7 +390,7 @@ TEST_F(DepsLogTest, Truncated) { DepsLog log; EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); if (!err.empty()) { - // At some point the log will be so short as to be unparseable. + // At some point the log will be so short as to be unparsable. break; } @@ -478,4 +478,31 @@ TEST_F(DepsLogTest, TruncatedRecovery) { } } +TEST_F(DepsLogTest, ReverseDepsNodes) { + State state; + DepsLog log; + string err; + EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + vector<Node*> deps; + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); + + deps.clear(); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar2.h", 0)); + log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); + + log.Close(); + + Node* rev_deps = log.GetFirstReverseDepsNode(state.GetNode("foo.h", 0)); + EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0) || + rev_deps == state.GetNode("out2.o", 0)); + + rev_deps = log.GetFirstReverseDepsNode(state.GetNode("bar.h", 0)); + EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0)); +} + } // anonymous namespace diff --git a/src/disk_interface.cc b/src/disk_interface.cc index a9497cb..e73d901 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -180,12 +180,13 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { dir = path; } - transform(dir.begin(), dir.end(), dir.begin(), ::tolower); + string dir_lowercase = dir; + transform(dir.begin(), dir.end(), dir_lowercase.begin(), ::tolower); transform(base.begin(), base.end(), base.begin(), ::tolower); - Cache::iterator ci = cache_.find(dir); + Cache::iterator ci = cache_.find(dir_lowercase); if (ci == cache_.end()) { - ci = cache_.insert(make_pair(dir, DirCache())).first; + ci = cache_.insert(make_pair(dir_lowercase, DirCache())).first; if (!StatAllFilesInDir(dir.empty() ? "." : dir, &ci->second, err)) { cache_.erase(ci); return -1; @@ -279,14 +280,31 @@ int RealDiskInterface::RemoveFile(const string& path) { // Skip error checking. If this fails, accept whatever happens below. SetFileAttributes(path.c_str(), attributes & ~FILE_ATTRIBUTE_READONLY); } - if (!DeleteFile(path.c_str())) { - DWORD win_err = GetLastError(); - if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { - return 1; + if (attributes & FILE_ATTRIBUTE_DIRECTORY) { + // remove() deletes both files and directories. On Windows we have to + // select the correct function (DeleteFile will yield Permission Denied when + // used on a directory) + // This fixes the behavior of ninja -t clean in some cases + // https://github.com/ninja-build/ninja/issues/828 + if (!RemoveDirectory(path.c_str())) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { + return 1; + } + // Report remove(), not RemoveDirectory(), for cross-platform consistency. + Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); + return -1; + } + } else { + if (!DeleteFile(path.c_str())) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { + return 1; + } + // Report as remove(), not DeleteFile(), for cross-platform consistency. + Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); + return -1; } - // Report as remove(), not DeleteFile(), for cross-platform consistency. - Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); - return -1; } #else if (remove(path.c_str()) < 0) { diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index b424243..5e952ed 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -219,6 +219,14 @@ TEST_F(DiskInterfaceTest, RemoveFile) { #endif } +TEST_F(DiskInterfaceTest, RemoveDirectory) { + const char* kDirectoryName = "directory-to-remove"; + EXPECT_TRUE(disk_.MakeDir(kDirectoryName)); + EXPECT_EQ(0, disk_.RemoveFile(kDirectoryName)); + EXPECT_EQ(1, disk_.RemoveFile(kDirectoryName)); + EXPECT_EQ(1, disk_.RemoveFile("does not exist")); +} + struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { StatTest() : scan_(&state_, NULL, NULL, this, NULL) {} @@ -264,7 +272,7 @@ TEST_F(StatTest, Simple) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_EQ(2u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_EQ("in", stats_[1]); @@ -280,7 +288,7 @@ TEST_F(StatTest, TwoStep) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_EQ(3u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_TRUE(GetNode("out")->dirty()); @@ -300,7 +308,7 @@ TEST_F(StatTest, Tree) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_EQ(1u + 6u, stats_.size()); ASSERT_EQ("mid1", stats_[1]); ASSERT_TRUE(GetNode("mid1")->dirty()); @@ -321,7 +329,7 @@ TEST_F(StatTest, Middle) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_FALSE(GetNode("in")->dirty()); ASSERT_TRUE(GetNode("mid")->dirty()); ASSERT_TRUE(GetNode("out")->dirty()); diff --git a/src/eval_env.h b/src/eval_env.h index ca7daa4..677dc21 100644 --- a/src/eval_env.h +++ b/src/eval_env.h @@ -55,7 +55,7 @@ private: TokenList parsed_; }; -/// An invokable build command and associated metadata (description, etc.). +/// An invocable build command and associated metadata (description, etc.). struct Rule { explicit Rule(const std::string& name) : name_(name) {} diff --git a/src/graph.cc b/src/graph.cc index c142f0c..8d58587 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -15,6 +15,7 @@ #include "graph.h" #include <algorithm> +#include <deque> #include <assert.h> #include <stdio.h> @@ -31,16 +32,57 @@ using namespace std; bool Node::Stat(DiskInterface* disk_interface, string* err) { - return (mtime_ = disk_interface->Stat(path_, err)) != -1; + METRIC_RECORD("node stat"); + mtime_ = disk_interface->Stat(path_, err); + if (mtime_ == -1) { + return false; + } + exists_ = (mtime_ != 0) ? ExistenceStatusExists : ExistenceStatusMissing; + return true; } -bool DependencyScan::RecomputeDirty(Node* node, string* err) { - vector<Node*> stack; - return RecomputeDirty(node, &stack, err); +void Node::UpdatePhonyMtime(TimeStamp mtime) { + if (!exists()) { + mtime_ = std::max(mtime_, mtime); + } } -bool DependencyScan::RecomputeDirty(Node* node, vector<Node*>* stack, +bool DependencyScan::RecomputeDirty(Node* initial_node, + std::vector<Node*>* validation_nodes, string* err) { + std::vector<Node*> stack; + std::vector<Node*> new_validation_nodes; + + std::deque<Node*> nodes(1, initial_node); + + // RecomputeNodeDirty might return new validation nodes that need to be + // checked for dirty state, keep a queue of nodes to visit. + while (!nodes.empty()) { + Node* node = nodes.front(); + nodes.pop_front(); + + stack.clear(); + new_validation_nodes.clear(); + + if (!RecomputeNodeDirty(node, &stack, &new_validation_nodes, err)) + return false; + nodes.insert(nodes.end(), new_validation_nodes.begin(), + new_validation_nodes.end()); + if (!new_validation_nodes.empty()) { + assert(validation_nodes && + "validations require RecomputeDirty to be called with validation_nodes"); + validation_nodes->insert(validation_nodes->end(), + new_validation_nodes.begin(), + new_validation_nodes.end()); + } + } + + return true; +} + +bool DependencyScan::RecomputeNodeDirty(Node* node, std::vector<Node*>* stack, + std::vector<Node*>* validation_nodes, + string* err) { Edge* edge = node->in_edge(); if (!edge) { // If we already visited this leaf node then we are done. @@ -84,7 +126,7 @@ bool DependencyScan::RecomputeDirty(Node* node, vector<Node*>* stack, // Later during the build the dyndep file will become ready and be // loaded to update this edge before it can possibly be scheduled. if (edge->dyndep_ && edge->dyndep_->dyndep_pending()) { - if (!RecomputeDirty(edge->dyndep_, stack, err)) + if (!RecomputeNodeDirty(edge->dyndep_, stack, validation_nodes, err)) return false; if (!edge->dyndep_->in_edge() || @@ -115,12 +157,20 @@ bool DependencyScan::RecomputeDirty(Node* node, vector<Node*>* stack, } } + // Store any validation nodes from the edge for adding to the initial + // nodes. Don't recurse into them, that would trigger the dependency + // cycle detector if the validation node depends on this node. + // RecomputeDirty will add the validation nodes to the initial nodes + // and recurse into them. + validation_nodes->insert(validation_nodes->end(), + edge->validations_.begin(), edge->validations_.end()); + // Visit all inputs; we're dirty if any of the inputs are dirty. Node* most_recent_input = NULL; for (vector<Node*>::iterator i = edge->inputs_.begin(); i != edge->inputs_.end(); ++i) { // Visit this input. - if (!RecomputeDirty(*i, stack, err)) + if (!RecomputeNodeDirty(*i, stack, validation_nodes, err)) return false; // If an input is not ready, neither are our outputs. @@ -237,6 +287,14 @@ bool DependencyScan::RecomputeOutputDirty(const Edge* edge, output->path().c_str()); return true; } + + // Update the mtime with the newest input. Dependents can thus call mtime() + // on the fake node and get the latest mtime of the dependencies + if (most_recent_input) { + output->UpdatePhonyMtime(most_recent_input->mtime()); + } + + // Phony edges are clean, nothing to do return false; } @@ -443,6 +501,13 @@ void Edge::Dump(const char* prefix) const { i != outputs_.end() && *i != NULL; ++i) { printf("%s ", (*i)->path().c_str()); } + if (!validations_.empty()) { + printf(" validations "); + for (std::vector<Node*>::const_iterator i = validations_.begin(); + i != validations_.end() && *i != NULL; ++i) { + printf("%s ", (*i)->path().c_str()); + } + } if (pool_) { if (!pool_->name().empty()) { printf("(in pool '%s')", pool_->name().c_str()); @@ -487,7 +552,7 @@ string Node::PathDecanonicalized(const string& path, uint64_t slash_bits) { void Node::Dump(const char* prefix) const { printf("%s <%s 0x%p> mtime: %" PRId64 "%s, (:%s), ", prefix, path().c_str(), this, - mtime(), mtime() ? "" : " (:missing)", + mtime(), exists() ? "" : " (:missing)", dirty() ? " dirty" : " clean"); if (in_edge()) { in_edge()->Dump("in-edge: "); @@ -499,6 +564,13 @@ void Node::Dump(const char* prefix) const { e != out_edges().end() && *e != NULL; ++e) { (*e)->Dump(" +- "); } + if (!validation_out_edges().empty()) { + printf(" validation out edges:\n"); + for (std::vector<Edge*>::const_iterator e = validation_out_edges().begin(); + e != validation_out_edges().end() && *e != NULL; ++e) { + (*e)->Dump(" +- "); + } + } } bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) { diff --git a/src/graph.h b/src/graph.h index bb4f10c..141b439 100644 --- a/src/graph.h +++ b/src/graph.h @@ -15,6 +15,7 @@ #ifndef NINJA_GRAPH_H_ #define NINJA_GRAPH_H_ +#include <algorithm> #include <set> #include <string> #include <vector> @@ -40,6 +41,7 @@ struct Node { : path_(path), slash_bits_(slash_bits), mtime_(-1), + exists_(ExistenceStatusUnknown), dirty_(false), dyndep_pending_(false), in_edge_(NULL), @@ -48,6 +50,9 @@ struct Node { /// Return false on error. bool Stat(DiskInterface* disk_interface, std::string* err); + /// If the file doesn't exist, set the mtime_ from its dependencies + void UpdatePhonyMtime(TimeStamp mtime); + /// Return false on error. bool StatIfNecessary(DiskInterface* disk_interface, std::string* err) { if (status_known()) @@ -58,20 +63,24 @@ struct Node { /// Mark as not-yet-stat()ed and not dirty. void ResetState() { mtime_ = -1; + exists_ = ExistenceStatusUnknown; dirty_ = false; } /// Mark the Node as already-stat()ed and missing. void MarkMissing() { - mtime_ = 0; + if (mtime_ == -1) { + mtime_ = 0; + } + exists_ = ExistenceStatusMissing; } bool exists() const { - return mtime_ != 0; + return exists_ == ExistenceStatusExists; } bool status_known() const { - return mtime_ != -1; + return exists_ != ExistenceStatusUnknown; } const std::string& path() const { return path_; } @@ -99,7 +108,9 @@ struct Node { void set_id(int id) { id_ = id; } const std::vector<Edge*>& out_edges() const { return out_edges_; } + const std::vector<Edge*>& validation_out_edges() const { return validation_out_edges_; } void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); } + void AddValidationOutEdge(Edge* edge) { validation_out_edges_.push_back(edge); } void Dump(const char* prefix="") const; @@ -113,9 +124,19 @@ private: /// Possible values of mtime_: /// -1: file hasn't been examined /// 0: we looked, and file doesn't exist - /// >0: actual file's mtime + /// >0: actual file's mtime, or the latest mtime of its dependencies if it doesn't exist TimeStamp mtime_; + enum ExistenceStatus { + /// The file hasn't been examined. + ExistenceStatusUnknown, + /// The file doesn't exist. mtime_ will be the latest mtime of its dependencies. + ExistenceStatusMissing, + /// The path is an actual file. mtime_ will be the file's mtime. + ExistenceStatusExists + }; + ExistenceStatus exists_; + /// Dirty is true when the underlying file is out-of-date. /// But note that Edge::outputs_ready_ is also used in judging which /// edges to build. @@ -132,6 +153,9 @@ private: /// All Edges that use this Node as an input. std::vector<Edge*> out_edges_; + /// All Edges that use this Node as a validation. + std::vector<Edge*> validation_out_edges_; + /// A dense integer id for the node, assigned and used by DepsLog. int id_; }; @@ -175,6 +199,7 @@ struct Edge { Pool* pool_; std::vector<Node*> inputs_; std::vector<Node*> outputs_; + std::vector<Node*> validations_; Node* dyndep_; BindingEnv* env_; VisitMark mark_; @@ -290,12 +315,14 @@ struct DependencyScan { dep_loader_(state, deps_log, disk_interface, depfile_parser_options), dyndep_loader_(state, disk_interface) {} - /// Update the |dirty_| state of the given node by inspecting its input edge. + /// Update the |dirty_| state of the given nodes by transitively inspecting + /// their input edges. /// Examine inputs, outputs, and command lines to judge whether an edge /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| /// state accordingly. + /// Appends any validation nodes found to the nodes parameter. /// Returns false on failure. - bool RecomputeDirty(Node* node, std::string* err); + bool RecomputeDirty(Node* node, std::vector<Node*>* validation_nodes, std::string* err); /// Recompute whether any output of the edge is dirty, if so sets |*dirty|. /// Returns false on failure. @@ -321,7 +348,8 @@ struct DependencyScan { bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const; private: - bool RecomputeDirty(Node* node, std::vector<Node*>* stack, std::string* err); + bool RecomputeNodeDirty(Node* node, std::vector<Node*>* stack, + std::vector<Node*>* validation_nodes, std::string* err); bool VerifyDAG(Node* node, std::vector<Node*>* stack, std::string* err); /// Recompute whether a given single output should be marked dirty. diff --git a/src/graph_test.cc b/src/graph_test.cc index 6b4bb51..5314bc5 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -33,7 +33,7 @@ TEST_F(GraphTest, MissingImplicit) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); // A missing implicit dep *should* make the output dirty. @@ -51,7 +51,7 @@ TEST_F(GraphTest, ModifiedImplicit) { fs_.Create("implicit", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); // A modified implicit dep should make the output dirty. @@ -71,7 +71,7 @@ TEST_F(GraphTest, FunkyMakefilePath) { fs_.Create("implicit.h", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); // implicit.h has changed, though our depfile refers to it with a @@ -94,7 +94,7 @@ TEST_F(GraphTest, ExplicitImplicit) { fs_.Create("data", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); // We have both an implicit and an explicit dep on implicit.h. @@ -122,7 +122,7 @@ TEST_F(GraphTest, ImplicitOutputMissing) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out")->dirty()); @@ -138,7 +138,7 @@ TEST_F(GraphTest, ImplicitOutputOutOfDate) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out")->dirty()); @@ -162,7 +162,7 @@ TEST_F(GraphTest, ImplicitOutputOnlyMissing) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.imp")->dirty()); @@ -176,7 +176,7 @@ TEST_F(GraphTest, ImplicitOutputOnlyOutOfDate) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.imp")->dirty()); @@ -193,7 +193,7 @@ TEST_F(GraphTest, PathWithCurrentDirectory) { fs_.Create("out.o", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); @@ -241,7 +241,7 @@ TEST_F(GraphTest, DepfileWithCanonicalizablePath) { fs_.Create("out.o", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); @@ -261,13 +261,13 @@ TEST_F(GraphTest, DepfileRemoved) { fs_.Create("out.o", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); state_.Reset(); fs_.RemoveFile("out.o.d"); - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.o")->dirty()); } @@ -314,7 +314,7 @@ TEST_F(GraphTest, NestedPhonyPrintsDone) { "build n2: phony n1\n" ); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), NULL, &err)); ASSERT_EQ("", err); Plan plan_; @@ -333,7 +333,7 @@ TEST_F(GraphTest, PhonySelfReferenceError) { parser_opts); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> a [-w phonycycle=err]", err); } @@ -345,7 +345,7 @@ TEST_F(GraphTest, DependencyCycle) { "build pre: cat out\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err); } @@ -353,7 +353,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes1) { string err; AssertParse(&state_, "build a b: cat a\n"); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> a", err); } @@ -361,7 +361,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes2) { string err; ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build b a: cat a\n")); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> a", err); } @@ -370,7 +370,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes3) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build a b: cat c\n" "build c: cat a\n")); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> c -> a", err); } @@ -382,7 +382,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes4) { "build b: cat a\n" "build a e: cat d\n" "build f: cat e\n")); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> d -> c -> b -> a", err); } @@ -398,7 +398,7 @@ TEST_F(GraphTest, CycleWithLengthZeroFromDepfile) { fs_.Create("dep.d", "a: b\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err)); ASSERT_EQ("dependency cycle: b -> b", err); // Despite the depfile causing edge to be a cycle (it has outputs a and b, @@ -423,7 +423,7 @@ TEST_F(GraphTest, CycleWithLengthOneFromDepfile) { fs_.Create("dep.d", "a: c\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err)); ASSERT_EQ("dependency cycle: b -> c -> b", err); // Despite the depfile causing edge to be a cycle (|edge| has outputs a and b, @@ -450,7 +450,7 @@ TEST_F(GraphTest, CycleWithLengthOneFromDepfileOneHopAway) { fs_.Create("dep.d", "a: c\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), NULL, &err)); ASSERT_EQ("dependency cycle: b -> c -> b", err); // Despite the depfile causing edge to be a cycle (|edge| has outputs a and b, @@ -705,7 +705,7 @@ TEST_F(GraphTest, DyndepFileMissing) { ); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("loading 'dd': No such file or directory", err); } @@ -721,7 +721,7 @@ TEST_F(GraphTest, DyndepFileError) { ); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("'out' not mentioned in its dyndep file 'dd'", err); } @@ -741,7 +741,7 @@ TEST_F(GraphTest, DyndepImplicitInputNewer) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("in")->dirty()); @@ -769,7 +769,7 @@ TEST_F(GraphTest, DyndepFileReady) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("in")->dirty()); @@ -794,7 +794,7 @@ TEST_F(GraphTest, DyndepFileNotClean) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("dd")->dirty()); @@ -820,7 +820,7 @@ TEST_F(GraphTest, DyndepFileNotReady) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("dd")->dirty()); @@ -848,7 +848,7 @@ TEST_F(GraphTest, DyndepFileSecondNotReady) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("dd1")->dirty()); @@ -877,7 +877,7 @@ TEST_F(GraphTest, DyndepFileCircular) { Edge* edge = GetNode("out")->in_edge(); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); EXPECT_EQ("dependency cycle: circ -> in -> circ", err); // Verify that "out.d" was loaded exactly once despite @@ -889,3 +889,58 @@ TEST_F(GraphTest, DyndepFileCircular) { EXPECT_EQ(1u, edge->implicit_deps_); EXPECT_EQ(1u, edge->order_only_deps_); } + +TEST_F(GraphTest, Validation) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out: cat in |@ validate\n" +"build validate: cat in\n")); + + fs_.Create("in", ""); + string err; + std::vector<Node*> validation_nodes; + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &validation_nodes, &err)); + ASSERT_EQ("", err); + + ASSERT_EQ(validation_nodes.size(), 1); + EXPECT_EQ(validation_nodes[0]->path(), "validate"); + + EXPECT_TRUE(GetNode("out")->dirty()); + EXPECT_TRUE(GetNode("validate")->dirty()); +} + +// Check that phony's dependencies' mtimes are propagated. +TEST_F(GraphTest, PhonyDepsMtimes) { + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"build in_ph: phony in1\n" +"build out1: touch in_ph\n" +)); + fs_.Create("in1", ""); + fs_.Create("out1", ""); + Node* out1 = GetNode("out1"); + Node* in1 = GetNode("in1"); + + EXPECT_TRUE(scan_.RecomputeDirty(out1, NULL, &err)); + EXPECT_TRUE(!out1->dirty()); + + // Get the mtime of out1 + ASSERT_TRUE(in1->Stat(&fs_, &err)); + ASSERT_TRUE(out1->Stat(&fs_, &err)); + TimeStamp out1Mtime1 = out1->mtime(); + TimeStamp in1Mtime1 = in1->mtime(); + + // Touch in1. This should cause out1 to be dirty + state_.Reset(); + fs_.Tick(); + fs_.Create("in1", ""); + + ASSERT_TRUE(in1->Stat(&fs_, &err)); + EXPECT_GT(in1->mtime(), in1Mtime1); + + EXPECT_TRUE(scan_.RecomputeDirty(out1, NULL, &err)); + EXPECT_GT(in1->mtime(), in1Mtime1); + EXPECT_EQ(out1->mtime(), out1Mtime1); + EXPECT_TRUE(out1->dirty()); +} diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 5d52943..081e364 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -48,7 +48,7 @@ bool IsPathSeparator(char c) { } // Return true if paths a and b are on the same windows drive. -// Return false if this funcation cannot check +// Return false if this function cannot check // whether or not on the same windows drive. bool SameDriveFast(StringPiece a, StringPiece b) { if (a.size() < 3 || b.size() < 3) { diff --git a/src/lexer.cc b/src/lexer.cc index 6e4a470..e5729f0 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 1.1.1 */ +/* Generated by re2c */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -85,6 +85,7 @@ const char* Lexer::TokenName(Token t) { case NEWLINE: return "newline"; case PIPE2: return "'||'"; case PIPE: return "'|'"; + case PIPEAT: return "'|@'"; case POOL: return "'pool'"; case RULE: return "'rule'"; case SUBNINJA: return "'subninja'"; @@ -291,7 +292,8 @@ yy25: goto yy14; yy26: yych = *++p; - if (yych == '|') goto yy42; + if (yych == '@') goto yy42; + if (yych == '|') goto yy44; { token = PIPE; break; } yy28: ++p; @@ -317,126 +319,129 @@ yy33: { continue; } yy36: yych = *++p; - if (yych == 'i') goto yy44; + if (yych == 'i') goto yy46; goto yy14; yy37: yych = *++p; - if (yych == 'f') goto yy45; + if (yych == 'f') goto yy47; goto yy14; yy38: yych = *++p; - if (yych == 'c') goto yy46; + if (yych == 'c') goto yy48; goto yy14; yy39: yych = *++p; - if (yych == 'o') goto yy47; + if (yych == 'o') goto yy49; goto yy14; yy40: yych = *++p; - if (yych == 'l') goto yy48; + if (yych == 'l') goto yy50; goto yy14; yy41: yych = *++p; - if (yych == 'b') goto yy49; + if (yych == 'b') goto yy51; goto yy14; yy42: ++p; - { token = PIPE2; break; } + { token = PIPEAT; break; } yy44: - yych = *++p; - if (yych == 'l') goto yy50; - goto yy14; -yy45: - yych = *++p; - if (yych == 'a') goto yy51; - goto yy14; + ++p; + { token = PIPE2; break; } yy46: yych = *++p; if (yych == 'l') goto yy52; goto yy14; yy47: yych = *++p; - if (yych == 'l') goto yy53; + if (yych == 'a') goto yy53; goto yy14; yy48: yych = *++p; - if (yych == 'e') goto yy55; + if (yych == 'l') goto yy54; goto yy14; yy49: yych = *++p; - if (yych == 'n') goto yy57; + if (yych == 'l') goto yy55; goto yy14; yy50: yych = *++p; - if (yych == 'd') goto yy58; + if (yych == 'e') goto yy57; goto yy14; yy51: yych = *++p; - if (yych == 'u') goto yy60; + if (yych == 'n') goto yy59; goto yy14; yy52: yych = *++p; - if (yych == 'u') goto yy61; + if (yych == 'd') goto yy60; goto yy14; yy53: yych = *++p; + if (yych == 'u') goto yy62; + goto yy14; +yy54: + yych = *++p; + if (yych == 'u') goto yy63; + goto yy14; +yy55: + yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = POOL; break; } -yy55: +yy57: yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = RULE; break; } -yy57: +yy59: yych = *++p; - if (yych == 'i') goto yy62; + if (yych == 'i') goto yy64; goto yy14; -yy58: +yy60: yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = BUILD; break; } -yy60: - yych = *++p; - if (yych == 'l') goto yy63; - goto yy14; -yy61: - yych = *++p; - if (yych == 'd') goto yy64; - goto yy14; yy62: yych = *++p; - if (yych == 'n') goto yy65; + if (yych == 'l') goto yy65; goto yy14; yy63: yych = *++p; - if (yych == 't') goto yy66; + if (yych == 'd') goto yy66; goto yy14; yy64: yych = *++p; - if (yych == 'e') goto yy68; + if (yych == 'n') goto yy67; goto yy14; yy65: yych = *++p; - if (yych == 'j') goto yy70; + if (yych == 't') goto yy68; goto yy14; yy66: yych = *++p; + if (yych == 'e') goto yy70; + goto yy14; +yy67: + yych = *++p; + if (yych == 'j') goto yy72; + goto yy14; +yy68: + yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = DEFAULT; break; } -yy68: +yy70: yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = INCLUDE; break; } -yy70: +yy72: yych = *++p; if (yych != 'a') goto yy14; yych = *++p; @@ -507,38 +512,38 @@ void Lexer::EatWhitespace() { }; yych = *p; if (yybm[0+yych] & 128) { - goto yy79; + goto yy81; } - if (yych <= 0x00) goto yy75; - if (yych == '$') goto yy82; - goto yy77; -yy75: - ++p; - { break; } + if (yych <= 0x00) goto yy77; + if (yych == '$') goto yy84; + goto yy79; yy77: ++p; -yy78: { break; } yy79: + ++p; +yy80: + { break; } +yy81: yych = *++p; if (yybm[0+yych] & 128) { - goto yy79; + goto yy81; } { continue; } -yy82: +yy84: yych = *(q = ++p); - if (yych == '\n') goto yy83; - if (yych == '\r') goto yy85; - goto yy78; -yy83: + if (yych == '\n') goto yy85; + if (yych == '\r') goto yy87; + goto yy80; +yy85: ++p; { continue; } -yy85: +yy87: yych = *++p; - if (yych == '\n') goto yy87; + if (yych == '\n') goto yy89; p = q; - goto yy78; -yy87: + goto yy80; +yy89: ++p; { continue; } } @@ -590,17 +595,17 @@ bool Lexer::ReadIdent(string* out) { }; yych = *p; if (yybm[0+yych] & 128) { - goto yy93; + goto yy95; } ++p; { last_token_ = start; return false; } -yy93: +yy95: yych = *++p; if (yybm[0+yych] & 128) { - goto yy93; + goto yy95; } { out->assign(start, p - start); @@ -660,33 +665,33 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { }; yych = *p; if (yybm[0+yych] & 16) { - goto yy100; + goto yy102; } if (yych <= '\r') { - if (yych <= 0x00) goto yy98; - if (yych <= '\n') goto yy103; - goto yy105; + if (yych <= 0x00) goto yy100; + if (yych <= '\n') goto yy105; + goto yy107; } else { - if (yych <= ' ') goto yy103; - if (yych <= '$') goto yy107; - goto yy103; + if (yych <= ' ') goto yy105; + if (yych <= '$') goto yy109; + goto yy105; } -yy98: +yy100: ++p; { last_token_ = start; return Error("unexpected EOF", err); } -yy100: +yy102: yych = *++p; if (yybm[0+yych] & 16) { - goto yy100; + goto yy102; } { eval->AddText(StringPiece(start, p - start)); continue; } -yy103: +yy105: ++p; { if (path) { @@ -699,112 +704,112 @@ yy103: continue; } } -yy105: +yy107: yych = *++p; - if (yych == '\n') goto yy108; + if (yych == '\n') goto yy110; { last_token_ = start; return Error(DescribeLastError(), err); } -yy107: +yy109: yych = *++p; if (yybm[0+yych] & 64) { - goto yy120; + goto yy122; } if (yych <= ' ') { if (yych <= '\f') { - if (yych == '\n') goto yy112; - goto yy110; + if (yych == '\n') goto yy114; + goto yy112; } else { - if (yych <= '\r') goto yy115; - if (yych <= 0x1F) goto yy110; - goto yy116; + if (yych <= '\r') goto yy117; + if (yych <= 0x1F) goto yy112; + goto yy118; } } else { if (yych <= '/') { - if (yych == '$') goto yy118; - goto yy110; + if (yych == '$') goto yy120; + goto yy112; } else { - if (yych <= ':') goto yy123; - if (yych <= '`') goto yy110; - if (yych <= '{') goto yy125; - goto yy110; + if (yych <= ':') goto yy125; + if (yych <= '`') goto yy112; + if (yych <= '{') goto yy127; + goto yy112; } } -yy108: +yy110: ++p; { if (path) p = start; break; } -yy110: +yy112: ++p; -yy111: +yy113: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy112: +yy114: yych = *++p; if (yybm[0+yych] & 32) { - goto yy112; + goto yy114; } { continue; } -yy115: +yy117: yych = *++p; - if (yych == '\n') goto yy126; - goto yy111; -yy116: + if (yych == '\n') goto yy128; + goto yy113; +yy118: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy118: +yy120: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy120: +yy122: yych = *++p; if (yybm[0+yych] & 64) { - goto yy120; + goto yy122; } { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy123: +yy125: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy125: +yy127: yych = *(q = ++p); if (yybm[0+yych] & 128) { - goto yy129; + goto yy131; } - goto yy111; -yy126: + goto yy113; +yy128: yych = *++p; - if (yych == ' ') goto yy126; + if (yych == ' ') goto yy128; { continue; } -yy129: +yy131: yych = *++p; if (yybm[0+yych] & 128) { - goto yy129; + goto yy131; } - if (yych == '}') goto yy132; + if (yych == '}') goto yy134; p = q; - goto yy111; -yy132: + goto yy113; +yy134: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); diff --git a/src/lexer.h b/src/lexer.h index 788d948..683fd6c 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -41,6 +41,7 @@ struct Lexer { NEWLINE, PIPE, PIPE2, + PIPEAT, POOL, RULE, SUBNINJA, diff --git a/src/lexer.in.cc b/src/lexer.in.cc index 88007e7..6f1d8e7 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -84,6 +84,7 @@ const char* Lexer::TokenName(Token t) { case NEWLINE: return "newline"; case PIPE2: return "'||'"; case PIPE: return "'|'"; + case PIPEAT: return "'|@'"; case POOL: return "'pool'"; case RULE: return "'rule'"; case SUBNINJA: return "'subninja'"; @@ -142,6 +143,7 @@ Lexer::Token Lexer::ReadToken() { "default" { token = DEFAULT; break; } "=" { token = EQUALS; break; } ":" { token = COLON; break; } + "|@" { token = PIPEAT; break; } "||" { token = PIPE2; break; } "|" { token = PIPE; break; } "include" { token = INCLUDE; break; } diff --git a/src/line_printer.cc b/src/line_printer.cc index 3138960..a3d0528 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -37,14 +37,9 @@ LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) { #ifndef _WIN32 smart_terminal_ = isatty(1) && term && string(term) != "dumb"; #else - // Disable output buffer. It'd be nice to use line buffering but - // MSDN says: "For some systems, [_IOLBF] provides line - // buffering. However, for Win32, the behavior is the same as _IOFBF - // - Full Buffering." if (term && string(term) == "dumb") { smart_terminal_ = false; } else { - setvbuf(stdout, NULL, _IONBF, 0); console_ = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO csbi; smart_terminal_ = GetConsoleScreenBufferInfo(console_, &csbi); diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 521edb4..8db6eb3 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -207,7 +207,7 @@ bool ManifestParser::ParseDefault(string* err) { } bool ManifestParser::ParseEdge(string* err) { - vector<EvalString> ins, outs; + vector<EvalString> ins, outs, validations; { EvalString out; @@ -288,6 +288,18 @@ bool ManifestParser::ParseEdge(string* err) { } } + // Add all validations, counting how many as we go. + if (lexer_.PeekToken(Lexer::PIPEAT)) { + for (;;) { + EvalString validation; + if (!lexer_.ReadPath(&validation, err)) + return false; + if (validation.empty()) + break; + validations.push_back(validation); + } + } + if (!ExpectToken(Lexer::NEWLINE, err)) return false; @@ -338,6 +350,7 @@ bool ManifestParser::ParseEdge(string* err) { } } } + if (edge->outputs_.empty()) { // All outputs of the edge are already created by other edges. Don't add // this edge. Do this check before input nodes are connected to the edge. @@ -359,6 +372,17 @@ bool ManifestParser::ParseEdge(string* err) { edge->implicit_deps_ = implicit; edge->order_only_deps_ = order_only; + edge->validations_.reserve(validations.size()); + for (std::vector<EvalString>::iterator v = validations.begin(); + v != validations.end(); ++v) { + string path = v->Evaluate(env); + if (path.empty()) + return lexer_.Error("empty path", err); + uint64_t slash_bits; + CanonicalizePath(&path, &slash_bits); + state_->AddValidation(edge, path, slash_bits); + } + if (options_.phony_cycle_action_ == kPhonyCycleActionWarn && edge->maybe_phonycycle_diagnostic()) { // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 5b0eddf..66b72e2 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -965,6 +965,16 @@ TEST_F(ParserTest, OrderOnly) { ASSERT_TRUE(edge->is_order_only(1)); } +TEST_F(ParserTest, Validations) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n command = cat $in > $out\n" +"build foo: cat bar |@ baz\n")); + + Edge* edge = state.LookupNode("foo")->in_edge(); + ASSERT_EQ(edge->validations_.size(), 1); + EXPECT_EQ(edge->validations_[0]->path(), "baz"); +} + TEST_F(ParserTest, ImplicitOutput) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule cat\n" diff --git a/src/missing_deps.cc b/src/missing_deps.cc index 78feb49..de76620 100644 --- a/src/missing_deps.cc +++ b/src/missing_deps.cc @@ -116,7 +116,7 @@ void MissingDependencyScanner::ProcessNodeDeps(Node* node, Node** dep_nodes, // rebuild this target when the build is reconfigured", but build.ninja is // often generated by a configuration tool like cmake or gn. The rest of // the build "implicitly" depends on the entire build being reconfigured, - // so a missing dep path to build.ninja is not an actual missing dependecy + // so a missing dep path to build.ninja is not an actual missing dependency // problem. if (deplog_node->path() == "build.ninja") return; diff --git a/src/missing_deps_test.cc b/src/missing_deps_test.cc index 7b62e6c..db66885 100644 --- a/src/missing_deps_test.cc +++ b/src/missing_deps_test.cc @@ -152,7 +152,7 @@ TEST_F(MissingDependencyScannerTest, CycleInGraph) { CreateInitialState(); CreateGraphDependencyBetween("compiled_object", "generated_header"); CreateGraphDependencyBetween("generated_header", "compiled_object"); - // The missing-deps tool doesn't deal with cycles in the graph, beacuse + // The missing-deps tool doesn't deal with cycles in the graph, because // there will be an error loading the graph before we get to the tool. // This test is to illustrate that. std::string err; diff --git a/src/ninja.cc b/src/ninja.cc index 1d94442..df39ba9 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -127,6 +127,7 @@ struct NinjaMain : public BuildLogUser { int ToolMSVC(const Options* options, int argc, char* argv[]); int ToolTargets(const Options* options, int argc, char* argv[]); int ToolCommands(const Options* options, int argc, char* argv[]); + int ToolInputs(const Options* options, int argc, char* argv[]); int ToolClean(const Options* options, int argc, char* argv[]); int ToolCleanDead(const Options* options, int argc, char* argv[]); int ToolCompilationDatabase(const Options* options, int argc, char* argv[]); @@ -306,15 +307,20 @@ Node* NinjaMain::CollectTarget(const char* cpath, string* err) { if (node) { if (first_dependent) { if (node->out_edges().empty()) { - *err = "'" + path + "' has no out edge"; - return NULL; - } - Edge* edge = node->out_edges()[0]; - if (edge->outputs_.empty()) { - edge->Dump(); - Fatal("edge has no outputs"); + Node* rev_deps = deps_log_.GetFirstReverseDepsNode(node); + if (!rev_deps) { + *err = "'" + path + "' has no out edge"; + return NULL; + } + node = rev_deps; + } else { + Edge* edge = node->out_edges()[0]; + if (edge->outputs_.empty()) { + edge->Dump(); + Fatal("edge has no outputs"); + } + node = edge->outputs_[0]; } - node = edge->outputs_[0]; } return node; } else { @@ -399,6 +405,13 @@ int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) { label = "|| "; printf(" %s%s\n", label, edge->inputs_[in]->path().c_str()); } + if (!edge->validations_.empty()) { + printf(" validations:\n"); + for (std::vector<Node*>::iterator validation = edge->validations_.begin(); + validation != edge->validations_.end(); ++validation) { + printf(" %s\n", (*validation)->path().c_str()); + } + } } printf(" outputs:\n"); for (vector<Edge*>::const_iterator edge = node->out_edges().begin(); @@ -408,6 +421,17 @@ int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) { printf(" %s\n", (*out)->path().c_str()); } } + const std::vector<Edge*> validation_edges = node->validation_out_edges(); + if (!validation_edges.empty()) { + printf(" validation for:\n"); + for (std::vector<Edge*>::const_iterator edge = validation_edges.begin(); + edge != validation_edges.end(); ++edge) { + for (vector<Node*>::iterator out = (*edge)->outputs_.begin(); + out != (*edge)->outputs_.end(); ++out) { + printf(" %s\n", (*out)->path().c_str()); + } + } + } } return 0; } @@ -679,7 +703,7 @@ void PrintCommands(Edge* edge, EdgeSet* seen, PrintCommandMode mode) { } int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) { - // The clean tool uses getopt, and expects argv[0] to contain the name of + // The commands tool uses getopt, and expects argv[0] to contain the name of // the tool, i.e. "commands". ++argc; --argv; @@ -720,6 +744,35 @@ int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) { return 0; } +void PrintInputs(Edge* edge, set<Edge*>* seen) { + if (!edge) + return; + if (!seen->insert(edge).second) + return; + + for (vector<Node*>::iterator in = edge->inputs_.begin(); + in != edge->inputs_.end(); ++in) + PrintInputs((*in)->in_edge(), seen); + + if (!edge->is_phony()) + puts(edge->GetBinding("in_newline").c_str()); +} + +int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) { + vector<Node*> nodes; + string err; + if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) { + Error("%s", err.c_str()); + return 1; + } + + set<Edge*> seen; + for (vector<Node*>::iterator in = nodes.begin(); in != nodes.end(); ++in) + PrintInputs((*in)->in_edge(), &seen); + + return 0; +} + int NinjaMain::ToolClean(const Options* options, int argc, char* argv[]) { // The clean tool uses getopt, and expects argv[0] to contain the name of // the tool, i.e. "clean". @@ -998,6 +1051,8 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOAD, &NinjaMain::ToolClean }, { "commands", "list all commands required to rebuild given targets", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCommands }, + { "inputs", "list all inputs required to rebuild given targets", + Tool::RUN_AFTER_LOAD, &NinjaMain::ToolInputs}, { "deps", "show dependencies stored in the deps log", Tool::RUN_AFTER_LOGS, &NinjaMain::ToolDeps }, { "missingdeps", "check deps log dependencies on generated files", @@ -1431,6 +1486,17 @@ NORETURN void real_main(int argc, char** argv) { exit((ninja.*options.tool->func)(&options, argc, argv)); } +#ifdef WIN32 + // It'd be nice to use line buffering but MSDN says: "For some systems, + // [_IOLBF] provides line buffering. However, for Win32, the behavior is the + // same as _IOFBF - Full Buffering." + // Buffering used to be disabled in the LinePrinter constructor but that + // now disables it too early and breaks -t deps performance (see issue #2018) + // so we disable it here instead, but only when not running a tool. + if (!options.tool) + setvbuf(stdout, NULL, _IONBF, 0); +#endif + // Limit number of rebuilds, to prevent infinite loops. const int kCycleLimit = 100; for (int cycle = 1; cycle <= kCycleLimit; ++cycle) { @@ -1481,7 +1547,7 @@ NORETURN void real_main(int argc, char** argv) { exit(result); } - status->Error("manifest '%s' still dirty after %d tries", + status->Error("manifest '%s' still dirty after %d tries, perhaps system time is not set", options.input_file, kCycleLimit); exit(1); } diff --git a/src/ninja_test.cc b/src/ninja_test.cc index b40e176..6720dec 100644 --- a/src/ninja_test.cc +++ b/src/ninja_test.cc @@ -67,7 +67,7 @@ void Usage() { "usage: ninja_tests [options]\n" "\n" "options:\n" -" --gtest_filter=POSTIVE_PATTERN[-NEGATIVE_PATTERN]\n" +" --gtest_filter=POSITIVE_PATTERN[-NEGATIVE_PATTERN]\n" " Run tests whose names match the positive but not the negative pattern.\n" " '*' matches any substring. (gtest's ':', '?' are not implemented).\n"); } diff --git a/src/state.cc b/src/state.cc index a33d5a8..556b0d8 100644 --- a/src/state.cc +++ b/src/state.cc @@ -19,7 +19,6 @@ #include "edit_distance.h" #include "graph.h" -#include "metrics.h" #include "util.h" using namespace std; @@ -104,7 +103,6 @@ Node* State::GetNode(StringPiece path, uint64_t slash_bits) { } Node* State::LookupNode(StringPiece path) const { - METRIC_RECORD("lookup node"); Paths::const_iterator i = paths_.find(path); if (i != paths_.end()) return i->second; @@ -143,6 +141,12 @@ bool State::AddOut(Edge* edge, StringPiece path, uint64_t slash_bits) { return true; } +void State::AddValidation(Edge* edge, StringPiece path, uint64_t slash_bits) { + Node* node = GetNode(path, slash_bits); + edge->validations_.push_back(node); + node->AddValidationOutEdge(edge); +} + bool State::AddDefault(StringPiece path, string* err) { Node* node = LookupNode(path); if (!node) { diff --git a/src/state.h b/src/state.h index 72c5b33..878ac6d 100644 --- a/src/state.h +++ b/src/state.h @@ -107,6 +107,7 @@ struct State { void AddIn(Edge* edge, StringPiece path, uint64_t slash_bits); bool AddOut(Edge* edge, StringPiece path, uint64_t slash_bits); + void AddValidation(Edge* edge, StringPiece path, uint64_t slash_bits); bool AddDefault(StringPiece path, std::string* error); /// Reset state. Keeps all nodes and edges, but restores them to the diff --git a/src/util.cc b/src/util.cc index 3dfa8dd..080883e 100644 --- a/src/util.cc +++ b/src/util.cc @@ -56,7 +56,6 @@ #endif #include "edit_distance.h" -#include "metrics.h" using namespace std; @@ -118,7 +117,6 @@ void Info(const char* msg, ...) { } void CanonicalizePath(string* path, uint64_t* slash_bits) { - METRIC_RECORD("canonicalize str"); size_t len = path->size(); char* str = 0; if (len > 0) @@ -138,7 +136,6 @@ static bool IsPathSeparator(char c) { void CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits) { // WARNING: this function is performance-critical; please benchmark // any changes you make to it. - METRIC_RECORD("canonicalize path"); if (*len == 0) { return; } |