From e5c22c0a4b93895334a10d412124ffff69c3fd25 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Nov 2015 16:17:33 -0500 Subject: Teach DependencyScan to load a dyndep file Add a LoadDyndeps method to load a dyndep file and update the edges that name it in their dyndep binding. --- configure.py | 1 + src/dyndep.cc | 124 ++++++++++++++++++++++++++++++++++++ src/dyndep.h | 28 ++++++++- src/graph.cc | 9 +++ src/graph.h | 12 +++- src/graph_test.cc | 183 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 355 insertions(+), 2 deletions(-) create mode 100644 src/dyndep.cc diff --git a/configure.py b/configure.py index b56ef89..850bb98 100755 --- a/configure.py +++ b/configure.py @@ -496,6 +496,7 @@ for name in ['build', 'depfile_parser', 'deps_log', 'disk_interface', + 'dyndep', 'dyndep_parser', 'edit_distance', 'eval_env', diff --git a/src/dyndep.cc b/src/dyndep.cc new file mode 100644 index 0000000..2aee601 --- /dev/null +++ b/src/dyndep.cc @@ -0,0 +1,124 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dyndep.h" + +#include +#include + +#include "debug_flags.h" +#include "disk_interface.h" +#include "dyndep_parser.h" +#include "graph.h" +#include "state.h" +#include "util.h" + +bool DyndepLoader::LoadDyndeps(Node* node, std::string* err) const { + DyndepFile ddf; + return LoadDyndeps(node, &ddf, err); +} + +bool DyndepLoader::LoadDyndeps(Node* node, DyndepFile* ddf, + std::string* err) const { + // We are loading the dyndep file now so it is no longer pending. + node->set_dyndep_pending(false); + + // Load the dyndep information from the file. + EXPLAIN("loading dyndep file '%s'", node->path().c_str()); + if (!LoadDyndepFile(node, ddf, err)) + return false; + + // Update each edge that specified this node as its dyndep binding. + std::vector const& out_edges = node->out_edges(); + for (std::vector::const_iterator oe = out_edges.begin(); + oe != out_edges.end(); ++oe) { + Edge* const edge = *oe; + if (edge->dyndep_ != node) + continue; + + DyndepFile::iterator ddi = ddf->find(edge); + if (ddi == ddf->end()) { + *err = ("'" + edge->outputs_[0]->path() + "' " + "not mentioned in its dyndep file " + "'" + node->path() + "'"); + return false; + } + + ddi->second.used_ = true; + Dyndeps const& dyndeps = ddi->second; + if (!UpdateEdge(edge, &dyndeps, err)) { + return false; + } + } + + // Reject extra outputs in dyndep file. + for (DyndepFile::const_iterator oe = ddf->begin(); oe != ddf->end(); + ++oe) { + if (!oe->second.used_) { + Edge* const edge = oe->first; + *err = ("dyndep file '" + node->path() + "' mentions output " + "'" + edge->outputs_[0]->path() + "' whose build statement " + "does not have a dyndep binding for the file"); + return false; + } + } + + return true; +} + +bool DyndepLoader::UpdateEdge(Edge* edge, Dyndeps const* dyndeps, + std::string* err) const { + // Add dyndep-discovered bindings to the edge. + // We know the edge already has its own binding + // scope because it has a "dyndep" binding. + if (dyndeps->restat_) + edge->env_->AddBinding("restat", "1"); + + // Add the dyndep-discovered outputs to the edge. + edge->outputs_.insert(edge->outputs_.end(), + dyndeps->implicit_outputs_.begin(), + dyndeps->implicit_outputs_.end()); + edge->implicit_outs_ += dyndeps->implicit_outputs_.size(); + + // Add this edge as incoming to each new output. + for (std::vector::const_iterator i = + dyndeps->implicit_outputs_.begin(); + i != dyndeps->implicit_outputs_.end(); ++i) { + if ((*i)->in_edge() != NULL) { + *err = "multiple rules generate " + (*i)->path(); + return false; + } + (*i)->set_in_edge(edge); + } + + // Add the dyndep-discovered inputs to the edge. + edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_, + dyndeps->implicit_inputs_.begin(), + dyndeps->implicit_inputs_.end()); + edge->implicit_deps_ += dyndeps->implicit_inputs_.size(); + + // Add this edge as outgoing from each new input. + for (std::vector::const_iterator i = + dyndeps->implicit_inputs_.begin(); + i != dyndeps->implicit_inputs_.end(); ++i) + (*i)->AddOutEdge(edge); + + return true; +} + +bool DyndepLoader::LoadDyndepFile(Node* file, DyndepFile* ddf, + std::string* err) const { + DyndepParser parser(state_, disk_interface_, ddf); + return parser.Load(file->path(), err); +} diff --git a/src/dyndep.h b/src/dyndep.h index 80c5d1b..907f921 100644 --- a/src/dyndep.h +++ b/src/dyndep.h @@ -16,14 +16,18 @@ #define NINJA_DYNDEP_LOADER_H_ #include +#include #include +struct DiskInterface; struct Edge; struct Node; +struct State; /// Store dynamically-discovered dependency information for one edge. struct Dyndeps { - Dyndeps() : restat_(false) {} + Dyndeps() : used_(false), restat_(false) {} + bool used_; bool restat_; std::vector implicit_inputs_; std::vector implicit_outputs_; @@ -35,4 +39,26 @@ struct Dyndeps { /// forward-declare it in other headers. struct DyndepFile: public std::map {}; +/// DyndepLoader loads dynamically discovered dependencies, as +/// referenced via the "dyndep" attribute in build files. +struct DyndepLoader { + DyndepLoader(State* state, DiskInterface* disk_interface) + : state_(state), disk_interface_(disk_interface) {} + + /// Load a dyndep file from the given node's path and update the + /// build graph with the new information. One overload accepts + /// a caller-owned 'DyndepFile' object in which to store the + /// information loaded from the dyndep file. + bool LoadDyndeps(Node* node, std::string* err) const; + bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const; + + private: + bool LoadDyndepFile(Node* file, DyndepFile* ddf, std::string* err) const; + + bool UpdateEdge(Edge* edge, Dyndeps const* dyndeps, std::string* err) const; + + State* state_; + DiskInterface* disk_interface_; +}; + #endif // NINJA_DYNDEP_LOADER_H_ diff --git a/src/graph.cc b/src/graph.cc index 2fbce84..a464299 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -276,6 +276,15 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, return false; } +bool DependencyScan::LoadDyndeps(Node* node, string* err) const { + return dyndep_loader_.LoadDyndeps(node, err); +} + +bool DependencyScan::LoadDyndeps(Node* node, DyndepFile* ddf, + string* err) const { + return dyndep_loader_.LoadDyndeps(node, ddf, err); +} + bool Edge::AllInputsReady() const { for (vector::const_iterator i = inputs_.begin(); i != inputs_.end(); ++i) { diff --git a/src/graph.h b/src/graph.h index 745297d..75edbc5 100644 --- a/src/graph.h +++ b/src/graph.h @@ -19,6 +19,7 @@ #include using namespace std; +#include "dyndep.h" #include "eval_env.h" #include "timestamp.h" #include "util.h" @@ -270,7 +271,8 @@ struct DependencyScan { DepfileParserOptions const* depfile_parser_options) : build_log_(build_log), disk_interface_(disk_interface), - dep_loader_(state, deps_log, disk_interface, depfile_parser_options) {} + dep_loader_(state, deps_log, disk_interface, depfile_parser_options), + dyndep_loader_(state, disk_interface) {} /// Update the |dirty_| state of the given node by inspecting its input edge. /// Examine inputs, outputs, and command lines to judge whether an edge @@ -295,6 +297,13 @@ struct DependencyScan { return dep_loader_.deps_log(); } + /// Load a dyndep file from the given node's path and update the + /// build graph with the new information. One overload accepts + /// a caller-owned 'DyndepFile' object in which to store the + /// information loaded from the dyndep file. + bool LoadDyndeps(Node* node, string* err) const; + bool LoadDyndeps(Node* node, DyndepFile* ddf, string* err) const; + private: bool RecomputeDirty(Node* node, vector* stack, string* err); bool VerifyDAG(Node* node, vector* stack, string* err); @@ -307,6 +316,7 @@ struct DependencyScan { BuildLog* build_log_; DiskInterface* disk_interface_; ImplicitDepLoader dep_loader_; + DyndepLoader dyndep_loader_; }; #endif // NINJA_GRAPH_H_ diff --git a/src/graph_test.cc b/src/graph_test.cc index 4a66831..f53c0e9 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -479,3 +479,186 @@ TEST_F(GraphTest, Decanonicalize) { EXPECT_EQ(root_nodes[3]->PathDecanonicalized(), "out4\\foo"); } #endif + +TEST_F(GraphTest, DyndepLoadTrivial) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("", err); + EXPECT_FALSE(GetNode("dd")->dyndep_pending()); + + Edge* edge = GetNode("out")->in_edge(); + ASSERT_EQ(1u, edge->outputs_.size()); + EXPECT_EQ("out", edge->outputs_[0]->path()); + ASSERT_EQ(2u, edge->inputs_.size()); + EXPECT_EQ("in", edge->inputs_[0]->path()); + EXPECT_EQ("dd", edge->inputs_[1]->path()); + EXPECT_EQ(0u, edge->implicit_deps_); + EXPECT_EQ(1u, edge->order_only_deps_); + EXPECT_FALSE(edge->GetBindingBool("restat")); +} + +TEST_F(GraphTest, DyndepLoadMissingFile) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" dyndep = dd\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("loading 'dd': No such file or directory", err); +} + +TEST_F(GraphTest, DyndepLoadMissingEntry) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("'out' not mentioned in its dyndep file 'dd'", err); +} + +TEST_F(GraphTest, DyndepLoadExtraEntry) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out: r in || dd\n" +" dyndep = dd\n" +"build out2: r in || dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +"build out2: dyndep\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("dyndep file 'dd' mentions output 'out2' whose build statement " + "does not have a dyndep binding for the file", err); +} + +TEST_F(GraphTest, DyndepLoadOutputWithMultipleRules1) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out1 | out-twice.imp: r in1\n" +"build out2: r in2 || dd\n" +" dyndep = dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out2 | out-twice.imp: dyndep\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("multiple rules generate out-twice.imp", err); +} + +TEST_F(GraphTest, DyndepLoadOutputWithMultipleRules2) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out1: r in1 || dd1\n" +" dyndep = dd1\n" +"build out2: r in2 || dd2\n" +" dyndep = dd2\n" + ); + fs_.Create("dd1", +"ninja_dyndep_version = 1\n" +"build out1 | out-twice.imp: dyndep\n" + ); + fs_.Create("dd2", +"ninja_dyndep_version = 1\n" +"build out2 | out-twice.imp: dyndep\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd1")->dyndep_pending()); + EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd1"), &err)); + EXPECT_EQ("", err); + ASSERT_TRUE(GetNode("dd2")->dyndep_pending()); + EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd2"), &err)); + EXPECT_EQ("multiple rules generate out-twice.imp", err); +} + +TEST_F(GraphTest, DyndepLoadMultiple) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out1: r in1 || dd\n" +" dyndep = dd\n" +"build out2: r in2 || dd\n" +" dyndep = dd\n" +"build outNot: r in3 || dd\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out1 | out1imp: dyndep | in1imp\n" +"build out2: dyndep | in2imp\n" +" restat = 1\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("", err); + EXPECT_FALSE(GetNode("dd")->dyndep_pending()); + + Edge* edge1 = GetNode("out1")->in_edge(); + ASSERT_EQ(2u, edge1->outputs_.size()); + EXPECT_EQ("out1", edge1->outputs_[0]->path()); + EXPECT_EQ("out1imp", edge1->outputs_[1]->path()); + EXPECT_EQ(1u, edge1->implicit_outs_); + ASSERT_EQ(3u, edge1->inputs_.size()); + EXPECT_EQ("in1", edge1->inputs_[0]->path()); + EXPECT_EQ("in1imp", edge1->inputs_[1]->path()); + EXPECT_EQ("dd", edge1->inputs_[2]->path()); + EXPECT_EQ(1u, edge1->implicit_deps_); + EXPECT_EQ(1u, edge1->order_only_deps_); + EXPECT_FALSE(edge1->GetBindingBool("restat")); + EXPECT_EQ(edge1, GetNode("out1imp")->in_edge()); + Node* in1imp = GetNode("in1imp"); + ASSERT_EQ(1u, in1imp->out_edges().size()); + EXPECT_EQ(edge1, in1imp->out_edges()[0]); + + Edge* edge2 = GetNode("out2")->in_edge(); + ASSERT_EQ(1u, edge2->outputs_.size()); + EXPECT_EQ("out2", edge2->outputs_[0]->path()); + EXPECT_EQ(0u, edge2->implicit_outs_); + ASSERT_EQ(3u, edge2->inputs_.size()); + EXPECT_EQ("in2", edge2->inputs_[0]->path()); + EXPECT_EQ("in2imp", edge2->inputs_[1]->path()); + EXPECT_EQ("dd", edge2->inputs_[2]->path()); + EXPECT_EQ(1u, edge2->implicit_deps_); + EXPECT_EQ(1u, edge2->order_only_deps_); + EXPECT_TRUE(edge2->GetBindingBool("restat")); + Node* in2imp = GetNode("in2imp"); + ASSERT_EQ(1u, in2imp->out_edges().size()); + EXPECT_EQ(edge2, in2imp->out_edges()[0]); +} -- cgit v0.12