Merge branch 'dep-pipeless'

This merges a new mechanism for tracking "depfile" dependencies that
is faster on all platforms but dramatically so on Windows.

1 files changed, 181 insertions, 98 deletions

diff --git a/src/build.cc b/src/build.cc
index 340a3d9..ab3d781 100644
--- a/src/build.cc
+++ b/src/build.cc
@@ -15,6 +15,7 @@
 #include "build.h"
 
 #include <assert.h>
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <functional>
@@ -24,8 +25,11 @@
 #endif
 
 #include "build_log.h"
+#include "depfile_parser.h"
+#include "deps_log.h"
 #include "disk_interface.h"
 #include "graph.h"
+#include "msvc_helper.h"
 #include "state.h"
 #include "subprocess.h"
 #include "util.h"
@@ -39,7 +43,7 @@ struct DryRunCommandRunner : public CommandRunner {
   // Overridden from CommandRunner:
   virtual bool CanRunMore();
   virtual bool StartCommand(Edge* edge);
-  virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */);
+  virtual bool WaitForCommand(Result* result);
 
  private:
   queue<Edge*> finished_;
@@ -54,16 +58,14 @@ bool DryRunCommandRunner::StartCommand(Edge* edge) {
   return true;
 }
 
-Edge* DryRunCommandRunner::WaitForCommand(ExitStatus* status,
-                                          string* /*output*/) {
-   if (finished_.empty()) {
-     *status = ExitFailure;
-     return NULL;
-   }
-   *status = ExitSuccess;
-   Edge* edge = finished_.front();
+bool DryRunCommandRunner::WaitForCommand(Result* result) {
+   if (finished_.empty())
+     return false;
+
+   result->status = ExitSuccess;
+   result->edge = finished_.front();
    finished_.pop();
-   return edge;
+   return true;
 }
 
 }  // namespace
@@ -427,7 +429,7 @@ void Plan::CleanNode(DependencyScan* scan, Node* node) {
         if (!(*ni)->dirty())
           continue;
 
-        if (scan->RecomputeOutputDirty(*ei, most_recent_input,
+        if (scan->RecomputeOutputDirty(*ei, most_recent_input, 0,
                                        command, *ni)) {
           (*ni)->MarkDirty();
           all_outputs_clean = false;
@@ -462,7 +464,7 @@ struct RealCommandRunner : public CommandRunner {
   virtual ~RealCommandRunner() {}
   virtual bool CanRunMore();
   virtual bool StartCommand(Edge* edge);
-  virtual Edge* WaitForCommand(ExitStatus* status, string* output);
+  virtual bool WaitForCommand(Result* result);
   virtual vector<Edge*> GetActiveEdges();
   virtual void Abort();
 
@@ -499,31 +501,30 @@ bool RealCommandRunner::StartCommand(Edge* edge) {
   return true;
 }
 
-Edge* RealCommandRunner::WaitForCommand(ExitStatus* status, string* output) {
+bool RealCommandRunner::WaitForCommand(Result* result) {
   Subprocess* subproc;
   while ((subproc = subprocs_.NextFinished()) == NULL) {
     bool interrupted = subprocs_.DoWork();
-    if (interrupted) {
-      *status = ExitInterrupted;
-      return 0;
-    }
+    if (interrupted)
+      return false;
   }
 
-  *status = subproc->Finish();
-  *output = subproc->GetOutput();
+  result->status = subproc->Finish();
+  result->output = subproc->GetOutput();
 
   map<Subprocess*, Edge*>::iterator i = subproc_to_edge_.find(subproc);
-  Edge* edge = i->second;
+  result->edge = i->second;
   subproc_to_edge_.erase(i);
 
   delete subproc;
-  return edge;
+  return true;
 }
 
 Builder::Builder(State* state, const BuildConfig& config,
-                 BuildLog* log, DiskInterface* disk_interface)
+                 BuildLog* build_log, DepsLog* deps_log,
+                 DiskInterface* disk_interface)
     : state_(state), config_(config), disk_interface_(disk_interface),
-      scan_(state, log, disk_interface) {
+      scan_(state, build_log, deps_log, disk_interface) {
   status_ = new BuildStatus(config);
 }
 
@@ -609,8 +610,6 @@ bool Builder::Build(string* err) {
   // First, we attempt to start as many commands as allowed by the
   // command runner.
   // Second, we attempt to wait for / reap the next finished command.
-  // If we can do neither of those, the build is stuck, and we report
-  // an error.
   while (plan_.more_to_do()) {
     // See if we can start any more commands.
     if (failures_allowed && command_runner_->CanRunMore()) {
@@ -620,10 +619,11 @@ bool Builder::Build(string* err) {
           return false;
         }
 
-        if (edge->is_phony())
-          FinishEdge(edge, true, "");
-        else
+        if (edge->is_phony()) {
+          plan_.EdgeFinished(edge);
+        } else {
           ++pending_commands;
+        }
 
         // We made some progress; go back to the main loop.
         continue;
@@ -632,34 +632,24 @@ bool Builder::Build(string* err) {
 
     // See if we can reap any finished commands.
     if (pending_commands) {
-      ExitStatus status;
-      string output;
-      Edge* edge = command_runner_->WaitForCommand(&status, &output);
-      if (edge && status != ExitInterrupted) {
-        bool success = (status == ExitSuccess);
-        --pending_commands;
-        FinishEdge(edge, success, output);
-        if (!success) {
-          if (failures_allowed)
-            failures_allowed--;
-        }
-
-        // We made some progress; start the main loop over.
-        continue;
-      }
-
-      if (status == ExitInterrupted) {
+      CommandRunner::Result result;
+      if (!command_runner_->WaitForCommand(&result) ||
+          result.status == ExitInterrupted) {
         status_->BuildFinished();
         *err = "interrupted by user";
         return false;
       }
-    }
 
-    // If we get here, we can neither enqueue new commands nor are any running.
-    if (pending_commands) {
-      status_->BuildFinished();
-      *err = "stuck: pending commands but none to wait for? [this is a bug]";
-      return false;
+      --pending_commands;
+      FinishCommand(&result);
+
+      if (!result.success()) {
+        if (failures_allowed)
+          failures_allowed--;
+      }
+
+      // We made some progress; start the main loop over.
+      continue;
     }
 
     // If we get here, we cannot make any more progress.
@@ -714,63 +704,156 @@ bool Builder::StartEdge(Edge* edge, string* err) {
   return true;
 }
 
-void Builder::FinishEdge(Edge* edge, bool success, const string& output) {
-  METRIC_RECORD("FinishEdge");
-  TimeStamp restat_mtime = 0;
+void Builder::FinishCommand(CommandRunner::Result* result) {
+  METRIC_RECORD("FinishCommand");
 
-  if (success) {
-    if (edge->GetBindingBool("restat") && !config_.dry_run) {
-      bool node_cleaned = false;
-
-      for (vector<Node*>::iterator i = edge->outputs_.begin();
-           i != edge->outputs_.end(); ++i) {
-        TimeStamp new_mtime = disk_interface_->Stat((*i)->path());
-        if ((*i)->mtime() == new_mtime) {
-          // The rule command did not change the output.  Propagate the clean
-          // state through the build graph.
-          // Note that this also applies to nonexistent outputs (mtime == 0).
-          plan_.CleanNode(&scan_, *i);
-          node_cleaned = true;
-        }
-      }
+  Edge* edge = result->edge;
 
-      if (node_cleaned) {
-        // If any output was cleaned, find the most recent mtime of any
-        // (existing) non-order-only input or the depfile.
-        for (vector<Node*>::iterator i = edge->inputs_.begin();
-             i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
-          TimeStamp input_mtime = disk_interface_->Stat((*i)->path());
-          if (input_mtime > restat_mtime)
-            restat_mtime = input_mtime;
-        }
+  // First try to extract dependencies from the result, if any.
+  // This must happen first as it filters the command output and
+  // can fail, which makes the command fail from a build perspective.
 
-        string depfile = edge->GetBinding("depfile");
-        if (restat_mtime != 0 && !depfile.empty()) {
-          TimeStamp depfile_mtime = disk_interface_->Stat(depfile);
-          if (depfile_mtime > restat_mtime)
-            restat_mtime = depfile_mtime;
-        }
+  vector<Node*> deps_nodes;
+  TimeStamp deps_mtime = 0;
+  string deps_type = edge->GetBinding("deps");
+  if (result->success() && !deps_type.empty()) {
+    string extract_err;
+    if (!ExtractDeps(result, deps_type, &deps_nodes, &deps_mtime,
+                     &extract_err)) {
+      if (!result->output.empty())
+        result->output.append("\n");
+      result->output.append(extract_err);
+      result->status = ExitFailure;
+    }
+  }
 
-        // The total number of edges in the plan may have changed as a result
-        // of a restat.
-        status_->PlanHasTotalEdges(plan_.command_edge_count());
+  int start_time, end_time;
+  status_->BuildEdgeFinished(edge, result->success(), result->output,
+                             &start_time, &end_time);
+
+  // The rest of this function only applies to successful commands.
+  if (!result->success())
+    return;
+
+  // Restat the edge outputs, if necessary.
+  TimeStamp restat_mtime = 0;
+  if (edge->GetBindingBool("restat") && !config_.dry_run) {
+    bool node_cleaned = false;
+
+    for (vector<Node*>::iterator i = edge->outputs_.begin();
+         i != edge->outputs_.end(); ++i) {
+      TimeStamp new_mtime = disk_interface_->Stat((*i)->path());
+      if ((*i)->mtime() == new_mtime) {
+        // The rule command did not change the output.  Propagate the clean
+        // state through the build graph.
+        // Note that this also applies to nonexistent outputs (mtime == 0).
+        plan_.CleanNode(&scan_, *i);
+        node_cleaned = true;
       }
     }
 
-    // Delete the response file on success (if exists)
-    string rspfile = edge->GetBinding("rspfile");
-    if (!rspfile.empty())
-      disk_interface_->RemoveFile(rspfile);
+    if (node_cleaned) {
+      // If any output was cleaned, find the most recent mtime of any
+      // (existing) non-order-only input or the depfile.
+      for (vector<Node*>::iterator i = edge->inputs_.begin();
+           i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
+        TimeStamp input_mtime = disk_interface_->Stat((*i)->path());
+        if (input_mtime > restat_mtime)
+          restat_mtime = input_mtime;
+      }
+
+      string depfile = edge->GetBinding("depfile");
+      if (restat_mtime != 0 && !depfile.empty()) {
+        TimeStamp depfile_mtime = disk_interface_->Stat(depfile);
+        if (depfile_mtime > restat_mtime)
+          restat_mtime = depfile_mtime;
+      }
 
-    plan_.EdgeFinished(edge);
+      // The total number of edges in the plan may have changed as a result
+      // of a restat.
+      status_->PlanHasTotalEdges(plan_.command_edge_count());
+    }
   }
 
-  if (edge->is_phony())
-    return;
+  plan_.EdgeFinished(edge);
+
+  // Delete any left over response file.
+  string rspfile = edge->GetBinding("rspfile");
+  if (!rspfile.empty())
+    disk_interface_->RemoveFile(rspfile);
+
+  if (scan_.build_log()) {
+    scan_.build_log()->RecordCommand(edge, start_time, end_time,
+                                     restat_mtime);
+  }
+
+  if (!deps_type.empty()) {
+    assert(edge->outputs_.size() == 1 && "should have been rejected by parser");
+    Node* out = edge->outputs_[0];
+    if (!deps_mtime) {
+      // On Windows there's no separate file to compare against; just reuse
+      // the output's mtime.
+      deps_mtime = disk_interface_->Stat(out->path());
+    }
+    scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes);
+  }
 
-  int start_time, end_time;
-  status_->BuildEdgeFinished(edge, success, output, &start_time, &end_time);
-  if (success && scan_.build_log())
-    scan_.build_log()->RecordCommand(edge, start_time, end_time, restat_mtime);
 }
 
+bool Builder::ExtractDeps(CommandRunner::Result* result,
+                          const string& deps_type,
+                          vector<Node*>* deps_nodes,
+                          TimeStamp* deps_mtime,
+                          string* err) {
+#ifdef _WIN32
+  if (deps_type == "msvc") {
+    CLParser parser;
+    result->output = parser.Parse(result->output);
+    for (set<string>::iterator i = parser.includes_.begin();
+         i != parser.includes_.end(); ++i) {
+      deps_nodes->push_back(state_->GetNode(*i));
+    }
+  } else
+#endif
+  if (deps_type == "gcc") {
+    string depfile = result->edge->GetBinding("depfile");
+    if (depfile.empty()) {
+      *err = string("edge with deps=gcc but no depfile makes no sense");
+      return false;
+    }
+
+    *deps_mtime = disk_interface_->Stat(depfile);
+    if (*deps_mtime <= 0) {
+      *err = string("unable to read depfile");
+      return false;
+    }
+
+    string content = disk_interface_->ReadFile(depfile, err);
+    if (!err->empty())
+      return false;
+    if (content.empty())
+      return true;
+
+    DepfileParser deps;
+    if (!deps.Parse(&content, err))
+      return false;
+
+    // XXX check depfile matches expected output.
+    deps_nodes->reserve(deps.ins_.size());
+    for (vector<StringPiece>::iterator i = deps.ins_.begin();
+         i != deps.ins_.end(); ++i) {
+      if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err))
+        return false;
+      deps_nodes->push_back(state_->GetNode(*i));
+    }
+
+    if (disk_interface_->RemoveFile(depfile) < 0) {
+      *err = string("deleting depfile: ") + strerror(errno) + string("\n");
+      return false;
+    }
+  } else {
+    Fatal("unknown deps type '%s'", deps_type.c_str());
+  }
+
+  return true;
+}