cmMakefile: Delay custom command creation

Move custom command creation to cmLocalGenerator and dispatch custom
commands in cmMakefile to generate time.  Generators add custom commands
using the new methods provided by cmLocalGenerator.

Issue: #12877

1 files changed, 170 insertions, 226 deletions

diff --git a/Source/cmMakefile.cxx b/Source/cmMakefile.cxx
index c6c8109..dc0b50f 100644
--- a/Source/cmMakefile.cxx
+++ b/Source/cmMakefile.cxx
@@ -14,7 +14,7 @@
 #include <utility>
 
 #include <cm/iterator>
-#include <cm/memory>
+#include <cm/optional>
 
 #include "cmsys/FStream.hxx"
 #include "cmsys/RegularExpression.hxx"
@@ -798,6 +798,8 @@ void cmMakefile::DoGenerate(cmLocalGenerator& lg)
     action.Value(lg, action.Backtrace);
   }
   this->GeneratorActionsInvoked = true;
+  this->DelayedOutputFiles.clear();
+  this->DelayedOutputFilesHaveGenex = false;
 
   // go through all configured files and see which ones still exist.
   // we don't want cmake to re-run if a configured file is created and deleted
@@ -828,6 +830,39 @@ void cmMakefile::Generate(cmLocalGenerator& lg)
   }
 }
 
+namespace {
+// There are still too many implicit backtraces through cmMakefile.  As a
+// workaround we reset the backtrace temporarily.
+struct BacktraceGuard
+{
+  BacktraceGuard(cmListFileBacktrace& lfbt, cmListFileBacktrace current)
+    : Backtrace(lfbt)
+    , Previous(lfbt)
+  {
+    this->Backtrace = std::move(current);
+  }
+
+  ~BacktraceGuard() { this->Backtrace = std::move(Previous); }
+
+private:
+  cmListFileBacktrace& Backtrace;
+  cmListFileBacktrace Previous;
+};
+
+cm::optional<std::string> MakeOptionalString(const char* str)
+{
+  if (str) {
+    return str;
+  }
+  return cm::nullopt;
+}
+
+const char* GetCStrOrNull(const cm::optional<std::string>& str)
+{
+  return str ? str->c_str() : nullptr;
+}
+}
+
 bool cmMakefile::ValidateCustomCommand(
   const cmCustomCommandLines& commandLines) const
 {
@@ -916,10 +951,10 @@ cmTarget* cmMakefile::AddCustomCommandToTarget(
   const cmCustomCommandLines& commandLines, cmCustomCommandType type,
   const char* comment, const char* workingDir, bool escapeOldStyle,
   bool uses_terminal, const std::string& depfile, const std::string& job_pool,
-  bool command_expand_lists, cmObjectLibraryCommands objLibCommands)
+  bool command_expand_lists)
 {
-  cmTarget* t =
-    this->GetCustomCommandTarget(target, objLibCommands, this->Backtrace);
+  cmTarget* t = this->GetCustomCommandTarget(
+    target, cmObjectLibraryCommands::Reject, this->Backtrace);
 
   // Validate custom commands.
   if (!t || !this->ValidateCustomCommand(commandLines)) {
@@ -927,176 +962,83 @@ cmTarget* cmMakefile::AddCustomCommandToTarget(
   }
 
   // Always create the byproduct sources and mark them generated.
-  this->CreateGeneratedSources(byproducts);
-
-  this->CommitCustomCommandToTarget(
-    t, byproducts, depends, commandLines, type, comment, workingDir,
-    escapeOldStyle, uses_terminal, depfile, job_pool, command_expand_lists);
+  this->CreateGeneratedByproducts(byproducts);
+
+  // Strings could be moved into the callback function with C++14.
+  cm::optional<std::string> commentStr = MakeOptionalString(comment);
+  cm::optional<std::string> workingStr = MakeOptionalString(workingDir);
+
+  // Dispatch command creation to allow generator expressions in outputs.
+  this->AddGeneratorAction([=](cmLocalGenerator& lg,
+                               const cmListFileBacktrace& lfbt) {
+    BacktraceGuard guard(this->Backtrace, lfbt);
+    detail::AddCustomCommandToTarget(
+      lg, lfbt, cmCommandOrigin::Project, t, byproducts, depends, commandLines,
+      type, GetCStrOrNull(commentStr), GetCStrOrNull(workingStr),
+      escapeOldStyle, uses_terminal, depfile, job_pool, command_expand_lists);
+  });
 
   return t;
 }
 
-void cmMakefile::CommitCustomCommandToTarget(
-  cmTarget* target, const std::vector<std::string>& byproducts,
-  const std::vector<std::string>& depends,
-  const cmCustomCommandLines& commandLines, cmCustomCommandType type,
-  const char* comment, const char* workingDir, bool escapeOldStyle,
-  bool uses_terminal, const std::string& depfile, const std::string& job_pool,
-  bool command_expand_lists)
-{
-  // Add the command to the appropriate build step for the target.
-  std::vector<std::string> no_output;
-  cmCustomCommand cc(no_output, byproducts, depends, commandLines,
-                     this->Backtrace, comment, workingDir);
-  cc.SetEscapeOldStyle(escapeOldStyle);
-  cc.SetEscapeAllowMakeVars(true);
-  cc.SetUsesTerminal(uses_terminal);
-  cc.SetCommandExpandLists(command_expand_lists);
-  cc.SetDepfile(depfile);
-  cc.SetJobPool(job_pool);
-  switch (type) {
-    case cmCustomCommandType::PRE_BUILD:
-      target->AddPreBuildCommand(std::move(cc));
-      break;
-    case cmCustomCommandType::PRE_LINK:
-      target->AddPreLinkCommand(std::move(cc));
-      break;
-    case cmCustomCommandType::POST_BUILD:
-      target->AddPostBuildCommand(std::move(cc));
-      break;
-  }
-
-  this->AddTargetByproducts(target, byproducts);
-}
-
-cmSourceFile* cmMakefile::AddCustomCommandToOutput(
+void cmMakefile::AddCustomCommandToOutput(
   const std::string& output, const std::vector<std::string>& depends,
   const std::string& main_dependency, const cmCustomCommandLines& commandLines,
-  const char* comment, const char* workingDir, bool replace,
-  bool escapeOldStyle, bool uses_terminal, bool command_expand_lists,
-  const std::string& depfile, const std::string& job_pool)
+  const char* comment, const char* workingDir,
+  const CommandSourceCallback& callback, bool replace, bool escapeOldStyle,
+  bool uses_terminal, bool command_expand_lists, const std::string& depfile,
+  const std::string& job_pool)
 {
-  std::vector<std::string> outputs;
-  outputs.push_back(output);
   std::vector<std::string> no_byproducts;
   cmImplicitDependsList no_implicit_depends;
-  return this->AddCustomCommandToOutput(
-    outputs, no_byproducts, depends, main_dependency, no_implicit_depends,
-    commandLines, comment, workingDir, replace, escapeOldStyle, uses_terminal,
-    command_expand_lists, depfile, job_pool);
+  this->AddCustomCommandToOutput(
+    { output }, no_byproducts, depends, main_dependency, no_implicit_depends,
+    commandLines, comment, workingDir, callback, replace, escapeOldStyle,
+    uses_terminal, command_expand_lists, depfile, job_pool);
 }
 
-cmSourceFile* cmMakefile::AddCustomCommandToOutput(
+void cmMakefile::AddCustomCommandToOutput(
   const std::vector<std::string>& outputs,
   const std::vector<std::string>& byproducts,
   const std::vector<std::string>& depends, const std::string& main_dependency,
   const cmImplicitDependsList& implicit_depends,
   const cmCustomCommandLines& commandLines, const char* comment,
-  const char* workingDir, bool replace, bool escapeOldStyle,
-  bool uses_terminal, bool command_expand_lists, const std::string& depfile,
-  const std::string& job_pool)
+  const char* workingDir, const CommandSourceCallback& callback, bool replace,
+  bool escapeOldStyle, bool uses_terminal, bool command_expand_lists,
+  const std::string& depfile, const std::string& job_pool)
 {
   // Make sure there is at least one output.
   if (outputs.empty()) {
     cmSystemTools::Error("Attempt to add a custom rule with no output!");
-    return nullptr;
+    return;
   }
 
   // Validate custom commands.
   if (!this->ValidateCustomCommand(commandLines)) {
-    return nullptr;
+    return;
   }
 
   // Always create the output sources and mark them generated.
-  this->CreateGeneratedSources(outputs);
-  this->CreateGeneratedSources(byproducts);
-
-  return this->CommitCustomCommandToOutput(
-    outputs, byproducts, depends, main_dependency, implicit_depends,
-    commandLines, comment, workingDir, replace, escapeOldStyle, uses_terminal,
-    command_expand_lists, depfile, job_pool);
-}
-
-cmSourceFile* cmMakefile::CommitCustomCommandToOutput(
-  const std::vector<std::string>& outputs,
-  const std::vector<std::string>& byproducts,
-  const std::vector<std::string>& depends, const std::string& main_dependency,
-  const cmImplicitDependsList& implicit_depends,
-  const cmCustomCommandLines& commandLines, const char* comment,
-  const char* workingDir, bool replace, bool escapeOldStyle,
-  bool uses_terminal, bool command_expand_lists, const std::string& depfile,
-  const std::string& job_pool)
-{
-  // Choose a source file on which to store the custom command.
-  cmSourceFile* file = nullptr;
-  if (!commandLines.empty() && !main_dependency.empty()) {
-    // The main dependency was specified.  Use it unless a different
-    // custom command already used it.
-    file = this->GetSource(main_dependency);
-    if (file && file->GetCustomCommand() && !replace) {
-      // The main dependency already has a custom command.
-      if (commandLines == file->GetCustomCommand()->GetCommandLines()) {
-        // The existing custom command is identical.  Silently ignore
-        // the duplicate.
-        return file;
-      }
-      // The existing custom command is different.  We need to
-      // generate a rule file for this new command.
-      file = nullptr;
-    } else if (!file) {
-      file = this->CreateSource(main_dependency);
-    }
-  }
-
-  // Generate a rule file if the main dependency is not available.
-  if (!file) {
-    cmGlobalGenerator* gg = this->GetGlobalGenerator();
-
-    // Construct a rule file associated with the first output produced.
-    std::string outName = gg->GenerateRuleFile(outputs[0]);
-
-    // Check if the rule file already exists.
-    file = this->GetSource(outName, cmSourceFileLocationKind::Known);
-    if (file && file->GetCustomCommand() && !replace) {
-      // The rule file already exists.
-      if (commandLines != file->GetCustomCommand()->GetCommandLines()) {
-        cmSystemTools::Error("Attempt to add a custom rule to output \"" +
-                             outName + "\" which already has a custom rule.");
-      }
-      return file;
+  this->CreateGeneratedOutputs(outputs);
+  this->CreateGeneratedByproducts(byproducts);
+
+  // Strings could be moved into the callback function with C++14.
+  cm::optional<std::string> commentStr = MakeOptionalString(comment);
+  cm::optional<std::string> workingStr = MakeOptionalString(workingDir);
+
+  // Dispatch command creation to allow generator expressions in outputs.
+  this->AddGeneratorAction([=](cmLocalGenerator& lg,
+                               const cmListFileBacktrace& lfbt) {
+    BacktraceGuard guard(this->Backtrace, lfbt);
+    cmSourceFile* sf = detail::AddCustomCommandToOutput(
+      lg, lfbt, cmCommandOrigin::Project, outputs, byproducts, depends,
+      main_dependency, implicit_depends, commandLines,
+      GetCStrOrNull(commentStr), GetCStrOrNull(workingStr), replace,
+      escapeOldStyle, uses_terminal, command_expand_lists, depfile, job_pool);
+    if (callback && sf) {
+      callback(sf);
     }
-
-    // Create a cmSourceFile for the rule file.
-    if (!file) {
-      file =
-        this->CreateSource(outName, true, cmSourceFileLocationKind::Known);
-    }
-    file->SetProperty("__CMAKE_RULE", "1");
-  }
-
-  // Attach the custom command to the file.
-  if (file) {
-    // Construct a complete list of dependencies.
-    std::vector<std::string> depends2(depends);
-    if (!main_dependency.empty()) {
-      depends2.push_back(main_dependency);
-    }
-
-    std::unique_ptr<cmCustomCommand> cc = cm::make_unique<cmCustomCommand>(
-      outputs, byproducts, depends2, commandLines, this->Backtrace, comment,
-      workingDir);
-    cc->SetEscapeOldStyle(escapeOldStyle);
-    cc->SetEscapeAllowMakeVars(true);
-    cc->SetImplicitDepends(implicit_depends);
-    cc->SetUsesTerminal(uses_terminal);
-    cc->SetCommandExpandLists(command_expand_lists);
-    cc->SetDepfile(depfile);
-    cc->SetJobPool(job_pool);
-    file->SetCustomCommand(std::move(cc));
-
-    this->AddSourceOutputs(file, outputs, byproducts);
-  }
-  return file;
+  });
 }
 
 void cmMakefile::AddCustomCommandOldStyle(
@@ -1144,11 +1086,8 @@ void cmMakefile::AddCustomCommandOldStyle(
   if (sourceFiles.find(source)) {
     // The source looks like a real file.  Use it as the main dependency.
     for (std::string const& output : outputs) {
-      cmSourceFile* sf = this->AddCustomCommandToOutput(
-        output, depends, source, commandLines, comment, nullptr);
-      if (sf) {
-        addRuleFileToTarget(sf);
-      }
+      this->AddCustomCommandToOutput(output, depends, source, commandLines,
+                                     comment, nullptr, addRuleFileToTarget);
     }
   } else {
     std::string no_main_dependency;
@@ -1157,11 +1096,9 @@ void cmMakefile::AddCustomCommandOldStyle(
 
     // The source may not be a real file.  Do not use a main dependency.
     for (std::string const& output : outputs) {
-      cmSourceFile* sf = this->AddCustomCommandToOutput(
-        output, depends2, no_main_dependency, commandLines, comment, nullptr);
-      if (sf) {
-        addRuleFileToTarget(sf);
-      }
+      this->AddCustomCommandToOutput(output, depends2, no_main_dependency,
+                                     commandLines, comment, nullptr,
+                                     addRuleFileToTarget);
     }
   }
 }
@@ -1178,29 +1115,18 @@ bool cmMakefile::AppendCustomCommandToOutput(
 
   // Validate custom commands.
   if (this->ValidateCustomCommand(commandLines)) {
-    // Add command factory to allow generator expressions in output.
-    this->CommitAppendCustomCommandToOutput(output, depends, implicit_depends,
-                                            commandLines);
+    // Dispatch command creation to allow generator expressions in outputs.
+    this->AddGeneratorAction(
+      [=](cmLocalGenerator& lg, const cmListFileBacktrace& lfbt) {
+        BacktraceGuard guard(this->Backtrace, lfbt);
+        detail::AppendCustomCommandToOutput(lg, lfbt, output, depends,
+                                            implicit_depends, commandLines);
+      });
   }
 
   return true;
 }
 
-void cmMakefile::CommitAppendCustomCommandToOutput(
-  const std::string& output, const std::vector<std::string>& depends,
-  const cmImplicitDependsList& implicit_depends,
-  const cmCustomCommandLines& commandLines)
-{
-  // Lookup an existing command.
-  if (cmSourceFile* sf = this->GetSourceFileWithOutput(output)) {
-    if (cmCustomCommand* cc = sf->GetCustomCommand()) {
-      cc->AppendCommands(commandLines);
-      cc->AppendDepends(depends);
-      cc->AppendImplicitDepends(implicit_depends);
-    }
-  }
-}
-
 cmUtilityOutput cmMakefile::GetUtilityOutput(cmTarget* target)
 {
   std::string force = cmStrCat(this->GetCurrentBinaryDirectory(),
@@ -1223,15 +1149,14 @@ cmUtilityOutput cmMakefile::GetUtilityOutput(cmTarget* target)
 }
 
 cmTarget* cmMakefile::AddUtilityCommand(
-  const std::string& utilityName, cmCommandOrigin origin, bool excludeFromAll,
-  const char* workingDirectory, const std::vector<std::string>& byproducts,
+  const std::string& utilityName, bool excludeFromAll, const char* workingDir,
+  const std::vector<std::string>& byproducts,
   const std::vector<std::string>& depends,
   const cmCustomCommandLines& commandLines, bool escapeOldStyle,
   const char* comment, bool uses_terminal, bool command_expand_lists,
   const std::string& job_pool)
 {
-  cmTarget* target =
-    this->AddNewUtilityTarget(utilityName, origin, excludeFromAll);
+  cmTarget* target = this->AddNewUtilityTarget(utilityName, excludeFromAll);
 
   // Validate custom commands.
   if ((commandLines.empty() && depends.empty()) ||
@@ -1244,45 +1169,26 @@ cmTarget* cmMakefile::AddUtilityCommand(
   this->GetOrCreateGeneratedSource(force.Name);
 
   // Always create the byproduct sources and mark them generated.
-  this->CreateGeneratedSources(byproducts);
-
-  if (!comment) {
-    // Use an empty comment to avoid generation of default comment.
-    comment = "";
-  }
-
-  this->CommitUtilityCommand(target, force, workingDirectory, byproducts,
-                             depends, commandLines, escapeOldStyle, comment,
-                             uses_terminal, command_expand_lists, job_pool);
+  this->CreateGeneratedByproducts(byproducts);
+
+  // Strings could be moved into the callback function with C++14.
+  cm::optional<std::string> commentStr = MakeOptionalString(comment);
+  cm::optional<std::string> workingStr = MakeOptionalString(workingDir);
+
+  // Dispatch command creation to allow generator expressions in outputs.
+  this->AddGeneratorAction(
+    [=](cmLocalGenerator& lg, const cmListFileBacktrace& lfbt) {
+      BacktraceGuard guard(this->Backtrace, lfbt);
+      detail::AddUtilityCommand(lg, lfbt, cmCommandOrigin::Project, target,
+                                force, GetCStrOrNull(workingStr), byproducts,
+                                depends, commandLines, escapeOldStyle,
+                                GetCStrOrNull(commentStr), uses_terminal,
+                                command_expand_lists, job_pool);
+    });
 
   return target;
 }
 
-void cmMakefile::CommitUtilityCommand(
-  cmTarget* target, const cmUtilityOutput& force, const char* workingDirectory,
-  const std::vector<std::string>& byproducts,
-  const std::vector<std::string>& depends,
-  const cmCustomCommandLines& commandLines, bool escapeOldStyle,
-  const char* comment, bool uses_terminal, bool command_expand_lists,
-  const std::string& job_pool)
-{
-  std::vector<std::string> forced;
-  forced.push_back(force.Name);
-  std::string no_main_dependency;
-  cmImplicitDependsList no_implicit_depends;
-  bool no_replace = false;
-  cmSourceFile* sf = this->AddCustomCommandToOutput(
-    forced, byproducts, depends, no_main_dependency, no_implicit_depends,
-    commandLines, comment, workingDirectory, no_replace, escapeOldStyle,
-    uses_terminal, command_expand_lists, /*depfile=*/"", job_pool);
-  if (!force.NameCMP0049.empty()) {
-    target->AddSource(force.NameCMP0049);
-  }
-  if (sf) {
-    this->AddTargetByproducts(target, byproducts);
-  }
-}
-
 static void s_AddDefineFlag(std::string const& flag, std::string& dflags)
 {
   // remove any \n\r
@@ -2095,11 +2001,9 @@ cmTarget* cmMakefile::AddNewTarget(cmStateEnums::TargetType type,
 }
 
 cmTarget* cmMakefile::AddNewUtilityTarget(const std::string& utilityName,
-                                          cmCommandOrigin origin,
                                           bool excludeFromAll)
 {
   cmTarget* target = this->AddNewTarget(cmStateEnums::UTILITY, utilityName);
-  target->SetIsGeneratorProvided(origin == cmCommandOrigin::Generator);
   if (excludeFromAll) {
     target->SetProperty("EXCLUDE_FROM_ALL", "TRUE");
   }
@@ -2217,8 +2121,8 @@ cmSourceFile* cmMakefile::GetSourceFileWithOutput(
       (!o->second.Sources.SourceIsByproduct ||
        kind == cmSourceOutputKind::OutputOrByproduct)) {
     // Source file could also be null pointer for example if we found the
-    // byproduct of a utility target or a PRE_BUILD, PRE_LINK, or POST_BUILD
-    // command of a target.
+    // byproduct of a utility target, a PRE_BUILD, PRE_LINK, or POST_BUILD
+    // command of a target, or a not yet created custom command.
     return o->second.Sources.Source;
   }
   return nullptr;
@@ -2226,12 +2130,20 @@ cmSourceFile* cmMakefile::GetSourceFileWithOutput(
 
 bool cmMakefile::MightHaveCustomCommand(const std::string& name) const
 {
-  // This will have to be changed for delaying custom command creation, because
-  // GetSourceFileWithOutput requires the command to be already created.
-  if (cmSourceFile* sf = this->GetSourceFileWithOutput(name)) {
-    if (sf->GetCustomCommand()) {
-      return true;
-    }
+  if (this->DelayedOutputFilesHaveGenex ||
+      cmGeneratorExpression::Find(name) != std::string::npos) {
+    // Could be more restrictive, but for now we assume that there could always
+    // be a match when generator expressions are involved.
+    return true;
+  }
+  // Also see LinearGetSourceFileWithOutput.
+  if (!cmSystemTools::FileIsFullPath(name)) {
+    return AnyOutputMatches(name, this->DelayedOutputFiles);
+  }
+  // Otherwise we use an efficient lookup map.
+  auto o = this->OutputToSource.find(name);
+  if (o != this->OutputToSource.end()) {
+    return o->second.SourceMightBeOutput;
   }
   return false;
 }
@@ -2284,6 +2196,7 @@ void cmMakefile::UpdateOutputToSourceMap(std::string const& output,
   SourceEntry entry;
   entry.Sources.Source = source;
   entry.Sources.SourceIsByproduct = byproduct;
+  entry.SourceMightBeOutput = !byproduct;
 
   auto pr = this->OutputToSource.emplace(output, entry);
   if (!pr.second) {
@@ -2293,6 +2206,7 @@ void cmMakefile::UpdateOutputToSourceMap(std::string const& output,
         (current.Sources.SourceIsByproduct && !byproduct)) {
       current.Sources.Source = source;
       current.Sources.SourceIsByproduct = false;
+      current.SourceMightBeOutput = true;
     } else {
       // Multiple custom commands produce the same output but may
       // be attached to a different source file (MAIN_DEPENDENCY).
@@ -3559,11 +3473,41 @@ cmSourceFile* cmMakefile::GetOrCreateGeneratedSource(
   return sf;
 }
 
-void cmMakefile::CreateGeneratedSources(
+void cmMakefile::CreateGeneratedOutputs(
   const std::vector<std::string>& outputs)
 {
-  for (std::string const& output : outputs) {
-    this->GetOrCreateGeneratedSource(output);
+  for (std::string const& o : outputs) {
+    if (cmGeneratorExpression::Find(o) == std::string::npos) {
+      this->GetOrCreateGeneratedSource(o);
+      this->AddDelayedOutput(o);
+    } else {
+      this->DelayedOutputFilesHaveGenex = true;
+    }
+  }
+}
+
+void cmMakefile::CreateGeneratedByproducts(
+  const std::vector<std::string>& byproducts)
+{
+  for (std::string const& o : byproducts) {
+    if (cmGeneratorExpression::Find(o) == std::string::npos) {
+      this->GetOrCreateGeneratedSource(o);
+    }
+  }
+}
+
+void cmMakefile::AddDelayedOutput(std::string const& output)
+{
+  // Note that this vector might contain the output names in a different order
+  // than in source file iteration order.
+  this->DelayedOutputFiles.push_back(output);
+
+  SourceEntry entry;
+  entry.SourceMightBeOutput = true;
+
+  auto pr = this->OutputToSource.emplace(output, entry);
+  if (!pr.second) {
+    pr.first->second.SourceMightBeOutput = true;
   }
 }