ninja: Add experimental infrastructure for C++20 module dependency scanning

Optionally enable this infrastructure through an undocumented
`CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP` variable.  Currently this is
experimental and intended for use by compiler writers to implement their
scanning tools.  Warn as such when the feature is activated.  Later when
compilers provide the needed scanning tools we can enable this variable
from our corresponding compiler information modules.  It is never meant
to be set by project code.

When enabled, generate a build graph similar to what we use for Fortran
module dependencies.  There are some differences needed because we can
scan dependencies without explicit preprocessing, and can directly
compile the original source afterward.

Co-Author: Ben Boeckel <ben.boeckel@kitware.com>

7 files changed, 150 insertions, 22 deletions

diff --git a/Help/dev/experimental.rst b/Help/dev/experimental.rst
index 4d2b076..4cf1c62 100644
--- a/Help/dev/experimental.rst
+++ b/Help/dev/experimental.rst
@@ -7,4 +7,44 @@ See documentation on `CMake Development`_ for more information.
 
 .. _`CMake Development`: README.rst
 
-No experimental features are under development in this version of CMake.
+C++20 Module Dependencies
+=========================
+
+The Ninja generator has experimental infrastructure supporting C++20 module
+dependency scanning.  This is similar to the Fortran modules support, but
+relies on external tools to scan C++20 translation units for module
+dependencies.  The approach is described by Kitware's `D1483r1`_ paper.
+
+The ``CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP`` variable can be set to ``1``
+in order to activate this undocumented experimental infrastructure.  This
+is **intended to make the functionality available to compiler writers** so
+they can use it to develop and test their dependency scanning tool.
+The ``CMAKE_EXPERIMENTAL_CXX_SCANDEP_SOURCE`` variable must also be set
+to tell CMake how to invoke the C++20 module dependency scanning tool.
+
+For example, add code like the following to a test project:
+
+.. code-block:: cmake
+
+  set(CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP 1)
+  string(CONCAT CMAKE_EXPERIMENTAL_CXX_SCANDEP_SOURCE
+    "<CMAKE_CXX_COMPILER> <DEFINES> <INCLUDES> <FLAGS> <SOURCE>"
+    " -MT <DYNDEP_FILE> -MD -MF <DEP_FILE>"
+    " ${flags_to_scan_deps} -fdep-file=<DYNDEP_FILE> -fdep-output=<OBJECT>"
+    )
+
+The tool specified by ``CMAKE_EXPERIMENTAL_CXX_SCANDEP_SOURCE`` is
+expected to process the translation unit, write preprocessor dependencies
+to the file specified by the ``<DEP_FILE>`` placeholder, and write module
+dependencies to the file specified by the ``<DYNDEP_FILE>`` placeholder.
+
+The module dependencies should be written in the format described
+by the `P1689r3`_ paper.
+
+Compiler writers may try out their scanning functionality using
+the `cxx-modules-sandbox`_ test project, modified to set variables
+as above for their compiler.
+
+.. _`D1483r1`: https://mathstuf.fedorapeople.org/fortran-modules/fortran-modules.html
+.. _`P1689r3`: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p1689r3.html
+.. _`cxx-modules-sandbox`: https://github.com/mathstuf/cxx-modules-sandbox
diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx
index b0b0cd6..16cdcd3 100644
--- a/Source/cmGlobalNinjaGenerator.cxx
+++ b/Source/cmGlobalNinjaGenerator.cxx
@@ -555,6 +555,7 @@ void cmGlobalNinjaGenerator::Generate()
   this->TargetAll = this->NinjaOutputPath("all");
   this->CMakeCacheFile = this->NinjaOutputPath("CMakeCache.txt");
   this->DisableCleandead = false;
+  this->DiagnosedCxxModuleSupport = false;
 
   this->PolicyCMP0058 =
     this->LocalGenerators[0]->GetMakefile()->GetPolicyStatus(
@@ -755,6 +756,37 @@ bool cmGlobalNinjaGenerator::CheckLanguages(
   return true;
 }
 
+bool cmGlobalNinjaGenerator::CheckCxxModuleSupport()
+{
+  bool const diagnose = !this->DiagnosedCxxModuleSupport &&
+    !this->CMakeInstance->GetIsInTryCompile();
+  if (diagnose) {
+    this->DiagnosedCxxModuleSupport = true;
+    this->GetCMakeInstance()->IssueMessage(
+      MessageType::AUTHOR_WARNING,
+      "C++20 modules support via CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP "
+      "is experimental.  It is meant only for compiler developers to try.");
+  }
+  if (this->NinjaSupportsDyndeps) {
+    return true;
+  }
+  if (diagnose) {
+    std::ostringstream e;
+    /* clang-format off */
+    e <<
+      "The Ninja generator does not support C++20 modules "
+      "using Ninja version \n"
+      "  " << this->NinjaVersion << "\n"
+      "due to lack of required features.  "
+      "Ninja " << RequiredNinjaVersionForDyndeps() << " or higher is required."
+      ;
+    /* clang-format on */
+    this->GetCMakeInstance()->IssueMessage(MessageType::FATAL_ERROR, e.str());
+    cmSystemTools::SetFatalErrorOccured();
+  }
+  return false;
+}
+
 bool cmGlobalNinjaGenerator::CheckFortran(cmMakefile* mf) const
 {
   if (this->NinjaSupportsDyndeps) {
diff --git a/Source/cmGlobalNinjaGenerator.h b/Source/cmGlobalNinjaGenerator.h
index 5e9defe..b21586b 100644
--- a/Source/cmGlobalNinjaGenerator.h
+++ b/Source/cmGlobalNinjaGenerator.h
@@ -445,6 +445,8 @@ public:
 
   bool IsSingleConfigUtility(cmGeneratorTarget const* target) const;
 
+  bool CheckCxxModuleSupport();
+
 protected:
   void Generate() override;
 
@@ -565,6 +567,8 @@ private:
   bool NinjaSupportsMultipleOutputs = false;
   bool NinjaSupportsMetadataOnRegeneration = false;
 
+  bool DiagnosedCxxModuleSupport = false;
+
 private:
   void InitOutputPathPrefix();
 
diff --git a/Source/cmNinjaTargetGenerator.cxx b/Source/cmNinjaTargetGenerator.cxx
index d3f4a00..124e885 100644
--- a/Source/cmNinjaTargetGenerator.cxx
+++ b/Source/cmNinjaTargetGenerator.cxx
@@ -35,6 +35,7 @@
 #include "cmRange.h"
 #include "cmRulePlaceholderExpander.h"
 #include "cmSourceFile.h"
+#include "cmStandardLevelResolver.h"
 #include "cmState.h"
 #include "cmStateTypes.h"
 #include "cmStringAlgorithms.h"
@@ -146,9 +147,26 @@ std::string cmNinjaTargetGenerator::LanguageDyndepRule(
     '_', config);
 }
 
-bool cmNinjaTargetGenerator::NeedDyndep(std::string const& lang) const
+bool cmNinjaTargetGenerator::NeedCxxModuleSupport(
+  std::string const& lang, std::string const& config) const
 {
-  return lang == "Fortran";
+  if (lang != "CXX") {
+    return false;
+  }
+  if (!this->Makefile->IsOn("CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP")) {
+    return false;
+  }
+  cmGeneratorTarget const* tgt = this->GetGeneratorTarget();
+  cmStandardLevelResolver standardResolver(this->Makefile);
+  bool const uses_cxx20 =
+    standardResolver.HaveStandardAvailable(tgt, "CXX", config, "cxx_std_20");
+  return uses_cxx20 && this->GetGlobalGenerator()->CheckCxxModuleSupport();
+}
+
+bool cmNinjaTargetGenerator::NeedDyndep(std::string const& lang,
+                                        std::string const& config) const
+{
+  return lang == "Fortran" || this->NeedCxxModuleSupport(lang, config);
 }
 
 std::string cmNinjaTargetGenerator::OrderDependsTargetForTarget(
@@ -531,6 +549,7 @@ cmNinjaRule GetScanRule(
   scanVars.Language = vars.Language;
   scanVars.Object = "$OBJ_FILE";
   scanVars.PreprocessedSource = "$out";
+  scanVars.DynDepFile = "$DYNDEP_INTERMEDIATE_FILE";
   scanVars.DependencyFile = rule.DepFile.c_str();
   scanVars.DependencyTarget = "$out";
 
@@ -585,7 +604,7 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang,
   cmMakefile* mf = this->GetMakefile();
 
   // For some cases we scan to dynamically discover dependencies.
-  bool const needDyndep = this->NeedDyndep(lang);
+  bool const needDyndep = this->NeedDyndep(lang, config);
   bool const compilationPreprocesses = !this->NeedExplicitPreprocessing(lang);
 
   std::string flags = "$FLAGS";
@@ -623,16 +642,26 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang,
     // Rule to scan dependencies of sources that need preprocessing.
     {
       std::vector<std::string> scanCommands;
-      std::string const& scanRuleName =
-        this->LanguagePreprocessAndScanRule(lang, config);
-      std::string const& ppCommmand = mf->GetRequiredDefinition(
-        cmStrCat("CMAKE_", lang, "_PREPROCESS_SOURCE"));
-      cmExpandList(ppCommmand, scanCommands);
-      for (std::string& i : scanCommands) {
-        i = cmStrCat(launcher, i);
+      std::string scanRuleName;
+      if (compilationPreprocesses) {
+        scanRuleName = this->LanguageScanRule(lang, config);
+        std::string const& scanCommand = mf->GetRequiredDefinition(
+          cmStrCat("CMAKE_EXPERIMENTAL_", lang, "_SCANDEP_SOURCE"));
+        cmExpandList(scanCommand, scanCommands);
+        for (std::string& i : scanCommands) {
+          i = cmStrCat(launcher, i);
+        }
+      } else {
+        scanRuleName = this->LanguagePreprocessAndScanRule(lang, config);
+        std::string const& ppCommmand = mf->GetRequiredDefinition(
+          cmStrCat("CMAKE_", lang, "_PREPROCESS_SOURCE"));
+        cmExpandList(ppCommmand, scanCommands);
+        for (std::string& i : scanCommands) {
+          i = cmStrCat(launcher, i);
+        }
+        scanCommands.emplace_back(GetScanCommand(cmakeCmd, tdi, lang, "$out",
+                                                 "$DYNDEP_INTERMEDIATE_FILE"));
       }
-      scanCommands.emplace_back(GetScanCommand(cmakeCmd, tdi, lang, "$out",
-                                               "$DYNDEP_INTERMEDIATE_FILE"));
 
       auto scanRule = GetScanRule(
         scanRuleName, vars, responseFlag, flags, rulePlaceholderExpander.get(),
@@ -640,12 +669,18 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang,
 
       scanRule.Comment =
         cmStrCat("Rule for generating ", lang, " dependencies.");
-      scanRule.Description = cmStrCat("Building ", lang, " preprocessed $out");
+      if (compilationPreprocesses) {
+        scanRule.Description =
+          cmStrCat("Scanning $in for ", lang, " dependencies");
+      } else {
+        scanRule.Description =
+          cmStrCat("Building ", lang, " preprocessed $out");
+      }
 
       this->GetGlobalGenerator()->AddRule(scanRule);
     }
 
-    {
+    if (!compilationPreprocesses) {
       // Compilation will not preprocess, so it does not need the defines
       // unless the compiler wants them for some other purpose.
       if (!this->CompileWithDefines(lang)) {
@@ -1258,7 +1293,7 @@ void cmNinjaTargetGenerator::WriteObjectBuildStatement(
   }
 
   // For some cases we scan to dynamically discover dependencies.
-  bool const needDyndep = this->NeedDyndep(language);
+  bool const needDyndep = this->NeedDyndep(language, config);
   bool const compilationPreprocesses =
     !this->NeedExplicitPreprocessing(language);
 
@@ -1440,17 +1475,26 @@ void cmNinjaTargetGenerator::WriteTargetDependInfo(std::string const& lang,
   tdi["compiler-id"] = this->Makefile->GetSafeDefinition(
     cmStrCat("CMAKE_", lang, "_COMPILER_ID"));
 
+  std::string mod_dir;
   if (lang == "Fortran") {
-    std::string mod_dir = this->GeneratorTarget->GetFortranModuleDirectory(
+    mod_dir = this->GeneratorTarget->GetFortranModuleDirectory(
       this->Makefile->GetHomeOutputDirectory());
-    if (mod_dir.empty()) {
-      mod_dir = this->Makefile->GetCurrentBinaryDirectory();
-    }
-    tdi["module-dir"] = mod_dir;
+  } else if (lang == "CXX") {
+    mod_dir =
+      cmSystemTools::CollapseFullPath(this->GeneratorTarget->ObjectDirectory);
+  }
+  if (mod_dir.empty()) {
+    mod_dir = this->Makefile->GetCurrentBinaryDirectory();
+  }
+  tdi["module-dir"] = mod_dir;
+
+  if (lang == "Fortran") {
     tdi["submodule-sep"] =
       this->Makefile->GetSafeDefinition("CMAKE_Fortran_SUBMODULE_SEP");
     tdi["submodule-ext"] =
       this->Makefile->GetSafeDefinition("CMAKE_Fortran_SUBMODULE_EXT");
+  } else if (lang == "CXX") {
+    // No extra information necessary.
   }
 
   tdi["dir-cur-bld"] = this->Makefile->GetCurrentBinaryDirectory();
diff --git a/Source/cmNinjaTargetGenerator.h b/Source/cmNinjaTargetGenerator.h
index 83a4342..79dc622 100644
--- a/Source/cmNinjaTargetGenerator.h
+++ b/Source/cmNinjaTargetGenerator.h
@@ -71,9 +71,11 @@ protected:
                                const std::string& config) const;
   std::string LanguageDyndepRule(std::string const& lang,
                                  const std::string& config) const;
-  bool NeedDyndep(std::string const& lang) const;
+  bool NeedDyndep(std::string const& lang, std::string const& config) const;
   bool NeedExplicitPreprocessing(std::string const& lang) const;
   bool CompileWithDefines(std::string const& lang) const;
+  bool NeedCxxModuleSupport(std::string const& lang,
+                            std::string const& config) const;
 
   std::string OrderDependsTargetForTarget(const std::string& config);
 
diff --git a/Source/cmRulePlaceholderExpander.cxx b/Source/cmRulePlaceholderExpander.cxx
index 5363fef..d00bbdf 100644
--- a/Source/cmRulePlaceholderExpander.cxx
+++ b/Source/cmRulePlaceholderExpander.cxx
@@ -44,6 +44,11 @@ std::string cmRulePlaceholderExpander::ExpandRuleVariable(
       return replaceValues.Source;
     }
   }
+  if (replaceValues.DynDepFile) {
+    if (variable == "DYNDEP_FILE") {
+      return replaceValues.DynDepFile;
+    }
+  }
   if (replaceValues.PreprocessedSource) {
     if (variable == "PREPROCESSED_SOURCE") {
       return replaceValues.PreprocessedSource;
diff --git a/Source/cmRulePlaceholderExpander.h b/Source/cmRulePlaceholderExpander.h
index 710f8a6..f8dc368 100644
--- a/Source/cmRulePlaceholderExpander.h
+++ b/Source/cmRulePlaceholderExpander.h
@@ -41,6 +41,7 @@ public:
     const char* Source = nullptr;
     const char* AssemblySource = nullptr;
     const char* PreprocessedSource = nullptr;
+    const char* DynDepFile = nullptr;
     const char* Output = nullptr;
     const char* Object = nullptr;
     const char* ObjectDir = nullptr;