Autogen: Refactor AUTOMOC and AUTOUIC and add source file parse data caching

New features
------------

CMake's `AUTOMOC` and `AUTOUIC` now cache information extracted when parsing
source files in `CMakeFiles/<ORIGIN>_autogen.dir/ParseCache.txt`.
This leads to faster `<ORIGIN>_autogen` target rebuilds, because source files
will be parsed again only if they're newer than the `ParseCache.txt` file.
The parse cache will be recomputed if it is older than the CMake executable.

`AUTOMOC` and `AUTOUIC` now check if `moc` or `uic` output files are older
than the `moc` or `uic` executable.  If an output file is older than the
compiler, it will be regenerated.  Therefore if a new `moc` or `uic` version
is installed, all output files will be regenerated.

`AUTOMOC` and `AUTOUIC` error and warning messages are more detailed.

Internal changes
----------------

`moc` and `uic` output file names are not computed in the `_autogen`
target anymore but in `cmQtAutoGenInitializer`.  This makes the available at
the configuration stage for improved dependency computations (to be done).

In `AutogenInfo.cmake`, equally sized lists for "source file names",
"source file flags" and "compiler output file names" are passed to the
`_autogen` target.  This replaces the separate file lists for
`AUTOMOC` and `AUTOUIC`.

Files times are read from the file system only once by using `cmFileTime`
instances instead of `cmQtAutoGenerator::FileSystem::FileIsOlderThan` calls.

All calls to not thread safe file system functions are moved to non concurrent
fence jobs (see `cmWorkerPool::JobT::IsFence()`).  This renders the
`cmQtAutoGenerator::FileSystem` wrapper class obsolete and it is removed.

Instead of composing a single large settings string that is fed to the
`cmCryptoHash`, now all setting sub strings are fed one by one to the
`cmCryptoHash` and the finalized result is stored.

The `std::mutex` in `cmQtAutoGenerator::Logger` is tagged `mutable` and most
`cmQtAutoGenerator::Logger` methods become `const`.

Outlook
-------

This patch provides the framework required to

- extract dependencies from `.ui` files in `AUTOUIC`.
  These will help to address issue
  #15420 "AUTOUIC: Track uic external inputs".

- generate adaptive `make` and `ninja` files in the `_autogen` target.
  These will help to address issue
  #16776 "AUTOUIC: Ninja needs two passes to correctly build Qt project".

- generate (possibly empty) `moc` and `uic` files for all headers instead of a
  `mocs_compilation.cpp` file.
  This will help to address issue
  #17277 "AUTOMOC: Provide a option to allow AUTOMOC to compile individual "
         "moc_x.cxx instead of including all in mocs_compilation.cxx"

1 files changed, 324 insertions, 161 deletions

diff --git a/Source/cmQtAutoMocUic.h b/Source/cmQtAutoMocUic.h
index 3902abb..8061c13 100644
--- a/Source/cmQtAutoMocUic.h
+++ b/Source/cmQtAutoMocUic.h
@@ -5,25 +5,28 @@
 
 #include "cmConfigure.h" // IWYU pragma: keep
 
+#include "cmFileTime.h"
 #include "cmQtAutoGen.h"
 #include "cmQtAutoGenerator.h"
 #include "cmWorkerPool.h"
 #include "cmsys/RegularExpression.hxx"
 
-#include <array>
 #include <atomic>
+#include <cstddef>
 #include <map>
 #include <memory> // IWYU pragma: keep
-#include <mutex>
 #include <set>
 #include <string>
+#include <unordered_map>
 #include <unordered_set>
 #include <utility>
 #include <vector>
 
 class cmMakefile;
 
-// @brief AUTOMOC and AUTOUIC generator
+/** \class cmQtAutoMocUic
+ * \brief AUTOMOC and AUTOUIC generator
+ */
 class cmQtAutoMocUic : public cmQtAutoGenerator
 {
 public:
@@ -35,10 +38,10 @@ public:
 
 public:
   // -- Types
-  typedef std::multimap<std::string, std::array<std::string, 2>> IncludesMap;
 
-  /// @brief Search key plus regular expression pair
-  ///
+  /**
+   * Search key plus regular expression pair
+   */
   struct KeyExpT
   {
     KeyExpT() = default;
@@ -59,35 +62,123 @@ public:
     cmsys::RegularExpression Exp;
   };
 
-  /// @brief Common settings
-  ///
-  class BaseSettingsT
+  /**
+   * Include string with sub parts
+   */
+  struct IncludeKeyT
+  {
+    IncludeKeyT(std::string const& key, std::size_t basePrefixLength);
+
+    std::string Key;  // Full include string
+    std::string Dir;  // Include directory
+    std::string Base; // Base part of the include file name
+  };
+
+  /**
+   * Source file parsing cache
+   */
+  class ParseCacheT
+  {
+  public:
+    // -- Types
+    /**
+     * Entry of the file parsing cache
+     */
+    struct FileT
+    {
+      void Clear();
+
+      struct MocT
+      {
+        std::string Macro;
+        struct IncludeT
+        {
+          std::vector<IncludeKeyT> Underscore;
+          std::vector<IncludeKeyT> Dot;
+        } Include;
+        std::vector<std::string> Depends;
+      } Moc;
+
+      struct UicT
+      {
+        std::vector<IncludeKeyT> Include;
+        std::vector<std::string> Depends;
+      } Uic;
+    };
+    typedef std::shared_ptr<FileT> FileHandleT;
+    typedef std::pair<FileHandleT, bool> GetOrInsertT;
+
+  public:
+    ParseCacheT();
+    ~ParseCacheT();
+
+    void Clear();
+
+    bool ReadFromFile(std::string const& fileName);
+    bool WriteToFile(std::string const& fileName);
+
+    //! Might return an invalid handle
+    FileHandleT Get(std::string const& fileName) const;
+    //! Always returns a valid handle
+    GetOrInsertT GetOrInsert(std::string const& fileName);
+
+  private:
+    std::unordered_map<std::string, FileHandleT> Map_;
+  };
+
+  /**
+   * Source file data
+   */
+  class SourceFileT
   {
   public:
-    // -- Volatile methods
-    BaseSettingsT(FileSystem* fileSystem)
-      : MultiConfig(false)
-      , IncludeProjectDirsBefore(false)
-      , QtVersionMajor(4)
-      , NumThreads(1)
-      , FileSys(fileSystem)
+    SourceFileT(std::string fileName)
+      : FileName(std::move(fileName))
     {
     }
 
+  public:
+    std::string FileName;
+    cmFileTime FileTime;
+    ParseCacheT::FileHandleT ParseData;
+    std::string BuildPath;
+    bool Moc = false;
+    bool Uic = false;
+  };
+  typedef std::shared_ptr<SourceFileT> SourceFileHandleT;
+  typedef std::map<std::string, SourceFileHandleT> SourceFileMapT;
+
+  /**
+   * Meta compiler file mapping information
+   */
+  struct MappingT
+  {
+    SourceFileHandleT SourceFile;
+    std::string OutputFile;
+    std::string IncludeString;
+    std::vector<SourceFileHandleT> IncluderFiles;
+  };
+  typedef std::shared_ptr<MappingT> MappingHandleT;
+  typedef std::map<std::string, MappingHandleT> MappingMapT;
+
+  /**
+   * Common settings
+   */
+  class BaseSettingsT
+  {
+  public:
+    // -- Constructors
+    BaseSettingsT();
+    ~BaseSettingsT();
+
     BaseSettingsT(BaseSettingsT const&) = delete;
     BaseSettingsT& operator=(BaseSettingsT const&) = delete;
 
-    // -- Const methods
-    std::string AbsoluteBuildPath(std::string const& relativePath) const;
-    bool FindHeader(std::string& header,
-                    std::string const& testBasePath) const;
-
     // -- Attributes
     // - Config
-    bool MultiConfig;
-    bool IncludeProjectDirsBefore;
-    unsigned int QtVersionMajor;
-    unsigned int NumThreads;
+    bool MultiConfig = false;
+    bool IncludeProjectDirsBefore = false;
+    unsigned int QtVersionMajor = 4;
     // - Directories
     std::string ProjectSourceDir;
     std::string ProjectBinaryDir;
@@ -96,37 +187,50 @@ public:
     std::string AutogenBuildDir;
     std::string AutogenIncludeDir;
     // - Files
+    std::string CMakeExecutable;
+    cmFileTime CMakeExecutableTime;
+    std::string ParseCacheFile;
     std::vector<std::string> HeaderExtensions;
-    // - File system
-    FileSystem* FileSys;
   };
 
-  /// @brief Moc settings
-  ///
+  /**
+   * Shared common variables
+   */
+  class BaseEvalT
+  {
+  public:
+    // -- Parse Cache
+    bool ParseCacheChanged = false;
+    cmFileTime ParseCacheTime;
+    ParseCacheT ParseCache;
+
+    // -- Sources
+    SourceFileMapT Headers;
+    SourceFileMapT Sources;
+  };
+
+  /**
+   * Moc settings
+   */
   class MocSettingsT
   {
   public:
-    MocSettingsT(FileSystem* fileSys)
-      : FileSys(fileSys)
-    {
-    }
+    // -- Constructors
+    MocSettingsT();
+    ~MocSettingsT();
 
     MocSettingsT(MocSettingsT const&) = delete;
     MocSettingsT& operator=(MocSettingsT const&) = delete;
 
     // -- Const methods
     bool skipped(std::string const& fileName) const;
-    std::string FindMacro(std::string const& content) const;
     std::string MacrosString() const;
-    std::string FindIncludedFile(std::string const& sourcePath,
-                                 std::string const& includeString) const;
-    void FindDependencies(std::string const& content,
-                          std::set<std::string>& depends) const;
 
     // -- Attributes
     bool Enabled = false;
     bool SettingsChanged = false;
     bool RelaxedMode = false;
+    cmFileTime ExecutableTime;
     std::string Executable;
     std::string CompFileAbs;
     std::string PredefsFileRel;
@@ -141,16 +245,35 @@ public:
     std::vector<KeyExpT> DependFilters;
     std::vector<KeyExpT> MacroFilters;
     cmsys::RegularExpression RegExpInclude;
-    // - File system
-    FileSystem* FileSys;
   };
 
-  /// @brief Uic settings
-  ///
+  /**
+   * Moc shared variables
+   */
+  class MocEvalT
+  {
+  public:
+    // -- predefines file
+    cmFileTime PredefsTime;
+    // -- Mappings
+    MappingMapT HeaderMappings;
+    MappingMapT SourceMappings;
+    MappingMapT Includes;
+    // -- Discovered files
+    SourceFileMapT HeadersDiscovered;
+    // -- Mocs compilation
+    bool CompUpdated = false;
+    std::vector<std::string> CompFiles;
+  };
+
+  /**
+   * Uic settings
+   */
   class UicSettingsT
   {
   public:
-    UicSettingsT() = default;
+    UicSettingsT();
+    ~UicSettingsT();
 
     UicSettingsT(UicSettingsT const&) = delete;
     UicSettingsT& operator=(UicSettingsT const&) = delete;
@@ -161,6 +284,7 @@ public:
     // -- Attributes
     bool Enabled = false;
     bool SettingsChanged = false;
+    cmFileTime ExecutableTime;
     std::string Executable;
     std::unordered_set<std::string> SkipList;
     std::vector<std::string> TargetOptions;
@@ -169,8 +293,19 @@ public:
     cmsys::RegularExpression RegExpInclude;
   };
 
-  /// @brief Abstract job class for concurrent job processing
-  ///
+  /**
+   * Uic shared variables
+   */
+  class UicEvalT
+  {
+  public:
+    SourceFileMapT UiFiles;
+    MappingMapT Includes;
+  };
+
+  /**
+   * Abstract job class for concurrent job processing
+   */
   class JobT : public cmWorkerPool::JobT
   {
   protected:
@@ -188,10 +323,14 @@ public:
       return static_cast<cmQtAutoMocUic*>(UserData());
     };
 
-    //! Get the file system interface. Only valid during Process() call!
-    FileSystem& FileSys() { return Gen()->FileSys(); }
-    //! Get the logger. Only valid during Process() call!
-    Logger& Log() { return Gen()->Log(); }
+    // -- Accessors. Only valid during Process() call!
+    Logger const& Log() const { return Gen()->Log(); }
+    BaseSettingsT const& BaseConst() const { return Gen()->BaseConst(); }
+    BaseEvalT& BaseEval() const { return Gen()->BaseEval(); }
+    MocSettingsT const& MocConst() const { return Gen()->MocConst(); }
+    MocEvalT& MocEval() const { return Gen()->MocEval(); }
+    UicSettingsT const& UicConst() const { return Gen()->UicConst(); }
+    UicEvalT& UicEval() const { return Gen()->UicEval(); }
 
     // -- Error logging with automatic abort
     void LogError(GenT genType, std::string const& message) const;
@@ -205,11 +344,13 @@ public:
      * @brief Run an external process. Use only during Process() call!
      */
     bool RunProcess(GenT genType, cmWorkerPool::ProcessResultT& result,
-                    std::vector<std::string> const& command);
+                    std::vector<std::string> const& command,
+                    std::string* infoMessage = nullptr);
   };
 
-  /// @brief Fence job utility class
-  ///
+  /**
+   * Fence job utility class
+   */
   class JobFenceT : public JobT
   {
   public:
@@ -220,121 +361,152 @@ public:
     void Process() override{};
   };
 
-  /// @brief Generate moc_predefs.h
-  ///
-  class JobMocPredefsT : public JobT
+  /**
+   * Generate moc_predefs.h
+   */
+  class JobMocPredefsT : public JobFenceT
   {
-  private:
     void Process() override;
+    bool Update(std::string* reason) const;
   };
 
-  /// @brief Parses a source file
-  ///
+  /**
+   * File parse job base class
+   */
   class JobParseT : public JobT
   {
   public:
-    JobParseT(std::string fileName, bool moc, bool uic, bool header = false)
-      : FileName(std::move(fileName))
-      , AutoMoc(moc)
-      , AutoUic(uic)
-      , Header(header)
+    JobParseT(SourceFileHandleT fileHandle)
+      : FileHandle(std::move(fileHandle))
     {
     }
 
-  private:
-    struct MetaT
-    {
-      std::string Content;
-      std::string FileDir;
-      std::string FileBase;
-    };
+  protected:
+    bool ReadFile();
+    void CreateKeys(std::vector<IncludeKeyT>& container,
+                    std::set<std::string> const& source,
+                    std::size_t basePrefixLength);
+    void MocMacro();
+    void MocDependecies();
+    void MocIncludes();
+    void UicIncludes();
+
+  protected:
+    SourceFileHandleT FileHandle;
+    std::string Content;
+  };
 
+  /**
+   * Header file parse job
+   */
+  class JobParseHeaderT : public JobParseT
+  {
+  public:
+    using JobParseT::JobParseT;
     void Process() override;
-    bool ParseMocSource(MetaT const& meta);
-    bool ParseMocHeader(MetaT const& meta);
-    std::string MocStringHeaders(std::string const& fileBase) const;
-    std::string MocFindIncludedHeader(std::string const& includerDir,
-                                      std::string const& includeBase);
-    bool ParseUic(MetaT const& meta);
-    bool ParseUicInclude(MetaT const& meta, std::string&& includeString);
-    std::string UicFindIncludedFile(MetaT const& meta,
-                                    std::string const& includeString);
+  };
 
-  private:
-    std::string FileName;
-    bool AutoMoc = false;
-    bool AutoUic = false;
-    bool Header = false;
+  /**
+   * Source file parse job
+   */
+  class JobParseSourceT : public JobParseT
+  {
+  public:
+    using JobParseT::JobParseT;
+    void Process() override;
   };
 
-  /// @brief Generates additional jobs after all files have been parsed
-  ///
-  class JobPostParseT : public JobFenceT
+  /**
+   * Evaluate parsed files
+   */
+  class JobEvaluateT : public JobFenceT
   {
-  private:
     void Process() override;
+
+    // -- Moc
+    bool MocEvalHeader(SourceFileHandleT source);
+    bool MocEvalSource(SourceFileHandleT const& source);
+    SourceFileHandleT MocFindIncludedHeader(
+      std::string const& includerDir, std::string const& includeBase) const;
+    SourceFileHandleT MocFindHeader(std::string const& basePath) const;
+    std::string MocMessageTestHeaders(std::string const& fileBase) const;
+    bool MocRegisterIncluded(std::string const& includeString,
+                             SourceFileHandleT includerFileHandle,
+                             SourceFileHandleT sourceFileHandle,
+                             bool sourceIsHeader) const;
+    void MocRegisterMapping(MappingHandleT mappingHandle,
+                            bool sourceIsHeader) const;
+
+    // -- Uic
+    bool UicEval(SourceFileMapT const& fileMap);
+    bool UicEvalFile(SourceFileHandleT sourceFileHandle);
+    SourceFileHandleT UicFindIncludedUi(std::string const& sourceFile,
+                                        std::string const& sourceDir,
+                                        IncludeKeyT const& incKey) const;
+    bool UicRegisterMapping(std::string const& includeString,
+                            SourceFileHandleT uiFileHandle,
+                            SourceFileHandleT includerFileHandle);
   };
 
-  /// @brief Generate mocs_compilation.cpp
-  ///
-  class JobMocsCompilationT : public JobFenceT
+  /**
+   * Generates moc/uic jobs
+   */
+  class JobGenerateT : public JobFenceT
   {
-  private:
     void Process() override;
+    // -- Moc
+    bool MocGenerate(MappingHandleT const& mapping, bool compFile) const;
+    bool MocUpdate(MappingT const& mapping, std::string* reason) const;
+    std::pair<std::string, cmFileTime> MocFindDependency(
+      std::string const& sourceDir, std::string const& includeString) const;
+    // -- Uic
+    bool UicGenerate(MappingHandleT const& mapping) const;
+    bool UicUpdate(MappingT const& mapping, std::string* reason) const;
   };
 
-  /// @brief Moc a file job
-  ///
-  class JobMocT : public JobT
+  /**
+   * File compiling base job
+   */
+  class JobCompileT : public JobT
   {
   public:
-    JobMocT(std::string sourceFile, std::string includerFile,
-            std::string includeString)
-      : SourceFile(std::move(sourceFile))
-      , IncluderFile(std::move(includerFile))
-      , IncludeString(std::move(includeString))
+    JobCompileT(MappingHandleT uicMapping, std::unique_ptr<std::string> reason)
+      : Mapping(std::move(uicMapping))
+      , Reason(std::move(reason))
     {
     }
 
-    void FindDependencies(std::string const& content);
+  protected:
+    MappingHandleT Mapping;
+    std::unique_ptr<std::string> Reason;
+  };
 
-  private:
+  /**
+   * moc compiles a file
+   */
+  class JobMocT : public JobCompileT
+  {
+  public:
+    using JobCompileT::JobCompileT;
     void Process() override;
-    bool UpdateRequired();
-    void GenerateMoc();
+  };
 
+  /**
+   * uic compiles a file
+   */
+  class JobUicT : public JobCompileT
+  {
   public:
-    std::string SourceFile;
-    std::string IncluderFile;
-    std::string IncludeString;
-    std::string BuildFile;
-    bool DependsValid = false;
-    std::set<std::string> Depends;
+    using JobCompileT::JobCompileT;
+    void Process() override;
   };
 
-  /// @brief Uic a file job
+  /// @brief Generate mocs_compilation.cpp
   ///
-  class JobUicT : public JobT
+  class JobMocsCompilationT : public JobFenceT
   {
-  public:
-    JobUicT(std::string sourceFile, std::string includerFile,
-            std::string includeString)
-      : SourceFile(std::move(sourceFile))
-      , IncluderFile(std::move(includerFile))
-      , IncludeString(std::move(includeString))
-    {
-    }
-
   private:
     void Process() override;
-    bool UpdateRequired();
-    void GenerateUic();
-
-  public:
-    std::string SourceFile;
-    std::string IncluderFile;
-    std::string IncludeString;
-    std::string BuildFile;
   };
 
   /// @brief The last job
@@ -346,39 +518,37 @@ public:
   };
 
   // -- Const settings interface
-  const BaseSettingsT& Base() const { return this->Base_; }
-  const MocSettingsT& Moc() const { return this->Moc_; }
-  const UicSettingsT& Uic() const { return this->Uic_; }
+  BaseSettingsT const& BaseConst() const { return this->BaseConst_; }
+  BaseEvalT& BaseEval() { return this->BaseEval_; }
+  MocSettingsT const& MocConst() const { return this->MocConst_; }
+  MocEvalT& MocEval() { return this->MocEval_; }
+  UicSettingsT const& UicConst() const { return this->UicConst_; }
+  UicEvalT& UicEval() { return this->UicEval_; }
 
   // -- Parallel job processing interface
   cmWorkerPool& WorkerPool() { return WorkerPool_; }
   void AbortError() { Abort(true); }
   void AbortSuccess() { Abort(false); }
-  bool ParallelJobPushMoc(cmWorkerPool::JobHandleT&& jobHandle);
-  bool ParallelJobPushUic(cmWorkerPool::JobHandleT&& jobHandle);
 
-  // -- Mocs compilation include file updated flag
-  void ParallelMocAutoUpdated() { MocAutoFileUpdated_.store(true); }
-  bool MocAutoFileUpdated() const { return MocAutoFileUpdated_.load(); }
-
-  // -- Mocs compilation file register
-  std::string ParallelMocAutoRegister(std::string const& baseName);
-  bool ParallelMocIncluded(std::string const& sourceFile);
-  std::set<std::string> const& MocAutoFiles() const
-  {
-    return this->MocAutoFiles_;
-  }
+  // -- Utility
+  std::string AbsoluteBuildPath(std::string const& relativePath) const;
+  std::string AbsoluteIncludePath(std::string const& relativePath) const;
+  template <class JOBTYPE>
+  void CreateParseJobs(SourceFileMapT const& sourceMap);
 
 private:
   // -- Utility accessors
-  Logger& Log() { return Logger_; }
-  FileSystem& FileSys() { return FileSys_; }
+  Logger const& Log() const { return Logger_; }
   // -- Abstract processing interface
   bool Init(cmMakefile* makefile) override;
+  void InitJobs();
   bool Process() override;
   // -- Settings file
   void SettingsFileRead();
   bool SettingsFileWrite();
+  // -- Parse cache
+  void ParseCacheRead();
+  bool ParseCacheWrite();
   // -- Thread processing
   void Abort(bool error);
   // -- Generation
@@ -387,25 +557,18 @@ private:
 private:
   // -- Utility
   Logger Logger_;
-  FileSystem FileSys_;
   // -- Settings
-  BaseSettingsT Base_;
-  MocSettingsT Moc_;
-  UicSettingsT Uic_;
-  // -- Moc meta
-  std::mutex MocMetaMutex_;
-  std::set<std::string> MocIncludedFiles_;
-  IncludesMap MocIncludes_;
-  std::set<std::string> MocAutoFiles_;
-  std::atomic<bool> MocAutoFileUpdated_ = ATOMIC_VAR_INIT(false);
-  // -- Uic meta
-  std::mutex UicMetaMutex_;
-  IncludesMap UicIncludes_;
+  BaseSettingsT BaseConst_;
+  BaseEvalT BaseEval_;
+  MocSettingsT MocConst_;
+  MocEvalT MocEval_;
+  UicSettingsT UicConst_;
+  UicEvalT UicEval_;
   // -- Settings file
   std::string SettingsFile_;
   std::string SettingsStringMoc_;
   std::string SettingsStringUic_;
-  // -- Thread pool and job queue
+  // -- Worker thread pool
   std::atomic<bool> JobError_ = ATOMIC_VAR_INIT(false);
   cmWorkerPool WorkerPool_;
 };