New option allowing processing using multiple threads

Introduces new option NUM_PROC_THREADS.
It specifies the number threads doxygen is allowed to use during
processing. When set to 0 doxygen will based this on the number of cores
available in the system. You can set it explicitly to a value larger than 0
to get more control over the balance between CPU load and processing speed.

At this moment only the input processing can be done using multiple threads.
I plan to extend this with more parallel processing in the future.

Since this is still an experimental feature the default is set to 1,
which efficively disables parallel processing. Please report any issues you
encounter that appear when changing the default.

Note that generating dot graphs in parallel is still controlled separately
by the DOT_NUM_THREADS setting.

1 files changed, 16 insertions, 17 deletions

diff --git a/src/doxygen.cpp b/src/doxygen.cpp
index 4bbabb2..0ee908a 100644
--- a/src/doxygen.cpp
+++ b/src/doxygen.cpp
@@ -9120,10 +9120,8 @@ static std::shared_ptr<Entry> parseFile(OutlineParserInterface &parser,
   return fileRoot;
 }
 
-#if MULTITHREADED_INPUT
-
 //! parse the list of input files
-static void parseFiles(const std::shared_ptr<Entry> &root)
+static void parseFilesMultiThreading(const std::shared_ptr<Entry> &root)
 {
 #if USE_LIBCLANG
   if (Doxygen::clangAssistedParsing)
@@ -9139,7 +9137,11 @@ static void parseFiles(const std::shared_ptr<Entry> &root)
 
     std::mutex processedFilesLock;
     // process source files (and their include dependencies)
-    std::size_t numThreads = std::thread::hardware_concurrency();
+    std::size_t numThreads = static_cast<std::size_t>(Config_getInt(NUM_PROC_THREADS));
+    if (numThreads==0)
+    {
+      numThreads = std::thread::hardware_concurrency();
+    }
     msg("Processing input using %lu threads.\n",numThreads);
     ThreadPool threadPool(numThreads);
     std::vector< std::future< std::vector< std::shared_ptr<Entry> > > > results;
@@ -9243,9 +9245,8 @@ static void parseFiles(const std::shared_ptr<Entry> &root)
       auto processFile = [s]() {
         bool ambig;
         FileDef *fd=findFileDef(Doxygen::inputNameLinkedMap,s.c_str(),ambig);
-        auto clangParser = ClangParser::instance()->createTUParser(fd);
         auto parser = getParserForFile(s.c_str());
-        auto fileRoot = parseFile(*parser.get(),fd,s.c_str(),clangParser.get(),true);
+        auto fileRoot = parseFile(*parser.get(),fd,s.c_str(),nullptr,true);
         return fileRoot;
       };
       // dispatch the work and collect the future results
@@ -9256,14 +9257,11 @@ static void parseFiles(const std::shared_ptr<Entry> &root)
     {
       root->moveToSubEntryAndKeep(f.get());
     }
-#warning "Multi-threaded input enabled. This is a highly experimental feature. Only use for doxygen development."
   }
 }
 
-#else // !MULTITHREADED_INPUT
-
 //! parse the list of input files
-static void parseFiles(const std::shared_ptr<Entry> &root)
+static void parseFilesSingleThreading(const std::shared_ptr<Entry> &root)
 {
 #if USE_LIBCLANG
   if (Doxygen::clangAssistedParsing)
@@ -9341,8 +9339,6 @@ static void parseFiles(const std::shared_ptr<Entry> &root)
   }
 }
 
-#endif
-
 // resolves a path that may include symlinks, if a recursive symlink is
 // found an empty string is returned.
 static QCString resolveSymlink(QCString path)
@@ -9411,9 +9407,7 @@ static QCString resolveSymlink(QCString path)
   return QDir::cleanDirPath(result).data();
 }
 
-#if MULTITHREADED_INPUT
 static std::mutex g_pathsVisitedMutex;
-#endif
 static StringUnorderedSet g_pathsVisited(1009);
 
 //----------------------------------------------------------------------------
@@ -9445,9 +9439,7 @@ static int readDir(QFileInfo *fi,
     dirName = resolveSymlink(dirName.data());
     if (dirName.isEmpty()) return 0;            // recursive symlink
 
-#if MULTITHREADED_INPUT
     std::lock_guard<std::mutex> lock(g_pathsVisitedMutex);
-#endif
     if (g_pathsVisited.find(dirName.str())!=g_pathsVisited.end()) return 0; // already visited path
     g_pathsVisited.insert(dirName.str());
   }
@@ -11041,7 +11033,14 @@ void parseInput()
   addSTLSupport(root);
 
   g_s.begin("Parsing files\n");
-  parseFiles(root);
+  if (Config_getInt(NUM_PROC_THREADS)==1)
+  {
+    parseFilesSingleThreading(root);
+  }
+  else
+  {
+    parseFilesMultiThreading(root);
+  }
   g_s.end();
 
   /**************************************************************************