New option allowing processing using multiple threads

Introduces new option NUM_PROC_THREADS.
It specifies the number threads doxygen is allowed to use during
processing. When set to 0 doxygen will based this on the number of cores
available in the system. You can set it explicitly to a value larger than 0
to get more control over the balance between CPU load and processing speed.

At this moment only the input processing can be done using multiple threads.
I plan to extend this with more parallel processing in the future.

Since this is still an experimental feature the default is set to 1,
which efficively disables parallel processing. Please report any issues you
encounter that appear when changing the default.

Note that generating dot graphs in parallel is still controlled separately
by the DOT_NUM_THREADS setting.

8 files changed, 31 insertions, 78 deletions

diff --git a/src/clangparser.cpp b/src/clangparser.cpp
index d0ed573..5f1a021 100644
--- a/src/clangparser.cpp
+++ b/src/clangparser.cpp
@@ -20,22 +20,16 @@
 #include "membername.h"
 #include "filename.h"
 #include "tooltip.h"
-#if MULTITHREADED_INPUT
 #include <mutex>
 #endif
-#endif
 
 //--------------------------------------------------------------------------
 
-#if MULTITHREADED_INPUT
 std::mutex g_clangMutex;
-#endif
 
 ClangParser *ClangParser::instance()
 {
-#if MULTITHREADED_INPUT
   std::lock_guard<std::mutex> lock(g_clangMutex);
-#endif
   if (!s_instance) s_instance = new ClangParser;
   return s_instance;
 }
diff --git a/src/commentscan.l b/src/commentscan.l
index a1dd0e1..d695f0a 100644
--- a/src/commentscan.l
+++ b/src/commentscan.l
@@ -409,11 +409,9 @@ struct commentscanYY_state
 };
 
 
-#if MULTITHREADED_INPUT
 static std::mutex g_sectionMutex;
 static std::mutex g_formulaMutex;
 static std::mutex g_citeMutex;
-#endif
 
 //-----------------------------------------------------------------------------
 
@@ -2822,9 +2820,7 @@ static void addXRefItem(yyscan_t yyscanner,
   if (listName==0) return;
   //printf("addXRefItem(%s,%s,%s,%d)\n",listName,itemTitle,listTitle,append);
 
-#if MULTITHREADED_INPUT
   std::unique_lock<std::mutex> lock(g_sectionMutex);
-#endif
 
   RefList *refList = RefListManager::instance().add(listName,listTitle,itemTitle);
   RefItem *item = 0;
@@ -2898,9 +2894,7 @@ static void addXRefItem(yyscan_t yyscanner,
 // not already added. Returns the label of the formula.
 static QCString addFormula(yyscan_t yyscanner)
 {
-#if MULTITHREADED_INPUT
   std::unique_lock<std::mutex> lock(g_formulaMutex);
-#endif
   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
   QCString formLabel;
   QCString fText=yyextra->formulaText.simplifyWhiteSpace();
@@ -2922,9 +2916,7 @@ static SectionType sectionLevelToType(int level)
 
 static void addSection(yyscan_t yyscanner)
 {
-#if MULTITHREADED_INPUT
   std::unique_lock<std::mutex> lock(g_sectionMutex);
-#endif
   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
   SectionManager &sm = SectionManager::instance();
   const SectionInfo *si = sm.find(yyextra->sectionLabel);
@@ -2957,9 +2949,7 @@ static void addSection(yyscan_t yyscanner)
 
 static void addCite(yyscan_t yyscanner)
 {
-#if MULTITHREADED_INPUT
   std::unique_lock<std::mutex> lock(g_citeMutex);
-#endif
   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
   QCString name=yytext;
   if (yytext[0] =='"')
@@ -3127,9 +3117,7 @@ static inline void setOutput(yyscan_t yyscanner,OutputContext ctx)
 
 static void addAnchor(yyscan_t yyscanner,const char *anchor)
 {
-#if MULTITHREADED_INPUT
   std::unique_lock<std::mutex> lock(g_sectionMutex);
-#endif
   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
   SectionManager &sm = SectionManager::instance();
   const SectionInfo *si = sm.find(anchor);
diff --git a/src/config.xml b/src/config.xml
index 81610c0..29f4068 100644
--- a/src/config.xml
+++ b/src/config.xml
@@ -812,6 +812,21 @@ Go to the <a href="commands.html">next</a> section or return to the
 ]]>
       </docs>
     </option>
+    <option type='int' id='NUM_PROC_THREADS' defval='1' minval='0' maxval='32'>
+      <docs>
+<![CDATA[
+ The \c NUM_PROC_THREADS specifies the number threads doxygen is allowed to use during
+ processing. When set to \c 0 doxygen will based this on the number of cores
+ available in the system. You can set it explicitly to a value larger than 0
+ to get more control over the balance between CPU load and processing speed.
+ At this moment only the input processing can be done using multiple threads.
+ Since this is still an experimental feature the default is set to 1,
+ which efficively disables parallel processing. Please report any issues you
+ encounter.
+ Generating dot graphs in parallel is controlled by the \c DOT_NUM_THREADS setting.
+]]>
+      </docs>
+    </option>
   </group>
   <group name='Build' docs='Build related configuration options'>
     <option type='bool' id='EXTRACT_ALL' defval='0'>
diff --git a/src/doxygen.cpp b/src/doxygen.cpp
index 4bbabb2..0ee908a 100644
--- a/src/doxygen.cpp
+++ b/src/doxygen.cpp
@@ -9120,10 +9120,8 @@ static std::shared_ptr<Entry> parseFile(OutlineParserInterface &parser,
   return fileRoot;
 }
 
-#if MULTITHREADED_INPUT
-
 //! parse the list of input files
-static void parseFiles(const std::shared_ptr<Entry> &root)
+static void parseFilesMultiThreading(const std::shared_ptr<Entry> &root)
 {
 #if USE_LIBCLANG
   if (Doxygen::clangAssistedParsing)
@@ -9139,7 +9137,11 @@ static void parseFiles(const std::shared_ptr<Entry> &root)
 
     std::mutex processedFilesLock;
     // process source files (and their include dependencies)
-    std::size_t numThreads = std::thread::hardware_concurrency();
+    std::size_t numThreads = static_cast<std::size_t>(Config_getInt(NUM_PROC_THREADS));
+    if (numThreads==0)
+    {
+      numThreads = std::thread::hardware_concurrency();
+    }
     msg("Processing input using %lu threads.\n",numThreads);
     ThreadPool threadPool(numThreads);
     std::vector< std::future< std::vector< std::shared_ptr<Entry> > > > results;
@@ -9243,9 +9245,8 @@ static void parseFiles(const std::shared_ptr<Entry> &root)
       auto processFile = [s]() {
         bool ambig;
         FileDef *fd=findFileDef(Doxygen::inputNameLinkedMap,s.c_str(),ambig);
-        auto clangParser = ClangParser::instance()->createTUParser(fd);
         auto parser = getParserForFile(s.c_str());
-        auto fileRoot = parseFile(*parser.get(),fd,s.c_str(),clangParser.get(),true);
+        auto fileRoot = parseFile(*parser.get(),fd,s.c_str(),nullptr,true);
         return fileRoot;
       };
       // dispatch the work and collect the future results
@@ -9256,14 +9257,11 @@ static void parseFiles(const std::shared_ptr<Entry> &root)
     {
       root->moveToSubEntryAndKeep(f.get());
     }
-#warning "Multi-threaded input enabled. This is a highly experimental feature. Only use for doxygen development."
   }
 }
 
-#else // !MULTITHREADED_INPUT
-
 //! parse the list of input files
-static void parseFiles(const std::shared_ptr<Entry> &root)
+static void parseFilesSingleThreading(const std::shared_ptr<Entry> &root)
 {
 #if USE_LIBCLANG
   if (Doxygen::clangAssistedParsing)
@@ -9341,8 +9339,6 @@ static void parseFiles(const std::shared_ptr<Entry> &root)
   }
 }
 
-#endif
-
 // resolves a path that may include symlinks, if a recursive symlink is
 // found an empty string is returned.
 static QCString resolveSymlink(QCString path)
@@ -9411,9 +9407,7 @@ static QCString resolveSymlink(QCString path)
   return QDir::cleanDirPath(result).data();
 }
 
-#if MULTITHREADED_INPUT
 static std::mutex g_pathsVisitedMutex;
-#endif
 static StringUnorderedSet g_pathsVisited(1009);
 
 //----------------------------------------------------------------------------
@@ -9445,9 +9439,7 @@ static int readDir(QFileInfo *fi,
     dirName = resolveSymlink(dirName.data());
     if (dirName.isEmpty()) return 0;            // recursive symlink
 
-#if MULTITHREADED_INPUT
     std::lock_guard<std::mutex> lock(g_pathsVisitedMutex);
-#endif
     if (g_pathsVisited.find(dirName.str())!=g_pathsVisited.end()) return 0; // already visited path
     g_pathsVisited.insert(dirName.str());
   }
@@ -11041,7 +11033,14 @@ void parseInput()
   addSTLSupport(root);
 
   g_s.begin("Parsing files\n");
-  parseFiles(root);
+  if (Config_getInt(NUM_PROC_THREADS)==1)
+  {
+    parseFilesSingleThreading(root);
+  }
+  else
+  {
+    parseFilesMultiThreading(root);
+  }
   g_s.end();
 
   /**************************************************************************
diff --git a/src/doxygen.h b/src/doxygen.h
index e5757c3..dc05750 100644
--- a/src/doxygen.h
+++ b/src/doxygen.h
@@ -30,19 +30,9 @@
 #include "memberlist.h"
 #include "define.h"
 
-#ifndef MULTITHREADED_INPUT
-#define MULTITHREADED_INPUT 0
-#endif
-
-#if MULTITHREADED_INPUT
 #define THREAD_LOCAL thread_local
 #define AtomicInt    std::atomic_int
 #define AtomicBool   std::atomic_bool
-#else
-#define THREAD_LOCAL
-#define AtomicInt    int
-#define AtomicBool   bool
-#endif
 
 class RefList;
 class PageSList;
diff --git a/src/message.cpp b/src/message.cpp
index 96c54a1..37204f5 100644
--- a/src/message.cpp
+++ b/src/message.cpp
@@ -35,9 +35,7 @@ static const char *error_str = "error: ";
 static FILE *warnFile = stderr;
 
 
-#if MULTITHREADED_INPUT
 static std::mutex g_mutex;
-#endif
 
 void initWarningFormat()
 {
@@ -112,9 +110,7 @@ void msg(const char *fmt, ...)
 {
   if (!Config_getBool(QUIET))
   {
-#if MULTITHREADED_INPUT
     std::unique_lock<std::mutex> lock(g_mutex);
-#endif
     if (Debug::isFlagSet(Debug::Time))
     {
       printf("%.3f sec: ",((double)Debug::elapsedTime()));
@@ -155,9 +151,7 @@ static void format_warn(const char *file,int line,const char *text)
   msgText += '\n';
 
   {
-#if MULTITHREADED_INPUT
     std::unique_lock<std::mutex> lock(g_mutex);
-#endif
     // print resulting message
     fwrite(msgText.data(),1,msgText.length(),warnFile);
   }
@@ -257,9 +251,7 @@ extern void err_full(const char *file,int line,const char *fmt, ...)
 void term(const char *fmt, ...)
 {
   {
-#if MULTITHREADED_INPUT
     std::unique_lock<std::mutex> lock(g_mutex);
-#endif
     va_list args;
     va_start(args, fmt);
     vfprintf(warnFile, (QCString(error_str) + fmt).data(), args);
@@ -284,9 +276,7 @@ void printlex(int dbg, bool enter, const char *lexName, const char *fileName)
     enter_txt_uc = "Finished";
   }
 
-#if MULTITHREADED_INPUT
   std::unique_lock<std::mutex> lock(g_mutex);
-#endif
   if (dbg)
   {
     if (fileName)
diff --git a/src/pre.l b/src/pre.l
index 18b4615..ff04162 100644
--- a/src/pre.l
+++ b/src/pre.l
@@ -32,10 +32,8 @@
 #include <deque>
 #include <algorithm>
 #include <utility>
-#if MULTITHREADED_INPUT
 #include <mutex>
 #include <thread>
-#endif
 
 #include <stdio.h>
 #include <assert.h>
@@ -221,13 +219,9 @@ class DefineManager
  *
  *	global state
  */
-#if MULTITHREADED_INPUT
 static std::mutex            g_debugMutex;
 static std::mutex            g_globalDefineMutex;
-//static std::mutex            g_addIncludeRelationMutex;
-//static std::mutex            g_macroDefinitionsMutex;
 static std::mutex            g_updateGlobals;
-#endif
 static DefineManager         g_defineManager;
 
 
@@ -1539,10 +1533,8 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
 
 					    yyextra->includeStack.pop_back();
 
-#if MULTITHREADED_INPUT
                                             {
                                               std::lock_guard<std::mutex> lock(g_globalDefineMutex);
-#endif
                                               // to avoid deadlocks we allow multiple threads to process the same header file.
                                               // The first one to finish will store the results globally. After that the
                                               // next time the same file is encountered, the stored data is used and the file
@@ -1560,9 +1552,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
                                                   Debug::print(Debug::Preprocessor,0,"#include %s: was already processed by another thread! not storing data...\n",qPrint(toFileName));
                                                 }
                                               }
-#if MULTITHREADED_INPUT
                                             }
-#endif
 					  }
   					}
 <*>"/*"/"*/"				|
@@ -1695,10 +1685,7 @@ static FileState *checkAndOpenFile(yyscan_t yyscanner,const QCString &fileName,b
     // global guard
     if (state->curlyCount==0) // not #include inside { ... }
     {
-#if MULTITHREADED_INPUT
       std::lock_guard<std::mutex> lock(g_globalDefineMutex);
-#endif
-
       if (g_defineManager.alreadyProcessed(absName.str()))
       {
         alreadyProcessed = TRUE;
@@ -2905,9 +2892,7 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
       if (alreadyProcessed) // if this header was already process we can just copy the stored macros
                            // in the local context
       {
-#if MULTITHREADED_INPUT
         std::lock_guard<std::mutex> lock(g_globalDefineMutex);
-#endif
         g_defineManager.retrieve(absIncFileName.str(),state->contextDefines);
       }
 
@@ -3318,9 +3303,7 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output
 
   if (Debug::isFlagSet(Debug::Preprocessor))
   {
-#if MULTITHREADED_INPUT
     std::lock_guard<std::mutex> lock(g_debugMutex);
-#endif
     char *orgPos=output.data()+orgOffset;
     char *newPos=output.data()+output.curPos();
     Debug::print(Debug::Preprocessor,0,"Preprocessor output of %s (size: %d bytes):\n",fileName,newPos-orgPos);
@@ -3350,9 +3333,7 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output
   }
 
   {
-#if MULTITHREADED_INPUT
     std::lock_guard<std::mutex> lock(g_updateGlobals);
-#endif
     for (const auto &inc : state->includeRelations)
     {
       if (inc->fromFileDef)
diff --git a/src/util.cpp b/src/util.cpp
index 1fed382..90b4af1 100644
--- a/src/util.cpp
+++ b/src/util.cpp
@@ -4479,18 +4479,14 @@ struct FindFileCacheElem
 
 static QCache<FindFileCacheElem> g_findFileDefCache(5000);
 
-#if MULTITHREADED_INPUT
 static std::mutex g_findFileDefMutex;
-#endif
 
 FileDef *findFileDef(const FileNameLinkedMap *fnMap,const char *n,bool &ambig)
 {
   ambig=FALSE;
   if (n==0) return 0;
 
-#if MULTITHREADED_INPUT
   std::unique_lock<std::mutex> lock(g_findFileDefMutex);
-#endif
 
   const int maxAddrSize = 20;
   char addr[maxAddrSize];