diff options
| -rw-r--r-- | src/clangparser.cpp | 6 | ||||
| -rw-r--r-- | src/commentscan.l | 12 | ||||
| -rw-r--r-- | src/config.xml | 15 | ||||
| -rw-r--r-- | src/doxygen.cpp | 33 | ||||
| -rw-r--r-- | src/doxygen.h | 10 | ||||
| -rw-r--r-- | src/message.cpp | 10 | ||||
| -rw-r--r-- | src/pre.l | 19 | ||||
| -rw-r--r-- | src/util.cpp | 4 |
8 files changed, 31 insertions, 78 deletions
diff --git a/src/clangparser.cpp b/src/clangparser.cpp index d0ed573..5f1a021 100644 --- a/src/clangparser.cpp +++ b/src/clangparser.cpp @@ -20,22 +20,16 @@ #include "membername.h" #include "filename.h" #include "tooltip.h" -#if MULTITHREADED_INPUT #include <mutex> #endif -#endif //-------------------------------------------------------------------------- -#if MULTITHREADED_INPUT std::mutex g_clangMutex; -#endif ClangParser *ClangParser::instance() { -#if MULTITHREADED_INPUT std::lock_guard<std::mutex> lock(g_clangMutex); -#endif if (!s_instance) s_instance = new ClangParser; return s_instance; } diff --git a/src/commentscan.l b/src/commentscan.l index a1dd0e1..d695f0a 100644 --- a/src/commentscan.l +++ b/src/commentscan.l @@ -409,11 +409,9 @@ struct commentscanYY_state }; -#if MULTITHREADED_INPUT static std::mutex g_sectionMutex; static std::mutex g_formulaMutex; static std::mutex g_citeMutex; -#endif //----------------------------------------------------------------------------- @@ -2822,9 +2820,7 @@ static void addXRefItem(yyscan_t yyscanner, if (listName==0) return; //printf("addXRefItem(%s,%s,%s,%d)\n",listName,itemTitle,listTitle,append); -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_sectionMutex); -#endif RefList *refList = RefListManager::instance().add(listName,listTitle,itemTitle); RefItem *item = 0; @@ -2898,9 +2894,7 @@ static void addXRefItem(yyscan_t yyscanner, // not already added. Returns the label of the formula. static QCString addFormula(yyscan_t yyscanner) { -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_formulaMutex); -#endif struct yyguts_t *yyg = (struct yyguts_t*)yyscanner; QCString formLabel; QCString fText=yyextra->formulaText.simplifyWhiteSpace(); @@ -2922,9 +2916,7 @@ static SectionType sectionLevelToType(int level) static void addSection(yyscan_t yyscanner) { -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_sectionMutex); -#endif struct yyguts_t *yyg = (struct yyguts_t*)yyscanner; SectionManager &sm = SectionManager::instance(); const SectionInfo *si = sm.find(yyextra->sectionLabel); @@ -2957,9 +2949,7 @@ static void addSection(yyscan_t yyscanner) static void addCite(yyscan_t yyscanner) { -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_citeMutex); -#endif struct yyguts_t *yyg = (struct yyguts_t*)yyscanner; QCString name=yytext; if (yytext[0] =='"') @@ -3127,9 +3117,7 @@ static inline void setOutput(yyscan_t yyscanner,OutputContext ctx) static void addAnchor(yyscan_t yyscanner,const char *anchor) { -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_sectionMutex); -#endif struct yyguts_t *yyg = (struct yyguts_t*)yyscanner; SectionManager &sm = SectionManager::instance(); const SectionInfo *si = sm.find(anchor); diff --git a/src/config.xml b/src/config.xml index 81610c0..29f4068 100644 --- a/src/config.xml +++ b/src/config.xml @@ -812,6 +812,21 @@ Go to the <a href="commands.html">next</a> section or return to the ]]> </docs> </option> + <option type='int' id='NUM_PROC_THREADS' defval='1' minval='0' maxval='32'> + <docs> +<![CDATA[ + The \c NUM_PROC_THREADS specifies the number threads doxygen is allowed to use during + processing. When set to \c 0 doxygen will based this on the number of cores + available in the system. You can set it explicitly to a value larger than 0 + to get more control over the balance between CPU load and processing speed. + At this moment only the input processing can be done using multiple threads. + Since this is still an experimental feature the default is set to 1, + which efficively disables parallel processing. Please report any issues you + encounter. + Generating dot graphs in parallel is controlled by the \c DOT_NUM_THREADS setting. +]]> + </docs> + </option> </group> <group name='Build' docs='Build related configuration options'> <option type='bool' id='EXTRACT_ALL' defval='0'> diff --git a/src/doxygen.cpp b/src/doxygen.cpp index 4bbabb2..0ee908a 100644 --- a/src/doxygen.cpp +++ b/src/doxygen.cpp @@ -9120,10 +9120,8 @@ static std::shared_ptr<Entry> parseFile(OutlineParserInterface &parser, return fileRoot; } -#if MULTITHREADED_INPUT - //! parse the list of input files -static void parseFiles(const std::shared_ptr<Entry> &root) +static void parseFilesMultiThreading(const std::shared_ptr<Entry> &root) { #if USE_LIBCLANG if (Doxygen::clangAssistedParsing) @@ -9139,7 +9137,11 @@ static void parseFiles(const std::shared_ptr<Entry> &root) std::mutex processedFilesLock; // process source files (and their include dependencies) - std::size_t numThreads = std::thread::hardware_concurrency(); + std::size_t numThreads = static_cast<std::size_t>(Config_getInt(NUM_PROC_THREADS)); + if (numThreads==0) + { + numThreads = std::thread::hardware_concurrency(); + } msg("Processing input using %lu threads.\n",numThreads); ThreadPool threadPool(numThreads); std::vector< std::future< std::vector< std::shared_ptr<Entry> > > > results; @@ -9243,9 +9245,8 @@ static void parseFiles(const std::shared_ptr<Entry> &root) auto processFile = [s]() { bool ambig; FileDef *fd=findFileDef(Doxygen::inputNameLinkedMap,s.c_str(),ambig); - auto clangParser = ClangParser::instance()->createTUParser(fd); auto parser = getParserForFile(s.c_str()); - auto fileRoot = parseFile(*parser.get(),fd,s.c_str(),clangParser.get(),true); + auto fileRoot = parseFile(*parser.get(),fd,s.c_str(),nullptr,true); return fileRoot; }; // dispatch the work and collect the future results @@ -9256,14 +9257,11 @@ static void parseFiles(const std::shared_ptr<Entry> &root) { root->moveToSubEntryAndKeep(f.get()); } -#warning "Multi-threaded input enabled. This is a highly experimental feature. Only use for doxygen development." } } -#else // !MULTITHREADED_INPUT - //! parse the list of input files -static void parseFiles(const std::shared_ptr<Entry> &root) +static void parseFilesSingleThreading(const std::shared_ptr<Entry> &root) { #if USE_LIBCLANG if (Doxygen::clangAssistedParsing) @@ -9341,8 +9339,6 @@ static void parseFiles(const std::shared_ptr<Entry> &root) } } -#endif - // resolves a path that may include symlinks, if a recursive symlink is // found an empty string is returned. static QCString resolveSymlink(QCString path) @@ -9411,9 +9407,7 @@ static QCString resolveSymlink(QCString path) return QDir::cleanDirPath(result).data(); } -#if MULTITHREADED_INPUT static std::mutex g_pathsVisitedMutex; -#endif static StringUnorderedSet g_pathsVisited(1009); //---------------------------------------------------------------------------- @@ -9445,9 +9439,7 @@ static int readDir(QFileInfo *fi, dirName = resolveSymlink(dirName.data()); if (dirName.isEmpty()) return 0; // recursive symlink -#if MULTITHREADED_INPUT std::lock_guard<std::mutex> lock(g_pathsVisitedMutex); -#endif if (g_pathsVisited.find(dirName.str())!=g_pathsVisited.end()) return 0; // already visited path g_pathsVisited.insert(dirName.str()); } @@ -11041,7 +11033,14 @@ void parseInput() addSTLSupport(root); g_s.begin("Parsing files\n"); - parseFiles(root); + if (Config_getInt(NUM_PROC_THREADS)==1) + { + parseFilesSingleThreading(root); + } + else + { + parseFilesMultiThreading(root); + } g_s.end(); /************************************************************************** diff --git a/src/doxygen.h b/src/doxygen.h index e5757c3..dc05750 100644 --- a/src/doxygen.h +++ b/src/doxygen.h @@ -30,19 +30,9 @@ #include "memberlist.h" #include "define.h" -#ifndef MULTITHREADED_INPUT -#define MULTITHREADED_INPUT 0 -#endif - -#if MULTITHREADED_INPUT #define THREAD_LOCAL thread_local #define AtomicInt std::atomic_int #define AtomicBool std::atomic_bool -#else -#define THREAD_LOCAL -#define AtomicInt int -#define AtomicBool bool -#endif class RefList; class PageSList; diff --git a/src/message.cpp b/src/message.cpp index 96c54a1..37204f5 100644 --- a/src/message.cpp +++ b/src/message.cpp @@ -35,9 +35,7 @@ static const char *error_str = "error: "; static FILE *warnFile = stderr; -#if MULTITHREADED_INPUT static std::mutex g_mutex; -#endif void initWarningFormat() { @@ -112,9 +110,7 @@ void msg(const char *fmt, ...) { if (!Config_getBool(QUIET)) { -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_mutex); -#endif if (Debug::isFlagSet(Debug::Time)) { printf("%.3f sec: ",((double)Debug::elapsedTime())); @@ -155,9 +151,7 @@ static void format_warn(const char *file,int line,const char *text) msgText += '\n'; { -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_mutex); -#endif // print resulting message fwrite(msgText.data(),1,msgText.length(),warnFile); } @@ -257,9 +251,7 @@ extern void err_full(const char *file,int line,const char *fmt, ...) void term(const char *fmt, ...) { { -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_mutex); -#endif va_list args; va_start(args, fmt); vfprintf(warnFile, (QCString(error_str) + fmt).data(), args); @@ -284,9 +276,7 @@ void printlex(int dbg, bool enter, const char *lexName, const char *fileName) enter_txt_uc = "Finished"; } -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_mutex); -#endif if (dbg) { if (fileName) @@ -32,10 +32,8 @@ #include <deque> #include <algorithm> #include <utility> -#if MULTITHREADED_INPUT #include <mutex> #include <thread> -#endif #include <stdio.h> #include <assert.h> @@ -221,13 +219,9 @@ class DefineManager * * global state */ -#if MULTITHREADED_INPUT static std::mutex g_debugMutex; static std::mutex g_globalDefineMutex; -//static std::mutex g_addIncludeRelationMutex; -//static std::mutex g_macroDefinitionsMutex; static std::mutex g_updateGlobals; -#endif static DefineManager g_defineManager; @@ -1539,10 +1533,8 @@ CHARLIT (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'")) yyextra->includeStack.pop_back(); -#if MULTITHREADED_INPUT { std::lock_guard<std::mutex> lock(g_globalDefineMutex); -#endif // to avoid deadlocks we allow multiple threads to process the same header file. // The first one to finish will store the results globally. After that the // next time the same file is encountered, the stored data is used and the file @@ -1560,9 +1552,7 @@ CHARLIT (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'")) Debug::print(Debug::Preprocessor,0,"#include %s: was already processed by another thread! not storing data...\n",qPrint(toFileName)); } } -#if MULTITHREADED_INPUT } -#endif } } <*>"/*"/"*/" | @@ -1695,10 +1685,7 @@ static FileState *checkAndOpenFile(yyscan_t yyscanner,const QCString &fileName,b // global guard if (state->curlyCount==0) // not #include inside { ... } { -#if MULTITHREADED_INPUT std::lock_guard<std::mutex> lock(g_globalDefineMutex); -#endif - if (g_defineManager.alreadyProcessed(absName.str())) { alreadyProcessed = TRUE; @@ -2905,9 +2892,7 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc) if (alreadyProcessed) // if this header was already process we can just copy the stored macros // in the local context { -#if MULTITHREADED_INPUT std::lock_guard<std::mutex> lock(g_globalDefineMutex); -#endif g_defineManager.retrieve(absIncFileName.str(),state->contextDefines); } @@ -3318,9 +3303,7 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output if (Debug::isFlagSet(Debug::Preprocessor)) { -#if MULTITHREADED_INPUT std::lock_guard<std::mutex> lock(g_debugMutex); -#endif char *orgPos=output.data()+orgOffset; char *newPos=output.data()+output.curPos(); Debug::print(Debug::Preprocessor,0,"Preprocessor output of %s (size: %d bytes):\n",fileName,newPos-orgPos); @@ -3350,9 +3333,7 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output } { -#if MULTITHREADED_INPUT std::lock_guard<std::mutex> lock(g_updateGlobals); -#endif for (const auto &inc : state->includeRelations) { if (inc->fromFileDef) diff --git a/src/util.cpp b/src/util.cpp index 1fed382..90b4af1 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -4479,18 +4479,14 @@ struct FindFileCacheElem static QCache<FindFileCacheElem> g_findFileDefCache(5000); -#if MULTITHREADED_INPUT static std::mutex g_findFileDefMutex; -#endif FileDef *findFileDef(const FileNameLinkedMap *fnMap,const char *n,bool &ambig) { ambig=FALSE; if (n==0) return 0; -#if MULTITHREADED_INPUT std::unique_lock<std::mutex> lock(g_findFileDefMutex); -#endif const int maxAddrSize = 20; char addr[maxAddrSize]; |