From b265433382b93625b75cbc1f10b0509489b0b6b7 Mon Sep 17 00:00:00 2001 From: Dimitri van Heesch Date: Sun, 31 May 2020 19:58:24 +0200 Subject: Added experimental multi-thread input processing support. This is disabled by default. It can be enabled by setting MULTITHREADED_INPUT to 1 in doxygen.h. Still has many data races, so don't use for anything other than development! --- qtools/qmap.h | 3 +- qtools/qregexp.cpp | 9 +++-- src/doxygen.cpp | 114 ++++++++++++++++++++++++++++++++++------------------ src/doxygen.h | 13 +++++- src/entry.cpp | 2 +- src/entry.h | 2 +- src/markdown.cpp | 18 ++++----- src/message.cpp | 42 ++++++++++++++----- src/parserintf.h | 65 ++++++++++++++++-------------- src/pre.l | 26 +++++++++++- src/threadpool.h | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/util.cpp | 18 +++++++-- src/vhdldocgen.cpp | 8 ++-- 13 files changed, 329 insertions(+), 107 deletions(-) create mode 100644 src/threadpool.h diff --git a/qtools/qmap.h b/qtools/qmap.h index f384a3d..0031e0d 100644 --- a/qtools/qmap.h +++ b/qtools/qmap.h @@ -1,5 +1,5 @@ /**************************************************************************** -** +** ** ** Definition of QMap class ** @@ -102,6 +102,7 @@ class Q_EXPORT QMapIterator QMapIterator() : node( 0 ) {} QMapIterator( QMapNode* p ) : node( p ) {} QMapIterator( const QMapIterator& it ) : node( it.node ) {} + QMapIterator &operator=(const QMapIterator &it) = default; bool operator==( const QMapIterator& it ) const { return node == it.node; } bool operator!=( const QMapIterator& it ) const { return node != it.node; } diff --git a/qtools/qregexp.cpp b/qtools/qregexp.cpp index d958f0a..8709858 100644 --- a/qtools/qregexp.cpp +++ b/qtools/qregexp.cpp @@ -1,5 +1,5 @@ /**************************************************************************** -** +** ** ** Implementation of QRegExp class ** @@ -901,8 +901,6 @@ static uint *dump( uint *p ) #endif // DEBUG -static const int maxlen = 1024; // max length of regexp array -static uint rxarray[ maxlen ]; // tmp regexp array /*! \internal @@ -913,6 +911,9 @@ static uint rxarray[ maxlen ]; // tmp regexp array void QRegExp::compile() { + const int maxlen = 1024; // max length of regexp array + uint rxarray[ maxlen ]; // tmp regexp array + if ( rxdata ) { // delete old data delete [] rxdata; rxdata = 0; @@ -1006,7 +1007,7 @@ void QRegExp::compile() numFields++; } if ( d >= rxarray + maxlen ) { // pattern too long - error = PatOverflow; + error = PatOverflow; return; } if ( !pl ) { // At least ']' should be left diff --git a/src/doxygen.cpp b/src/doxygen.cpp index c7c9b45..1824ceb 100644 --- a/src/doxygen.cpp +++ b/src/doxygen.cpp @@ -103,6 +103,7 @@ #include "emoji.h" #include "plantuml.h" #include "stlsupport.h" +#include "threadpool.h" // provided by the generated file resources.cpp extern void initResources(); @@ -159,7 +160,6 @@ QCString Doxygen::spaces; bool Doxygen::generatingXmlOutput = FALSE; bool Doxygen::markdownSupport = TRUE; GenericsSDict *Doxygen::genericsDict; -Preprocessor *Doxygen::preprocessor = 0; DefineList Doxygen::macroDefinitions; // locally accessible globals @@ -9034,7 +9034,7 @@ static void generateDiskNames() //---------------------------------------------------------------------------- -static OutlineParserInterface &getParserForFile(const char *fn) +static std::unique_ptr getParserForFile(const char *fn) { QCString fileName=fn; QCString extension; @@ -9052,8 +9052,8 @@ static OutlineParserInterface &getParserForFile(const char *fn) return Doxygen::parserManager->getOutlineParser(extension); } -static void parseFile(OutlineParserInterface &parser, - const std::shared_ptr &root,FileDef *fd,const char *fn, +static std::shared_ptr parseFile(OutlineParserInterface &parser, + FileDef *fd,const char *fn, bool sameTu,QStrList &filesInSameTu) { #if USE_LIBCLANG @@ -9079,10 +9079,18 @@ static void parseFile(OutlineParserInterface &parser, if (Config_getBool(ENABLE_PREPROCESSING) && parser.needsPreprocessing(extension)) { + Preprocessor preprocessor; + QStrList &includePath = Config_getList(INCLUDE_PATH); + char *s=includePath.first(); + while (s) + { + preprocessor.addSearchDir(QFileInfo(s).absFilePath().utf8()); + s=includePath.next(); + } BufStr inBuf(fi.size()+4096); msg("Preprocessing %s...\n",fn); readInputFile(fileName,inBuf); - Doxygen::preprocessor->processFile(fileName,inBuf,preBuf); + preprocessor.processFile(fileName,inBuf,preBuf); } else // no preprocessing { @@ -9110,7 +9118,7 @@ static void parseFile(OutlineParserInterface &parser, // use language parse to parse the file parser.parseInput(fileName,convBuf.data(),fileRoot,sameTu,filesInSameTu); fileRoot->setFileDef(fd); - root->moveToSubEntryAndKeep(fileRoot); + return fileRoot; } //! parse the list of input files @@ -9138,9 +9146,10 @@ static void parseFiles(const std::shared_ptr &root) if (fd->isSource() && !fd->isReference()) // this is a source file { QStrList filesInSameTu; - OutlineParserInterface &parser = getParserForFile(s.c_str()); - parser.startTranslationUnit(s.c_str()); - parseFile(parser,root,fd,s.c_str(),FALSE,filesInSameTu); + std::unique_ptr parser { getParserForFile(s.c_str()) }; + parser->startTranslationUnit(s.c_str()); + std::shared_ptr fileRoot = parseFile(*parser.get(),fd,s.c_str(),FALSE,filesInSameTu); + root->moveToSubEntryAndKeep(fileRoot); //printf(" got %d extra files in tu\n",filesInSameTu.count()); // Now process any include files in the same translation unit @@ -9155,13 +9164,14 @@ static void parseFiles(const std::shared_ptr &root) { QStrList moreFiles; //printf(" Processing %s in same translation unit as %s\n",incFile,s->c_str()); - parseFile(parser,root,ifd,incFile,TRUE,moreFiles); + fileRoot = parseFile(*parser.get(),ifd,incFile,TRUE,moreFiles); + root->moveToSubEntryAndKeep(fileRoot); g_processedFiles.insert(incFile,(void*)0x8); } } incFile = filesInSameTu.next(); } - parser.finishTranslationUnit(); + parser->finishTranslationUnit(); g_processedFiles.insert(s.c_str(),(void*)0x8); } } @@ -9174,10 +9184,11 @@ static void parseFiles(const std::shared_ptr &root) QStrList filesInSameTu; FileDef *fd=findFileDef(Doxygen::inputNameLinkedMap,s.c_str(),ambig); ASSERT(fd!=0); - OutlineParserInterface &parser = getParserForFile(s.c_str()); - parser.startTranslationUnit(s.c_str()); - parseFile(parser,root,fd,s.c_str(),FALSE,filesInSameTu); - parser.finishTranslationUnit(); + std::unique_ptr parser { getParserForFile(s.c_str()) }; + parser->startTranslationUnit(s.c_str()); + std::shared_ptr fileRoot = parseFile(*parser.get(),fd,s.c_str(),FALSE,filesInSameTu); + root->moveToSubEntryAndKeep(fileRoot); + parser->finishTranslationUnit(); g_processedFiles.insert(s.c_str(),(void*)0x8); } } @@ -9185,16 +9196,46 @@ static void parseFiles(const std::shared_ptr &root) else // normal processing #endif { +#if !MULTITHREADED_INPUT for (const auto &s : g_inputFiles) { bool ambig; QStrList filesInSameTu; FileDef *fd=findFileDef(Doxygen::inputNameLinkedMap,s.c_str(),ambig); ASSERT(fd!=0); - OutlineParserInterface &parser = getParserForFile(s.c_str()); - parser.startTranslationUnit(s.c_str()); - parseFile(parser,root,fd,s.c_str(),FALSE,filesInSameTu); + std::unique_ptr parser { getParserForFile(s.c_str()) }; + parser->startTranslationUnit(s.c_str()); + std::shared_ptr fileRoot = parseFile(*parser.get(),fd,s.c_str(),FALSE,filesInSameTu); + root->moveToSubEntryAndKeep(fileRoot); + } +#else + std::size_t numThreads = std::thread::hardware_concurrency(); + msg("Processing input using %lu threads.\n",numThreads); + ThreadPool threadPool(numThreads); + std::vector< std::future< std::shared_ptr > > results; + for (const auto &s : g_inputFiles) + { + // lambda representing the work to executed by a thread + auto processFile = [s]() { + bool ambig; + QStrList filesInSameTu; + FileDef *fd=findFileDef(Doxygen::inputNameLinkedMap,s.c_str(),ambig); + ASSERT(fd!=0); + std::unique_ptr parser { getParserForFile(s.c_str()) }; + parser->startTranslationUnit(s.c_str()); + std::shared_ptr fileRoot = parseFile(*parser.get(),fd,s.c_str(),FALSE,filesInSameTu); + return fileRoot; + }; + // dispatch the work and collect the future results + results.emplace_back(threadPool.queue(processFile)); } + // synchronise with the Entry results produced and add them to the root + for (auto &f : results) + { + root->moveToSubEntryAndKeep(f.get()); + } +#warning "Multi-threaded input enabled. This is a highly experimental feature. Only use for doxygen development." +#endif } } @@ -9676,6 +9717,10 @@ class NullOutlineParser : public OutlineParserInterface }; +template std::function< std::unique_ptr() > make_output_parser_factory() +{ + return []() { return std::make_unique(); }; +} void initDoxygen() { @@ -9689,27 +9734,25 @@ void initDoxygen() Portable::correct_path(); Debug::startTimer(); - Doxygen::preprocessor = new Preprocessor(); - - Doxygen::parserManager = new ParserManager( std::make_unique(), + Doxygen::parserManager = new ParserManager( make_output_parser_factory(), std::make_unique()); - Doxygen::parserManager->registerParser("c", std::make_unique(), + Doxygen::parserManager->registerParser("c", make_output_parser_factory(), std::make_unique()); - Doxygen::parserManager->registerParser("python", std::make_unique(), + Doxygen::parserManager->registerParser("python", make_output_parser_factory(), std::make_unique()); - Doxygen::parserManager->registerParser("fortran", std::make_unique(), + Doxygen::parserManager->registerParser("fortran", make_output_parser_factory(), std::make_unique()); - Doxygen::parserManager->registerParser("fortranfree", std::make_unique(), + Doxygen::parserManager->registerParser("fortranfree", make_output_parser_factory(), std::make_unique()); - Doxygen::parserManager->registerParser("fortranfixed", std::make_unique(), + Doxygen::parserManager->registerParser("fortranfixed", make_output_parser_factory(), std::make_unique()); - Doxygen::parserManager->registerParser("vhdl", std::make_unique(), + Doxygen::parserManager->registerParser("vhdl", make_output_parser_factory(), std::make_unique()); - Doxygen::parserManager->registerParser("xml", std::make_unique(), + Doxygen::parserManager->registerParser("xml", make_output_parser_factory(), std::make_unique()); - Doxygen::parserManager->registerParser("sql", std::make_unique(), + Doxygen::parserManager->registerParser("sql", make_output_parser_factory(), std::make_unique()); - Doxygen::parserManager->registerParser("md", std::make_unique(), + Doxygen::parserManager->registerParser("md", make_output_parser_factory(), std::make_unique()); // register any additional parsers here... @@ -9788,7 +9831,6 @@ void cleanUpDoxygen() delete Doxygen::exampleSDict; delete Doxygen::globalScope; delete Doxygen::parserManager; - delete Doxygen::preprocessor; delete theTranslator; delete g_outputList; Mappers::freeMappers(); @@ -10250,14 +10292,6 @@ void adjustConfiguration() warn_uncond("Output language %s not supported! Using English instead.\n", outputLanguage.data()); } - QStrList &includePath = Config_getList(INCLUDE_PATH); - char *s=includePath.first(); - while (s) - { - QFileInfo fi(s); - Doxygen::preprocessor->addSearchDir(fi.absFilePath().utf8()); - s=includePath.next(); - } /* Set the global html file extension. */ Doxygen::htmlFileExtension = Config_getString(HTML_FILE_EXTENSION); @@ -10312,7 +10346,7 @@ void adjustConfiguration() // add predefined macro name to a dictionary QStrList &expandAsDefinedList =Config_getList(EXPAND_AS_DEFINED); - s=expandAsDefinedList.first(); + char *s=expandAsDefinedList.first(); while (s) { Doxygen::expandAsDefinedSet.insert(s); diff --git a/src/doxygen.h b/src/doxygen.h index b824b54..99b5d6f 100644 --- a/src/doxygen.h +++ b/src/doxygen.h @@ -30,6 +30,18 @@ #include "memberlist.h" #include "define.h" +#define MULTITHREADED_INPUT 0 + +#if MULTITHREADED_INPUT +#define THREAD_LOCAL thread_local +#define AtomicInt std::atomic_int +#define AtomicBool std::atomic_bool +#else +#define THREAD_LOCAL +#define AtomicInt int +#define AtomicBool bool +#endif + class RefList; class PageSList; class PageSDict; @@ -138,7 +150,6 @@ class Doxygen static bool generatingXmlOutput; static bool markdownSupport; static GenericsSDict *genericsDict; - static Preprocessor *preprocessor; static DefineList macroDefinitions; }; diff --git a/src/entry.cpp b/src/entry.cpp index 0c7da99..e5f6d90 100644 --- a/src/entry.cpp +++ b/src/entry.cpp @@ -150,7 +150,7 @@ void Entry::moveToSubEntryAndKeep(Entry *current) m_sublist.emplace_back(current); } -void Entry::moveToSubEntryAndKeep(std::shared_ptr ¤t) +void Entry::moveToSubEntryAndKeep(std::shared_ptr current) { current->m_parent=this; m_sublist.push_back(current); diff --git a/src/entry.h b/src/entry.h index bdc8ab8..c3d1558 100644 --- a/src/entry.h +++ b/src/entry.h @@ -207,7 +207,7 @@ class Entry * @{ */ void moveToSubEntryAndKeep(Entry* e); - void moveToSubEntryAndKeep(std::shared_ptr &e); + void moveToSubEntryAndKeep(std::shared_ptr e); /*! @} */ /*! @name add entry as a child, pass ownership and reinitialize entry */ diff --git a/src/markdown.cpp b/src/markdown.cpp index b04ab2a..c4dd9fb 100644 --- a/src/markdown.cpp +++ b/src/markdown.cpp @@ -39,6 +39,8 @@ #include //#define USE_ORIGINAL_TABLES +#include + #include "markdown.h" #include "growbuf.h" #include "debug.h" @@ -1312,7 +1314,7 @@ static QCString extractTitleId(QCString &title, int level) } if ((level > 0) && (level <= Config_getInt(TOC_INCLUDE_HEADINGS))) { - static int autoId = 0; + static AtomicInt autoId { 0 }; QCString id; id.sprintf("autotoc_md%d",autoId++); //printf("auto-generated id='%s' title='%s'\n",id.data(),title.data()); @@ -2469,7 +2471,7 @@ static QCString extractPageTitle(QCString &docs,QCString &id) static QCString detab(const QCString &s,int &refIndent) { - static int tabSize = Config_getInt(TAB_SIZE); + int tabSize = Config_getInt(TAB_SIZE); int size = s.length(); GrowBuf out(size); const char *data = s.data(); @@ -2539,7 +2541,7 @@ static QCString detab(const QCString &s,int &refIndent) QCString processMarkdown(const QCString &fileName,const int lineNr,Entry *e,const QCString &input) { - static bool init=FALSE; + static AtomicBool init { FALSE }; if (!init) { // setup callback table for special characters @@ -2562,7 +2564,7 @@ QCString processMarkdown(const QCString &fileName,const int lineNr,Entry *e,cons g_current = e; g_fileName = fileName; g_lineNr = lineNr; - static GrowBuf out; + GrowBuf out; if (input.isEmpty()) return input; out.clear(); int refIndent; @@ -2657,7 +2659,7 @@ void MarkdownOutlineParser::parseInput(const char *fileName, g_indentLevel=title.isEmpty() ? 0 : -1; QCString titleFn = QFileInfo(fileName).baseName().utf8(); QCString fn = QFileInfo(fileName).fileName().utf8(); - static QCString mdfileAsMainPage = Config_getString(USE_MDFILE_AS_MAINPAGE); + QCString mdfileAsMainPage = Config_getString(USE_MDFILE_AS_MAINPAGE); bool wasEmpty = id.isEmpty(); if (wasEmpty) id = markdownFileNameToId(fileName); if (!isExplicitPage(docs)) @@ -2729,11 +2731,7 @@ void MarkdownOutlineParser::parseInput(const char *fileName, void MarkdownOutlineParser::parsePrototype(const char *text) { - OutlineParserInterface &intf = Doxygen::parserManager->getOutlineParser("*.cpp"); - if (&intf!=this) - { - intf.parsePrototype(text); - } + Doxygen::parserManager->getOutlineParser("*.cpp")->parsePrototype(text); } //------------------------------------------------------------------------ diff --git a/src/message.cpp b/src/message.cpp index a787357..bbf578b 100644 --- a/src/message.cpp +++ b/src/message.cpp @@ -18,6 +18,9 @@ #include "debug.h" #include "portable.h" #include "message.h" +#include "doxygen.h" + +#include static QCString outputFormat; static const char *warning_str = "warning: "; @@ -31,6 +34,11 @@ static const char *error_str = "error: "; static FILE *warnFile = stderr; + +#if MULTITHREADED_INPUT +static std::mutex g_mutex; +#endif + void initWarningFormat() { // int filePos = Config_getString(WARN_FORMAT).find("$file"); @@ -104,6 +112,9 @@ void msg(const char *fmt, ...) { if (!Config_getBool(QUIET)) { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_mutex); +#endif if (Debug::isFlagSet(Debug::Time)) { printf("%.3f sec: ",((double)Debug::elapsedTime())/1000.0); @@ -143,8 +154,13 @@ static void format_warn(const char *file,int line,const char *text) } msgText += '\n'; - // print resulting message - fwrite(msgText.data(),1,msgText.length(),warnFile); + { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_mutex); +#endif + // print resulting message + fwrite(msgText.data(),1,msgText.length(),warnFile); + } if (warnAsError) { exit(1); @@ -240,14 +256,19 @@ extern void err_full(const char *file,int line,const char *fmt, ...) void term(const char *fmt, ...) { - va_list args; - va_start(args, fmt); - vfprintf(warnFile, (QCString(error_str) + fmt).data(), args); - va_end(args); - if (warnFile != stderr) { - for (int i = 0; i < (int)strlen(error_str); i++) fprintf(warnFile, " "); - fprintf(warnFile, "%s\n", "Exiting..."); +#if MULTITHREADED_INPUT + std::unique_lock lock(g_mutex); +#endif + va_list args; + va_start(args, fmt); + vfprintf(warnFile, (QCString(error_str) + fmt).data(), args); + va_end(args); + if (warnFile != stderr) + { + for (int i = 0; i < (int)strlen(error_str); i++) fprintf(warnFile, " "); + fprintf(warnFile, "%s\n", "Exiting..."); + } } exit(1); } @@ -263,6 +284,9 @@ void printlex(int dbg, bool enter, const char *lexName, const char *fileName) enter_txt_uc = "Finished"; } +#if MULTITHREADED_INPUT + std::unique_lock lock(g_mutex); +#endif if (dbg) { if (fileName) diff --git a/src/parserintf.h b/src/parserintf.h index 6dc9569..f11352e 100644 --- a/src/parserintf.h +++ b/src/parserintf.h @@ -1,12 +1,12 @@ /****************************************************************************** * - * + * * * Copyright (C) 1997-2015 by Dimitri van Heesch. * * Permission to use, copy, modify, and distribute this software and its - * documentation under the terms of the GNU General Public License is hereby - * granted. No representations are made about the suitability of this software + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software * for any purpose. It is provided "as is" without express or implied warranty. * See the GNU General Public License for more details. * @@ -20,6 +20,7 @@ #include +#include #include #include #include @@ -35,7 +36,7 @@ class Definition; /** \brief Abstract interface for outline parsers. * * By implementing the methods of this interface one can add - * a new language parser to doxygen. The parser implementation can make use of the + * a new language parser to doxygen. The parser implementation can make use of the * comment block parser to parse the contents of special comment blocks. */ class OutlineParserInterface @@ -46,21 +47,21 @@ class OutlineParserInterface /** Starts processing a translation unit (source files + headers). * After this call parseInput() is called with sameTranslationUnit * set to FALSE. If parseInput() returns additional include files, - * these are also processed using parseInput() with + * these are also processed using parseInput() with * sameTranslationUnit set to TRUE. After that * finishTranslationUnit() is called. */ virtual void startTranslationUnit(const char *fileName) = 0; - /** Called after all files in a translation unit have been + /** Called after all files in a translation unit have been * processed. */ virtual void finishTranslationUnit() = 0; - /** Parses a single input file with the goal to build an Entry tree. + /** Parses a single input file with the goal to build an Entry tree. * @param[in] fileName The full name of the file. * @param[in] fileBuf The contents of the file (zero terminated). - * @param[in,out] root The root of the tree of Entry *nodes + * @param[in,out] root The root of the tree of Entry *nodes * representing the information extracted from the file. * @param[in] sameTranslationUnit TRUE if this file was found in the same * translation unit (in the filesInSameTranslationUnit list @@ -80,7 +81,7 @@ class OutlineParserInterface * @see parseInput() */ virtual bool needsPreprocessing(const QCString &extension) const = 0; - + /** Callback function called by the comment block scanner. * It provides a string \a text containing the prototype of a function * or variable. The parser should parse this and store the information @@ -110,14 +111,14 @@ class CodeParserInterface * @param[in] input Actual code in the form of a string * @param[in] isExampleBlock TRUE iff the code is part of an example. * @param[in] exampleName Name of the example. - * @param[in] fileDef File definition to which the code + * @param[in] fileDef File definition to which the code * is associated. - * @param[in] startLine Starting line in case of a code fragment. + * @param[in] startLine Starting line in case of a code fragment. * @param[in] endLine Ending line of the code fragment. - * @param[in] inlineFragment Code fragment that is to be shown inline + * @param[in] inlineFragment Code fragment that is to be shown inline * as part of the documentation. * @param[in] memberDef Member definition to which the code - * is associated (non null in case of an inline fragment + * is associated (non null in case of an inline fragment * for a member). * @param[in] showLineNumbers if set to TRUE and also fileDef is not 0, * line numbers will be added to the source fragment @@ -151,29 +152,31 @@ class CodeParserInterface //----------------------------------------------------------------------------- +using OutlineParserFactory = std::function()>; + /** \brief Manages programming language parsers. * - * This class manages the language parsers in the system. One can + * This class manages the language parsers in the system. One can * register parsers, and obtain a parser given a file extension. */ class ParserManager { public: + struct ParserPair { - ParserPair(std::unique_ptr oli, - std::unique_ptr cpi) - : outlineParser(std::move(oli)), codeParser(std::move(cpi)) + ParserPair(OutlineParserFactory opf, std::unique_ptr cpi) + : outlineParserFactory(opf), codeParserInterface(std::move(cpi)) { } - std::unique_ptr outlineParser; - std::unique_ptr codeParser; + OutlineParserFactory outlineParserFactory; + std::unique_ptr codeParserInterface; }; - ParserManager(std::unique_ptr outlineParser, - std::unique_ptr codeParser) - : m_defaultParsers(std::move(outlineParser),std::move(codeParser)) + ParserManager(OutlineParserFactory outlineParserFactory, + std::unique_ptr codeParserInterface) + : m_defaultParsers(outlineParserFactory,std::move(codeParserInterface)) { } @@ -185,14 +188,14 @@ class ParserManager * @param[in] codeParser The code parser that is to be used for the * given name. */ - void registerParser(const char *name,std::unique_ptr outlineParser, - std::unique_ptr codeParser) + void registerParser(const char *name,OutlineParserFactory outlineParserFactory, + std::unique_ptr codeParserInterface) { m_parsers.emplace(std::string(name), - ParserPair(std::move(outlineParser),std::move(codeParser))); + ParserPair(outlineParserFactory,std::move(codeParserInterface))); } - /** Registers a file \a extension with a parser with name \a parserName. + /** Registers a file \a extension with a parser with name \a parserName. * Returns TRUE if the extension was successfully registered. */ bool registerExtension(const char *extension, const char *parserName) @@ -212,21 +215,21 @@ class ParserManager } /** Gets the interface to the parser associated with given \a extension. - * If there is no parser explicitly registered for the supplied extension, + * If there is no parser explicitly registered for the supplied extension, * the interface to the default parser will be returned. */ - OutlineParserInterface &getOutlineParser(const char *extension) + std::unique_ptr getOutlineParser(const char *extension) { - return *getParsers(extension).outlineParser; + return getParsers(extension).outlineParserFactory(); } /** Gets the interface to the parser associated with given \a extension. - * If there is no parser explicitly registered for the supplied extension, + * If there is no parser explicitly registered for the supplied extension, * the interface to the default parser will be returned. */ CodeParserInterface &getCodeParser(const char *extension) { - return *getParsers(extension).codeParser; + return *getParsers(extension).codeParserInterface; } private: diff --git a/src/pre.l b/src/pre.l index ef97f0a..5981147 100644 --- a/src/pre.l +++ b/src/pre.l @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -161,6 +162,9 @@ class DefineManager */ void startContext(const char *fileName) { +#if MULTITHREADED_INPUT + std::unique_lock lock(m_mutex); +#endif //printf("DefineManager::startContext()\n"); m_contextDefines.clear(); if (fileName==0) return; @@ -177,6 +181,9 @@ class DefineManager */ void endContext() { +#if MULTITHREADED_INPUT + std::unique_lock lock(m_mutex); +#endif //printf("DefineManager::endContext()\n"); m_contextDefines.clear(); } @@ -188,6 +195,10 @@ class DefineManager void addFileToContext(const char *fileName) { if (fileName==0) return; +#if MULTITHREADED_INPUT + std::unique_lock lock(m_mutex); +#endif + //printf("DefineManager::addFileToContext(%s)\n",fileName); auto it = m_fileMap.find(fileName); if (it==m_fileMap.end()) @@ -210,6 +221,10 @@ class DefineManager void addDefine(const char *fileName,std::unique_ptr &&def) { if (fileName==0) return; + +#if MULTITHREADED_INPUT + std::unique_lock lock(m_mutex); +#endif //printf("DefineManager::addDefine(%s,%s)\n",fileName,def->name.data()); m_contextDefines[def->name.data()] = def.get(); @@ -233,6 +248,9 @@ class DefineManager */ void addInclude(const char *fromFileName,const char *toFileName) { +#if MULTITHREADED_INPUT + std::unique_lock lock(m_mutex); +#endif //printf("DefineManager::addInclude(%s,%s)\n",fromFileName,toFileName); if (fromFileName==0 || toFileName==0) return; auto it = m_fileMap.find(fromFileName); @@ -252,6 +270,9 @@ class DefineManager */ Define *isDefined(const char *name) { +#if MULTITHREADED_INPUT + std::unique_lock lock(m_mutex); +#endif Define *d=0; auto it = m_contextDefines.find(name); if (it!=m_contextDefines.end()) @@ -295,6 +316,9 @@ class DefineManager std::map< std::string,std::unique_ptr > m_fileMap; DefineMapRef m_contextDefines; +#if MULTITHREADED_INPUT + std::mutex m_mutex; +#endif }; @@ -2848,7 +2872,7 @@ static inline void outputArray(yyscan_t yyscanner,const char *a,int len) static void readIncludeFile(yyscan_t yyscanner,const QCString &inc) { YY_EXTRA_TYPE state = preYYget_extra(yyscanner); - static bool searchIncludes = Config_getBool(SEARCH_INCLUDES); + bool searchIncludes = Config_getBool(SEARCH_INCLUDES); uint i=0; // find the start of the include file name diff --git a/src/threadpool.h b/src/threadpool.h new file mode 100644 index 0000000..3acba4e --- /dev/null +++ b/src/threadpool.h @@ -0,0 +1,116 @@ +#ifndef THREADPOOL_H +#define THREADPOOL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// Class managing a pool of worker threads. +/// Work can be queued by passing a function to queue(). When the +/// work is done the result of the function will be passed back via a future. +class ThreadPool +{ + public: + /// start N threads in the thread pool. + ThreadPool(std::size_t N=1) + { + for (std::size_t i = 0; i < N; ++i) + { + // each thread is a std::async running thread_task(): + m_finished.push_back( + std::async( + std::launch::async, + [this]{ threadTask(); } + ) + ); + } + } + /// deletes the thread pool by finishing all threads + ~ThreadPool() + { + finish(); + } + + /// Queue the lambda 'task' for the threads to execute. + /// A future of the return type of the lambda is returned to capture the result. + template > + std::future queue(F&& f) + { + // wrap the function object into a packaged task, splitting + // execution from the return value: + std::packaged_task p(std::forward(f)); + + auto r=p.get_future(); // get the return value before we hand off the task + { + std::unique_lock l(m_mutex); + m_work.emplace_back(std::move(p)); // store the task as a task + m_cond.notify_one(); // wake a thread to work on the task + } + + return r; // return the future result of the task + } + + /// finish enques a "stop the thread" message for every thread, + /// then waits for them to finish + void finish() + { + { + std::unique_lock l(m_mutex); + for(auto&& u : m_finished) + { + unused_variable(u); + m_work.push_back({}); + } + } + m_cond.notify_all(); + m_finished.clear(); + } + private: + + // helper to silence the compiler warning about unused variables + template + void unused_variable(Args&& ...args) { (void)(sizeof...(args)); } + + // the work that a worker thread does: + void threadTask() + { + while(true) + { + // pop a task off the queue: + std::packaged_task f; + { + // usual thread-safe queue code: + std::unique_lock l(m_mutex); + if (m_work.empty()) + { + m_cond.wait(l,[&]{return !m_work.empty();}); + } + f = std::move(m_work.front()); + m_work.pop_front(); + } + // if the task is invalid, it means we are asked to abort + if (!f.valid()) return; + // otherwise, run the task + f(); + } + } + + // the mutex, condition variable and deque form a single + // thread-safe triggered queue of tasks: + std::mutex m_mutex; + std::condition_variable m_cond; + // note that a packaged_task can store a packaged_task: + std::deque< std::packaged_task > m_work; + + // this holds futures representing the worker threads being done: + std::vector< std::future > m_finished; +}; + +#endif + diff --git a/src/util.cpp b/src/util.cpp index d198a5d..86435fa 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -22,6 +22,7 @@ #include #include +#include #include "md5.h" @@ -1664,15 +1665,16 @@ static CharAroundSpace g_charAroundSpace; // Note: this function is not reentrant due to the use of static buffer! QCString removeRedundantWhiteSpace(const QCString &s) { - static bool cliSupport = Config_getBool(CPP_CLI_SUPPORT); - static bool vhdl = Config_getBool(OPTIMIZE_OUTPUT_VHDL); + bool cliSupport = Config_getBool(CPP_CLI_SUPPORT); + bool vhdl = Config_getBool(OPTIMIZE_OUTPUT_VHDL); if (s.isEmpty() || vhdl) return s; // We use a static character array to // improve the performance of this function - static char *growBuf = 0; - static int growBufLen = 0; + // and thread_local is needed to make it multi-thread safe + static THREAD_LOCAL char *growBuf = 0; + static THREAD_LOCAL int growBufLen = 0; if ((int)s.length()*3>growBufLen) // For input character we produce at most 3 output characters, { growBufLen = s.length()*3; @@ -4480,11 +4482,19 @@ struct FindFileCacheElem static QCache g_findFileDefCache(5000); +#if MULTITHREADED_INPUT +static std::mutex g_findFileDefMutex; +#endif + FileDef *findFileDef(const FileNameLinkedMap *fnMap,const char *n,bool &ambig) { ambig=FALSE; if (n==0) return 0; +#if MULTITHREADED_INPUT + std::unique_lock lock(g_findFileDefMutex); +#endif + const int maxAddrSize = 20; char addr[maxAddrSize]; qsnprintf(addr,maxAddrSize,"%p:",(void*)fnMap); diff --git a/src/vhdldocgen.cpp b/src/vhdldocgen.cpp index 380c77b..9a48e14 100644 --- a/src/vhdldocgen.cpp +++ b/src/vhdldocgen.cpp @@ -2936,13 +2936,13 @@ void VhdlDocGen::createFlowChart(const MemberDef *mdef) bool b=readCodeFragment( fd->absFilePath().data(), actualStart,actualEnd,codeFragment); if (!b) return; - VHDLOutlineParser &intf =dynamic_cast(Doxygen::parserManager->getOutlineParser(".vhd")); + auto parser { Doxygen::parserManager->getOutlineParser(".vhd") }; VhdlDocGen::setFlowMember(mdef); std::shared_ptr root = std::make_shared(); QStrList filesInSameTu; - intf.startTranslationUnit(""); - intf.parseInput("",codeFragment.data(),root,FALSE,filesInSameTu); - intf.finishTranslationUnit(); + parser->startTranslationUnit(""); + parser->parseInput("",codeFragment.data(),root,FALSE,filesInSameTu); + parser->finishTranslationUnit(); } void VhdlDocGen::resetCodeVhdlParserState() -- cgit v0.12