From d0f24b762e23b1b0c069209978aa403acc027cdf Mon Sep 17 00:00:00 2001 From: Dimitri van Heesch Date: Wed, 17 Jun 2020 20:49:11 +0200 Subject: Multi-threaded parsing: added locks around global data --- src/commentscan.l | 52 +++++++++++++++++++++++++++++++++++++++------------- src/doxygen.cpp | 3 --- src/doxygen.h | 1 - src/entry.cpp | 21 +++++++++------------ src/entry.h | 3 --- src/markdown.cpp | 8 -------- src/pagedef.cpp | 9 --------- src/pre.l | 28 +++++++++++++++++++++++++++- src/util.cpp | 38 ++++++++++++++++++++------------------ 9 files changed, 95 insertions(+), 68 deletions(-) diff --git a/src/commentscan.l b/src/commentscan.l index f521528..12391cb 100644 --- a/src/commentscan.l +++ b/src/commentscan.l @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -401,6 +402,13 @@ struct commentscanYY_state bool markdownSupport = TRUE; }; + +#if MULTITHREADED_INPUT +static std::mutex g_sectionMutex; +static std::mutex g_formulaMutex; +static std::mutex g_citeMutex; +#endif + //----------------------------------------------------------------------------- static QCString stripQuotes(const char *s); @@ -2786,6 +2794,10 @@ static void addXRefItem(yyscan_t yyscanner, if (listName==0) return; //printf("addXRefItem(%s,%s,%s,%d)\n",listName,itemTitle,listTitle,append); +#if MULTITHREADED_INPUT + std::unique_lock lock(g_sectionMutex); +#endif + RefList *refList = RefListManager::instance().add(listName,listTitle,itemTitle); RefItem *item = 0; for (RefItem *i : yyextra->current->sli) @@ -2826,26 +2838,28 @@ static void addXRefItem(yyscan_t yyscanner, yyextra->current->doc += cmdString; } - SectionManager &sm = SectionManager::instance(); - const SectionInfo *si = sm.find(anchorLabel); - if (si) { - if (si->lineNr() != -1) + SectionManager &sm = SectionManager::instance(); + const SectionInfo *si = sm.find(anchorLabel); + if (si) { - warn(listName,yyextra->lineNr,"multiple use of section label '%s', (first occurrence: %s, line %d)",anchorLabel.data(),si->fileName().data(),si->lineNr()); + if (si->lineNr() != -1) + { + warn(listName,yyextra->lineNr,"multiple use of section label '%s', (first occurrence: %s, line %d)",anchorLabel.data(),si->fileName().data(),si->lineNr()); + } + else + { + warn(listName,yyextra->lineNr,"multiple use of section label '%s', (first occurrence: %s)",anchorLabel.data(),si->fileName().data()); + } } else { - warn(listName,yyextra->lineNr,"multiple use of section label '%s', (first occurrence: %s)",anchorLabel.data(),si->fileName().data()); + si = sm.add(anchorLabel,listName,yyextra->lineNr, + yyextra->sectionTitle,SectionType::Anchor, + yyextra->sectionLevel); + yyextra->current->anchors.push_back(si); } } - else - { - si = sm.add(anchorLabel,listName,yyextra->lineNr, - yyextra->sectionTitle,SectionType::Anchor, - yyextra->sectionLevel); - yyextra->current->anchors.push_back(si); - } } yyextra->outputXRef.resize(0); } @@ -2856,6 +2870,9 @@ static void addXRefItem(yyscan_t yyscanner, // not already added. Returns the label of the formula. static QCString addFormula(yyscan_t yyscanner) { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_formulaMutex); +#endif struct yyguts_t *yyg = (struct yyguts_t*)yyscanner; QCString formLabel; QCString fText=yyextra->formulaText.simplifyWhiteSpace(); @@ -2877,6 +2894,9 @@ static SectionType sectionLevelToType(int level) static void addSection(yyscan_t yyscanner) { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_sectionMutex); +#endif struct yyguts_t *yyg = (struct yyguts_t*)yyscanner; SectionManager &sm = SectionManager::instance(); const SectionInfo *si = sm.find(yyextra->sectionLabel); @@ -2909,6 +2929,9 @@ static void addSection(yyscan_t yyscanner) static void addCite(yyscan_t yyscanner) { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_citeMutex); +#endif struct yyguts_t *yyg = (struct yyguts_t*)yyscanner; QCString name=yytext; if (yytext[0] =='"') @@ -3060,6 +3083,9 @@ static inline void setOutput(yyscan_t yyscanner,OutputContext ctx) static void addAnchor(yyscan_t yyscanner,const char *anchor) { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_sectionMutex); +#endif struct yyguts_t *yyg = (struct yyguts_t*)yyscanner; SectionManager &sm = SectionManager::instance(); const SectionInfo *si = sm.find(anchor); diff --git a/src/doxygen.cpp b/src/doxygen.cpp index 188d920..d8436bb 100644 --- a/src/doxygen.cpp +++ b/src/doxygen.cpp @@ -158,7 +158,6 @@ int Doxygen::subpageNestingLevel = 0; bool Doxygen::userComments = FALSE; QCString Doxygen::spaces; bool Doxygen::generatingXmlOutput = FALSE; -//bool Doxygen::markdownSupport = TRUE; GenericsSDict *Doxygen::genericsDict; DefineList Doxygen::macroDefinitions; @@ -10299,8 +10298,6 @@ void adjustConfiguration() Config_getBool(REFERENCES_RELATION) || Config_getBool(REFERENCED_BY_RELATION); - //Doxygen::markdownSupport = Config_getBool(MARKDOWN_SUPPORT); - /************************************************************************** * Add custom extension mappings **************************************************************************/ diff --git a/src/doxygen.h b/src/doxygen.h index 0746bbd..3198c82 100644 --- a/src/doxygen.h +++ b/src/doxygen.h @@ -148,7 +148,6 @@ class Doxygen static int subpageNestingLevel; static QCString spaces; static bool generatingXmlOutput; - //static bool markdownSupport; static GenericsSDict *genericsDict; static DefineList macroDefinitions; }; diff --git a/src/entry.cpp b/src/entry.cpp index e5f6d90..6e343b0 100644 --- a/src/entry.cpp +++ b/src/entry.cpp @@ -24,18 +24,15 @@ #include "doxygen.h" #include "arguments.h" #include "config.h" -//------------------------------------------------------------------ - -#define HEADER ('D'<<24)+('O'<<16)+('X'<<8)+'!' //------------------------------------------------------------------ -int Entry::num=0; +static AtomicInt g_num; Entry::Entry() { //printf("Entry::Entry(%p)\n",this); - num++; + g_num++; m_parent=0; section = EMPTY_SEC; //printf("Entry::Entry() tArgList=0\n"); @@ -50,7 +47,7 @@ Entry::Entry() Entry::Entry(const Entry &e) { //printf("Entry::Entry(%p):copy\n",this); - num++; + g_num++; section = e.section; type = e.type; name = e.name; @@ -123,11 +120,11 @@ Entry::Entry(const Entry &e) Entry::~Entry() { - //printf("Entry::~Entry(%p) num=%d\n",this,num); + //printf("Entry::~Entry(%p) num=%d\n",this,g_num); //printf("Deleting entry %d name %s type %x children %d\n", // num,name.data(),section,sublist->count()); - num--; + g_num--; } void Entry::moveToSubEntryAndRefresh(Entry *¤t) @@ -183,10 +180,10 @@ void Entry::removeSubEntry(const Entry *e) void Entry::reset() { - static bool entryCallGraph = Config_getBool(CALL_GRAPH); - static bool entryCallerGraph = Config_getBool(CALLER_GRAPH); - static bool entryReferencedByRelation = Config_getBool(REFERENCED_BY_RELATION); - static bool entryReferencesRelation = Config_getBool(REFERENCES_RELATION); + bool entryCallGraph = Config_getBool(CALL_GRAPH); + bool entryCallerGraph = Config_getBool(CALLER_GRAPH); + bool entryReferencedByRelation = Config_getBool(REFERENCED_BY_RELATION); + bool entryReferencesRelation = Config_getBool(REFERENCES_RELATION); //printf("Entry::reset()\n"); name.resize(0); type.resize(0); diff --git a/src/entry.h b/src/entry.h index c3d1558..d6c0936 100644 --- a/src/entry.h +++ b/src/entry.h @@ -297,9 +297,6 @@ class Entry LocalToc localToc; QCString metaData; //!< Slice metadata - - static int num; //!< counts the total number of entries - /// return the command name used to define GROUPDOC_SEC const char *groupDocCmd() const { diff --git a/src/markdown.cpp b/src/markdown.cpp index 2cf4e0a..583ad17 100644 --- a/src/markdown.cpp +++ b/src/markdown.cpp @@ -2574,11 +2574,6 @@ void MarkdownOutlineParser::parseInput(const char *fileName, } int lineNr=1; - // even without markdown support enabled, we still - // parse markdown files as such - //bool markdownEnabled = Doxygen::markdownSupport; - //Doxygen::markdownSupport = TRUE; - Protection prot=Public; bool needsEntry = FALSE; int position=0; @@ -2610,9 +2605,6 @@ void MarkdownOutlineParser::parseInput(const char *fileName, { root->moveToSubEntryAndKeep(current); } - - // restore setting - //Doxygen::markdownSupport = markdownEnabled; } void MarkdownOutlineParser::parsePrototype(const char *text) diff --git a/src/pagedef.cpp b/src/pagedef.cpp index 823e87d..09152de 100644 --- a/src/pagedef.cpp +++ b/src/pagedef.cpp @@ -289,13 +289,6 @@ void PageDefImpl::writeDocumentation(OutputList &ol) void PageDefImpl::writePageDocumentation(OutputList &ol) { - - //bool markdownEnabled = Doxygen::markdownSupport; - //if (getLanguage()==SrcLangExt_Markdown) - //{ - // Doxygen::markdownSupport = TRUE; - //} - ol.startTextBlock(); QCString docStr = documentation()+inbodyDocumentation(); if (hasBriefDescription() && !SectionManager::instance().find(name())) @@ -320,8 +313,6 @@ void PageDefImpl::writePageDocumentation(OutputList &ol) ); ol.endTextBlock(); - //Doxygen::markdownSupport = markdownEnabled; - if (hasSubPages()) { // for printed documentation we write subpages as section's of the diff --git a/src/pre.l b/src/pre.l index b78908f..4ec76ab 100644 --- a/src/pre.l +++ b/src/pre.l @@ -360,6 +360,11 @@ void DefineManager::DefinesPerFile::collectDefines( * * global state */ +#if MULTITHREADED_INPUT +static std::mutex g_allIncludesMutex; +static std::mutex g_addIncludeRelationMutex; +static std::mutex g_macroDefinitionsMutex; +#endif static StringUnorderedSet g_allIncludes; static DefineManager g_defineManager; @@ -1804,6 +1809,10 @@ static FileState *checkAndOpenFile(yyscan_t yyscanner,const QCString &fileName,b // global guard if (state->curlyCount==0) // not #include inside { ... } { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_allIncludesMutex); +#endif + if (g_allIncludes.find(absName.data())!=g_allIncludes.end()) { alreadyIncluded = TRUE; @@ -2854,7 +2863,12 @@ static void addMacroDefinition(yyscan_t yyscanner) { define->definition = litTextStripped; } - Doxygen::macroDefinitions.push_back(std::move(define)); + { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_macroDefinitionsMutex); +#endif + Doxygen::macroDefinitions.push_back(std::move(define)); + } } static inline void outputChar(yyscan_t yyscanner,char c) @@ -2959,6 +2973,9 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc) } if (oldFileDef) { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_addIncludeRelationMutex); +#endif // add include dependency to the file in which the #include was found bool ambig; // change to absolute name for bug 641336 @@ -2973,6 +2990,9 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc) } else if (state->inputFileDef) { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_addIncludeRelationMutex); +#endif state->inputFileDef->addIncludeDependency(0,absIncFileName,localInclude,state->isImported,TRUE); } struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; @@ -3002,6 +3022,9 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc) //printf(" calling findFile(%s) alreadyInc=%d\n",incFileName.data(),alreadyIncluded); if (oldFileDef) { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_addIncludeRelationMutex); +#endif bool ambig; // change to absolute name for bug 641336 @@ -3018,6 +3041,9 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc) } else if (state->inputFileDef) { +#if MULTITHREADED_INPUT + std::unique_lock lock(g_addIncludeRelationMutex); +#endif state->inputFileDef->addIncludeDependency(0,absIncFileName,localInclude,state->isImported,TRUE); } if (Debug::isFlagSet(Debug::Preprocessor)) diff --git a/src/util.cpp b/src/util.cpp index a571ffe..710de06 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -23,6 +23,7 @@ #include #include +#include #include "md5.h" @@ -7019,9 +7020,8 @@ QCString parseCommentAsText(const Definition *scope,const MemberDef *md, //-------------------------------------------------------------------------------------- -static QDict aliasesProcessed; - -static QCString expandAliasRec(const QCString s,bool allowRecursion=FALSE); +static QCString expandAliasRec(StringUnorderedSet &aliasesProcessed, + const QCString s,bool allowRecursion=FALSE); struct Marker { @@ -7063,7 +7063,8 @@ static int findEndOfCommand(const char *s) * with the corresponding values found in the comma separated argument * list \a argList and the returns the result after recursive alias expansion. */ -static QCString replaceAliasArguments(const QCString &aliasValue,const QCString &argList) +static QCString replaceAliasArguments(StringUnorderedSet &aliasesProcessed, + const QCString &aliasValue,const QCString &argList) { //printf("----- replaceAliasArguments(val=[%s],args=[%s])\n",aliasValue.data(),argList.data()); @@ -7143,7 +7144,7 @@ static QCString replaceAliasArguments(const QCString &aliasValue,const QCString //printf("part before marker %d: '%s'\n",i,aliasValue.mid(p,m->pos-p).data()); if (m->number>0 && m->number<=(int)args.count()) // valid number { - result+=expandAliasRec(*args.at(m->number-1),TRUE); + result+=expandAliasRec(aliasesProcessed,*args.at(m->number-1),TRUE); //printf("marker index=%d pos=%d number=%d size=%d replacement %s\n",i,m->pos,m->number,m->size, // args.at(m->number-1)->data()); } @@ -7155,7 +7156,7 @@ static QCString replaceAliasArguments(const QCString &aliasValue,const QCString // expand the result again result = substitute(result,"\\{","{"); result = substitute(result,"\\}","}"); - result = expandAliasRec(substitute(result,"\\,",",")); + result = expandAliasRec(aliasesProcessed,substitute(result,"\\,",",")); return result; } @@ -7182,7 +7183,7 @@ static QCString escapeCommas(const QCString &s) return result.data(); } -static QCString expandAliasRec(const QCString s,bool allowRecursion) +static QCString expandAliasRec(StringUnorderedSet &aliasesProcessed,const QCString s,bool allowRecursion) { QCString result; static QRegExp cmdPat("[\\\\@][a-z_A-Z][a-z_A-Z0-9]*"); @@ -7215,19 +7216,20 @@ static QCString expandAliasRec(const QCString s,bool allowRecursion) } //printf("Found command s='%s' cmd='%s' numArgs=%d args='%s' aliasText=%s\n", // s.data(),cmd.data(),numArgs,args.data(),aliasText?aliasText->data():""); - if ((allowRecursion || aliasesProcessed.find(cmd)==0) && aliasText) // expand the alias + if ((allowRecursion || aliasesProcessed.find(cmd.str())==aliasesProcessed.end()) && + aliasText) // expand the alias { //printf("is an alias!\n"); - if (!allowRecursion) aliasesProcessed.insert(cmd,(void *)0x8); + if (!allowRecursion) aliasesProcessed.insert(cmd.str()); QCString val = *aliasText; if (hasArgs) { - val = replaceAliasArguments(val,args); + val = replaceAliasArguments(aliasesProcessed,val,args); //printf("replace '%s'->'%s' args='%s'\n", // aliasText->data(),val.data(),args.data()); } - result+=expandAliasRec(val); - if (!allowRecursion) aliasesProcessed.remove(cmd); + result+=expandAliasRec(aliasesProcessed,val); + if (!allowRecursion) aliasesProcessed.erase(cmd.str()); p=i+l; if (hasArgs) p+=argsLen+2; } @@ -7297,9 +7299,9 @@ QCString extractAliasArgs(const QCString &args,int pos) QCString resolveAliasCmd(const QCString aliasCmd) { QCString result; - aliasesProcessed.clear(); + StringUnorderedSet aliasesProcessed; //printf("Expanding: '%s'\n",aliasCmd.data()); - result = expandAliasRec(aliasCmd); + result = expandAliasRec(aliasesProcessed,aliasCmd); //printf("Expanding result: '%s'->'%s'\n",aliasCmd.data(),result.data()); return result; } @@ -7307,12 +7309,12 @@ QCString resolveAliasCmd(const QCString aliasCmd) QCString expandAlias(const QCString &aliasName,const QCString &aliasValue) { QCString result; - aliasesProcessed.clear(); + StringUnorderedSet aliasesProcessed; // avoid expanding this command recursively - aliasesProcessed.insert(aliasName,(void *)0x8); + aliasesProcessed.insert(aliasName.str()); // expand embedded commands //printf("Expanding: '%s'->'%s'\n",aliasName.data(),aliasValue.data()); - result = expandAliasRec(aliasValue); + result = expandAliasRec(aliasesProcessed,aliasValue); //printf("Expanding result: '%s'->'%s'\n",aliasName.data(),result.data()); return result; } @@ -7520,7 +7522,7 @@ QCString filterTitle(const QCString &title) bool patternMatch(const QFileInfo &fi,const StringVector &patList) { - static bool caseSenseNames = Config_getBool(CASE_SENSE_NAMES); + bool caseSenseNames = Config_getBool(CASE_SENSE_NAMES); bool found = FALSE; // For platforms where the file system is non case sensitive overrule the setting -- cgit v0.12