Multi-threaded parsing: added locks around global data

9 files changed, 95 insertions, 68 deletions

diff --git a/src/commentscan.l b/src/commentscan.l
index f521528..12391cb 100644
--- a/src/commentscan.l
+++ b/src/commentscan.l
@@ -30,6 +30,7 @@
 #include <map>
 #include <stack>
 #include <string>
+#include <mutex>
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -401,6 +402,13 @@ struct commentscanYY_state
   bool             markdownSupport = TRUE;
 };
 
+
+#if MULTITHREADED_INPUT
+static std::mutex g_sectionMutex;
+static std::mutex g_formulaMutex;
+static std::mutex g_citeMutex;
+#endif
+
 //-----------------------------------------------------------------------------
 
 static QCString stripQuotes(const char *s);
@@ -2786,6 +2794,10 @@ static void addXRefItem(yyscan_t yyscanner,
   if (listName==0) return;
   //printf("addXRefItem(%s,%s,%s,%d)\n",listName,itemTitle,listTitle,append);
 
+#if MULTITHREADED_INPUT
+  std::unique_lock<std::mutex> lock(g_sectionMutex);
+#endif
+
   RefList *refList = RefListManager::instance().add(listName,listTitle,itemTitle);
   RefItem *item = 0;
   for (RefItem *i : yyextra->current->sli)
@@ -2826,26 +2838,28 @@ static void addXRefItem(yyscan_t yyscanner,
       yyextra->current->doc += cmdString;
     }
 
-    SectionManager &sm = SectionManager::instance();
-    const SectionInfo *si = sm.find(anchorLabel);
-    if (si)
     {
-      if (si->lineNr() != -1)
+      SectionManager &sm = SectionManager::instance();
+      const SectionInfo *si = sm.find(anchorLabel);
+      if (si)
       {
-        warn(listName,yyextra->lineNr,"multiple use of section label '%s', (first occurrence: %s, line %d)",anchorLabel.data(),si->fileName().data(),si->lineNr());
+        if (si->lineNr() != -1)
+        {
+          warn(listName,yyextra->lineNr,"multiple use of section label '%s', (first occurrence: %s, line %d)",anchorLabel.data(),si->fileName().data(),si->lineNr());
+        }
+        else
+        {
+          warn(listName,yyextra->lineNr,"multiple use of section label '%s', (first occurrence: %s)",anchorLabel.data(),si->fileName().data());
+        }
       }
       else
       {
-        warn(listName,yyextra->lineNr,"multiple use of section label '%s', (first occurrence: %s)",anchorLabel.data(),si->fileName().data());
+        si = sm.add(anchorLabel,listName,yyextra->lineNr,
+            yyextra->sectionTitle,SectionType::Anchor,
+            yyextra->sectionLevel);
+        yyextra->current->anchors.push_back(si);
       }
     }
-    else
-    {
-      si = sm.add(anchorLabel,listName,yyextra->lineNr,
-                   yyextra->sectionTitle,SectionType::Anchor,
-                   yyextra->sectionLevel);
-      yyextra->current->anchors.push_back(si);
-    }
   }
   yyextra->outputXRef.resize(0);
 }
@@ -2856,6 +2870,9 @@ static void addXRefItem(yyscan_t yyscanner,
 // not already added. Returns the label of the formula.
 static QCString addFormula(yyscan_t yyscanner)
 {
+#if MULTITHREADED_INPUT
+  std::unique_lock<std::mutex> lock(g_formulaMutex);
+#endif
   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
   QCString formLabel;
   QCString fText=yyextra->formulaText.simplifyWhiteSpace();
@@ -2877,6 +2894,9 @@ static SectionType sectionLevelToType(int level)
 
 static void addSection(yyscan_t yyscanner)
 {
+#if MULTITHREADED_INPUT
+  std::unique_lock<std::mutex> lock(g_sectionMutex);
+#endif
   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
   SectionManager &sm = SectionManager::instance();
   const SectionInfo *si = sm.find(yyextra->sectionLabel);
@@ -2909,6 +2929,9 @@ static void addSection(yyscan_t yyscanner)
 
 static void addCite(yyscan_t yyscanner)
 {
+#if MULTITHREADED_INPUT
+  std::unique_lock<std::mutex> lock(g_citeMutex);
+#endif
   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
   QCString name=yytext;
   if (yytext[0] =='"')
@@ -3060,6 +3083,9 @@ static inline void setOutput(yyscan_t yyscanner,OutputContext ctx)
 
 static void addAnchor(yyscan_t yyscanner,const char *anchor)
 {
+#if MULTITHREADED_INPUT
+  std::unique_lock<std::mutex> lock(g_sectionMutex);
+#endif
   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
   SectionManager &sm = SectionManager::instance();
   const SectionInfo *si = sm.find(anchor);
diff --git a/src/doxygen.cpp b/src/doxygen.cpp
index 188d920..d8436bb 100644
--- a/src/doxygen.cpp
+++ b/src/doxygen.cpp
@@ -158,7 +158,6 @@ int              Doxygen::subpageNestingLevel = 0;
 bool             Doxygen::userComments = FALSE;
 QCString         Doxygen::spaces;
 bool             Doxygen::generatingXmlOutput = FALSE;
-//bool             Doxygen::markdownSupport = TRUE;
 GenericsSDict   *Doxygen::genericsDict;
 DefineList       Doxygen::macroDefinitions;
 
@@ -10299,8 +10298,6 @@ void adjustConfiguration()
                                 Config_getBool(REFERENCES_RELATION) ||
                                 Config_getBool(REFERENCED_BY_RELATION);
 
-  //Doxygen::markdownSupport = Config_getBool(MARKDOWN_SUPPORT);
-
   /**************************************************************************
    *            Add custom extension mappings
    **************************************************************************/
diff --git a/src/doxygen.h b/src/doxygen.h
index 0746bbd..3198c82 100644
--- a/src/doxygen.h
+++ b/src/doxygen.h
@@ -148,7 +148,6 @@ class Doxygen
     static int                       subpageNestingLevel;
     static QCString                  spaces;
     static bool                      generatingXmlOutput;
-    //static bool                      markdownSupport;
     static GenericsSDict            *genericsDict;
     static DefineList                macroDefinitions;
 };
diff --git a/src/entry.cpp b/src/entry.cpp
index e5f6d90..6e343b0 100644
--- a/src/entry.cpp
+++ b/src/entry.cpp
@@ -24,18 +24,15 @@
 #include "doxygen.h"
 #include "arguments.h"
 #include "config.h"
-//------------------------------------------------------------------
-
-#define HEADER ('D'<<24)+('O'<<16)+('X'<<8)+'!'
 
 //------------------------------------------------------------------
 
-int Entry::num=0;
+static AtomicInt g_num;
 
 Entry::Entry()
 {
   //printf("Entry::Entry(%p)\n",this);
-  num++;
+  g_num++;
   m_parent=0;
   section = EMPTY_SEC;
   //printf("Entry::Entry() tArgList=0\n");
@@ -50,7 +47,7 @@ Entry::Entry()
 Entry::Entry(const Entry &e)
 {
   //printf("Entry::Entry(%p):copy\n",this);
-  num++;
+  g_num++;
   section     = e.section;
   type        = e.type;
   name        = e.name;
@@ -123,11 +120,11 @@ Entry::Entry(const Entry &e)
 
 Entry::~Entry()
 {
-  //printf("Entry::~Entry(%p) num=%d\n",this,num);
+  //printf("Entry::~Entry(%p) num=%d\n",this,g_num);
   //printf("Deleting entry %d name %s type %x children %d\n",
   //       num,name.data(),section,sublist->count());
 
-  num--;
+  g_num--;
 }
 
 void Entry::moveToSubEntryAndRefresh(Entry *&current)
@@ -183,10 +180,10 @@ void Entry::removeSubEntry(const Entry *e)
 
 void Entry::reset()
 {
-  static bool entryCallGraph   = Config_getBool(CALL_GRAPH);
-  static bool entryCallerGraph = Config_getBool(CALLER_GRAPH);
-  static bool entryReferencedByRelation = Config_getBool(REFERENCED_BY_RELATION);
-  static bool entryReferencesRelation   = Config_getBool(REFERENCES_RELATION);
+  bool entryCallGraph   = Config_getBool(CALL_GRAPH);
+  bool entryCallerGraph = Config_getBool(CALLER_GRAPH);
+  bool entryReferencedByRelation = Config_getBool(REFERENCED_BY_RELATION);
+  bool entryReferencesRelation   = Config_getBool(REFERENCES_RELATION);
   //printf("Entry::reset()\n");
   name.resize(0);
   type.resize(0);
diff --git a/src/entry.h b/src/entry.h
index c3d1558..d6c0936 100644
--- a/src/entry.h
+++ b/src/entry.h
@@ -297,9 +297,6 @@ class Entry
     LocalToc    localToc;
     QCString    metaData;     //!< Slice metadata
 
-
-    static int  num;          //!< counts the total number of entries
-
     /// return the command name used to define GROUPDOC_SEC
     const char *groupDocCmd() const
     {
diff --git a/src/markdown.cpp b/src/markdown.cpp
index 2cf4e0a..583ad17 100644
--- a/src/markdown.cpp
+++ b/src/markdown.cpp
@@ -2574,11 +2574,6 @@ void MarkdownOutlineParser::parseInput(const char *fileName,
   }
   int lineNr=1;
 
-  // even without markdown support enabled, we still
-  // parse markdown files as such
-  //bool markdownEnabled = Doxygen::markdownSupport;
-  //Doxygen::markdownSupport = TRUE;
-
   Protection prot=Public;
   bool needsEntry = FALSE;
   int position=0;
@@ -2610,9 +2605,6 @@ void MarkdownOutlineParser::parseInput(const char *fileName,
   {
     root->moveToSubEntryAndKeep(current);
   }
-
-  // restore setting
-  //Doxygen::markdownSupport = markdownEnabled;
 }
 
 void MarkdownOutlineParser::parsePrototype(const char *text)
diff --git a/src/pagedef.cpp b/src/pagedef.cpp
index 823e87d..09152de 100644
--- a/src/pagedef.cpp
+++ b/src/pagedef.cpp
@@ -289,13 +289,6 @@ void PageDefImpl::writeDocumentation(OutputList &ol)
 
 void PageDefImpl::writePageDocumentation(OutputList &ol)
 {
-
-  //bool markdownEnabled = Doxygen::markdownSupport;
-  //if (getLanguage()==SrcLangExt_Markdown)
-  //{
-  //  Doxygen::markdownSupport = TRUE;
-  //}
-
   ol.startTextBlock();
   QCString docStr = documentation()+inbodyDocumentation();
   if (hasBriefDescription() && !SectionManager::instance().find(name()))
@@ -320,8 +313,6 @@ void PageDefImpl::writePageDocumentation(OutputList &ol)
       );
   ol.endTextBlock();
 
-  //Doxygen::markdownSupport = markdownEnabled;
-
   if (hasSubPages())
   {
     // for printed documentation we write subpages as section's of the
diff --git a/src/pre.l b/src/pre.l
index b78908f..4ec76ab 100644
--- a/src/pre.l
+++ b/src/pre.l
@@ -360,6 +360,11 @@ void DefineManager::DefinesPerFile::collectDefines(
  *
  *	global state
  */
+#if MULTITHREADED_INPUT
+static std::mutex            g_allIncludesMutex;
+static std::mutex            g_addIncludeRelationMutex;
+static std::mutex            g_macroDefinitionsMutex;
+#endif
 static StringUnorderedSet    g_allIncludes;
 static DefineManager         g_defineManager;
 
@@ -1804,6 +1809,10 @@ static FileState *checkAndOpenFile(yyscan_t yyscanner,const QCString &fileName,b
     // global guard
     if (state->curlyCount==0) // not #include inside { ... }
     {
+#if MULTITHREADED_INPUT
+      std::unique_lock<std::mutex> lock(g_allIncludesMutex);
+#endif
+
       if (g_allIncludes.find(absName.data())!=g_allIncludes.end())
       {
         alreadyIncluded = TRUE;
@@ -2854,7 +2863,12 @@ static void addMacroDefinition(yyscan_t yyscanner)
   {
     define->definition = litTextStripped;
   }
-  Doxygen::macroDefinitions.push_back(std::move(define));
+  {
+#if MULTITHREADED_INPUT
+    std::unique_lock<std::mutex> lock(g_macroDefinitionsMutex);
+#endif
+    Doxygen::macroDefinitions.push_back(std::move(define));
+  }
 }
 
 static inline void outputChar(yyscan_t yyscanner,char c)
@@ -2959,6 +2973,9 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
       }
       if (oldFileDef)
       {
+#if MULTITHREADED_INPUT
+        std::unique_lock<std::mutex> lock(g_addIncludeRelationMutex);
+#endif
         // add include dependency to the file in which the #include was found
 	bool ambig;
 	// change to absolute name for bug 641336 
@@ -2973,6 +2990,9 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
       }
       else if (state->inputFileDef)
       {
+#if MULTITHREADED_INPUT
+        std::unique_lock<std::mutex> lock(g_addIncludeRelationMutex);
+#endif
         state->inputFileDef->addIncludeDependency(0,absIncFileName,localInclude,state->isImported,TRUE);
       }
       struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
@@ -3002,6 +3022,9 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
       //printf("  calling findFile(%s) alreadyInc=%d\n",incFileName.data(),alreadyIncluded);
       if (oldFileDef)
       {
+#if MULTITHREADED_INPUT
+        std::unique_lock<std::mutex> lock(g_addIncludeRelationMutex);
+#endif
 	bool ambig;
 
 	// change to absolute name for bug 641336 
@@ -3018,6 +3041,9 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
       }
       else if (state->inputFileDef)
       {
+#if MULTITHREADED_INPUT
+        std::unique_lock<std::mutex> lock(g_addIncludeRelationMutex);
+#endif
         state->inputFileDef->addIncludeDependency(0,absIncFileName,localInclude,state->isImported,TRUE);
       }
       if (Debug::isFlagSet(Debug::Preprocessor))
diff --git a/src/util.cpp b/src/util.cpp
index a571ffe..710de06 100644
--- a/src/util.cpp
+++ b/src/util.cpp
@@ -23,6 +23,7 @@
 #include <string.h>
 
 #include <mutex>
+#include <unordered_set>
 
 #include "md5.h"
 
@@ -7019,9 +7020,8 @@ QCString parseCommentAsText(const Definition *scope,const MemberDef *md,
 
 //--------------------------------------------------------------------------------------
 
-static QDict<void> aliasesProcessed;
-
-static QCString expandAliasRec(const QCString s,bool allowRecursion=FALSE);
+static QCString expandAliasRec(StringUnorderedSet &aliasesProcessed,
+                               const QCString s,bool allowRecursion=FALSE);
 
 struct Marker
 {
@@ -7063,7 +7063,8 @@ static int findEndOfCommand(const char *s)
  *  with the corresponding values found in the comma separated argument
  *  list \a argList and the returns the result after recursive alias expansion.
  */
-static QCString replaceAliasArguments(const QCString &aliasValue,const QCString &argList)
+static QCString replaceAliasArguments(StringUnorderedSet &aliasesProcessed,
+                                      const QCString &aliasValue,const QCString &argList)
 {
   //printf("----- replaceAliasArguments(val=[%s],args=[%s])\n",aliasValue.data(),argList.data());
 
@@ -7143,7 +7144,7 @@ static QCString replaceAliasArguments(const QCString &aliasValue,const QCString
     //printf("part before marker %d: '%s'\n",i,aliasValue.mid(p,m->pos-p).data());
     if (m->number>0 && m->number<=(int)args.count()) // valid number
     {
-      result+=expandAliasRec(*args.at(m->number-1),TRUE);
+      result+=expandAliasRec(aliasesProcessed,*args.at(m->number-1),TRUE);
       //printf("marker index=%d pos=%d number=%d size=%d replacement %s\n",i,m->pos,m->number,m->size,
       //    args.at(m->number-1)->data());
     }
@@ -7155,7 +7156,7 @@ static QCString replaceAliasArguments(const QCString &aliasValue,const QCString
   // expand the result again
   result = substitute(result,"\\{","{");
   result = substitute(result,"\\}","}");
-  result = expandAliasRec(substitute(result,"\\,",","));
+  result = expandAliasRec(aliasesProcessed,substitute(result,"\\,",","));
 
   return result;
 }
@@ -7182,7 +7183,7 @@ static QCString escapeCommas(const QCString &s)
   return result.data();
 }
 
-static QCString expandAliasRec(const QCString s,bool allowRecursion)
+static QCString expandAliasRec(StringUnorderedSet &aliasesProcessed,const QCString s,bool allowRecursion)
 {
   QCString result;
   static QRegExp cmdPat("[\\\\@][a-z_A-Z][a-z_A-Z0-9]*");
@@ -7215,19 +7216,20 @@ static QCString expandAliasRec(const QCString s,bool allowRecursion)
     }
     //printf("Found command s='%s' cmd='%s' numArgs=%d args='%s' aliasText=%s\n",
     //    s.data(),cmd.data(),numArgs,args.data(),aliasText?aliasText->data():"<none>");
-    if ((allowRecursion || aliasesProcessed.find(cmd)==0) && aliasText) // expand the alias
+    if ((allowRecursion || aliasesProcessed.find(cmd.str())==aliasesProcessed.end()) &&
+        aliasText) // expand the alias
     {
       //printf("is an alias!\n");
-      if (!allowRecursion) aliasesProcessed.insert(cmd,(void *)0x8);
+      if (!allowRecursion) aliasesProcessed.insert(cmd.str());
       QCString val = *aliasText;
       if (hasArgs)
       {
-        val = replaceAliasArguments(val,args);
+        val = replaceAliasArguments(aliasesProcessed,val,args);
         //printf("replace '%s'->'%s' args='%s'\n",
         //       aliasText->data(),val.data(),args.data());
       }
-      result+=expandAliasRec(val);
-      if (!allowRecursion) aliasesProcessed.remove(cmd);
+      result+=expandAliasRec(aliasesProcessed,val);
+      if (!allowRecursion) aliasesProcessed.erase(cmd.str());
       p=i+l;
       if (hasArgs) p+=argsLen+2;
     }
@@ -7297,9 +7299,9 @@ QCString extractAliasArgs(const QCString &args,int pos)
 QCString resolveAliasCmd(const QCString aliasCmd)
 {
   QCString result;
-  aliasesProcessed.clear();
+  StringUnorderedSet aliasesProcessed;
   //printf("Expanding: '%s'\n",aliasCmd.data());
-  result = expandAliasRec(aliasCmd);
+  result = expandAliasRec(aliasesProcessed,aliasCmd);
   //printf("Expanding result: '%s'->'%s'\n",aliasCmd.data(),result.data());
   return result;
 }
@@ -7307,12 +7309,12 @@ QCString resolveAliasCmd(const QCString aliasCmd)
 QCString expandAlias(const QCString &aliasName,const QCString &aliasValue)
 {
   QCString result;
-  aliasesProcessed.clear();
+  StringUnorderedSet aliasesProcessed;
   // avoid expanding this command recursively
-  aliasesProcessed.insert(aliasName,(void *)0x8);
+  aliasesProcessed.insert(aliasName.str());
   // expand embedded commands
   //printf("Expanding: '%s'->'%s'\n",aliasName.data(),aliasValue.data());
-  result = expandAliasRec(aliasValue);
+  result = expandAliasRec(aliasesProcessed,aliasValue);
   //printf("Expanding result: '%s'->'%s'\n",aliasName.data(),result.data());
   return result;
 }
@@ -7520,7 +7522,7 @@ QCString filterTitle(const QCString &title)
 
 bool patternMatch(const QFileInfo &fi,const StringVector &patList)
 {
-  static bool caseSenseNames = Config_getBool(CASE_SENSE_NAMES);
+  bool caseSenseNames = Config_getBool(CASE_SENSE_NAMES);
   bool found = FALSE;
 
   // For platforms where the file system is non case sensitive overrule the setting