issue #7954: The Doxygen uses too much memory (or has probably a memory leak)

1 files changed, 121 insertions, 74 deletions

diff --git a/src/pre.l b/src/pre.l
index ff04162..982e1b0 100644
--- a/src/pre.l
+++ b/src/pre.l
@@ -109,7 +109,7 @@ struct PreIncludeInfo
 };
 
 /** A dictionary of managed Define objects. */
-typedef std::map< std::string,std::unique_ptr<Define> > DefineMapOwning;
+typedef std::map< std::string, Define > DefineMap;
 
 /** @brief Class that manages the defines available while 
  *  preprocessing files. 
@@ -130,19 +130,19 @@ class DefineManager
         {
           m_includedFiles.insert(fileName);
         }
-        void store(const DefineMapOwning &fromMap)
+        void store(const DefineMap &fromMap)
         {
           for (auto &kv : fromMap)
           {
-            m_defines.emplace(kv.first,std::make_unique<Define>(*kv.second.get()));
+            m_defines.emplace(kv.first,kv.second);
           }
         }
-        void retrieve(DefineMapOwning &toMap)
+        void retrieve(DefineMap &toMap)
         {
           StringSet includeStack;
           retrieveRec(toMap,includeStack);
         }
-        void retrieveRec(DefineMapOwning &toMap,StringSet &includeStack)
+        void retrieveRec(DefineMap &toMap,StringSet &includeStack)
         {
           for (auto incFile : m_includedFiles)
           {
@@ -156,12 +156,12 @@ class DefineManager
           }
           for (auto &kv : m_defines)
           {
-            toMap.emplace(kv.first,std::make_unique<Define>(*kv.second.get()));
+            toMap.emplace(kv.first,kv.second);
           }
         }
       private:
         DefineManager *m_parent;
-        DefineMapOwning m_defines;
+        DefineMap m_defines;
         StringSet m_includedFiles;
     };
 
@@ -178,8 +178,9 @@ class DefineManager
       }
     }
 
-    void store(std::string fileName,const DefineMapOwning &fromMap)
+    void store(std::string fileName,const DefineMap &fromMap)
     {
+      //printf("DefineManager::store(%s,#=%zu)\n",fileName.c_str(),fromMap.size());
       auto it = m_fileMap.find(fileName);
       if (it==m_fileMap.end())
       {
@@ -188,8 +189,9 @@ class DefineManager
       it->second->store(fromMap);
     }
 
-    void retrieve(std::string fileName,DefineMapOwning &toMap)
+    void retrieve(std::string fileName,DefineMap &toMap)
     {
+      //printf("DefineManager::retrieve(%s,#=%zu)\n",fileName.c_str(),toMap.size());
       auto it = m_fileMap.find(fileName);
       if (it!=m_fileMap.end())
       {
@@ -277,6 +279,7 @@ struct preYY_state
   yy_size_t          fenceSize      = 0;
   bool               ccomment       = false;
   QCString           delimiter;
+  bool               isSpecialComment = false;
   StringVector                             pathList;
   IntMap                                   argMap;
   BoolStack                                levelGuard;
@@ -285,7 +288,8 @@ struct preYY_state
   std::unordered_map<std::string,Define*>  expandedDict;
   StringUnorderedSet                       expanded;
   ConstExpressionParser                    constExpParser;
-  DefineMapOwning                          contextDefines;
+  DefineMap                                contextDefines; // macros imported from other files
+  DefineMap                                localDefines;   // macros defined in this file
   DefineList                               macroDefinitions;
   LinkedMap<PreIncludeInfo>                includeRelations;
 };
@@ -1131,7 +1135,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
   					}
 <SkipCComment>^({B}*"*"+)?{B}{0,3}"~~~"[~]*   {
                                           bool markdownSupport = Config_getBool(MARKDOWN_SUPPORT);
-                                          if (!markdownSupport)
+                                          if (!markdownSupport || !yyextra->isSpecialComment)
                                           {
                                             REJECT;
                                           }
@@ -1144,7 +1148,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
                                         }
 <SkipCComment>^({B}*"*"+)?{B}{0,3}"```"[`]*            {
                                           bool markdownSupport = Config_getBool(MARKDOWN_SUPPORT);
-                                          if (!markdownSupport)
+                                          if (!markdownSupport || !yyextra->isSpecialComment)
                                           {
                                             REJECT;
                                           }
@@ -1543,7 +1547,18 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
                                               {
                                                 // now that the file is completely processed, prevent it from processing it again
                                                 g_defineManager.addInclude(yyextra->yyFileName.str(),toFileName.str());
-                                                g_defineManager.store(toFileName.str(),yyextra->contextDefines);
+                                                g_defineManager.store(toFileName.str(),yyextra->localDefines);
+                                                // move the local macros definitions for in this file to the translation unit context
+                                                for (const auto &kv : yyextra->localDefines)
+                                                {
+                                                  auto pair = yyextra->contextDefines.insert(kv);
+                                                  if (!pair.second) // define already in context -> replace with local version
+                                                  {
+                                                    yyextra->contextDefines.erase(pair.first);
+                                                    yyextra->contextDefines.insert(kv);
+                                                  }
+                                                }
+                                                yyextra->localDefines.clear();
                                               }
                                               else
                                               {
@@ -1556,7 +1571,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
 					  }
   					}
 <*>"/*"/"*/"				|
-<*>"/*"[*]?				{
+<*>"/*"[*!]?				{
                                           if (YY_START==SkipVerbatim || YY_START==SkipCond)
                                           {
                                             REJECT;
@@ -1566,11 +1581,19 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
 					    outputArray(yyscanner,yytext,(int)yyleng);
   					    yyextra->lastCContext=YY_START;
 					    yyextra->commentCount=1;
-					    if (yyleng==3) yyextra->lastGuardName.resize(0); // reset guard in case the #define is documented!
+					    if (yyleng==3) 
+                                            {
+                                              yyextra->isSpecialComment = true;
+                                              yyextra->lastGuardName.resize(0); // reset guard in case the #define is documented!
+                                            }
+                                            else
+                                            {
+                                              yyextra->isSpecialComment = false;
+                                            }
 					    BEGIN(SkipCComment);
                                           }
   					}
-<*>"//"[/]?				{
+<*>"//"[/!]?				{
                                           if (YY_START==SkipVerbatim || YY_START==SkipCond || getLanguageFromFileName(yyextra->yyFileName)==SrcLangExt_Fortran)
                                           {
                                             REJECT;
@@ -1579,7 +1602,15 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
                                           {
 					    outputArray(yyscanner,yytext,(int)yyleng);
   					    yyextra->lastCPPContext=YY_START;
-					    if (yyleng==3) yyextra->lastGuardName.resize(0); // reset guard in case the #define is documented!
+					    if (yyleng==3)
+                                            {
+                                              yyextra->isSpecialComment = true;
+                                              yyextra->lastGuardName.resize(0); // reset guard in case the #define is documented!
+                                            }
+                                            else
+                                            {
+                                              yyextra->isSpecialComment = false;
+                                            }
 					    BEGIN(SkipCPPComment);
                                           }
 					}
@@ -2676,29 +2707,29 @@ static QCString expandMacro(yyscan_t yyscanner,const QCString &name)
 static void addDefine(yyscan_t yyscanner)
 {
   YY_EXTRA_TYPE state = preYYget_extra(yyscanner);
-  std::unique_ptr<Define> def = std::make_unique<Define>();
-  def->name       = state->defName;
-  def->definition = state->defText.stripWhiteSpace();
-  def->nargs      = state->defArgs;
-  def->fileName   = state->yyFileName; 
-  def->fileDef    = state->yyFileDef;
-  def->lineNr     = state->yyLineNr-state->yyMLines;
-  def->columnNr   = state->yyColNr;
-  def->varArgs    = state->defVarArgs;
+  Define def;
+  def.name       = state->defName;
+  def.definition = state->defText.stripWhiteSpace();
+  def.nargs      = state->defArgs;
+  def.fileName   = state->yyFileName; 
+  def.fileDef    = state->yyFileDef;
+  def.lineNr     = state->yyLineNr-state->yyMLines;
+  def.columnNr   = state->yyColNr;
+  def.varArgs    = state->defVarArgs;
   //printf("newDefine: %s %s file: %s\n",def->name.data(),def->definition.data(),
   //    def->fileDef ? def->fileDef->name().data() : def->fileName.data());
   //printf("newDefine: '%s'->'%s'\n",def->name.data(),def->definition.data());
-  if (!def->name.isEmpty() && 
-      Doxygen::expandAsDefinedSet.find(def->name.str())!=Doxygen::expandAsDefinedSet.end())
+  if (!def.name.isEmpty() && 
+      Doxygen::expandAsDefinedSet.find(def.name.str())!=Doxygen::expandAsDefinedSet.end())
   {
-    def->isPredefined=TRUE;
+    def.isPredefined=TRUE;
   }
-  auto it = state->contextDefines.find(def->name.str());
-  if (it!=state->contextDefines.end()) // redefine
+  auto it = state->localDefines.find(def.name.str());
+  if (it!=state->localDefines.end()) // redefine
   {
-    state->contextDefines.erase(it);
+    state->localDefines.erase(it);
   }
-  state->contextDefines.insert(std::make_pair(toStdString(def->name),std::move(def)));
+  state->localDefines.insert(std::make_pair(def.name.str(),def));
 }
 
 static void addMacroDefinition(yyscan_t yyscanner)
@@ -2707,13 +2738,13 @@ static void addMacroDefinition(yyscan_t yyscanner)
   if (state->skip) return; // do not add this define as it is inside a 
                       // conditional section (cond command) that is disabled.
 
-  auto define = std::make_unique<Define>();
-  define->fileName = state->yyFileName;
-  define->lineNr   = state->yyLineNr - state->yyMLines;
-  define->columnNr = state->yyColNr;
-  define->name     = state->defName;
-  define->args     = state->defArgsStr;
-  define->fileDef  = state->inputFileDef;
+  Define define;
+  define.fileName = state->yyFileName;
+  define.lineNr   = state->yyLineNr - state->yyMLines;
+  define.columnNr = state->yyColNr;
+  define.name     = state->defName;
+  define.args     = state->defArgsStr;
+  define.fileDef  = state->inputFileDef;
 
   QCString litText = state->defLitText;
   int l=litText.find('\n');
@@ -2734,14 +2765,14 @@ static void addMacroDefinition(yyscan_t yyscanner)
   QCString litTextStripped = state->defLitText.stripWhiteSpace();
   if (litTextStripped.contains('\n')>=1)
   {
-    define->definition = litText;
+    define.definition = litText;
   }
   else
   {
-    define->definition = litTextStripped;
+    define.definition = litTextStripped;
   }
   {
-    state->macroDefinitions.push_back(std::move(define));
+    state->macroDefinitions.push_back(define);
   }
 }
 
@@ -2844,7 +2875,7 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
         {
           Debug::print(Debug::Preprocessor,0,"  ");
         }
-        //msg("#include %s: parsing...\n",incFileName.data());
+        Debug::print(Debug::Preprocessor,0,"#include %s: parsing...\n",incFileName.data());
       }
 
       if (state->includeStack.empty() && oldFileDef)
@@ -2915,6 +2946,10 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
 
       if (Debug::isFlagSet(Debug::Preprocessor))
       {
+        for (i=0;i<state->includeStack.size();i++) 
+        {
+          Debug::print(Debug::Preprocessor,0,"  ");
+        }
 	if (alreadyProcessed)
 	{
           Debug::print(Debug::Preprocessor,0,"#include %s: already processed! skipping...\n",qPrint(incFileName));
@@ -2929,7 +2964,6 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
       {
 	warn(state->yyFileName,state->yyLineNr,"include file %s not found, perhaps you forgot to add its directory to INCLUDE_PATH?",incFileName.data());
       }
-
     }
   }
 }
@@ -3084,17 +3118,30 @@ static void unputChar(yyscan_t yyscanner,const QCString &expr,QCString *rest,uin
 static Define *isDefined(yyscan_t yyscanner,const char *name)
 {
   YY_EXTRA_TYPE state = preYYget_extra(yyscanner);
-  Define *d=0;
-  auto it = state->contextDefines.find(name);
-  if (it!=state->contextDefines.end())
+
+  bool undef = false;
+  auto findDefine = [&undef,&name](DefineMap &map)
   {
-    d = it->second.get();
-    if (d->undef)
+    Define *d=0;
+    auto it = map.find(name);
+    if (it!=map.end())
     {
-      d=0;
+      d = &it->second;
+      if (d->undef)
+      {
+        undef=true;
+        d=0;
+      }
     }
+    return d;
+  };
+
+  Define *def = findDefine(state->localDefines);
+  if (def==0 && !undef)
+  {
+    def = findDefine(state->contextDefines);
   }
-  return d;
+  return def;
 }
 
 static void initPredefined(yyscan_t yyscanner,const char *fileName)
@@ -3168,15 +3215,15 @@ static void initPredefined(yyscan_t yyscanner,const char *fileName)
       QCString dname = ds.left(i_obrace);
       if (!dname.isEmpty())
       {
-        std::unique_ptr<Define> def = std::make_unique<Define>();
-        def->name         = dname;
-        def->definition   = definition; 
-        def->nargs        = count;
-        def->isPredefined = TRUE;
-        def->nonRecursive = nonRecursive;
-        def->fileDef      = state->yyFileDef;
-        def->fileName     = fileName;
-        state->contextDefines.insert(std::make_pair(toStdString(def->name),std::move(def)));
+        Define def;
+        def.name         = dname;
+        def.definition   = definition;
+        def.nargs        = count;
+        def.isPredefined = TRUE;
+        def.nonRecursive = nonRecursive;
+        def.fileDef      = state->yyFileDef;
+        def.fileName     = fileName;
+        state->contextDefines.insert(std::make_pair(def.name.str(),def));
 
         //printf("#define '%s' '%s' #nargs=%d\n",
         //  def->name.data(),def->definition.data(),def->nargs);
@@ -3189,26 +3236,26 @@ static void initPredefined(yyscan_t yyscanner,const char *fileName)
         ) // predefined non-function macro definition
     {
       //printf("predefined normal macro '%s'\n",defStr);
-      std::unique_ptr<Define> def = std::make_unique<Define>();
+      Define def;
       if (i_equals==-1) // simple define without argument
       {
-        def->name = ds;
-        def->definition = "1"; // substitute occurrences by 1 (true)
+        def.name = ds;
+        def.definition = "1"; // substitute occurrences by 1 (true)
       }
       else // simple define with argument
       {
         int ine=i_equals - (nonRecursive ? 1 : 0);
-        def->name = ds.left(ine);
-        def->definition = ds.right(ds.length()-i_equals-1);
+        def.name = ds.left(ine);
+        def.definition = ds.right(ds.length()-i_equals-1);
       }
-      if (!def->name.isEmpty())
+      if (!def.name.isEmpty())
       {
-        def->nargs = -1;
-        def->isPredefined = TRUE;
-        def->nonRecursive = nonRecursive;
-        def->fileDef      = state->yyFileDef;
-        def->fileName     = fileName;
-        state->contextDefines.insert(std::make_pair(toStdString(def->name),std::move(def)));
+        def.nargs = -1;
+        def.isPredefined = TRUE;
+        def.nonRecursive = nonRecursive;
+        def.fileDef      = state->yyFileDef;
+        def.fileName     = fileName;
+        state->contextDefines.insert(std::make_pair(def.name.str(),def));
       }
     }
   }
@@ -3322,7 +3369,7 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output
       Debug::print(Debug::Preprocessor,0,"---------\n");
       for (auto &kv : yyextra->contextDefines)
       {
-        Debug::print(Debug::Preprocessor,0,"%s ",qPrint(kv.second->name));
+        Debug::print(Debug::Preprocessor,0,"%s ",qPrint(kv.second.name));
       }
       Debug::print(Debug::Preprocessor,0,"\n---------\n");
     }