From 1efb1b23ce1547f5a38b7078d5e6ec69cc40f263 Mon Sep 17 00:00:00 2001 From: Dimitri van Heesch Date: Mon, 17 Aug 2020 22:45:22 +0200 Subject: issue #7954: The Doxygen uses too much memory (or has probably a memory leak) (part 2) --- src/pre.l | 62 ++++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/src/pre.l b/src/pre.l index 982e1b0..e36d1ec 100644 --- a/src/pre.l +++ b/src/pre.l @@ -136,6 +136,8 @@ class DefineManager { m_defines.emplace(kv.first,kv.second); } + //printf(" m_defines.size()=%zu\n",m_defines.size()); + m_stored=true; } void retrieve(DefineMap &toMap) { @@ -144,14 +146,15 @@ class DefineManager } void retrieveRec(DefineMap &toMap,StringSet &includeStack) { + //printf(" retrieveRec #includedFiles=%zu\n",m_includedFiles.size()); for (auto incFile : m_includedFiles) { DefinesPerFile *dpf = m_parent->find(incFile); if (dpf && includeStack.find(incFile)==includeStack.end()) { - //printf(" processing include %s\n",incFile.data()); includeStack.insert(incFile); dpf->retrieveRec(toMap,includeStack); + //printf(" retrieveRec: processing include %s: #toMap=%zu\n",incFile.data(),toMap.size()); } } for (auto &kv : m_defines) @@ -159,10 +162,12 @@ class DefineManager toMap.emplace(kv.first,kv.second); } } + bool stored() const { return m_stored; } private: DefineManager *m_parent; DefineMap m_defines; StringSet m_includedFiles; + bool m_stored = false; }; friend class DefinesPerFile; @@ -170,12 +175,14 @@ class DefineManager void addInclude(std::string fromFileName,std::string toFileName) { + //printf("DefineManager::addInclude('%s'->'%s')\n",fromFileName.c_str(),toFileName.c_str()); auto it = m_fileMap.find(fromFileName); - if (it!=m_fileMap.end()) + if (it==m_fileMap.end()) { - auto &dpf = it->second; - dpf->addInclude(toFileName); + it = m_fileMap.emplace(fromFileName,std::make_unique(this)).first; } + auto &dpf = it->second; + dpf->addInclude(toFileName); } void store(std::string fileName,const DefineMap &fromMap) @@ -191,18 +198,23 @@ class DefineManager void retrieve(std::string fileName,DefineMap &toMap) { - //printf("DefineManager::retrieve(%s,#=%zu)\n",fileName.c_str(),toMap.size()); auto it = m_fileMap.find(fileName); if (it!=m_fileMap.end()) { auto &dpf = it->second; dpf->retrieve(toMap); } + //printf("DefineManager::retrieve(%s,#=%zu)\n",fileName.c_str(),toMap.size()); } bool alreadyProcessed(std::string fileName) const { - return m_fileMap.find(fileName)!=m_fileMap.end(); + auto it = m_fileMap.find(fileName); + if (it!=m_fileMap.end()) + { + return it->second->stored(); + } + return false; } private: @@ -1548,17 +1560,6 @@ CHARLIT (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'")) // now that the file is completely processed, prevent it from processing it again g_defineManager.addInclude(yyextra->yyFileName.str(),toFileName.str()); g_defineManager.store(toFileName.str(),yyextra->localDefines); - // move the local macros definitions for in this file to the translation unit context - for (const auto &kv : yyextra->localDefines) - { - auto pair = yyextra->contextDefines.insert(kv); - if (!pair.second) // define already in context -> replace with local version - { - yyextra->contextDefines.erase(pair.first); - yyextra->contextDefines.insert(kv); - } - } - yyextra->localDefines.clear(); } else { @@ -1568,6 +1569,17 @@ CHARLIT (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'")) } } } + // move the local macros definitions for in this file to the translation unit context + for (const auto &kv : yyextra->localDefines) + { + auto pair = yyextra->contextDefines.insert(kv); + if (!pair.second) // define already in context -> replace with local version + { + yyextra->contextDefines.erase(pair.first); + yyextra->contextDefines.insert(kv); + } + } + yyextra->localDefines.clear(); } } <*>"/*"/"*/" | @@ -2716,9 +2728,9 @@ static void addDefine(yyscan_t yyscanner) def.lineNr = state->yyLineNr-state->yyMLines; def.columnNr = state->yyColNr; def.varArgs = state->defVarArgs; - //printf("newDefine: %s %s file: %s\n",def->name.data(),def->definition.data(), - // def->fileDef ? def->fileDef->name().data() : def->fileName.data()); - //printf("newDefine: '%s'->'%s'\n",def->name.data(),def->definition.data()); + //printf("newDefine: %s %s file: %s\n",def.name.data(),def.definition.data(), + // def.fileDef ? def.fileDef->name().data() : def.fileName.data()); + //printf("newDefine: '%s'->'%s'\n",def.name.data(),def.definition.data()); if (!def.name.isEmpty() && Doxygen::expandAsDefinedSet.find(def.name.str())!=Doxygen::expandAsDefinedSet.end()) { @@ -2868,6 +2880,11 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc) //printf("calling findFile(%s)\n",incFileName.data()); if ((fs=findFile(yyscanner,incFileName,localInclude,alreadyProcessed))) // see if the include file can be found { + { + std::lock_guard lock(g_globalDefineMutex); + g_defineManager.addInclude(oldFileName.str(),absIncFileName.str()); + } + //printf("Found include file!\n"); if (Debug::isFlagSet(Debug::Preprocessor)) { @@ -2924,6 +2941,7 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc) // in the local context { std::lock_guard lock(g_globalDefineMutex); + g_defineManager.addInclude(state->yyFileName.str(),absIncFileName.str()); g_defineManager.retrieve(absIncFileName.str(),state->contextDefines); } @@ -3371,6 +3389,10 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output { Debug::print(Debug::Preprocessor,0,"%s ",qPrint(kv.second.name)); } + for (auto &kv : yyextra->localDefines) + { + Debug::print(Debug::Preprocessor,0,"%s ",qPrint(kv.second.name)); + } Debug::print(Debug::Preprocessor,0,"\n---------\n"); } else -- cgit v0.12