Refactor: make preprocessor run in parallel

And at the same time make sure it gives the same results as when
processed using a single thread.

10 files changed, 408 insertions, 481 deletions

diff --git a/src/context.cpp b/src/context.cpp
index f0b697a..83e8e7a 100644
--- a/src/context.cpp
+++ b/src/context.cpp
@@ -1774,10 +1774,7 @@ class IncludeInfoListContext::Private : public GenericNodeListContext
       IncludeInfo *ii;
       for (li.toFirst();(ii=li.current());++li)
       {
-        if (!ii->indirect)
-        {
-          append(IncludeInfoContext::alloc(ii,lang));
-        }
+        append(IncludeInfoContext::alloc(ii,lang));
       }
     }
 };
diff --git a/src/define.h b/src/define.h
index 0a3d62c..015d399 100644
--- a/src/define.h
+++ b/src/define.h
@@ -16,11 +16,13 @@
 #ifndef DEFINE_H
 #define DEFINE_H
 
-#include <map>
-#include <string>
+#include <vector>
 #include <memory>
+#include <string>
+#include <unordered_map>
 
 #include <qcstring.h>
+#include "containers.h"
 
 class FileDef;
 
@@ -44,5 +46,6 @@ class Define
 
 /** List of all macro definitions */
 using DefineList = std::vector< std::unique_ptr<Define> >;
+using DefinesPerFileList = std::unordered_map< std::string, DefineList >;
 
 #endif
diff --git a/src/doxygen.cpp b/src/doxygen.cpp
index d8436bb..01b47f5 100644
--- a/src/doxygen.cpp
+++ b/src/doxygen.cpp
@@ -159,7 +159,7 @@ bool             Doxygen::userComments = FALSE;
 QCString         Doxygen::spaces;
 bool             Doxygen::generatingXmlOutput = FALSE;
 GenericsSDict   *Doxygen::genericsDict;
-DefineList       Doxygen::macroDefinitions;
+DefinesPerFileList Doxygen::macroDefinitions;
 
 // locally accessible globals
 static std::unordered_map< std::string, const Entry* > g_classEntries;
@@ -7701,28 +7701,35 @@ static void addSourceReferences()
 // add the macro definitions found during preprocessing as file members
 static void buildDefineList()
 {
-  for (const auto &def : Doxygen::macroDefinitions)
+  for (const auto &s : g_inputFiles)
   {
-    std::unique_ptr<MemberDef> md { createMemberDef(
-      def->fileName,def->lineNr,def->columnNr,
-      "#define",def->name,def->args,0,
-      Public,Normal,FALSE,Member,MemberType_Define,
-      ArgumentList(),ArgumentList(),"") };
-
-    if (!def->args.isEmpty())
+    auto it = Doxygen::macroDefinitions.find(s);
+    if (it!=Doxygen::macroDefinitions.end())
     {
-      md->moveArgumentList(stringToArgumentList(SrcLangExt_Cpp, def->args));
-    }
-    md->setInitializer(def->definition);
-    md->setFileDef(def->fileDef);
-    md->setDefinition("#define "+def->name);
+      for (const auto &def : it->second)
+      {
+        std::unique_ptr<MemberDef> md { createMemberDef(
+            def->fileName,def->lineNr,def->columnNr,
+            "#define",def->name,def->args,0,
+            Public,Normal,FALSE,Member,MemberType_Define,
+            ArgumentList(),ArgumentList(),"") };
 
-    MemberName *mn=Doxygen::functionNameLinkedMap->add(def->name);
-    if (def->fileDef)
-    {
-      def->fileDef->insertMember(md.get());
+        if (!def->args.isEmpty())
+        {
+          md->moveArgumentList(stringToArgumentList(SrcLangExt_Cpp, def->args));
+        }
+        md->setInitializer(def->definition);
+        md->setFileDef(def->fileDef);
+        md->setDefinition("#define "+def->name);
+
+        MemberName *mn=Doxygen::functionNameLinkedMap->add(def->name);
+        if (def->fileDef)
+        {
+          def->fileDef->insertMember(md.get());
+        }
+        mn->push_back(std::move(md));
+      }
     }
-    mn->push_back(std::move(md));
   }
 }
 
diff --git a/src/doxygen.h b/src/doxygen.h
index 3198c82..2978052 100644
--- a/src/doxygen.h
+++ b/src/doxygen.h
@@ -30,7 +30,9 @@
 #include "memberlist.h"
 #include "define.h"
 
+#ifndef MULTITHREADED_INPUT
 #define MULTITHREADED_INPUT 0
+#endif
 
 #if MULTITHREADED_INPUT
 #define THREAD_LOCAL thread_local
@@ -149,7 +151,7 @@ class Doxygen
     static QCString                  spaces;
     static bool                      generatingXmlOutput;
     static GenericsSDict            *genericsDict;
-    static DefineList                macroDefinitions;
+    static DefinesPerFileList        macroDefinitions;
 };
 
 void initDoxygen();
diff --git a/src/filedef.cpp b/src/filedef.cpp
index 91a7835..18e5cef 100644
--- a/src/filedef.cpp
+++ b/src/filedef.cpp
@@ -117,7 +117,7 @@ class FileDefImpl : public DefinitionImpl, public FileDef
     virtual void combineUsingRelations();
     virtual bool generateSourceFile() const;
     virtual void sortMemberLists();
-    virtual void addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported,bool indirect);
+    virtual void addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported);
     virtual void addIncludedByDependency(FileDef *fd,const char *incName,bool local,bool imported);
     virtual void addMembersToMemberGroup();
     virtual void distributeMemberGroupDocumentation();
@@ -357,25 +357,22 @@ void FileDefImpl::writeTagFile(FTextStream &tagFile)
     IncludeInfo *ii;
     for (;(ii=ili.current());++ili)
     {
-      if (!ii->indirect)
+      FileDef *fd=ii->fileDef;
+      if (fd && fd->isLinkable() && !fd->isReference())
       {
-        FileDef *fd=ii->fileDef;
-        if (fd && fd->isLinkable() && !fd->isReference())
-        {
-          bool isIDLorJava = FALSE;
-          SrcLangExt lang = fd->getLanguage();
-          isIDLorJava = lang==SrcLangExt_IDL || lang==SrcLangExt_Java;
-          const char *locStr = (ii->local    || isIDLorJava) ? "yes" : "no";
-          const char *impStr = (ii->imported || isIDLorJava) ? "yes" : "no";
-          tagFile << "    <includes id=\""
-                  << convertToXML(fd->getOutputFileBase()) << "\" "
-                  << "name=\"" << convertToXML(fd->name()) << "\" "
-                  << "local=\"" << locStr << "\" "
-                  << "imported=\"" << impStr << "\">"
-                  << convertToXML(ii->includeName)
-                  << "</includes>"
-                  << endl;
-        }
+        bool isIDLorJava = FALSE;
+        SrcLangExt lang = fd->getLanguage();
+        isIDLorJava = lang==SrcLangExt_IDL || lang==SrcLangExt_Java;
+        const char *locStr = (ii->local    || isIDLorJava) ? "yes" : "no";
+        const char *impStr = (ii->imported || isIDLorJava) ? "yes" : "no";
+        tagFile << "    <includes id=\""
+          << convertToXML(fd->getOutputFileBase()) << "\" "
+          << "name=\"" << convertToXML(fd->name()) << "\" "
+          << "local=\"" << locStr << "\" "
+          << "imported=\"" << impStr << "\">"
+          << convertToXML(ii->includeName)
+          << "</includes>"
+          << endl;
       }
     }
   }
@@ -600,61 +597,58 @@ void FileDefImpl::writeIncludeFiles(OutputList &ol)
     IncludeInfo *ii;
     for (;(ii=ili.current());++ili)
     {
-      if (!ii->indirect)
+      FileDef *fd=ii->fileDef;
+      bool isIDLorJava = FALSE;
+      if (fd)
       {
-        FileDef *fd=ii->fileDef;
-        bool isIDLorJava = FALSE;
-        if (fd)
-        {
-          SrcLangExt lang   = fd->getLanguage();
-          isIDLorJava = lang==SrcLangExt_IDL || lang==SrcLangExt_Java;
-        }
-        ol.startTypewriter();
-        if (isIDLorJava) // IDL/Java include
-        {
-          ol.docify("import ");
-        }
-        else if (ii->imported) // Objective-C include
-        {
-          ol.docify("#import ");
-        }
-        else // C/C++ include
-        {
-          ol.docify("#include ");
-        }
-        if (ii->local || isIDLorJava)
-          ol.docify("\"");
-        else
-          ol.docify("<");
-        ol.disable(OutputGenerator::Html);
-        ol.docify(ii->includeName);
-        ol.enableAll();
-        ol.disableAllBut(OutputGenerator::Html);
-
-        // Here we use the include file name as it appears in the file.
-        // we could also we the name as it is used within doxygen,
-        // then we should have used fd->docName() instead of ii->includeName
-        if (fd && fd->isLinkable())
-        {
-          ol.writeObjectLink(fd->getReference(),
-              fd->generateSourceFile() ? fd->includeName() : fd->getOutputFileBase(),
-              0,ii->includeName);
-        }
-        else
-        {
-          ol.docify(ii->includeName);
-        }
+        SrcLangExt lang   = fd->getLanguage();
+        isIDLorJava = lang==SrcLangExt_IDL || lang==SrcLangExt_Java;
+      }
+      ol.startTypewriter();
+      if (isIDLorJava) // IDL/Java include
+      {
+        ol.docify("import ");
+      }
+      else if (ii->imported) // Objective-C include
+      {
+        ol.docify("#import ");
+      }
+      else // C/C++ include
+      {
+        ol.docify("#include ");
+      }
+      if (ii->local || isIDLorJava)
+        ol.docify("\"");
+      else
+        ol.docify("<");
+      ol.disable(OutputGenerator::Html);
+      ol.docify(ii->includeName);
+      ol.enableAll();
+      ol.disableAllBut(OutputGenerator::Html);
 
-        ol.enableAll();
-        if (ii->local || isIDLorJava)
-          ol.docify("\"");
-        else
-          ol.docify(">");
-        if (isIDLorJava)
-          ol.docify(";");
-        ol.endTypewriter();
-        ol.lineBreak();
+      // Here we use the include file name as it appears in the file.
+      // we could also we the name as it is used within doxygen,
+      // then we should have used fd->docName() instead of ii->includeName
+      if (fd && fd->isLinkable())
+      {
+        ol.writeObjectLink(fd->getReference(),
+            fd->generateSourceFile() ? fd->includeName() : fd->getOutputFileBase(),
+            0,ii->includeName);
+      }
+      else
+      {
+        ol.docify(ii->includeName);
       }
+
+      ol.enableAll();
+      if (ii->local || isIDLorJava)
+        ol.docify("\"");
+      else
+        ol.docify(">");
+      if (isIDLorJava)
+        ol.docify(";");
+      ol.endTypewriter();
+      ol.lineBreak();
     }
     ol.endTextBlock();
   }
@@ -1544,8 +1538,7 @@ void FileDefImpl::addUsingDeclaration(Definition *d)
   }
 }
 
-void FileDefImpl::addIncludeDependency(FileDef *fd,const char *incName,bool local,
-                                   bool imported,bool indirect)
+void FileDefImpl::addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported)
 {
   //printf("FileDefImpl::addIncludeDependency(%p,%s,%d)\n",fd,incName,local);
   QCString iName = fd ? fd->absFilePath().data() : incName;
@@ -1562,7 +1555,6 @@ void FileDefImpl::addIncludeDependency(FileDef *fd,const char *incName,bool loca
     ii->includeName = incName;
     ii->local       = local;
     ii->imported    = imported;
-    ii->indirect    = indirect;
     m_includeList->append(ii);
     m_includeDict->insert(iName,ii);
   }
@@ -1659,7 +1651,6 @@ void FileDefImpl::addIncludedByDependency(FileDef *fd,const char *incName,
     ii->includeName = incName;
     ii->local       = local;
     ii->imported    = imported;
-    ii->indirect    = FALSE;
     m_includedByList->append(ii);
     m_includedByDict->insert(iName,ii);
   }
diff --git a/src/filedef.h b/src/filedef.h
index 2510d85..c03e7ef 100644
--- a/src/filedef.h
+++ b/src/filedef.h
@@ -44,13 +44,12 @@ class FTextStream;
 /** Class representing the data associated with a \#include statement. */
 struct IncludeInfo
 {
-  IncludeInfo() : fileDef(0), local(FALSE), imported(FALSE), indirect(FALSE) {}
+  IncludeInfo() : fileDef(0), local(FALSE), imported(FALSE) {}
   ~IncludeInfo() {}
   FileDef *fileDef;
   QCString includeName;
   bool local;
   bool imported;
-  bool indirect;
 };
 
 /** A model of a file symbol.
@@ -170,7 +169,7 @@ class FileDef : virtual public Definition
     virtual bool generateSourceFile() const = 0;
     virtual void sortMemberLists() = 0;
 
-    virtual void addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported,bool indirect) = 0;
+    virtual void addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported) = 0;
     virtual void addIncludedByDependency(FileDef *fd,const char *incName,bool local,bool imported) = 0;
 
     virtual void addMembersToMemberGroup() = 0;
diff --git a/src/pre.l b/src/pre.l
index 4ec76ab..9b27ff8 100644
--- a/src/pre.l
+++ b/src/pre.l
@@ -26,11 +26,16 @@
  *	includes
  */
 
+#include "doxygen.h"
+
 #include <stack>
 #include <deque>
 #include <algorithm>
 #include <utility>
+#if MULTITHREADED_INPUT
 #include <mutex>
+#include <thread>
+#endif
 
 #include <stdio.h>
 #include <assert.h>
@@ -45,7 +50,6 @@
 #include "pre.h"
 #include "constexp.h"
 #include "define.h"
-#include "doxygen.h"
 #include "message.h"
 #include "util.h"
 #include "defargs.h"
@@ -92,280 +96,138 @@ struct FileState
   QCString fileName;
 };
 
-/** A dictionary of references to Define objects. */
-typedef std::map< std::string,Define* > DefineMapRef;
+struct PreIncludeInfo
+{
+  PreIncludeInfo(const char *fn,FileDef *srcFd, FileDef *dstFd,const char *iName,bool loc, bool imp)
+    : fileName(fn), fromFileDef(srcFd), toFileDef(dstFd), includeName(iName), local(loc), imported(imp)
+  {
+  }
+  QCString fileName;    // file name in which the include statement was found
+  FileDef *fromFileDef; // filedef in which the include statement was found
+  FileDef *toFileDef;   // filedef to which the include is pointing
+  QCString includeName; // name used in the #include statement
+  bool local;           // is it a "local" or <global> include
+  bool imported;        // include via "import" keyword (Objective-C)
+};
 
 /** A dictionary of managed Define objects. */
 typedef std::map< std::string,std::unique_ptr<Define> > DefineMapOwning;
 
-/** @brief Singleton that manages the defines available while 
+/** @brief Class that manages the defines available while 
  *  preprocessing files. 
  */
 class DefineManager
 {
-  /** Local class used to hold the defines for a single file */
-  class DefinesPerFile
-  {
-    public:
-      /** Creates an empty container for defines */
-      DefinesPerFile(DefineManager *parent) 
-         : m_parent(parent)
-      {
-      }
-      /** Destroys the object */
-      virtual ~DefinesPerFile()
-      {
-      }
-      /** Adds a define in the context of a file. Will replace 
-       *  an existing define with the same name (redefinition)
-       *  @param def The Define object to add. Ownership will be transferred.
-       */
-      void addDefine(std::unique_ptr<Define> &&def)
-      {
-	auto it = m_defines.find(def->name.data());
-	if (it!=m_defines.end()) // redefine
-	{
-	  m_defines.erase(it);
-	}
-	m_defines.insert(std::make_pair(toStdString(def->name),std::move(def)));
-      }
-      /** Adds an include file for this file 
-       *  @param fileName The name of the include file
-       */
-      void addInclude(const char *fileName)
-      {
-	m_includedFiles.insert(fileName);
-      }
-      void collectDefines(DefineMapRef &map,StringSet &includeStack);
-    private:
-      DefineManager *m_parent;
-      DefineMapOwning m_defines;
-      StringSet m_includedFiles;
-  };
-
-  public:
-    friend class DefinesPerFile;
-
-    /** Creates a new DefineManager object */
-    DefineManager()
+  private:
+    /** Local class used to hold the defines for a single file */
+    class DefinesPerFile
     {
-    }
+      public:
+        /** Creates an empty container for defines */
+        DefinesPerFile(DefineManager *parent) 
+          : m_parent(parent)
+        {
+        }
+        void addInclude(std::string fileName)
+        {
+          m_includedFiles.insert(fileName);
+        }
+        void store(const DefineMapOwning &fromMap)
+        {
+          for (auto &kv : fromMap)
+          {
+            m_defines.emplace(kv.first,std::make_unique<Define>(*kv.second.get()));
+          }
+        }
+        void retrieve(DefineMapOwning &toMap)
+        {
+          StringSet includeStack;
+          retrieveRec(toMap,includeStack);
+        }
+        void retrieveRec(DefineMapOwning &toMap,StringSet &includeStack)
+        {
+          for (auto incFile : m_includedFiles)
+          {
+            DefinesPerFile *dpf = m_parent->find(incFile);
+            if (dpf && includeStack.find(incFile)==includeStack.end())
+            {
+              //printf("  processing include %s\n",incFile.data());
+              includeStack.insert(incFile);
+              dpf->retrieveRec(toMap,includeStack);
+            }
+          }
+          for (auto &kv : m_defines)
+          {
+            toMap.emplace(kv.first,std::make_unique<Define>(*kv.second.get()));
+          }
+        }
+      private:
+        DefineManager *m_parent;
+        DefineMapOwning m_defines;
+        StringSet m_includedFiles;
+    };
 
-    /** Destroys the object */
-    virtual ~DefineManager() 
-    {
-    }
+    friend class DefinesPerFile;
+  public:
 
-    /** Starts a context in which defines are collected. 
-     *  Called at the start of a new file that is preprocessed.
-     *  @param fileName the name of the file to process.
-     */
-    void startContext(const char *fileName)
-    {
-#if MULTITHREADED_INPUT
-      std::unique_lock<std::mutex> lock(m_mutex);
-#endif
-      //printf("DefineManager::startContext()\n");
-      m_contextDefines.clear();
-      if (fileName==0) return;
-      //DefinesPerFile *dpf = m_fileMap.find(fileName);
-      auto it = m_fileMap.find(fileName);
-      if (it==m_fileMap.end())
-      {
-	//printf("New file!\n");
-	m_fileMap.emplace(toStdString(fileName),std::make_unique<DefinesPerFile>(this));
-      }
-    }
-    /** Ends the context started with startContext() freeing any
-     *  defines collected within in this context.
-     */
-    void endContext()
+    void addInclude(std::string fromFileName,std::string toFileName)
     {
-#if MULTITHREADED_INPUT
-      std::unique_lock<std::mutex> lock(m_mutex);
-#endif
-      //printf("DefineManager::endContext()\n");
-      m_contextDefines.clear();
-    }
-    /** Add an included file to the current context.
-     *  If the file has been pre-processed already, all defines are added
-     *  to the context.
-     *  @param fileName The name of the include file to add to the context.
-     */
-    void addFileToContext(const char *fileName)
-    {
-      if (fileName==0) return;
-#if MULTITHREADED_INPUT
-      std::unique_lock<std::mutex> lock(m_mutex);
-#endif
-
-      //printf("DefineManager::addFileToContext(%s)\n",fileName);
-      auto it = m_fileMap.find(fileName);
-      if (it==m_fileMap.end())
-      {
-	//printf("New file!\n");
-	m_fileMap.emplace(toStdString(fileName),std::make_unique<DefinesPerFile>(this));
-      }
-      else
+      auto it = m_fileMap.find(fromFileName);
+      if (it!=m_fileMap.end())
       {
-	//printf("existing file!\n");
-        StringSet includeStack;
-	it->second->collectDefines(m_contextDefines,includeStack);
+        auto &dpf = it->second;
+        dpf->addInclude(toFileName);
       }
     }
 
-    /** Add a define to the manager object.
-     *  @param fileName The file in which the define was found
-     *  @param def The Define object to add. Ownership will be transferred.
-     */
-    void addDefine(const char *fileName,std::unique_ptr<Define> &&def)
+    void store(std::string fileName,const DefineMapOwning &fromMap)
     {
-      if (fileName==0) return;
-
-#if MULTITHREADED_INPUT
-      std::unique_lock<std::mutex> lock(m_mutex);
-#endif
-      //printf("DefineManager::addDefine(%s,%s)\n",fileName,def->name.data());
-
-      m_contextDefines[def->name.data()] = def.get();
-
       auto it = m_fileMap.find(fileName);
       if (it==m_fileMap.end())
       {
-        auto ptr = std::make_unique<DefinesPerFile>(this);
-        ptr->addDefine(std::move(def));
-	m_fileMap.emplace(toStdString(fileName),std::move(ptr));
-      }
-      else
-      {
-        it->second->addDefine(std::move(def));
+        it = m_fileMap.emplace(fileName,std::make_unique<DefinesPerFile>(this)).first;
       }
+      it->second->store(fromMap);
     }
 
-    /** Add an include relation to the manager object.
-     *  @param fromFileName file name in which the include was found.
-     *  @param toFileName file name that is included.
-     */
-    void addInclude(const char *fromFileName,const char *toFileName)
+    void retrieve(std::string fileName,DefineMapOwning &toMap)
     {
-#if MULTITHREADED_INPUT
-      std::unique_lock<std::mutex> lock(m_mutex);
-#endif
-      //printf("DefineManager::addInclude(%s,%s)\n",fromFileName,toFileName);
-      if (fromFileName==0 || toFileName==0) return;
-      auto it = m_fileMap.find(fromFileName);
-      if (it==m_fileMap.end())
-      {
-        auto ptr = std::make_unique<DefinesPerFile>(this);
-        ptr->addInclude(toFileName);
-	m_fileMap.emplace(toStdString(fromFileName),std::move(ptr));
-      }
-      else
-      {
-        it->second->addInclude(toFileName);
-      }
-    }
-    /** Returns a reference to a Define object given its name or 0 if the Define does
-     *  not exist.
-     */
-    Define *isDefined(const char *name)
-    {
-#if MULTITHREADED_INPUT
-      std::unique_lock<std::mutex> lock(m_mutex);
-#endif
-      Define *d=0;
-      auto it = m_contextDefines.find(name);
-      if (it!=m_contextDefines.end())
+      auto it = m_fileMap.find(fileName);
+      if (it!=m_fileMap.end())
       {
-        d = it->second;
-        if (d->undef)
-        {
-          d=0;
-        }
+        auto &dpf = it->second;
+        dpf->retrieve(toMap);
       }
-      //printf("isDefined(%s)=%p\n",name,d);
-      return d;
     }
 
-    /** Returns a reference to the defines found in the current context. */
-    const DefineMapRef &defineContext() const
+    bool alreadyProcessed(std::string fileName) const
     {
-      return m_contextDefines;
+      return m_fileMap.find(fileName)!=m_fileMap.end();
     }
 
   private:
-    /** Helper function to collect all define for a given file */
-    void collectDefinesForFile(const char *fileName,DefineMapRef &map)
-    {
-      if (fileName==0) return;
-      auto it = m_fileMap.find(fileName);
-      if (it!=m_fileMap.end())
-      {
-        StringSet includeStack;
-	it->second->collectDefines(map,includeStack);
-      }
-    }
-
     /** Helper function to return the DefinesPerFile object for a given file name. */
-    DefinesPerFile *find(const char *fileName) const
+    DefinesPerFile *find(std::string fileName) const
     {
-      if (fileName==0) return nullptr;
       auto it = m_fileMap.find(fileName);
       return it!=m_fileMap.end() ? it->second.get() : nullptr;
     }
 
-    std::map< std::string,std::unique_ptr<DefinesPerFile> > m_fileMap;
-    DefineMapRef m_contextDefines;
-#if MULTITHREADED_INPUT
-    std::mutex m_mutex;
-#endif
+    std::unordered_map< std::string, std::unique_ptr<DefinesPerFile> > m_fileMap;
 };
 
 
-/** Collects all defines for a file and all files that the file includes.
- *  This function will recursively call itself for each file.
- *  @param dict The dictionary to fill with the defines. A redefine will
- *         replace a previous definition.
- *  @param includeStack The stack of includes, used to stop recursion in
- *         case there is a cyclic include dependency.
- */
-void DefineManager::DefinesPerFile::collectDefines(
-                     DefineMapRef &map,StringSet &includeStack)
-{
-  //printf("DefinesPerFile::collectDefines #defines=%d\n",m_defines.count());
-  {
-    for (auto incFile : m_includedFiles)
-    {
-      DefinesPerFile *dpf = m_parent->find(incFile.c_str());
-      if (dpf && includeStack.find(incFile)==includeStack.end())
-      {
-        //printf("  processing include %s\n",incFile.data());
-	includeStack.insert(incFile);
-	dpf->collectDefines(map,includeStack);
-      }
-    }
-  }
-  {
-    for (auto &kv : m_defines)
-    {
-      const std::unique_ptr<Define> &def = kv.second;
-      map[def->name.data()] = def.get();
-      //printf("  adding define %s\n",def->name.data());
-    }
-  }
-}
-
-
 /* -----------------------------------------------------------------
  *
  *	global state
  */
 #if MULTITHREADED_INPUT
-static std::mutex            g_allIncludesMutex;
-static std::mutex            g_addIncludeRelationMutex;
-static std::mutex            g_macroDefinitionsMutex;
+static std::mutex            g_debugMutex;
+static std::mutex            g_globalDefineMutex;
+//static std::mutex            g_addIncludeRelationMutex;
+//static std::mutex            g_macroDefinitionsMutex;
+static std::mutex            g_updateGlobals;
 #endif
-static StringUnorderedSet    g_allIncludes;
 static DefineManager         g_defineManager;
 
 
@@ -429,6 +291,9 @@ struct preYY_state
   std::unordered_map<std::string,Define*>  expandedDict;
   StringUnorderedSet                       expanded;
   ConstExpressionParser                    constExpParser;
+  DefineMapOwning                          contextDefines;
+  DefineList                               macroDefinitions;
+  LinkedMap<PreIncludeInfo>                includeRelations;
 };
 
 // stateless functions
@@ -449,9 +314,10 @@ static bool  computeExpression(yyscan_t yyscanner,const QCString &expr);
 static void   startCondSection(yyscan_t yyscanner,const char *sectId);
 static void     endCondSection(yyscan_t yyscanner);
 static void addMacroDefinition(yyscan_t yyscanner);
-static std::unique_ptr<Define> newDefine(yyscan_t yyscanner);
+static void          addDefine(yyscan_t yyscanner);
 static void        setFileName(yyscan_t yyscanner,const char *name);
 static yy_size_t        yyread(yyscan_t yyscanner,char *buf,yy_size_t max_size);
+static Define *      isDefined(yyscan_t yyscanner,const char *name);
 
 /* ----------------------------------------------------------------- */
 
@@ -544,7 +410,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
 					      !(
 					         (yyextra->includeStack.empty() || yyextra->curlyCount>0) &&
 					         yyextra->macroExpansion &&
-					         (def=g_defineManager.isDefined(name)) &&
+					         (def=isDefined(yyscanner,name)) &&
 						 /*macroIsAccessible(def) &&*/
 					         (!yyextra->expandOnlyPredef || def->isPredefined)
 					       )
@@ -670,7 +536,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
   					  yyextra->expectGuard = FALSE;
   					  Define *def=0;
 					  //def=yyextra->globalDefineDict->find(yytext);
-					  //def=g_defineManager.isDefined(yytext);
+					  //def=isDefined(yyscanner,yytext);
 					  //printf("Search for define %s found=%d yyextra->includeStack.empty()=%d "
 					  //       "yyextra->curlyCount=%d yyextra->macroExpansion=%d yyextra->expandOnlyPredef=%d "
 					  //	 "isPreDefined=%d\n",yytext,def ? 1 : 0,
@@ -679,7 +545,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
 					  //	);
 					  if ((yyextra->includeStack.empty() || yyextra->curlyCount>0) &&
 					      yyextra->macroExpansion &&
-					      (def=g_defineManager.isDefined(yytext)) &&
+					      (def=isDefined(yyscanner,yytext)) &&
 				              /*(def->isPredefined || macroIsAccessible(def)) && */
 					      (!yyextra->expandOnlyPredef || def->isPredefined)
 					     )
@@ -707,7 +573,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
                                           Define *def=0;
   					  if ((yyextra->includeStack.empty() || yyextra->curlyCount>0) && 
 					      yyextra->macroExpansion &&
-					      (def=g_defineManager.isDefined(yytext)) &&
+					      (def=isDefined(yyscanner,yytext)) &&
 					      def->nargs==-1 &&
 				              /*(def->isPredefined || macroIsAccessible(def)) &&*/
 					      (!yyextra->expandOnlyPredef || def->isPredefined)
@@ -924,7 +790,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
 <Command>. {yyextra->yyColNr+=(int)yyleng;}
 <UndefName>{ID}				{
   					  Define *def;
-  					  if ((def=g_defineManager.isDefined(yytext)) 
+  					  if ((def=isDefined(yyscanner,yytext)) 
 					      /*&& !def->isPredefined*/
 					      && !def->nonRecursive
 					     )
@@ -966,7 +832,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
   					}
 <DefinedExpr1,DefinedExpr2>\\\n		{ yyextra->yyLineNr++; outputChar(yyscanner,'\n'); }
 <DefinedExpr1>{ID}			{
-  					  if (g_defineManager.isDefined(yytext) || yyextra->guardName==yytext)
+  					  if (isDefined(yyscanner,yytext) || yyextra->guardName==yytext)
 					    yyextra->guardExpr+=" 1L ";
 					  else
 					    yyextra->guardExpr+=" 0L ";
@@ -974,7 +840,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
 					  BEGIN(Guard);
   					}
 <DefinedExpr2>{ID}			{
-  					  if (g_defineManager.isDefined(yytext) || yyextra->guardName==yytext)
+  					  if (isDefined(yyscanner,yytext) || yyextra->guardName==yytext)
 					    yyextra->guardExpr+=" 1L ";
 					  else
 					    yyextra->guardExpr+=" 0L ";
@@ -1575,17 +1441,11 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
 					  {
 					    addMacroDefinition(yyscanner);
 					  }
-					  def=g_defineManager.isDefined(yyextra->defName);
+					  def=isDefined(yyscanner,yyextra->defName);
 					  if (def==0) // new define
 					  {
 					    //printf("new define '%s'!\n",yyextra->defName.data());
-					    g_defineManager.addDefine(yyextra->yyFileName,newDefine(yyscanner));
-
-					    // also add it to the local file list if it is a source file
-					    //if (yyextra->isSource && yyextra->includeStack.empty())
-					    //{
-					    //  yyextra->fileDefineDict->insert(yyextra->defName,nd);
-					    //}
+					    addDefine(yyscanner);
 					  }
 					  else if (def /*&& macroIsAccessible(def)*/)
 					       // name already exists
@@ -1657,6 +1517,7 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
 					  }
 					  else
 					  {
+                                            QCString toFileName = yyextra->yyFileName;
                                             const std::unique_ptr<FileState> &fs=yyextra->includeStack.back();
 					    //fileDefineCache->merge(yyextra->yyFileName,fs->fileName);
 					    YY_BUFFER_STATE oldBuf = YY_CURRENT_BUFFER;
@@ -1677,6 +1538,31 @@ CHARLIT   (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'"))
                                             outputArray(yyscanner,lineStr.data(),lineStr.length());
 
 					    yyextra->includeStack.pop_back();
+
+#if MULTITHREADED_INPUT
+                                            {
+                                              std::lock_guard<std::mutex> lock(g_globalDefineMutex);
+#endif
+                                              // to avoid deadlocks we allow multiple threads to process the same header file.
+                                              // The first one to finish will store the results globally. After that the
+                                              // next time the same file is encountered, the stored data is used and the file
+                                              // is not processed again.
+                                              if (!g_defineManager.alreadyProcessed(toFileName.str()))
+                                              {
+                                                // now that the file is completely processed, prevent it from processing it again
+                                                g_defineManager.addInclude(yyextra->yyFileName.str(),toFileName.str());
+                                                g_defineManager.store(toFileName.str(),yyextra->contextDefines);
+                                              }
+                                              else
+                                              {
+                                                if (Debug::isFlagSet(Debug::Preprocessor))
+                                                {
+                                                  Debug::print(Debug::Preprocessor,0,"#include %s: was already processed by another thread! not storing data...\n",qPrint(toFileName));
+                                                }
+                                              }
+#if MULTITHREADED_INPUT
+                                            }
+#endif
 					  }
   					}
 <*>"/*"/"*/"				|
@@ -1792,10 +1678,10 @@ static void setCaseDone(yyscan_t yyscanner,bool value)
 }
 
 
-static FileState *checkAndOpenFile(yyscan_t yyscanner,const QCString &fileName,bool &alreadyIncluded)
+static FileState *checkAndOpenFile(yyscan_t yyscanner,const QCString &fileName,bool &alreadyProcessed)
 {
   YY_EXTRA_TYPE state = preYYget_extra(yyscanner);
-  alreadyIncluded = FALSE;
+  alreadyProcessed = FALSE;
   FileState *fs = 0;
   //printf("checkAndOpenFile(%s)\n",fileName.data());
   QFileInfo fi(fileName);
@@ -1810,27 +1696,26 @@ static FileState *checkAndOpenFile(yyscan_t yyscanner,const QCString &fileName,b
     if (state->curlyCount==0) // not #include inside { ... }
     {
 #if MULTITHREADED_INPUT
-      std::unique_lock<std::mutex> lock(g_allIncludesMutex);
+      std::lock_guard<std::mutex> lock(g_globalDefineMutex);
 #endif
 
-      if (g_allIncludes.find(absName.data())!=g_allIncludes.end())
+      if (g_defineManager.alreadyProcessed(absName.str()))
       {
-        alreadyIncluded = TRUE;
+        alreadyProcessed = TRUE;
         //printf("  already included 1\n");
         return 0; // already done
       }
-      g_allIncludes.insert(absName.data());
     }
     // check include stack for absName
 
-    alreadyIncluded = std::any_of(
+    alreadyProcessed = std::any_of(
       state->includeStack.begin(),
       state->includeStack.end(),
       [absName](const std::unique_ptr<FileState> &lfs)
         { return lfs->fileName==absName; }
     );
 
-    if (alreadyIncluded)
+    if (alreadyProcessed)
     {
       //printf("  already included 2\n");
       return 0;
@@ -1853,20 +1738,20 @@ static FileState *checkAndOpenFile(yyscan_t yyscanner,const QCString &fileName,b
   return fs;
 }
 
-static FileState *findFile(yyscan_t yyscanner, const char *fileName,bool localInclude,bool &alreadyIncluded)
+static FileState *findFile(yyscan_t yyscanner, const char *fileName,bool localInclude,bool &alreadyProcessed)
 {
   YY_EXTRA_TYPE state = preYYget_extra(yyscanner);
   //printf("** findFile(%s,%d) state->yyFileName=%s\n",fileName,localInclude,state->yyFileName.data());
   if (Portable::isAbsolutePath(fileName))
   {
-    FileState *fs = checkAndOpenFile(yyscanner,fileName,alreadyIncluded);
+    FileState *fs = checkAndOpenFile(yyscanner,fileName,alreadyProcessed);
     if (fs)
     {
       setFileName(yyscanner,fileName);
       state->yyLineNr=1;
       return fs;
     }
-    else if (alreadyIncluded)
+    else if (alreadyProcessed)
     {
       return 0;
     }
@@ -1877,14 +1762,14 @@ static FileState *findFile(yyscan_t yyscanner, const char *fileName,bool localIn
     if (fi.exists())
     {
       QCString absName = QCString(fi.dirPath(TRUE).data())+"/"+fileName;
-      FileState *fs = checkAndOpenFile(yyscanner,absName,alreadyIncluded);
+      FileState *fs = checkAndOpenFile(yyscanner,absName,alreadyProcessed);
       if (fs)
       {
 	setFileName(yyscanner,absName);
 	state->yyLineNr=1;
 	return fs;
       }
-      else if (alreadyIncluded)
+      else if (alreadyProcessed)
       {
 	return 0;
       }
@@ -1898,7 +1783,7 @@ static FileState *findFile(yyscan_t yyscanner, const char *fileName,bool localIn
   {
     std::string absName = path+"/"+fileName;
     //printf("  Looking for %s in %s\n",fileName,path.c_str());
-    FileState *fs = checkAndOpenFile(yyscanner,absName.c_str(),alreadyIncluded);
+    FileState *fs = checkAndOpenFile(yyscanner,absName.c_str(),alreadyProcessed);
     if (fs)
     {
       setFileName(yyscanner,absName.c_str());
@@ -1906,7 +1791,7 @@ static FileState *findFile(yyscan_t yyscanner, const char *fileName,bool localIn
       //printf("  -> found it\n");
       return fs;
     }
-    else if (alreadyIncluded)
+    else if (alreadyProcessed)
     {
       return 0;
     }
@@ -2448,7 +2333,7 @@ static bool expandExpression(yyscan_t yyscanner,QCString &expr,QCString *rest,in
     {
       if (state->expandedDict.find(macroName.data())==state->expandedDict.end()) // expand macro
       {
-	Define *def=g_defineManager.isDefined(macroName);
+	Define *def=isDefined(yyscanner,macroName);
         if (macroName=="defined")
         {
   	  //printf("found defined inside macro definition '%s'\n",expr.right(expr.length()-p).data());
@@ -2801,7 +2686,7 @@ static QCString expandMacro(yyscan_t yyscanner,const QCString &name)
   return n;
 }
 
-static std::unique_ptr<Define> newDefine(yyscan_t yyscanner)
+static void addDefine(yyscan_t yyscanner)
 {
   YY_EXTRA_TYPE state = preYYget_extra(yyscanner);
   std::unique_ptr<Define> def = std::make_unique<Define>();
@@ -2817,11 +2702,16 @@ static std::unique_ptr<Define> newDefine(yyscan_t yyscanner)
   //    def->fileDef ? def->fileDef->name().data() : def->fileName.data());
   //printf("newDefine: '%s'->'%s'\n",def->name.data(),def->definition.data());
   if (!def->name.isEmpty() && 
-      Doxygen::expandAsDefinedSet.find(def->name.data())!=Doxygen::expandAsDefinedSet.end())
+      Doxygen::expandAsDefinedSet.find(def->name.str())!=Doxygen::expandAsDefinedSet.end())
   {
     def->isPredefined=TRUE;
   }
-  return def;
+  auto it = state->contextDefines.find(def->name.str());
+  if (it!=state->contextDefines.end()) // redefine
+  {
+    state->contextDefines.erase(it);
+  }
+  state->contextDefines.insert(std::make_pair(toStdString(def->name),std::move(def)));
 }
 
 static void addMacroDefinition(yyscan_t yyscanner)
@@ -2864,10 +2754,7 @@ static void addMacroDefinition(yyscan_t yyscanner)
     define->definition = litTextStripped;
   }
   {
-#if MULTITHREADED_INPUT
-    std::unique_lock<std::mutex> lock(g_macroDefinitionsMutex);
-#endif
-    Doxygen::macroDefinitions.push_back(std::move(define));
+    state->macroDefinitions.push_back(std::move(define));
   }
 }
 
@@ -2883,10 +2770,47 @@ static inline void outputArray(yyscan_t yyscanner,const char *a,int len)
   if (state->includeStack.empty() || state->curlyCount>0) state->outputBuf->addArray(a,len);
 }
 
+static QCString determineAbsoluteIncludeName(const QCString &curFile,const QCString &incFileName)
+{
+  bool searchIncludes = Config_getBool(SEARCH_INCLUDES);
+  QCString absIncFileName = incFileName;
+  QFileInfo fi(curFile);
+  if (fi.exists())
+  {
+    QCString absName = QCString(fi.dirPath(TRUE).data())+"/"+incFileName;
+    QFileInfo fi2(absName);
+    if (fi2.exists())
+    {
+      absIncFileName=fi2.absFilePath().utf8();
+    }
+    else if (searchIncludes) // search in INCLUDE_PATH as well
+    {
+      const StringVector &includePath = Config_getList(INCLUDE_PATH);
+      for (const auto &incPath : includePath)
+      {
+        QFileInfo fi3(incPath.c_str());
+        if (fi3.exists() && fi3.isDir())
+        {
+          absName = QCString(fi3.absFilePath().utf8())+"/"+incFileName;
+          //printf("trying absName=%s\n",absName.data());
+          QFileInfo fi4(absName);
+          if (fi4.exists())
+          {
+            absIncFileName=fi4.absFilePath().utf8();
+            break;
+          }
+          //printf( "absIncFileName = %s\n", absIncFileName.data() );
+        }
+      }
+    }
+    //printf( "absIncFileName = %s\n", absIncFileName.data() );
+  }
+  return absIncFileName;
+}
+
 static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
 {
   YY_EXTRA_TYPE state = preYYget_extra(yyscanner);
-  bool searchIncludes = Config_getBool(SEARCH_INCLUDES);
   uint i=0;
 
   // find the start of the include file name
@@ -2918,49 +2842,13 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
     int oldLineNr        = state->yyLineNr;
     //printf("Searching for '%s'\n",incFileName.data());
 
-    // absIncFileName avoids difficulties for incFileName starting with "../" (bug 641336)
-    QCString absIncFileName = incFileName;
-    {
-      QFileInfo fi(state->yyFileName);
-      if (fi.exists())
-      {
-	QCString absName = QCString(fi.dirPath(TRUE).data())+"/"+incFileName;
-        QFileInfo fi2(absName);
-        if (fi2.exists())
-        {
-	  absIncFileName=fi2.absFilePath().utf8();
-	}
-	else if (searchIncludes) // search in INCLUDE_PATH as well
-	{
-	  const StringVector &includePath = Config_getList(INCLUDE_PATH);
-          for (const auto &incPath : includePath)
-	  {
-	    QFileInfo fi3(incPath.c_str());
-	    if (fi3.exists() && fi3.isDir())
-	    {
-	      absName = QCString(fi3.absFilePath().utf8())+"/"+incFileName;
-	      //printf("trying absName=%s\n",absName.data());
-	      QFileInfo fi4(absName);
-	      if (fi4.exists())
-	      {
-		absIncFileName=fi4.absFilePath().utf8();
-		break;
-	      }
-	      //printf( "absIncFileName = %s\n", absIncFileName.data() );
-	    }
-	  }
-	}
-	//printf( "absIncFileName = %s\n", absIncFileName.data() );
-      }
-    }
-    g_defineManager.addInclude(state->yyFileName,absIncFileName);
-    g_defineManager.addFileToContext(absIncFileName);
+    QCString absIncFileName = determineAbsoluteIncludeName(state->yyFileName,incFileName);
 
     // findFile will overwrite state->yyFileDef if found
     FileState *fs;
-    bool alreadyIncluded = FALSE;
+    bool alreadyProcessed = FALSE;
     //printf("calling findFile(%s)\n",incFileName.data());
-    if ((fs=findFile(yyscanner,incFileName,localInclude,alreadyIncluded))) // see if the include file can be found
+    if ((fs=findFile(yyscanner,incFileName,localInclude,alreadyProcessed))) // see if the include file can be found
     {
       //printf("Found include file!\n");
       if (Debug::isFlagSet(Debug::Preprocessor))
@@ -2971,30 +2859,25 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
         }
         //msg("#include %s: parsing...\n",incFileName.data());
       }
-      if (oldFileDef)
+
+      if (state->includeStack.empty() && oldFileDef)
       {
-#if MULTITHREADED_INPUT
-        std::unique_lock<std::mutex> lock(g_addIncludeRelationMutex);
-#endif
-        // add include dependency to the file in which the #include was found
-	bool ambig;
-	// change to absolute name for bug 641336 
-        FileDef *incFd = findFileDef(Doxygen::inputNameLinkedMap,absIncFileName,ambig);
-        oldFileDef->addIncludeDependency(ambig ? 0 : incFd,incFileName,localInclude,state->isImported,FALSE);
-        // add included by dependency
-        if (state->yyFileDef)
+        PreIncludeInfo *ii = state->includeRelations.find(absIncFileName);
+        if (ii==0)
         {
-          //printf("Adding include dependency %s->%s\n",oldFileDef->name().data(),incFileName.data());
-          state->yyFileDef->addIncludedByDependency(oldFileDef,oldFileDef->docName(),localInclude,state->isImported);
+          bool ambig;
+          FileDef *incFd = findFileDef(Doxygen::inputNameLinkedMap,absIncFileName,ambig);
+          state->includeRelations.add(
+              absIncFileName,
+              oldFileDef,
+              ambig?nullptr:incFd,
+              incFileName,
+              localInclude,
+              state->isImported
+              );
         }
       }
-      else if (state->inputFileDef)
-      {
-#if MULTITHREADED_INPUT
-        std::unique_lock<std::mutex> lock(g_addIncludeRelationMutex);
-#endif
-        state->inputFileDef->addIncludeDependency(0,absIncFileName,localInclude,state->isImported,TRUE);
-      }
+
       struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
       fs->bufState = YY_CURRENT_BUFFER;
       fs->lineNr   = oldLineNr;
@@ -3019,38 +2902,37 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
     }
     else
     {
-      //printf("  calling findFile(%s) alreadyInc=%d\n",incFileName.data(),alreadyIncluded);
-      if (oldFileDef)
+      if (alreadyProcessed) // if this header was already process we can just copy the stored macros
+                           // in the local context
       {
 #if MULTITHREADED_INPUT
-        std::unique_lock<std::mutex> lock(g_addIncludeRelationMutex);
+        std::lock_guard<std::mutex> lock(g_globalDefineMutex);
 #endif
-	bool ambig;
-
-	// change to absolute name for bug 641336 
-	FileDef *fd = findFileDef(Doxygen::inputNameLinkedMap,absIncFileName,ambig);
-	//printf("%s::findFileDef(%s)=%p\n",oldFileDef->name().data(),incFileName.data(),fd);
-	// add include dependency to the file in which the #include was found
-	oldFileDef->addIncludeDependency(ambig ? 0 : fd,incFileName,localInclude,state->isImported,FALSE);
-	// add included by dependency
-        if (fd)
-        {
-          //printf("Adding include dependency (2) %s->%s ambig=%d\n",oldFileDef->name().data(),fd->name().data(),ambig);
-          fd->addIncludedByDependency(oldFileDef,oldFileDef->docName(),localInclude,state->isImported);
-        }
+        g_defineManager.retrieve(absIncFileName.str(),state->contextDefines);
       }
-      else if (state->inputFileDef)
+
+      if (state->includeStack.empty() && oldFileDef)
       {
-#if MULTITHREADED_INPUT
-        std::unique_lock<std::mutex> lock(g_addIncludeRelationMutex);
-#endif
-        state->inputFileDef->addIncludeDependency(0,absIncFileName,localInclude,state->isImported,TRUE);
+        PreIncludeInfo *ii = state->includeRelations.find(absIncFileName);
+        if (ii==0)
+        {
+          bool ambig;
+          FileDef *incFd = findFileDef(Doxygen::inputNameLinkedMap,absIncFileName,ambig);
+          ii = state->includeRelations.add(absIncFileName,
+              oldFileDef,
+              ambig?0:incFd,
+              incFileName,
+              localInclude,
+              state->isImported
+              );
+        }
       }
+
       if (Debug::isFlagSet(Debug::Preprocessor))
       {
-	if (alreadyIncluded)
+	if (alreadyProcessed)
 	{
-          Debug::print(Debug::Preprocessor,0,"#include %s: already included! skipping...\n",qPrint(incFileName));
+          Debug::print(Debug::Preprocessor,0,"#include %s: already processed! skipping...\n",qPrint(incFileName));
 	}
 	else
 	{
@@ -3058,10 +2940,11 @@ static void readIncludeFile(yyscan_t yyscanner,const QCString &inc)
 	}
         //printf("error: include file %s not found\n",yytext);
       }
-      if (state->curlyCount>0 && !alreadyIncluded) // failed to find #include inside { ... }
+      if (state->curlyCount>0 && !alreadyProcessed) // failed to find #include inside { ... }
       {
 	warn(state->yyFileName,state->yyLineNr,"include file %s not found, perhaps you forgot to add its directory to INCLUDE_PATH?",incFileName.data());
       }
+
     }
   }
 }
@@ -3210,6 +3093,25 @@ static void unputChar(yyscan_t yyscanner,const QCString &expr,QCString *rest,uin
   //printf("result: unputChar(%s,%s,%d,%c)\n",expr.data(),rest ? rest->data() : 0,pos,c);
 }
 
+/** Returns a reference to a Define object given its name or 0 if the Define does
+ *  not exist.
+ */
+static Define *isDefined(yyscan_t yyscanner,const char *name)
+{
+  YY_EXTRA_TYPE state = preYYget_extra(yyscanner);
+  Define *d=0;
+  auto it = state->contextDefines.find(name);
+  if (it!=state->contextDefines.end())
+  {
+    d = it->second.get();
+    if (d->undef)
+    {
+      d=0;
+    }
+  }
+  return d;
+}
+
 static void initPredefined(yyscan_t yyscanner,const char *fileName)
 {
   YY_EXTRA_TYPE state = preYYget_extra(yyscanner);
@@ -3289,7 +3191,7 @@ static void initPredefined(yyscan_t yyscanner,const char *fileName)
         def->nonRecursive = nonRecursive;
         def->fileDef      = state->yyFileDef;
         def->fileName     = fileName;
-        g_defineManager.addDefine(state->yyFileName,std::move(def));
+        state->contextDefines.insert(std::make_pair(state->yyFileName.str(),std::move(def)));
 
         //printf("#define '%s' '%s' #nargs=%d\n",
         //  def->name.data(),def->definition.data(),def->nargs);
@@ -3321,7 +3223,7 @@ static void initPredefined(yyscan_t yyscanner,const char *fileName)
         def->nonRecursive = nonRecursive;
         def->fileDef      = state->yyFileDef;
         def->fileName     = fileName;
-        g_defineManager.addDefine(state->yyFileName,std::move(def));
+        state->contextDefines.insert(std::make_pair(state->yyFileName.str(),std::move(def)));
       }
     }
   }
@@ -3355,6 +3257,7 @@ Preprocessor::~Preprocessor()
 
 void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output)
 {
+//  printf("Preprocessor::processFile(%s)\n",fileName);
   yyscan_t yyscanner = p->yyscanner;
   YY_EXTRA_TYPE state = preYYget_extra(p->yyscanner);
   struct yyguts_t *yyg = (struct yyguts_t*)p->yyscanner;
@@ -3378,12 +3281,13 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output
   state->outputBuf=&output;
   state->includeStack.clear();
   state->expandedDict.clear();
+  state->contextDefines.clear();
   while (!state->condStack.empty()) state->condStack.pop();
-  //state->fileDefineDict->clear();
 
   setFileName(yyscanner,fileName);
+
   state->inputFileDef = state->yyFileDef;
-  g_defineManager.startContext(state->yyFileName);
+  //yyextra->defineManager.startContext(state->yyFileName);
   
   initPredefined(yyscanner,fileName);
  
@@ -3414,6 +3318,9 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output
 
   if (Debug::isFlagSet(Debug::Preprocessor))
   {
+#if MULTITHREADED_INPUT
+    std::lock_guard<std::mutex> lock(g_debugMutex);
+#endif
     char *orgPos=output.data()+orgOffset;
     char *newPos=output.data()+output.curPos();
     Debug::print(Debug::Preprocessor,0,"Preprocessor output of %s (size: %d bytes):\n",fileName,newPos-orgPos);
@@ -3426,14 +3333,13 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output
       orgPos++;
     }
     Debug::print(Debug::Preprocessor,0,"\n---------\n");
-    if (g_defineManager.defineContext().size()>0)
+    if (yyextra->contextDefines.size()>0)
     {
       Debug::print(Debug::Preprocessor,0,"Macros accessible in this file (%s):\n", fileName);
       Debug::print(Debug::Preprocessor,0,"---------\n");
-      for (auto &kv : g_defineManager.defineContext())
+      for (auto &kv : yyextra->contextDefines)
       {
-        Define *def = kv.second;
-        Debug::print(Debug::Preprocessor,0,"%s ",qPrint(def->name));
+        Debug::print(Debug::Preprocessor,0,"%s ",qPrint(kv.second->name));
       }
       Debug::print(Debug::Preprocessor,0,"\n---------\n");
     }
@@ -3442,8 +3348,29 @@ void Preprocessor::processFile(const char *fileName,BufStr &input,BufStr &output
       Debug::print(Debug::Preprocessor,0,"No macros accessible in this file (%s).\n", fileName);
     }
   }
-  g_defineManager.endContext();
+
+  {
+#if MULTITHREADED_INPUT
+    std::lock_guard<std::mutex> lock(g_updateGlobals);
+#endif
+    for (const auto &inc : state->includeRelations)
+    {
+      if (inc->fromFileDef)
+      {
+        inc->fromFileDef->addIncludeDependency(inc->toFileDef,inc->includeName,inc->local,inc->imported);
+      }
+      if (inc->toFileDef)
+      {
+        inc->toFileDef->addIncludedByDependency(inc->fromFileDef,inc->includeName,inc->local,inc->imported);
+      }
+    }
+    // add the macro definition for this file to the global map
+    Doxygen::macroDefinitions.emplace(std::make_pair(state->yyFileName,std::move(state->macroDefinitions)));
+  }
+
+  //yyextra->defineManager.endContext();
   printlex(yy_flex_debug, FALSE, __FILE__, fileName);
+//  printf("Preprocessor::processFile(%s) finished\n",fileName);
 }
 
 #if USE_STATE2STRING
diff --git a/src/scanner.l b/src/scanner.l
index 4f64503..cc5428d 100644
--- a/src/scanner.l
+++ b/src/scanner.l
@@ -2045,7 +2045,7 @@ OPERATOR  "operator"{B}*({ARITHOP}|{ASSIGNOP}|{LOGICOP}|{BITOP})
 				          BEGIN(FindMembers);
 			 	        }
 <FindMembers,FindMemberName>{SCOPENAME}	{
-                                          if (yyextra->insideCpp || yyextra->insideObjC) 
+                                          if (Config_getBool(CLANG_ASSISTED_PARSING) && (yyextra->insideCpp || yyextra->insideObjC))
                                           {
                                             yyextra->current->id = ClangParser::instance()->lookup(yyextra->yyLineNr,yytext);
                                           }
@@ -2348,7 +2348,7 @@ OPERATOR  "operator"{B}*({ARITHOP}|{ASSIGNOP}|{LOGICOP}|{BITOP})
   */
 <Define>{ID}				{
   					  //printf("Define '%s' without args\n",yytext);
-                                          if (yyextra->insideCpp || yyextra->insideObjC) 
+                                          if (Config_getBool(CLANG_ASSISTED_PARSING) && (yyextra->insideCpp || yyextra->insideObjC))
                                           {
                                             yyextra->current->id = ClangParser::instance()->lookup(yyextra->yyLineNr,yytext);
                                           }
@@ -2359,7 +2359,6 @@ OPERATOR  "operator"{B}*({ARITHOP}|{ASSIGNOP}|{LOGICOP}|{BITOP})
   					}
 <DefineEnd>\n				{
   					  //printf("End define: doc=%s docFile=%s docLine=%d\n",yyextra->current->doc.data(),yyextra->current->docFile.data(),yyextra->current->docLine);
-                                          lineCount(yyscanner);
 					  yyextra->current->fileName   = yyextra->yyFileName;
 					  yyextra->current->startLine  = yyextra->yyLineNr;
 					  yyextra->current->startColumn = yyextra->yyColNr;
@@ -2368,6 +2367,7 @@ OPERATOR  "operator"{B}*({ARITHOP}|{ASSIGNOP}|{LOGICOP}|{BITOP})
   					  yyextra->current->name       = yyextra->current->name.stripWhiteSpace();
 					  yyextra->current->section    = Entry::DEFINE_SEC;
 					  yyextra->current_root->moveToSubEntryAndRefresh(yyextra->current);
+                                          lineCount(yyscanner);
 					  initEntry(yyscanner);
 					  BEGIN(yyextra->lastDefineContext);
   					}
@@ -3463,7 +3463,7 @@ OPERATOR  "operator"{B}*({ARITHOP}|{ASSIGNOP}|{LOGICOP}|{BITOP})
 					}
 <Sharp>.				{ yyextra->current->type += *yytext ; }
 <FindFields>{ID}			{
-                                          if (yyextra->insideCpp || yyextra->insideObjC) 
+                                          if (Config_getBool(CLANG_ASSISTED_PARSING) && (yyextra->insideCpp || yyextra->insideObjC))
                                           {
                                             yyextra->current->id = ClangParser::instance()->lookup(yyextra->yyLineNr,yytext);
                                           }
@@ -5334,7 +5334,7 @@ OPERATOR  "operator"{B}*({ARITHOP}|{ASSIGNOP}|{LOGICOP}|{BITOP})
   					}
 <CompoundName>{SCOPENAME}		{ 
 					  yyextra->current->name = yytext ;
-                                          if (yyextra->insideCpp || yyextra->insideObjC) 
+                                          if (Config_getBool(CLANG_ASSISTED_PARSING) && (yyextra->insideCpp || yyextra->insideObjC))
                                           {
                                             yyextra->current->id = ClangParser::instance()->lookup(yyextra->yyLineNr,yytext);
                                           }
@@ -5399,7 +5399,7 @@ OPERATOR  "operator"{B}*({ARITHOP}|{ASSIGNOP}|{LOGICOP}|{BITOP})
 					  }
 					}
 <ClassVar>{ID}				{
-                                          if (yyextra->insideCpp || yyextra->insideObjC) 
+                                          if (Config_getBool(CLANG_ASSISTED_PARSING) && (yyextra->insideCpp || yyextra->insideObjC))
                                           {
                                             yyextra->current->id = ClangParser::instance()->lookup(yyextra->yyLineNr,yytext);
                                           }
@@ -7255,16 +7255,19 @@ static void parseMain(yyscan_t yyscanner,
   yyextra->yyBegColNr    = 0;
   yyextra->yyFileName = fileName;
   setContext(yyscanner);
-  bool processWithClang = yyextra->insideCpp || yyextra->insideObjC;
-  if (processWithClang)
+  if (Config_getBool(CLANG_ASSISTED_PARSING))
   {
-    if (!sameTranslationUnit) // new file
+    bool processWithClang = yyextra->insideCpp || yyextra->insideObjC;
+    if (processWithClang)
     {
-      ClangParser::instance()->start(fileName,filesInSameTranslationUnit);
-    }
-    else
-    {
-      ClangParser::instance()->switchToFile(fileName);
+      if (!sameTranslationUnit) // new file
+      {
+        ClangParser::instance()->start(fileName,filesInSameTranslationUnit);
+      }
+      else
+      {
+        ClangParser::instance()->switchToFile(fileName);
+      }
     }
   }
   rt->lang = yyextra->language;
diff --git a/src/sqlite3gen.cpp b/src/sqlite3gen.cpp
index 70c6561..77aab68 100644
--- a/src/sqlite3gen.cpp
+++ b/src/sqlite3gen.cpp
@@ -1995,7 +1995,6 @@ static void generateSqlite3ForClass(const ClassDef *cd)
       DBG_CTX(("-----> ClassDef includeInfo for %s\n", nm.data()));
       DBG_CTX(("       local    : %d\n", ii->local));
       DBG_CTX(("       imported : %d\n", ii->imported));
-      DBG_CTX(("       indirect : %d\n", ii->indirect));
       DBG_CTX(("header: %s\n", ii->fileDef->absFilePath().data()));
       DBG_CTX(("       file_id  : %d\n", file_id));
       DBG_CTX(("       header_id: %d\n", header_id));
@@ -2211,7 +2210,6 @@ static void generateSqlite3ForFile(const FileDef *fd)
       DBG_CTX(("-----> FileDef includeInfo for %s\n", ii->includeName.data()));
       DBG_CTX(("       local:    %d\n", ii->local));
       DBG_CTX(("       imported: %d\n", ii->imported));
-      DBG_CTX(("       indirect: %d\n", ii->indirect));
       if(ii->fileDef)
       {
         DBG_CTX(("include: %s\n", ii->fileDef->absFilePath().data()));
diff --git a/src/tagreader.cpp b/src/tagreader.cpp
index 12010e8..3c493b4 100644
--- a/src/tagreader.cpp
+++ b/src/tagreader.cpp
@@ -1526,7 +1526,7 @@ void TagFileParser::addIncludes()
                 //        ifd->getOutputFileBase().data(),ii->id.data());
                 if (ifd->getOutputFileBase()==QCString(ii->id))
                 {
-                  fd->addIncludeDependency(ifd.get(),ii->text,ii->isLocal,ii->isImported,FALSE);
+                  fd->addIncludeDependency(ifd.get(),ii->text,ii->isLocal,ii->isImported);
                 }
               }
             }