Debug output for lexical analyzer

In case of error messages like:
    input buffer overflow, can't enlarge buffer because scanner uses REJECT
it is not always directly clear from which lexical analyzer (.l file) this problem comes.
This patch helps to find these problems and does the following things:
- when using the option -d lex with doxygen each time a lexical analyzer is called at the start a line like the following line will be given:
    Entering lexical analyzer: pre.l (for: ..../file.c)
  and at the end:
    Finished lexical analyzer: pre.l (for: ..../file.c)
- in case the lexical analyzer has been translated with the -d option of lex / flex the above mentioned lines will be given as part of the lexical analyzer output (to stderr) and look like:
    --entering lexical analyzer: pre.l (for: ..../file.c)
    --finished lexical analyzer: pre.l (for: ..../file.c)

20 files changed, 79 insertions, 2 deletions

diff --git a/src/code.l b/src/code.l
index 2417bf9..b0c7104 100644
--- a/src/code.l
+++ b/src/code.l
@@ -3546,7 +3546,11 @@ void parseCCode(CodeOutputInterface &od,const char *className,const QCString &s,
 {
   //printf("***parseCode() exBlock=%d exName=%s fd=%p className=%s searchCtx=%s\n",
   //      exBlock,exName,fd,className,searchCtx?searchCtx->name().data():"<none>");
+
   if (s.isEmpty()) return;
+
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
+
   TooltipManager::instance()->clearTooltips();
   if (g_codeClassSDict==0)
   {
@@ -3637,6 +3641,8 @@ void parseCCode(CodeOutputInterface &od,const char *className,const QCString &s,
     delete g_sourceFileDef;
     g_sourceFileDef=0;
   }
+
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
   return;
 }
 
diff --git a/src/commentcnv.l b/src/commentcnv.l
index 1b48da5..d59ef51 100644
--- a/src/commentcnv.l
+++ b/src/commentcnv.l
@@ -952,6 +952,7 @@ void convertCppComments(BufStr *inBuf,BufStr *outBuf,const char *fileName)
   g_condStack.clear();
   g_condStack.setAutoDelete(TRUE);
 
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   isFixedForm = FALSE;
   if (g_lang==SrcLangExt_Fortran)
   {
@@ -981,6 +982,7 @@ void convertCppComments(BufStr *inBuf,BufStr *outBuf,const char *fileName)
     g_outBuf->at(g_outBuf->curPos())='\0';
     msg("-------------\n%s\n-------------\n",g_outBuf->data());
   }
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 
diff --git a/src/commentscan.l b/src/commentscan.l
index 22d6444..229cc2b 100644
--- a/src/commentscan.l
+++ b/src/commentscan.l
@@ -2818,6 +2818,7 @@ bool parseCommentBlock(/* in */     ParserInterface *parser,
   g_spaceBeforeCmd.resize(0);
   g_spaceBeforeIf.resize(0);
 
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName ? fileName.data(): NULL);
   if (!current->inbodyDocs.isEmpty() && isInbody) // separate in body fragments
   {
     current->inbodyDocs+="\n\n";
@@ -2894,6 +2895,7 @@ bool parseCommentBlock(/* in */     ParserInterface *parser,
   //printf("position=%d parseMore=%d newEntryNeeded=%d\n",
   //  position,parseMore,newEntryNeeded);
 
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName ? fileName.data(): NULL);
   return parseMore;
 }
 
diff --git a/src/constexp.l b/src/constexp.l
index 85ae7fd..b333c72 100644
--- a/src/constexp.l
+++ b/src/constexp.l
@@ -21,6 +21,7 @@
 #include "constexp.h"  
 #include "cppvalue.h"
 #include "ce_parse.h" // generated header file
+#include "message.h"
 
 #define YY_NEVER_INTERACTIVE 1
 #define YY_NO_INPUT 1
@@ -105,6 +106,7 @@ CONSTSUFFIX ([uU][lL]?[lL]?)|([lL][lL]?[uU]?)
 
 bool parseconstexp(const char *fileName,int lineNr,const QCString &s)
 {
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   //printf("Expression: `%s'\n",s.data());
   g_constExpFileName = fileName;
   g_constExpLineNr = lineNr;
@@ -113,6 +115,7 @@ bool parseconstexp(const char *fileName,int lineNr,const QCString &s)
   constexpYYrestart( constexpYYin );
   constexpYYparse();
   //printf("Result: %ld\n",(long)g_resultValue);
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
   return (long)g_resultValue!=0;
 }
 
diff --git a/src/debug.cpp b/src/debug.cpp
index c67ce22..1942355 100644
--- a/src/debug.cpp
+++ b/src/debug.cpp
@@ -46,6 +46,7 @@ static LabelMap s_labels[] =
   { "extcmd",       Debug::ExtCmd       },
   { "markdown",     Debug::Markdown     },
   { "filteroutput", Debug::FilterOutput },
+  { "lex",          Debug::Lex },
   { 0,             (Debug::DebugMask)0  }
 };
 
diff --git a/src/debug.h b/src/debug.h
index 0893278..29f91bd 100644
--- a/src/debug.h
+++ b/src/debug.h
@@ -36,7 +36,8 @@ class Debug
                      Time         = 0x00000200,
                      ExtCmd       = 0x00000400,
                      Markdown     = 0x00000800,
-                     FilterOutput = 0x00001000
+                     FilterOutput = 0x00001000,
+                     Lex          = 0x00002000
                    };
     static void print(DebugMask mask,int prio,const char *fmt,...);
     static int  setFlag(const char *label);
diff --git a/src/declinfo.l b/src/declinfo.l
index 3e4ad4a..58ecb05 100644
--- a/src/declinfo.l
+++ b/src/declinfo.l
@@ -231,6 +231,7 @@ ID	"$"?([a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF]*)|(@[0-9]+)
 void parseFuncDecl(const QCString &decl,bool objC,QCString &cl,QCString &t,
                    QCString &n,QCString &a,QCString &ftl,QCString &exc)
 {
+  printlex(yy_flex_debug, TRUE, __FILE__, NULL);
   inputString   = decl;
   //printf("Input=`%s'\n",inputString);
   if (inputString==0) return;
@@ -307,6 +308,7 @@ void parseFuncDecl(const QCString &decl,bool objC,QCString &cl,QCString &t,
   //printf("type=`%s' class=`%s' name=`%s' args=`%s'\n",
   //        t.data(),cl.data(),n.data(),a.data());
 
+  printlex(yy_flex_debug, FALSE, __FILE__, NULL);
   return;
   
   
diff --git a/src/defargs.l b/src/defargs.l
index 6d6136a..5f64b66 100644
--- a/src/defargs.l
+++ b/src/defargs.l
@@ -55,6 +55,7 @@
 #include "entry.h"
 #include "util.h"
 #include "arguments.h"
+#include "message.h"
   
 #define YY_NEVER_INTERACTIVE 1
 #define YY_NO_INPUT 1
@@ -527,6 +528,7 @@ void stringToArgumentList(const char *argsString,ArgumentList* al,QCString *extr
 {
   if (al==0) return; 
   if (argsString==0) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, NULL);
 
   g_copyArgValue=0;
   g_curArgDocs.resize(0);
@@ -549,6 +551,7 @@ void stringToArgumentList(const char *argsString,ArgumentList* al,QCString *extr
   defargsYYlex();
   if (extraTypeChars) *extraTypeChars=g_extraTypeChars;
   //printf("stringToArgumentList(%s) result=%s\n",argsString,argListToString(al).data());
+  printlex(yy_flex_debug, FALSE, __FILE__, NULL);
 }
 
 #if !defined(YY_FLEX_SUBMINOR_VERSION) 
diff --git a/src/doctokenizer.l b/src/doctokenizer.l
index 5cedff3..deded6a 100644
--- a/src/doctokenizer.l
+++ b/src/doctokenizer.l
@@ -1189,6 +1189,7 @@ void doctokenizerYYFindSections(const char *input,Definition *d,
                                 MemberGroup *mg,const char *fileName)
 {
   if (input==0) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   g_inputString = input;
   //printf("parsing --->`%s'<---\n",input);
   g_inputPos    = 0;
@@ -1198,6 +1199,7 @@ void doctokenizerYYFindSections(const char *input,Definition *d,
   BEGIN(St_Sections);
   doctokenizerYYlineno = 1;
   doctokenizerYYlex();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void doctokenizerYYinit(const char *input,const char *fileName)
diff --git a/src/fortrancode.l b/src/fortrancode.l
index 9ba4ba8..842debf 100644
--- a/src/fortrancode.l
+++ b/src/fortrancode.l
@@ -1115,6 +1115,7 @@ void parseFortranCode(CodeOutputInterface &od,const char *className,const QCStri
   (void)className;
 
   if (s.isEmpty()) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
   TooltipManager::instance()->clearTooltips();
   g_code = &od;
   g_inputString   = s;
@@ -1174,6 +1175,7 @@ void parseFortranCode(CodeOutputInterface &od,const char *className,const QCStri
     delete g_sourceFileDef;
     g_sourceFileDef=0;
   }
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
   return;
 }
 
diff --git a/src/fortranscanner.l b/src/fortranscanner.l
index 2102a20..3cafc76 100644
--- a/src/fortranscanner.l
+++ b/src/fortranscanner.l
@@ -2339,7 +2339,12 @@ void FortranLanguageScanner::parseInput(const char *fileName,
                                         QStrList & /*filesInSameTranslationUnit*/)
 {
   g_thisParser = this;
+
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
+
   ::parseMain(fileName,fileBuf,root);
+
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void FortranLanguageScanner::parseCode(CodeOutputInterface & codeOutIntf,
diff --git a/src/message.cpp b/src/message.cpp
index 7c9c02b..babb282 100644
--- a/src/message.cpp
+++ b/src/message.cpp
@@ -211,3 +211,31 @@ void err(const char *fmt, ...)
   vfprintf(warnFile, (QCString(error_str) + fmt).data(), args);
   va_end(args); 
 }
+
+void printlex(int dbg, bool enter, const char *lexName, const char *fileName)
+{
+  char *enter_txt = "entering";
+  char *enter_txt_uc = "Entering";
+
+  if (!enter)
+  {
+    enter_txt = "finished";
+    enter_txt_uc = "Finished";
+  }
+
+  if (dbg)
+  {
+    if (fileName)
+      fprintf(stderr,"--%s lexical analyzer: %s (for: %s)\n",enter_txt, lexName, fileName);
+    else
+      fprintf(stderr,"--%s lexical analyzer: %s\n",enter_txt, lexName);
+  }
+  else
+  {
+    if (fileName)
+      Debug::print(Debug::Lex,0,"%s lexical analyzer: %s (for: %s)\n",enter_txt_uc, lexName, fileName);
+    else
+      Debug::print(Debug::Lex,0,"%s lexical analyzer: %s\n",enter_txt_uc, lexName);
+  }
+
+}
diff --git a/src/message.h b/src/message.h
index e01b8be..b7bdfe4 100644
--- a/src/message.h
+++ b/src/message.h
@@ -29,4 +29,5 @@ extern void warn_uncond(const char *fmt, ...);
 extern void err(const char *fmt, ...);
 void initWarningFormat();
 
+extern void printlex(int dbg, bool enter, const char *lexName, const char *fileName);
 #endif
diff --git a/src/pre.l b/src/pre.l
index e0b45df..775d448 100644
--- a/src/pre.l
+++ b/src/pre.l
@@ -2969,6 +2969,7 @@ void cleanUpPreprocessor()
 
 void preprocessFile(const char *fileName,BufStr &input,BufStr &output)
 {
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   uint orgOffset=output.curPos();
   //printf("##########################\n%s\n####################\n",
   //    input.data());
@@ -3177,6 +3178,7 @@ void preprocessFile(const char *fileName,BufStr &input,BufStr &output)
     }
   }
   DefineManager::instance().endContext();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void preFreeScanner()
diff --git a/src/pycode.l b/src/pycode.l
index 448068c..8512711 100644
--- a/src/pycode.l
+++ b/src/pycode.l
@@ -1434,6 +1434,7 @@ void parsePythonCode(CodeOutputInterface &od,const char * /*className*/,
   
   //--------------------------------------
   if (s.isEmpty()) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
   TooltipManager::instance()->clearTooltips();
   g_code = &od;
   g_inputString   = s;
@@ -1494,6 +1495,7 @@ void parsePythonCode(CodeOutputInterface &od,const char * /*className*/,
     delete g_sourceFileDef;
     g_sourceFileDef=0;
   }
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
   return;
 }
 
diff --git a/src/pyscanner.l b/src/pyscanner.l
index c113b58..671d330 100644
--- a/src/pyscanner.l
+++ b/src/pyscanner.l
@@ -1686,7 +1686,9 @@ void PythonLanguageScanner::parseInput(const char *fileName,
                                        QStrList & /*filesInSameTranslationUnit*/)
 {
   g_thisParser = this;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
   ::parseMain(fileName,fileBuf,root);
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 
   // May print the AST for debugging purposes
   // printAST(global_root);
diff --git a/src/scanner.l b/src/scanner.l
index 0df0a8a..6160158 100644
--- a/src/scanner.l
+++ b/src/scanner.l
@@ -6893,8 +6893,13 @@ void CLanguageScanner::parseInput(const char *fileName,
                                   QStrList & filesInSameTranslationUnit)
 {
   g_thisParser = this;
+
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
+
   ::parseMain(fileName,fileBuf,root,
               sameTranslationUnit,filesInSameTranslationUnit);
+
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void CLanguageScanner::parseCode(CodeOutputInterface & codeOutIntf,
diff --git a/src/tclscanner.l b/src/tclscanner.l
index 5bbb396..1fd20cd 100644
--- a/src/tclscanner.l
+++ b/src/tclscanner.l
@@ -127,7 +127,7 @@ int TclFindElement(
     goto done;
   }
 
-  if (*p == '{') 
+  if (*p == '{')   /* } to keep vi happy */
   {
     openBraces = 1;
     p++;
@@ -2541,6 +2541,7 @@ tcl_inf("%s\n",fileName);
 
   tcl.input_string = input;
   if (tcl.input_string.length()<1) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
 
   msg("Parsing %s...\n",fileName);
   groupEnterFile(fileName,yylineno);
@@ -2554,6 +2555,7 @@ tcl_inf("%s\n",fileName);
   groupLeaveFile(tcl.file_name,yylineno);
   root->program.resize(0);
   myFile.close();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 //! Parse file and codify.
@@ -2583,6 +2585,7 @@ void TclLanguageScanner::parseCode(CodeOutputInterface & codeOutIntf,
   (void)collectXRefs;
 
   if (input.length()<1) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fileDef ? fileDef->fileName().data(): NULL);
   tcl.input_string = input;
 
   QCString myNs="";
@@ -2655,6 +2658,7 @@ tcl_inf("%s (%d,%d) %d %d\n",myStr.ascii(),startLine,endLine,isExampleBlock,inli
   tcl.cl.clear();
   tcl.fn.clear();
   tcl.entry.clear();
+  printlex(yy_flex_debug, FALSE, __FILE__, fileDef ? fileDef->fileName().data(): NULL);
 }
 
 bool TclLanguageScanner::needsPreprocessing(const QCString &extension)
diff --git a/src/vhdlcode.l b/src/vhdlcode.l
index 8381853..94ba3c0 100644
--- a/src/vhdlcode.l
+++ b/src/vhdlcode.l
@@ -1517,6 +1517,7 @@ void parseVhdlCode(CodeOutputInterface &od,const char *className,const QCString
 {
   //printf("***parseCode() exBlock=%d exName=%s fd=%p\n",exBlock,exName,fd);
   if (s.isEmpty()) return;
+  printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
   TooltipManager::instance()->clearTooltips();
   if (memberDef)
   {
@@ -1599,6 +1600,7 @@ void parseVhdlCode(CodeOutputInterface &od,const char *className,const QCString
     g_sourceFileDef=0;
   }
   g_startCode=FALSE;
+  printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
 }
 
 void codeFreeVhdlScanner()
diff --git a/src/vhdlscanner.l b/src/vhdlscanner.l
index e24fd6c..716c0c0 100644
--- a/src/vhdlscanner.l
+++ b/src/vhdlscanner.l
@@ -763,6 +763,7 @@ void VHDLLanguageScanner::parseInput(const char *fileName,
     VhdlDocGen::parseUCF(fileBuf,root,yyFileName,TRUE); 
     return; 
   }
+  printlex(yy_flex_debug, TRUE, __FILE__, fileName);
 
   ::parserInit();
   yycont=getVhdlCont();
@@ -791,6 +792,7 @@ void VHDLLanguageScanner::parseInput(const char *fileName,
   free(lineIndex);
   inputFile.close();
   mapLibPackage(root);
+  printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
 
 void VHDLLanguageScanner::parseCode(CodeOutputInterface &codeOutIntf,