Miscounting lines in doctokinizer

When having a file like:
```
@page md_aa aa

Last \error2

- Install \error3
```
we get the warnings:
```
aa.md:3: warning: Found unknown command '\error2'
aa.md:7: warning: Found unknown command '\error3'
```
instead of
```
aa.md:3: warning: Found unknown command '\error2'
aa.md:5: warning: Found unknown command '\error3'

```

Investigation lead to that this is due to the fact that with a `REJECT` the line counter in `doctokinizer.l` is not reset. By counting the lines ourselves we can properly count the lines.
(Other lexers don't have this problem as here we already do the counting ourselves)

1 files changed, 85 insertions, 10 deletions

diff --git a/src/doctokenizer.l b/src/doctokenizer.l
index c20ae1a..a08d909 100644
--- a/src/doctokenizer.l
+++ b/src/doctokenizer.l
@@ -82,6 +82,11 @@ struct DocLexerContext
 
 static QStack<DocLexerContext> g_lexerStack;
 
+static int g_yyLineNr = 0;
+
+static void lineCount();
+static void lineCount(const char* text);
+
 #if USE_STATE2STRING
 static const char *stateToString(int state);
 #endif
@@ -187,7 +192,7 @@ static void processSection()
   }
   else
   {
-    warn(g_fileName,yylineno,"Found section/anchor %s without context\n",g_secLabel.data());
+    warn(g_fileName,g_yyLineNr,"Found section/anchor %s without context\n",g_secLabel.data());
   }
   SectionInfo *si = SectionManager::instance().find(g_secLabel);
   if (si)
@@ -441,7 +446,6 @@ REFWORD        {FILEMASK}|{LABELID}|{REFWORD2}|{REFWORD3}|{REFWORD4}
 REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 
 %option noyywrap
-%option yylineno
 
 %x St_Para
 %x St_Comment
@@ -494,6 +498,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 %%
 <St_Para>\r            /* skip carriage return */
 <St_Para>^{LISTITEM}   { /* list item */
+                         lineCount(yytext);
                          QCString text=yytext;
                          int dashPos = text.findRev('-');
                          g_token->isEnumList = text.at(dashPos+1)=='#';
@@ -508,6 +513,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          }
                          else
                          {
+                           lineCount(yytext);
                            QCString text=yytext;
                            static QRegExp re("[*+]");
                            int listPos = text.findRev(re);
@@ -535,6 +541,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          }
                        }
 <St_Para>{BLANK}*(\n|"\\ilinebr"){LISTITEM}     { /* list item on next line */
+                         lineCount(yytext);
                          QCString text=extractPartAfterNewLine(yytext);
                          int dashPos = text.findRev('-');
                          g_token->isEnumList = text.at(dashPos+1)=='#';
@@ -549,6 +556,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          }
                          else
                          {
+                           lineCount(yytext);
                            QCString text=extractPartAfterNewLine(yytext);
                            static QRegExp re("[*+]");
                            int markPos = text.findRev(re);
@@ -559,12 +567,14 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          }
                        }
 <St_Para>{BLANK}*(\n|"\\ilinebr"){OLISTITEM}     { /* list item on next line */
+                         lineCount(yytext);
                          if (!g_markdownSupport || g_insidePre)
                          {
                            REJECT;
                          }
                          else
                          {
+                           lineCount(yytext);
                            QCString text=extractPartAfterNewLine(yytext);
                            static QRegExp re("[1-9]");
                            int digitPos = text.find(re);
@@ -576,11 +586,13 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          }
                        }
 <St_Para>^{ENDLIST}       { /* end list */
+                         lineCount(yytext);
                          int dotPos = QCString(yytext).findRev('.');
                          g_token->indent     = computeIndent(yytext,dotPos);
                          return TK_ENDLIST;
                        }
 <St_Para>{BLANK}*(\n|"\\ilinebr"){ENDLIST}      { /* end list on next line */
+                         lineCount(yytext);
                          QCString text=extractPartAfterNewLine(yytext);
                          int dotPos = text.findRev('.');
                          g_token->indent = computeIndent(text,dotPos);
@@ -595,7 +607,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          return TK_COMMAND_AT;
                        }
 <St_Para>"@_fakenl"    { // artificial new line
-                         //yylineno++;
+                         //g_yyLineNr++;
                        }
 <St_Para>{SPCMD3}      {
                          g_token->name = "_form";
@@ -605,7 +617,8 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          return TK_COMMAND_SEL();
                        }
 <St_Para>{CMD}"n"\n    { /* \n followed by real newline */
-                         //yylineno++;
+                         lineCount(yytext);
+                         //g_yyLineNr++;
                          g_token->name = yytext+1;
                          g_token->name = g_token->name.stripWhiteSpace();
                          g_token->paramDir=TokenInfo::Unspecified;
@@ -704,6 +717,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          for (int i=value.length()-1;i>=0;i--) unput(value.at(i));
                        }
 <St_Para>{HTMLTAG}     { /* html tag */
+                         lineCount(yytext);
                          handleHtmlTag();
                          return TK_HTMLTAG;
                        }
@@ -739,6 +753,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_Para,St_Text>[\-+0-9] |
 <St_Para,St_Text>{WORD1} |
 <St_Para,St_Text>{WORD2} { /* function call */
+                         lineCount(yytext);
                          if (yytext[0]=='%') // strip % if present
                            g_token->name = &yytext[1];
                          else
@@ -765,6 +780,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 
 <St_Para,St_Text>{BLANK}+      |
 <St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
+                         lineCount(yytext);
                          g_token->chars=yytext;
                          return TK_WHITESPACE;
                        }
@@ -777,20 +793,24 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          {
                            REJECT;
                          }
+                         lineCount(yytext);
                        }
 <St_Para>({BLANK}*\n)+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
                          if (!g_markdownSupport || g_insidePre || g_autoListLevel==0)
                          {
                            REJECT;
                          }
+                         lineCount(yytext);
                        }
 <St_Para>({BLANK}*\n)+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
                          if (!g_markdownSupport || g_insidePre || g_autoListLevel==0)
                          {
                            REJECT;
                          }
+                         lineCount(yytext);
                        }
 <St_Para>({BLANK}*\n)+{BLANK}*\n{BLANK}* {
+                         lineCount(yytext);
                          g_token->indent=computeIndent(yytext,(int)yyleng);
                          int i;
                          // put back the indentation (needed for list items)
@@ -817,14 +837,17 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          BEGIN(St_Code);
                        }
 <St_Code>{WS}*{CMD}"endcode" {
+                         lineCount(yytext);
                          return RetVal_OK;
                        }
 <St_XmlCode>{WS}*"</code>" {
+                         lineCount(yytext);
                          return RetVal_OK;
                        }
 <St_Code,St_XmlCode>[^\\@\n<]+  |
 <St_Code,St_XmlCode>\n          |
 <St_Code,St_XmlCode>.           {
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
@@ -845,6 +868,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_HtmlOnly>[^\\@\n$]+    |
 <St_HtmlOnly>\n            |
 <St_HtmlOnly>.             {
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_ManOnly>{CMD}"endmanonly" {
@@ -853,6 +877,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_ManOnly>[^\\@\n$]+    |
 <St_ManOnly>\n            |
 <St_ManOnly>.             {
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_RtfOnly>{CMD}"endrtfonly" {
@@ -861,6 +886,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_RtfOnly>[^\\@\n$]+    |
 <St_RtfOnly>\n            |
 <St_RtfOnly>.             {
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_LatexOnly>{CMD}"endlatexonly" {
@@ -869,6 +895,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_LatexOnly>[^\\@\n]+     |
 <St_LatexOnly>\n            |
 <St_LatexOnly>.             {
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_XmlOnly>{CMD}"endxmlonly" {
@@ -877,6 +904,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_XmlOnly>[^\\@\n]+  |
 <St_XmlOnly>\n         |
 <St_XmlOnly>.          {
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_DbOnly>{CMD}"enddocbookonly" {
@@ -885,6 +913,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_DbOnly>[^\\@\n]+  |
 <St_DbOnly>\n         |
 <St_DbOnly>.          {
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_Verbatim>{CMD}"endverbatim" {
@@ -894,6 +923,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_Verbatim>[^\\@\n]+ |
 <St_Verbatim>\n        |
 <St_Verbatim>.         { /* Verbatim text */
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_Dot>{CMD}"enddot"  {
@@ -902,6 +932,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_Dot>[^\\@\n]+      |
 <St_Dot>\n             |
 <St_Dot>.              { /* dot text */
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_Msc>{CMD}("endmsc")  {
@@ -910,6 +941,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_Msc>[^\\@\n]+      |
 <St_Msc>\n             |
 <St_Msc>.              { /* msc text */
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_PlantUMLOpt>{BLANK}*"{"[^}]*"}" { // case 1: file name is specified as {filename}
@@ -947,6 +979,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_PlantUML>[^\\@\n]+ |
 <St_PlantUML>\n        |
 <St_PlantUML>.         { /* plantuml text */
+                         lineCount(yytext);
                          g_token->verb+=yytext;
                        }
 <St_Title>"\""         { // quoted title
@@ -973,6 +1006,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          return TK_SYMBOL;
                        }
 <St_TitleN>{HTMLTAG}   {
+                         lineCount(yytext);
                        }
 <St_TitleN>\n          { /* new line => end of title */
                          unput(*yytext);
@@ -998,6 +1032,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_TitleN>[\-+0-9]    |
 <St_TitleN>{WORD1}     |
 <St_TitleN>{WORD2}     { /* word */
+                         lineCount(yytext);
                          if (yytext[0]=='%') // strip % if present
                            g_token->name = &yytext[1];
                          else
@@ -1041,6 +1076,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          BEGIN(St_TitleV);
                        }
 <St_TitleV>[^ \t\r\n]+ { // attribute value
+                         lineCount(yytext);
                          g_token->chars = yytext;
                          BEGIN(St_TitleN);
                          return TK_WORD;
@@ -1055,6 +1091,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                        }
 
 <St_Anchor>{LABELID}{WS}? { // anchor
+                         lineCount(yytext);
                          g_token->name = QCString(yytext).stripWhiteSpace();
                          return TK_WORD;
                        }
@@ -1103,6 +1140,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          return 0;
                        }
 <St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
+                         lineCount(yytext);
                          BEGIN(St_Ref2);
                        }
 <St_Ref>(\n|"\\ilinebr") { // new line
@@ -1153,6 +1191,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          return TK_SYMBOL;
                        }
 <St_Ref2>"\""|\n|"\\ilinebr" { /* " or \n => end of title */
+                         lineCount(yytext);
                          return 0;
                        }
 <St_Ref2>{SPCMD1}      |
@@ -1193,6 +1232,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          return TK_WORD;
                        }
 <St_Param>({PHPTYPE}{BLANK}*("["{BLANK}*"]")*{BLANK}*"|"{BLANK}*)*{PHPTYPE}{BLANK}*("["{BLANK}*"]")*{WS}+("&")?"$"{LABELID} {
+                         lineCount(yytext);
                          QCString params = yytext;
                          int j = params.find('&');
                          int i = params.find('$');
@@ -1211,6 +1251,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                        }
 <St_Param>{WS}*","{WS}*  /* param separator */
 <St_Param>{WS}         {
+                         lineCount(yytext);
                          g_token->chars=yytext;
                          return TK_WHITESPACE;
                        }
@@ -1219,6 +1260,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                        }
 <St_Options>{WS}*","{WS}*
 <St_Options>{WS}       { /* option separator */
+                         lineCount(yytext);
                          g_token->name+=",";
                        }
 <St_Options>"}"        {
@@ -1254,6 +1296,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          return TK_WORD;
                        }
 <St_Pattern>\n         {
+                         lineCount(yytext);
                          g_token->name = g_token->name.stripWhiteSpace();
                          return TK_WORD;
                        }
@@ -1273,13 +1316,17 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
      /* State for skipping title (all chars until the end of the line) */
 
 <St_SkipTitle>.
-<St_SkipTitle>(\n|"\\ilinebr") { return 0; }
+<St_SkipTitle>(\n|"\\ilinebr") {
+                         lineCount(yytext);
+                         return 0;
+                         }
 
      /* State for the pass used to find the anchors and sections */
 
 <St_Sections>[^\n@\\<]+
 <St_Sections>{CMD}("<"|{CMD})
 <St_Sections>"<"{CAPTION}({WS}+{ATTRIB})*">" {
+                                      lineCount(yytext);
                                       QCString tag=yytext;
                                       int s=tag.find("id=");
                                       if (s!=-1) // command has id attribute
@@ -1383,6 +1430,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_Sections>.
 <St_Sections>(\n|"\\ilinebr")
 <St_SecLabel1>{LABELID} {
+                         lineCount(yytext);
                          g_secLabel = yytext;
                          processSection();
                          BEGIN(St_Sections);
@@ -1395,6 +1443,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                        }
 <St_SecTitle>[^\n]+    |
 <St_SecTitle>[^\n]*\n  {
+                         lineCount(yytext);
                          g_secTitle = yytext;
                          g_secTitle = g_secTitle.stripWhiteSpace();
                          if (g_secTitle.right(8)=="\\ilinebr")
@@ -1405,7 +1454,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          BEGIN(St_Sections);
                        }
 <St_SecTitle,St_SecLabel1,St_SecLabel2>. {
-                         warn(g_fileName,yylineno,"Unexpected character '%s' while looking for section label or title",yytext);
+                         warn(g_fileName,g_yyLineNr,"Unexpected character '%s' while looking for section label or title",yytext);
                        }
 
 <St_Snippet>[^\\\n]+   {
@@ -1415,23 +1464,25 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          g_token->name += yytext;
                        }
 <St_Snippet>(\n|"\\ilinebr")  {
+                         lineCount(yytext);
                          g_token->name = g_token->name.stripWhiteSpace();
                          return TK_WORD;
                        }
 
      /* Generic rules that work for all states */
 <*>\n                  {
-                         warn(g_fileName,yylineno,"Unexpected new line character");
+                         lineCount(yytext);
+                         warn(g_fileName,g_yyLineNr,"Unexpected new line character");
                        }
 <*>"\\ilinebr"         {
                        }
 <*>[\\@<>&$#%~"=]      { /* unescaped special character */
-                         //warn(g_fileName,yylineno,"Unexpected character '%s', assuming command \\%s was meant.",yytext,yytext);
+                         //warn(g_fileName,g_yyLineNr,"Unexpected character '%s', assuming command \\%s was meant.",yytext,yytext);
                          g_token->name = yytext;
                          return TK_COMMAND_SEL();
                        }
 <*>.                   {
-                         warn(g_fileName,yylineno,"Unexpected character '%s'",yytext);
+                         warn(g_fileName,g_yyLineNr,"Unexpected character '%s'",yytext);
                        }
 %%
 
@@ -1448,7 +1499,7 @@ void doctokenizerYYFindSections(const char *input,const Definition *d,
   g_definition  = d;
   g_fileName    = fileName;
   BEGIN(St_Sections);
-  doctokenizerYYlineno = 1;
+  g_yyLineNr = 1;
   doctokenizerYYlex();
   printlex(yy_flex_debug, FALSE, __FILE__, fileName);
 }
@@ -1686,6 +1737,30 @@ void doctokenizerYYendAutoList()
 //  return retval;
 //}
 
+void setDoctokinizerLineNr(int lineno)
+{
+  g_yyLineNr = lineno;
+}
+
+int getDoctokinizerLineNr(void)
+{
+  return g_yyLineNr;
+}
+
+static void lineCount()
+{
+  g_yyLineNr++;
+}
+
+static void lineCount(const char* text)
+{
+  for (const char* c=text ; *c ; ++c )
+  {
+    if (*c == '\n') g_yyLineNr++;
+  }
+}
+
+
 #if USE_STATE2STRING
 #include "doctokenizer.l.h"
 #endif