path: root/src/doctokenizer.l

/******************************************************************************
 *
 * $Id: $
 *
 *
 * Copyright (C) 1997-2015 by Dimitri van Heesch.
 *
 * Permission to use, copy, modify, and distribute this software and its
 * documentation under the terms of the GNU General Public License is hereby
 * granted. No representations are made about the suitability of this software
 * for any purpose. It is provided "as is" without express or implied warranty.
 * See the GNU General Public License for more details.
 *
 * Documents produced by Doxygen are derivative works derived from the
 * input used in their production; they are not affected by this license.
 *
 */

%option never-interactive
%option prefix="doctokenizerYY"
%top{
#include <stdint.h>
}

%{

#include <ctype.h>
#include <stack>
#include <string>
#include <cassert>

#include "doctokenizer.h"
#include "cmdmapper.h"
#include "config.h"
#include "message.h"
#include "section.h"
#include "membergroup.h"
#include "definition.h"
#include "doxygen.h"
#include "portable.h"
#include "cite.h"
#include "regex.h"

#define YY_NO_INPUT 1
#define YY_NO_UNISTD_H 1

#define USE_STATE2STRING 0

#define TK_COMMAND_SEL() (yytext[0] == '@' ? TK_COMMAND_AT : TK_COMMAND_BS)

//--------------------------------------------------------------------------

// context for tokenizer phase
static int g_commentState;
TokenInfo *g_token = 0;
static yy_size_t g_inputPos = 0;
static const char *g_inputString;
static QCString g_fileName;
static bool g_insidePre;
static int g_sharpCount=0;
static bool g_markdownSupport=TRUE;

// context for section finding phase
static const Definition  *g_definition;
static QCString     g_secLabel;
static QCString     g_secTitle;
static SectionType  g_secType;
static QCString     g_endMarker;
static int          g_autoListLevel;

struct DocLexerContext
{
  DocLexerContext(TokenInfo *tk,int r,int lvl,yy_size_t pos,const char *s,YY_BUFFER_STATE bs)
    : token(tk), rule(r), autoListLevel(lvl), inputPos(pos), inputString(s), state(bs) {}
  TokenInfo *token;
  int rule;
  int autoListLevel;
  yy_size_t inputPos;
  const char *inputString;
  YY_BUFFER_STATE state;
};

static std::stack< std::unique_ptr<DocLexerContext> > g_lexerStack;

static int g_yyLineNr = 0;

#define lineCount(s,len) do { for(int i=0;i<(int)len;i++) if (s[i]=='\n') g_yyLineNr++; } while(0)


#if USE_STATE2STRING
static const char *stateToString(int state);
#endif
//--------------------------------------------------------------------------

void doctokenizerYYpushContext()
{
  g_lexerStack.push(
      std::make_unique<DocLexerContext>(
        g_token,YY_START,g_autoListLevel,g_inputPos,g_inputString,YY_CURRENT_BUFFER));
  yy_switch_to_buffer(yy_create_buffer(doctokenizerYYin, YY_BUF_SIZE));
}

bool doctokenizerYYpopContext()
{
  if (g_lexerStack.empty()) return FALSE;
  const auto &ctx = g_lexerStack.top();
  g_autoListLevel = ctx->autoListLevel;
  g_inputPos = ctx->inputPos;
  g_inputString = ctx->inputString;
  yy_delete_buffer(YY_CURRENT_BUFFER);
  yy_switch_to_buffer(ctx->state);
  BEGIN(ctx->rule);
  g_lexerStack.pop();
  return TRUE;
}

QCString extractPartAfterNewLine(const QCString &text)
{
  int nl1 = text.find('\n');
  int nl2 = text.find("\\ilinebr");
  if (nl1!=-1 && nl1<nl2)
  {
    return text.mid(nl1+1);
  }
  if (nl2!=-1)
  {
    if (text.at(nl2+8)==' ') nl2++; // skip space after \\ilinebr
    return text.mid(nl2+8);
  }
  return text;
}

//--------------------------------------------------------------------------

const char *tokToString(int token)
{
  switch (token)
  {
    case 0:              return "TK_EOF";
    case TK_WORD:        return "TK_WORD";
    case TK_LNKWORD:     return "TK_LNKWORD";
    case TK_WHITESPACE:  return "TK_WHITESPACE";
    case TK_LISTITEM:    return "TK_LISTITEM";
    case TK_ENDLIST:     return "TK_ENDLIST";
    case TK_COMMAND_AT:  return "TK_COMMAND_AT";
    case TK_HTMLTAG:     return "TK_HTMLTAG";
    case TK_SYMBOL:      return "TK_SYMBOL";
    case TK_NEWPARA:     return "TK_NEWPARA";
    case TK_RCSTAG:      return "TK_RCSTAG";
    case TK_URL:         return "TK_URL";
    case TK_COMMAND_BS:  return "TK_COMMAND_BS";
  }
  return "ERROR";
}

static int computeIndent(const char *str,size_t length)
{
  if (str==0 || length==std::string::npos) return 0;
  size_t i;
  int indent=0;
  static int tabSize=Config_getInt(TAB_SIZE);
  for (i=0;i<length;i++)
  {
    if (str[i]=='\t')
    {
      indent+=tabSize - (indent%tabSize);
    }
    else if (str[i]=='\n')
    {
      indent=0;
    }
    else
    {
      indent++;
    }
  }
  return indent;
}

//--------------------------------------------------------------------------

static void processSection()
{
  //printf("%s: found section/anchor with name '%s'\n",qPrint(g_fileName),qPrint(g_secLabel));
  QCString file;
  if (g_definition)
  {
    file = g_definition->getOutputFileBase();
  }
  else
  {
    warn(g_fileName,g_yyLineNr,"Found section/anchor %s without context\n",qPrint(g_secLabel));
  }
  SectionInfo *si = SectionManager::instance().find(g_secLabel);
  if (si)
  {
    si->setFileName(file);
    si->setType(g_secType);
  }
}

static void handleHtmlTag()
{
  QCString tagText(yytext);
  g_token->attribs.clear();
  g_token->endTag = FALSE;
  g_token->emptyTag = FALSE;

  // Check for end tag
  int startNamePos=1;
  if (tagText.at(1)=='/')
  {
    g_token->endTag = TRUE;
    startNamePos++;
  }

  // Parse the name portion
  int i = startNamePos;
  for (i=startNamePos; i < (int)yyleng; i++)
  {
    // Check for valid HTML/XML name chars (including namespaces)
    char c = tagText.at(i);
    if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
  }
  g_token->name = tagText.mid(startNamePos,i-startNamePos);

  // Parse the attributes. Each attribute is a name, value pair
  // The result is stored in g_token->attribs.
  int startName,endName,startAttrib,endAttrib;
  int startAttribList = i;
  while (i<(int)yyleng)
  {
    char c=tagText.at(i);
    // skip spaces
    while (i<(int)yyleng && isspace((uchar)c)) { c=tagText.at(++i); }
    // check for end of the tag
    if (c == '>') break;
    // Check for XML style "empty" tag.
    if (c == '/')
    {
      g_token->emptyTag = TRUE;
      break;
    }
    startName=i;
    // search for end of name
    while (i<(int)yyleng && !isspace((uchar)c) && c!='=' && c!= '>') { c=tagText.at(++i); }
    endName=i;
    HtmlAttrib opt;
    opt.name  = tagText.mid(startName,endName-startName).lower();
    // skip spaces
    while (i<(int)yyleng && isspace((uchar)c)) { c=tagText.at(++i); }
    if (tagText.at(i)=='=') // option has value
    {
      c=tagText.at(++i);
      // skip spaces
      while (i<(int)yyleng && isspace((uchar)c)) { c=tagText.at(++i); }
      if (tagText.at(i)=='\'') // option '...'
      {
        c=tagText.at(++i);
        startAttrib=i;

        // search for matching quote
        while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
        endAttrib=i;
        if (i<(int)yyleng) { c=tagText.at(++i);}
      }
      else if (tagText.at(i)=='"') // option "..."
      {
        c=tagText.at(++i);
        startAttrib=i;
        // search for matching quote
        while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
        endAttrib=i;
        if (i<(int)yyleng) { c=tagText.at(++i);}
      }
      else // value without any quotes
      {
        startAttrib=i;
        // search for separator or end symbol
        while (i<(int)yyleng && !isspace((uchar)c) && c!='>') { c=tagText.at(++i); }
        endAttrib=i;
        if (i<(int)yyleng) { c=tagText.at(++i);}
      }
      opt.value  = tagText.mid(startAttrib,endAttrib-startAttrib);
      if (opt.name == "align") opt.value = opt.value.lower();
      else if (opt.name == "valign")
      {
        opt.value = opt.value.lower();
        if (opt.value == "center") opt.value="middle";
      }
    }
    else // start next option
    {
    }
    //printf("=====> Adding option name=<%s> value=<%s>\n",
    //    qPrint(opt.name),qPrint(opt.value));
    g_token->attribs.push_back(opt);
  }
  g_token->attribsStr = tagText.mid(startAttribList,i-startAttribList);
}

static QCString stripEmptyLines(const QCString &s)
{
  if (s.isEmpty()) return QCString();
  int end=s.length();
  int start=0,p=0;
  // skip leading empty lines
  for (;;)
  {
    int c;
    while ((c=s[p]) && (c==' ' || c=='\t')) p++;
    if (s[p]=='\n')
    {
      start=++p;
    }
    else
    {
      break;
    }
  }
  // skip trailing empty lines
  p=end-1;
  if (p>=start && s.at(p)=='\n') p--;
  while (p>=start)
  {
    int c;
    while ((c=s[p]) && (c==' ' || c=='\t')) p--;
    if (s[p]=='\n')
    {
      end=p;
    }
    else
    {
      break;
    }
    p--;
  }
  //printf("stripEmptyLines(%d-%d)\n",start,end);
  return s.mid(start,end-start);
}

#define unput_string(yytext,yyleng) do { for (int i=(int)yyleng-1;i>=0;i--) unput(yytext[i]); } while(0)
//--------------------------------------------------------------------------

#undef  YY_INPUT
#define YY_INPUT(buf,result,max_size) result=yyread(buf,max_size);

static yy_size_t yyread(char *buf,yy_size_t max_size)
{
  yy_size_t c=0;
  const char *src=g_inputString+g_inputPos;
  while ( c < max_size && *src ) *buf++ = *src++, c++;
  g_inputPos+=c;
  return c;
}

//--------------------------------------------------------------------------
//#define REAL_YY_DECL int doctokenizerYYlex (void)
//#define YY_DECL static int local_doctokenizer(void)
//#define LOCAL_YY_DECL local_doctokenizer()

%}

CMD       ("\\"|"@")
WS        [ \t\r\n]
NONWS     [^ \t\r\n]
BLANK     [ \t\r]
BLANKopt  {BLANK}*
ID        "$"?[a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF]*
LABELID   [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-]*
PHPTYPE   [\\:a-z_A-Z0-9\x80-\xFF\-]+
CITESCHAR [a-z_A-Z0-9\x80-\xFF\-\?]
CITEECHAR [a-z_A-Z0-9\x80-\xFF\-\+:\/\?]
CITEID    {CITESCHAR}{CITEECHAR}*("."{CITESCHAR}{CITEECHAR}*)*|"\""{CITESCHAR}{CITEECHAR}*("."{CITESCHAR}{CITEECHAR}*)*"\""
MAILADDR  ("mailto:")?[a-z_A-Z0-9.+-]+"@"[a-z_A-Z0-9-]+("."[a-z_A-Z0-9\-]+)+[a-z_A-Z0-9\-]+
MAILWS    [\t a-z_A-Z0-9+-]
MAILADDR2 {MAILWS}+{BLANK}+("at"|"AT"|"_at_"|"_AT_"){BLANK}+{MAILWS}+("dot"|"DOT"|"_dot_"|"_DOT_"){BLANK}+{MAILWS}+
OPTSTARS  ("/""/"{BLANK}*)?"*"*{BLANK}*
LISTITEM  {BLANK}*[-]("#")?{WS}
MLISTITEM {BLANK}*[+*]{WS}
OLISTITEM {BLANK}*[1-9][0-9]*"."{BLANK}
ENDLIST   {BLANK}*"."{BLANK}*\n
ATTRNAME  [a-z_A-Z\x80-\xFF][:a-z_A-Z0-9\x80-\xFF\-]*
ATTRIB    {ATTRNAME}{WS}*("="{WS}*(("\""[^\"]*"\"")|("'"[^\']*"'")|[^ \t\r\n'"><]+))?
URLCHAR   [a-z_A-Z0-9\!\~\,\:\;\'\$\?\@\&\%\#\.\-\+\/\=\x80-\xFF]
URLMASK   ({URLCHAR}+([({]{URLCHAR}*[)}])?)+
URLPROTOCOL ("http:"|"https:"|"ftp:"|"ftps:"|"sftp:"|"file:"|"news:"|"irc:"|"ircs:")
FILESCHAR [a-z_A-Z0-9\\:\\\/\-\+&#@]
FILEECHAR [a-z_A-Z0-9\-\+&#@]
HFILEMASK ("."{FILESCHAR}*{FILEECHAR}+)+
FILEMASK  ({FILESCHAR}*{FILEECHAR}+("."{FILESCHAR}*{FILEECHAR}+)*)|{HFILEMASK}
LINKMASK  [^ \t\n\r\\@<&${}]+("("[^\n)]*")")?({BLANK}*("const"|"volatile"){BLANK}+)?
VERBATIM  "verbatim"{BLANK}*
SPCMD1    {CMD}([a-z_A-Z][a-z_A-Z0-9]*|{VERBATIM}|"--"|"---")
SPCMD2    {CMD}[\\@<>&$#%~".+=|-]
SPCMD3    {CMD}_form#[0-9]+
SPCMD4    {CMD}"::"
SPCMD5    {CMD}":"
INOUT     "in"|"out"|("in"{BLANK}*","?{BLANK}*"out")|("out"{BLANK}*","?{BLANK}*"in")
PARAMIO   {CMD}param{BLANK}*"["{BLANK}*{INOUT}{BLANK}*"]"
VARARGS   "..."
TEMPCHAR  [a-z_A-Z0-9.,: \t\*\&\(\)\[\]]
FUNCCHAR  [a-z_A-Z0-9,:\<\> \t\^\*\&\[\]]|{VARARGS}
FUNCPART  {FUNCCHAR}*("("{FUNCCHAR}*")"{FUNCCHAR}*)?
SCOPESEP  "::"|"#"|"."
TEMPLPART "<"{TEMPCHAR}*">"
ANONNS    "anonymous_namespace{"[^}]*"}"
SCOPEPRE  (({ID}{TEMPLPART}?)|{ANONNS}){SCOPESEP}
SCOPEKEYS ":"({ID}":")*
SCOPECPP  {SCOPEPRE}*(~)?{ID}{TEMPLPART}?
SCOPEOBJC {SCOPEPRE}?{ID}{SCOPEKEYS}?
SCOPEMASK {SCOPECPP}|{SCOPEOBJC}
FUNCARG   "("{FUNCPART}")"({BLANK}*("volatile"|"const"){BLANK})?
FUNCARG2  "("{FUNCPART}")"({BLANK}*("volatile"|"const"))?
OPNEW     {BLANK}+"new"({BLANK}*"[]")?
OPDEL     {BLANK}+"delete"({BLANK}*"[]")?
OPNORM    {OPNEW}|{OPDEL}|"+"|"-"|"*"|"/"|"%"|"^"|"&"|"|"|"~"|"!"|"="|"<"|">"|"+="|"-="|"*="|"/="|"%="|"^="|"&="|"|="|"<<"|">>"|"<<="|">>="|"=="|"!="|"<="|">="|"&&"|"||"|"++"|"--"|","|"->*"|"->"|"[]"|"()"|"<=>"
OPCAST    {BLANK}+[^<(\r\n.,][^(\r\n.,]*
OPMASK    ({BLANK}*{OPNORM}{FUNCARG})
OPMASKOPT ({BLANK}*{OPNORM}{FUNCARG}?)|({OPCAST}{FUNCARG})
OPMASKOP2 ({BLANK}*{OPNORM}{FUNCARG2}?)|({OPCAST}{FUNCARG2})
LNKWORD1  ("::"|"#")?{SCOPEMASK}
CVSPEC    {BLANK}*("const"|"volatile")
LNKWORD2  (({SCOPEPRE}*"operator"{OPMASK})|({SCOPEPRE}"operator"{OPMASKOPT})|(("::"|"#"){SCOPEPRE}*"operator"{OPMASKOPT})){CVSPEC}?
LNKWORD3  ([0-9a-z_A-Z\-]+("/"|"\\"))*[0-9a-z_A-Z\-]+("."[0-9a-z_A-Z]+)+
CHARWORDQ [^ \t\n\r\\@<>()\[\]:;\?{}&%$#,."=']
ESCWORD   ("%"{ID}(("::"|"."){ID})*)|("%'")
CHARWORDQ1 [^ \-+0-9\t\n\r\\@<>()\[\]:;\?{}&%$#,."=']
WORD1     {ESCWORD}|{CHARWORDQ1}{CHARWORDQ}*|"{"|"}"|"'\"'"|("\""([^"\n]*\n?)*[^"\n]*"\"")
WORD2     "."|","|"("|")"|"["|"]"|"::"|":"|";"|"\?"|"="|"'"
WORD1NQ   {ESCWORD}|{CHARWORDQ}+|"{"|"}"
WORD2NQ   "."|","|"("|")"|"["|"]"|"::"|":"|";"|"\?"|"="|"'"
CAPTION   [cC][aA][pP][tT][iI][oO][nN]
HTMLTAG   "<"(("/")?){ID}({WS}+{ATTRIB})*{WS}*(("/")?)">"
HTMLKEYL  "strong"|"center"|"table"|"caption"|"small"|"code"|"dfn"|"var"|"img"|"pre"|"sub"|"sup"|"tr"|"td"|"th"|"ol"|"ul"|"li"|"tt"|"kbd"|"em"|"hr"|"dl"|"dt"|"dd"|"br"|"i"|"a"|"b"|"p"|"strike"|"u"|"del"|"ins"|"s"
HTMLKEYU  "STRONG"|"CENTER"|"TABLE"|"CAPTION"|"SMALL"|"CODE"|"DFN"|"VAR"|"IMG"|"PRE"|"SUB"|"SUP"|"TR"|"TD"|"TH"|"OL"|"UL"|"LI"|"TT"|"KBD"|"EM"|"HR"|"DL"|"DT"|"DD"|"BR"|"I"|"A"|"B"|"P"|"STRIKE"|"U"|"DEL"|"INS"|"S"
HTMLKEYW  {HTMLKEYL}|{HTMLKEYU}
REFWORD2_PRE   ("#"|"::")?((({ID}{TEMPLPART}?)|{ANONNS})("."|"#"|"::"|"-"|"/"))*({ID}{TEMPLPART}?(":")?)
REFWORD2       {REFWORD2_PRE}{FUNCARG2}?
REFWORD2_NOCV  {REFWORD2_PRE}("("{FUNCPART}")")?
REFWORD3       ({ID}":")*{ID}":"?
REFWORD4_NOCV  (({SCOPEPRE}*"operator"{OPMASKOP2})|(("::"|"#"){SCOPEPRE}*"operator"{OPMASKOP2}))
REFWORD4       {REFWORD4_NOCV}{CVSPEC}?
REFWORD        {FILEMASK}|{LABELID}|{REFWORD2}|{REFWORD3}|{REFWORD4}
REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
RCSID "$"("Author"|"Date"|"Header"|"Id"|"Locker"|"Log"|"Name"|"RCSfile"|"Revision"|"Source"|"State")":"[^:\n$][^\n$]*"$"

%option noyywrap

%x St_Para
%x St_Comment
%x St_Title
%x St_TitleN
%x St_TitleQ
%x St_TitleA
%x St_TitleV
%x St_Code
%x St_CodeOpt
%x St_XmlCode
%x St_HtmlOnly
%x St_HtmlOnlyOption
%x St_ManOnly
%x St_LatexOnly
%x St_RtfOnly
%x St_XmlOnly
%x St_DbOnly
%x St_Verbatim
%x St_Dot
%x St_Msc
%x St_PlantUMLOpt
%x St_PlantUML
%x St_Param
%x St_XRefItem
%x St_XRefItem2
%x St_File
%x St_Pattern
%x St_Link
%x St_Cite
%x St_Ref
%x St_Ref2
%x St_IntRef
%x St_Text
%x St_SkipTitle
%x St_Anchor
%x St_Snippet
%x St_SetScope
%x St_SetScopeEnd
%x St_Options
%x St_Block
%x St_Emoji

%x St_Sections
%s St_SecLabel1
%s St_SecLabel2
%s St_SecTitle
%x St_SecSkip

%%
<St_Para>\r            /* skip carriage return */
<St_Para>^{LISTITEM}   { /* list item */
                         lineCount(yytext,yyleng);
                         QCString text(yytext);
                         size_t dashPos = static_cast<size_t>(text.findRev('-'));
                         assert(dashPos!=std::string::npos);
                         g_token->isEnumList = text.at(dashPos+1)=='#';
                         g_token->id         = -1;
                         g_token->indent     = computeIndent(yytext,dashPos);
                         return TK_LISTITEM;
                       }
<St_Para>^{MLISTITEM}  { /* list item */
                         if (!g_markdownSupport || g_insidePre)
                         {
                           REJECT;
                         }
                         else
                         {
                           lineCount(yytext,yyleng);
                           std::string text(yytext);
                           static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
                           reg::Match match;
                           reg::search(text,match,re);
                           size_t listPos = match.position();
                           assert(listPos!=std::string::npos);
                           g_token->isEnumList = FALSE;
                           g_token->id         = -1;
                           g_token->indent     = computeIndent(yytext,listPos);
                           return TK_LISTITEM;
                         }
                       }
<St_Para>^{OLISTITEM}  { /* numbered list item */
                         if (!g_markdownSupport || g_insidePre)
                         {
                           REJECT;
                         }
                         else
                         {
                           std::string text(yytext);
                           static const reg::Ex re(R"(\d+)");
                           reg::Match match;
                           reg::search(text,match,re);
                           size_t markPos = match.position();
                           assert(markPos!=std::string::npos);
                           g_token->isEnumList = true;
                           g_token->id         = std::stoul(match.str());
                           g_token->indent     = computeIndent(yytext,markPos);
                           return TK_LISTITEM;
                         }
                       }
<St_Para>{BLANK}*(\n|"\\ilinebr"){LISTITEM}     { /* list item on next line */
                         lineCount(yytext,yyleng);
                         QCString text=extractPartAfterNewLine(QCString(yytext));
                         size_t dashPos = static_cast<size_t>(text.findRev('-'));
                         assert(dashPos!=std::string::npos);
                         g_token->isEnumList = text.at(dashPos+1)=='#';
                         g_token->id         = -1;
                         g_token->indent     = computeIndent(text.data(),dashPos);
                         return TK_LISTITEM;
                       }
<St_Para>{BLANK}*(\n|"\\ilinebr"){MLISTITEM}     { /* list item on next line */
                         if (!g_markdownSupport || g_insidePre)
                         {
                           REJECT;
                         }
                         else
                         {
                           lineCount(yytext,yyleng);
                           std::string text=extractPartAfterNewLine(QCString(yytext)).str();
                           static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
                           reg::Match match;
                           reg::search(text,match,re);
                           size_t markPos = match.position();
                           assert(markPos!=std::string::npos);
                           g_token->isEnumList = FALSE;
                           g_token->id         = -1;
                           g_token->indent     = computeIndent(text.c_str(),markPos);
                           return TK_LISTITEM;
                         }
                       }
<St_Para>{BLANK}*(\n|"\\ilinebr"){OLISTITEM}     { /* list item on next line */
                         if (!g_markdownSupport || g_insidePre)
                         {
                           REJECT;
                         }
                         else
                         {
                           lineCount(yytext,yyleng);
                           std::string text=extractPartAfterNewLine(QCString(yytext)).str();
                           static const reg::Ex re(R"(\d+)");
                           reg::Match match;
                           reg::search(text,match,re);
                           size_t markPos = match.position();
                           assert(markPos!=std::string::npos);
                           g_token->isEnumList = true;
                           g_token->id         = std::stoul(match.str());
                           g_token->indent     = computeIndent(text.c_str(),markPos);
                           return TK_LISTITEM;
                         }
                       }
<St_Para>^{ENDLIST}       { /* end list */
                         lineCount(yytext,yyleng);
                         int dotPos = QCString(yytext).findRev('.');
                         assert(dotPos!=-1);
                         g_token->indent     = computeIndent(yytext,dotPos);
                         return TK_ENDLIST;
                       }
<St_Para>{BLANK}*(\n|"\\ilinebr"){ENDLIST}      { /* end list on next line */
                         lineCount(yytext,yyleng);
                         QCString text=extractPartAfterNewLine(QCString(yytext));
                         int dotPos = text.findRev('.');
                         g_token->indent = computeIndent(text.data(),dotPos);
                         return TK_ENDLIST;
                       }
<St_Para>"{"{BLANK}*"@link"/{WS}+ {
                         g_token->name = "javalink";
                         return TK_COMMAND_AT;
                       }
<St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
                         g_token->name = "inheritdoc";
                         return TK_COMMAND_AT;
                       }
<St_Para>"@_fakenl"    { // artificial new line
                         //g_yyLineNr++;
                       }
<St_Para>{SPCMD3}      {
                         g_token->name = "_form";
                         bool ok;
                         g_token->id = QCString(yytext).right((int)yyleng-7).toInt(&ok);
                         ASSERT(ok);
                         return TK_COMMAND_SEL();
                       }
<St_Para>{CMD}"n"\n    { /* \n followed by real newline */
                         lineCount(yytext,yyleng);
                         //g_yyLineNr++;
                         g_token->name = yytext+1;
                         g_token->name = g_token->name.stripWhiteSpace();
                         g_token->paramDir=TokenInfo::Unspecified;
                         return TK_COMMAND_SEL();
                       }
<St_Para>"\\ilinebr"   {
                       }
<St_Para>{SPCMD1}      |
<St_Para>{SPCMD2}      |
<St_Para>{SPCMD5}      |
<St_Para>{SPCMD4}      { /* special command */
                         g_token->name = yytext+1;
                         g_token->name = g_token->name.stripWhiteSpace();
                         g_token->paramDir=TokenInfo::Unspecified;
                         return TK_COMMAND_SEL();
                       }
<St_Para>{PARAMIO}     { /* param [in,out] command */
                         g_token->name = "param";
                         QCString s(yytext);
                         bool isIn  = s.find("in")!=-1;
                         bool isOut = s.find("out")!=-1;
                         if (isIn)
                         {
                           if (isOut)
                           {
                             g_token->paramDir=TokenInfo::InOut;
                           }
                           else
                           {
                             g_token->paramDir=TokenInfo::In;
                           }
                         }
                         else if (isOut)
                         {
                           g_token->paramDir=TokenInfo::Out;
                         }
                         else
                         {
                           g_token->paramDir=TokenInfo::Unspecified;
                         }
                         return TK_COMMAND_SEL();
                       }
<St_Para>{URLPROTOCOL}{URLMASK}/[,\.] { // URL, or URL.
                         g_token->name=yytext;
                         g_token->isEMailAddr=FALSE;
                         return TK_URL;
                       }
<St_Para>{URLPROTOCOL}{URLMASK} { // URL
                         g_token->name=yytext;
                         g_token->isEMailAddr=FALSE;
                         return TK_URL;
                       }
<St_Para>"<"{URLPROTOCOL}{URLMASK}">" { // URL
                         g_token->name=yytext;
                         g_token->name = g_token->name.mid(1,g_token->name.length()-2);
                         g_token->isEMailAddr=FALSE;
                         return TK_URL;
                       }
<St_Para>{MAILADDR}    { // Mail address
                         g_token->name=yytext;
                         g_token->name.stripPrefix("mailto:");
                         g_token->isEMailAddr=TRUE;
                         return TK_URL;
                       }
<St_Para>"<"{MAILADDR}">" { // Mail address
                         g_token->name=yytext;
                         g_token->name = g_token->name.mid(1,g_token->name.length()-2);
                         g_token->name.stripPrefix("mailto:");
                         g_token->isEMailAddr=TRUE;
                         return TK_URL;
                       }
<St_Para>"<"{MAILADDR2}">" { // anti spam mail address
                         g_token->name=yytext;
                         return TK_WORD;
                       }
<St_Para>{RCSID} { /* RCS tag */
                         QCString tagName(yytext+1);
                         int index=tagName.find(':');
                         g_token->name = tagName.left(index);
                         int text_begin = index+2;
                         int text_end = tagName.length()-1;
                         if (tagName[text_begin-1]==':') /* check for Subversion fixed-length keyword */
                         {
                                 ++text_begin;
                                 if (tagName[text_end-1]=='#')
                                         --text_end;
                         }
                         g_token->text = tagName.mid(text_begin,text_end-text_begin);
                         return TK_RCSTAG;
                       }
<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}")"   | /* environment variable */
<St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}"("{ID}"))"   { /* environment variable */
                         QCString name(&yytext[2]);
                         name = name.left(name.length()-1);
                         QCString value = Portable::getenv(name);
                         for (int i=value.length()-1;i>=0;i--) unput(value.at(i));
                       }
<St_Para>{HTMLTAG}     { /* html tag */
                         lineCount(yytext,yyleng);
                         handleHtmlTag();
                         return TK_HTMLTAG;
                       }
<St_Para,St_Text>"&"{ID}";" { /* special symbol */
                         g_token->name = yytext;
                         return TK_SYMBOL;
                       }

  /********* patterns for linkable words ******************/

<St_Para>{ID}/"<"{HTMLKEYW}">" { /* this rule is to prevent opening html
                                  * tag to be recognized as a templated classes
                                  */
                         g_token->name = yytext;
                         return TK_LNKWORD;
                        }
<St_Para>{LNKWORD1}/"<br>"           | // prevent <br> html tag to be parsed as template arguments
<St_Para>{LNKWORD1}                  |
<St_Para>{LNKWORD1}{FUNCARG}         |
<St_Para>{LNKWORD2}                  |
<St_Para>{LNKWORD3}    {
                         g_token->name = yytext;
                         return TK_LNKWORD;
                       }
<St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
                         g_token->name = yytext;
                         g_token->name = g_token->name.left(g_token->name.length()-1);
                         unput(yytext[(int)yyleng-1]);
                         return TK_LNKWORD;
                       }
  /********* patterns for normal words ******************/

<St_Para,St_Text>[\-+0-9] |
<St_Para,St_Text>{WORD1} |
<St_Para,St_Text>{WORD2} { /* function call */
                         if (QCString(yytext).find("\\ilinebr")!=-1) REJECT; // see issue #8311
                         lineCount(yytext,yyleng);
                         if (yytext[0]=='%') // strip % if present
                           g_token->name = &yytext[1];
                         else
                           g_token->name = yytext;
                         return TK_WORD;

                         /* the following is dummy code to please the
                          * compiler, removing this results in a warning
                          * on my machine
                          */
                         goto find_rule;
                       }
<St_Text>({ID}".")+{ID} {
                          g_token->name = yytext;
                          return TK_WORD;
                        }
<St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
                                                          // avoid interpretation as "operator <"
                           g_token->name = yytext;
                           return TK_WORD;
                         }

  /*******************************************************/

<St_Para,St_Text>{BLANK}+      |
<St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
                         lineCount(yytext,yyleng);
                         g_token->chars=yytext;
                         return TK_WHITESPACE;
                       }
<St_Text>[\\@<>&$#%~]  {
                         g_token->name = yytext;
                         return TK_COMMAND_SEL();
                       }
<St_Para>({BLANK}*\n)+{BLANK}*\n/{LISTITEM} { /* skip trailing paragraph followed by new list item */
                         if (g_insidePre || g_autoListLevel==0)
                         {
                           REJECT;
                         }
                         lineCount(yytext,yyleng);
                       }
<St_Para>({BLANK}*\n)+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
                         if (!g_markdownSupport || g_insidePre || g_autoListLevel==0)
                         {
                           REJECT;
                         }
                         lineCount(yytext,yyleng);
                       }
<St_Para>({BLANK}*\n)+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
                         if (!g_markdownSupport || g_insidePre || g_autoListLevel==0)
                         {
                           REJECT;
                         }
                         lineCount(yytext,yyleng);
                       }
<St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}* {
                         lineCount(yytext,yyleng);
                         if (g_insidePre)
                         {
                           g_token->chars=yytext;
                           return TK_WHITESPACE;
                         }
                         else
                         {
                           g_token->indent=computeIndent(yytext,yyleng);
                           int i;
                           // put back the indentation (needed for list items)
                           for (i=0;i<g_token->indent;i++)
                           {
                             unput(' ');
                           }
                           // tell flex that after putting the last indent
                           // back we are at the beginning of the line
                           YY_CURRENT_BUFFER->yy_at_bol=1;
                           // start of a new paragraph
                           return TK_NEWPARA;
                         }
                       }
<St_CodeOpt>{BLANK}*"{"(".")?{LABELID}"}" {
                         g_token->name = yytext;
                         int i=g_token->name.find('{'); /* } to keep vi happy */
                         g_token->name = g_token->name.mid(i+1,g_token->name.length()-i-2);
                         BEGIN(St_Code);
                       }
<St_CodeOpt>"\\ilinebr" |
<St_CodeOpt>\n         |
<St_CodeOpt>.          {
                         unput_string(yytext,yyleng);
                         BEGIN(St_Code);
                       }
<St_Code>{WS}*{CMD}"endcode" {
                         lineCount(yytext,yyleng);
                         return RetVal_OK;
                       }
<St_XmlCode>{WS}*"</code>" {
                         lineCount(yytext,yyleng);
                         return RetVal_OK;
                       }
<St_Code,St_XmlCode>[^\\@\n<]+  |
<St_Code,St_XmlCode>\n          |
<St_Code,St_XmlCode>.           {
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
                         g_token->name="block";
                         BEGIN(St_HtmlOnly);
                        }
<St_HtmlOnlyOption>.|\n {
                         unput(*yytext);
                         BEGIN(St_HtmlOnly);
                        }
<St_HtmlOnlyOption>"\\ilinebr" {
                         unput_string(yytext,yyleng);
                         BEGIN(St_HtmlOnly);
                        }
<St_HtmlOnly>{CMD}"endhtmlonly" {
                         return RetVal_OK;
                       }
<St_HtmlOnly>[^\\@\n$]+    |
<St_HtmlOnly>\n            |
<St_HtmlOnly>.             {
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_ManOnly>{CMD}"endmanonly" {
                         return RetVal_OK;
                       }
<St_ManOnly>[^\\@\n$]+    |
<St_ManOnly>\n            |
<St_ManOnly>.             {
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_RtfOnly>{CMD}"endrtfonly" {
                         return RetVal_OK;
                       }
<St_RtfOnly>[^\\@\n$]+    |
<St_RtfOnly>\n            |
<St_RtfOnly>.             {
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_LatexOnly>{CMD}"endlatexonly" {
                         return RetVal_OK;
                       }
<St_LatexOnly>[^\\@\n]+     |
<St_LatexOnly>\n            |
<St_LatexOnly>.             {
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_XmlOnly>{CMD}"endxmlonly" {
                         return RetVal_OK;
                       }
<St_XmlOnly>[^\\@\n]+  |
<St_XmlOnly>\n         |
<St_XmlOnly>.          {
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_DbOnly>{CMD}"enddocbookonly" {
                         return RetVal_OK;
                       }
<St_DbOnly>[^\\@\n]+  |
<St_DbOnly>\n         |
<St_DbOnly>.          {
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_Verbatim>{CMD}"endverbatim" {
                         g_token->verb=stripEmptyLines(g_token->verb);
                         return RetVal_OK;
                       }
<St_Verbatim>[^\\@\n]+ |
<St_Verbatim>\n        |
<St_Verbatim>.         { /* Verbatim text */
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_Dot>{CMD}"enddot"  {
                         return RetVal_OK;
                       }
<St_Dot>[^\\@\n]+      |
<St_Dot>\n             |
<St_Dot>.              { /* dot text */
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_Msc>{CMD}("endmsc")  {
                         return RetVal_OK;
                       }
<St_Msc>[^\\@\n]+      |
<St_Msc>\n             |
<St_Msc>.              { /* msc text */
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_PlantUMLOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
                         g_token->sectionId = QCString(yytext).stripWhiteSpace();
                         return RetVal_OK;
                       }
<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
                         g_token->sectionId = QCString(yytext).stripWhiteSpace();
                         return RetVal_OK;
                       }
<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
                         g_token->sectionId = QCString(yytext).stripWhiteSpace();
                         return RetVal_OK;
                       }
<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
                         g_token->sectionId = QCString(yytext).stripWhiteSpace();
                         return RetVal_OK;
                       }
<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
                         g_token->sectionId = QCString(yytext).stripWhiteSpace();
                         return RetVal_OK;
                       }
<St_PlantUMLOpt>"\\ilinebr" |
<St_PlantUMLOpt>"\n"   |
<St_PlantUMLOpt>.      {
                         g_token->sectionId = "";
                         unput_string(yytext,yyleng);
                         return RetVal_OK;
                       }
<St_PlantUML>{CMD}"enduml"  {
                         return RetVal_OK;
                       }
<St_PlantUML>[^\\@\n]+ |
<St_PlantUML>\n        |
<St_PlantUML>.         { /* plantuml text */
                         lineCount(yytext,yyleng);
                         g_token->verb+=yytext;
                       }
<St_Title>"\""         { // quoted title
                         BEGIN(St_TitleQ);
                       }
<St_Title>[ \t]+       {
                         g_token->chars=yytext;
                         return TK_WHITESPACE;
                       }
<St_Title>.            { // non-quoted title
                         unput(*yytext);
                         BEGIN(St_TitleN);
                       }
<St_Title>\n           {
                         unput(*yytext);
                         return 0;
                       }
<St_Title>"\\ilinebr"  {
                         unput_string(yytext,yyleng);
                         return 0;
                       }
<St_TitleN>"&"{ID}";"  { /* symbol */
                         g_token->name = yytext;
                         return TK_SYMBOL;
                       }
<St_TitleN>{HTMLTAG}   {
                         lineCount(yytext,yyleng);
                       }
<St_TitleN>\n          { /* new line => end of title */
                         unput(*yytext);
                         return 0;
                       }
<St_TitleN>"\\ilinebr" { /* new line => end of title */
                         unput_string(yytext,yyleng);
                         return 0;
                       }
<St_TitleN>{SPCMD1}    |
<St_TitleN>{SPCMD2}    { /* special command */
                         g_token->name = yytext+1;
                         g_token->paramDir=TokenInfo::Unspecified;
                         return TK_COMMAND_SEL();
                       }
<St_TitleN>{ID}"="     { /* attribute */
                         if (yytext[0]=='%') // strip % if present
                           g_token->name = &yytext[1];
                         else
                           g_token->name = yytext;
                         return TK_WORD;
                       }
<St_TitleN>[\-+0-9]    |
<St_TitleN>{WORD1}     |
<St_TitleN>{WORD2}     { /* word */
                         if (QCString(yytext).find("\\ilinebr")!=-1) REJECT; // see issue #8311
                         lineCount(yytext,yyleng);
                         if (yytext[0]=='%') // strip % if present
                           g_token->name = &yytext[1];
                         else
                           g_token->name = yytext;
                         return TK_WORD;
                       }
<St_TitleN>[ \t]+      {
                         g_token->chars=yytext;
                         return TK_WHITESPACE;
                       }
<St_TitleQ>"&"{ID}";"  { /* symbol */
                         g_token->name = yytext;
                         return TK_SYMBOL;
                       }
<St_TitleQ>(\n|"\\ilinebr") { /* new line => end of title */
                         unput_string(yytext,yyleng);
                         return 0;
                       }
<St_TitleQ>{SPCMD1}    |
<St_TitleQ>{SPCMD2}    { /* special command */
                         g_token->name = yytext+1;
                         g_token->paramDir=TokenInfo::Unspecified;
                         return TK_COMMAND_SEL();
                       }
<St_TitleQ>{WORD1NQ}   |
<St_TitleQ>{WORD2NQ}   { /* word */
                         g_token->name = yytext;
                         return TK_WORD;
                       }
<St_TitleQ>[ \t]+      {
                         g_token->chars=yytext;
                         return TK_WHITESPACE;
                       }
<St_TitleQ>"\""        { /* closing quote => end of title */
                         BEGIN(St_TitleA);
                         return 0;
                       }
<St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
                         g_token->name = yytext;
                         g_token->name = g_token->name.left(g_token->name.find('=')).stripWhiteSpace();
                         BEGIN(St_TitleV);
                       }
<St_TitleV>[^ \t\r\n]+ { // attribute value
                         lineCount(yytext,yyleng);
                         g_token->chars = yytext;
                         BEGIN(St_TitleN);
                         return TK_WORD;
                       }
<St_TitleV,St_TitleA>. {
                         unput(*yytext);
                         return 0;
                       }
<St_TitleV,St_TitleA>(\n|"\\ilinebr") {
                         unput_string(yytext,yyleng);
                         return 0;
                       }

<St_Anchor>{LABELID}{WS}? { // anchor
                         lineCount(yytext,yyleng);
                         g_token->name = QCString(yytext).stripWhiteSpace();
                         return TK_WORD;
                       }
<St_Anchor>.           {
                         unput(*yytext);
                         return 0;
                       }
<St_Cite>{CITEID}      { // label to cite
                         if (yytext[0] =='"')
                         {
                           g_token->name=yytext+1;
                           g_token->name=g_token->name.left(static_cast<uint>(yyleng)-2);
                         }
                         else
                         {
                           g_token->name=yytext;
                         }
                         return TK_WORD;
                       }
<St_Cite>{BLANK}       { // white space
                         unput(' ');
                         return 0;
                       }
<St_Cite>(\n|"\\ilinebr")  { // new line
                         unput_string(yytext,yyleng);
                         return 0;
                       }
<St_Cite>.             { // any other character
                         unput(*yytext);
                         return 0;
                       }
<St_Ref>{REFWORD_NOCV}/{BLANK}("const")[a-z_A-Z0-9] { // see bug776988
                         g_token->name=yytext;
                         return TK_WORD;
                       }
<St_Ref>{REFWORD_NOCV}/{BLANK}("volatile")[a-z_A-Z0-9] { // see bug776988
                         g_token->name=yytext;
                         return TK_WORD;
                       }
<St_Ref>{REFWORD}      { // label to refer to
                         g_token->name=yytext;
                         return TK_WORD;
                       }
<St_Ref>{BLANK}        { // white space
                         unput(' ');
                         return 0;
                       }
<St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
                         lineCount(yytext,yyleng);
                         BEGIN(St_Ref2);
                       }
<St_Ref>(\n|"\\ilinebr") { // new line
                         unput_string(yytext,yyleng);
                         return 0;
                       }
<St_Ref>.              { // any other character
                         unput(*yytext);
                         return 0;
                       }
<St_IntRef>[A-Z_a-z0-9.:/#\-\+\(\)]+ {
                         g_token->name = yytext;
                         return TK_WORD;
                       }
<St_IntRef>{BLANK}+"\"" {
                         BEGIN(St_Ref2);
                       }
<St_SetScope>({SCOPEMASK}|{ANONNS}){BLANK}|{FILEMASK} {
                         g_token->name = yytext;
                         g_token->name = g_token->name.stripWhiteSpace();
                         return TK_WORD;
                       }
<St_SetScope>{SCOPEMASK}"<" {
                         g_token->name = yytext;
                         g_token->name = g_token->name.stripWhiteSpace();
                         g_sharpCount=1;
                         BEGIN(St_SetScopeEnd);
                       }
<St_SetScope>{BLANK}   {
                       }
<St_SetScopeEnd>"<"    {
                         g_token->name += yytext;
                         g_sharpCount++;
                       }
<St_SetScopeEnd>">"    {
                         g_token->name += yytext;
                         g_sharpCount--;
                         if (g_sharpCount<=0)
                         {
                           return TK_WORD;
                         }
                       }
<St_SetScopeEnd>.      {
                         g_token->name += yytext;
                       }
<St_Ref2>"&"{ID}";"    { /* symbol */
                         g_token->name = yytext;
                         return TK_SYMBOL;
                       }
<St_Ref2>"\""|\n|"\\ilinebr" { /* " or \n => end of title */
                         lineCount(yytext,yyleng);
                         return 0;
                       }
<St_Ref2>{SPCMD1}      |
<St_Ref2>{SPCMD2}      { /* special command */
                         g_token->name = yytext+1;
                         g_token->paramDir=TokenInfo::Unspecified;
                         return TK_COMMAND_SEL();
                       }
<St_Ref2>{WORD1NQ}     |
<St_Ref2>{WORD2NQ}     {
                         /* word */
                         g_token->name = yytext;
                         return TK_WORD;
                       }
<St_Ref2>[ \t]+        {
                         g_token->chars=yytext;
                         return TK_WHITESPACE;
                       }
<St_XRefItem>{LABELID} {
                         g_token->name=yytext;
                       }
<St_XRefItem>" "       {
                         BEGIN(St_XRefItem2);
                       }
<St_XRefItem2>[0-9]+"." {
                         QCString numStr(yytext);
                         numStr=numStr.left((int)yyleng-1);
                         g_token->id=numStr.toInt();
                         return RetVal_OK;
                       }
<St_Para,St_Title,St_Ref2>"<!--"     { /* html style comment block */
                         g_commentState = YY_START;
                         BEGIN(St_Comment);
                       }
<St_Param>"\""[^\n\"]+"\"" {
                         g_token->name = yytext+1;
                         g_token->name = g_token->name.left((int)yyleng-2);
                         return TK_WORD;
                       }
<St_Param>({PHPTYPE}{BLANK}*("["{BLANK}*"]")*{BLANK}*"|"{BLANK}*)*{PHPTYPE}{BLANK}*("["{BLANK}*"]")*{WS}+("&")?"$"{LABELID} {
                         lineCount(yytext,yyleng);
                         QCString params(yytext);
                         int j = params.find('&');
                         int i = params.find('$');
                         if (j<i && j!=-1) i=j;
                         QCString types = params.left(i).stripWhiteSpace();
                         g_token->name = types+"#"+params.mid(i);
                         return TK_WORD;
                       }
<St_Param>[^ \t\n,@\\]+  {
                         g_token->name = yytext;
                         if (g_token->name.at(static_cast<uint>(yyleng)-1)==':')
                         {
                           g_token->name=g_token->name.left(static_cast<uint>(yyleng)-1);
                         }
                         return TK_WORD;
                       }
<St_Param>{WS}*","{WS}*  /* param separator */
<St_Param>{WS}         {
                         lineCount(yytext,yyleng);
                         g_token->chars=yytext;
                         return TK_WHITESPACE;
                       }
<St_Options>{ID}       {
                         g_token->name+=yytext;
                       }
<St_Options>{WS}*","{WS}*
<St_Options>{WS}       { /* option separator */
                         lineCount(yytext,yyleng);
                         g_token->name+=",";
                       }
<St_Options>"}"        {
                         return TK_WORD;
                       }
<St_Block>{ID}         {
                         g_token->name+=yytext;
                       }
<St_Block>"]"          {
                         return TK_WORD;
                       }
<St_Emoji>[:0-9_a-z+-]+  {
                         g_token->name=yytext;
                         return TK_WORD;
                       }
<St_Emoji>.            {
                         return 0;
                       }
<St_File>{FILEMASK}    {
                         g_token->name = yytext;
                         return TK_WORD;
                       }
<St_File>"\""[^\n\"]+"\"" {
                         QCString text(yytext);
                         g_token->name = text.mid(1,text.length()-2);
                         return TK_WORD;
                       }
<St_Pattern>[^\\\r\n]+   {
                         g_token->name += yytext;
                       }
<St_Pattern>"\\ilinebr" {
                         g_token->name = g_token->name.stripWhiteSpace();
                         return TK_WORD;
                       }
<St_Pattern>\n         {
                         lineCount(yytext,yyleng);
                         g_token->name = g_token->name.stripWhiteSpace();
                         return TK_WORD;
                       }
<St_Pattern>.          {
                         g_token->name += yytext;
                       }
<St_Link>{LINKMASK}|{REFWORD}    {
                         g_token->name = yytext;
                         return TK_WORD;
                       }
<St_Comment>"-->"      { /* end of html comment */
                         BEGIN(g_commentState);
                       }
<St_Comment>[^-]+      /* inside html comment */
<St_Comment>.          /* inside html comment */

     /* State for skipping title (all chars until the end of the line) */

<St_SkipTitle>.
<St_SkipTitle>(\n|"\\ilinebr") {
                         lineCount(yytext,yyleng);
                         return 0;
                         }

     /* State for the pass used to find the anchors and sections */

<St_Sections>[^\n@\\<]+
<St_Sections>{CMD}("<"|{CMD})
<St_Sections>"<"{CAPTION}({WS}+{ATTRIB})*">" {
                                      lineCount(yytext,yyleng);
                                      QCString tag(yytext);
                                      int s=tag.find("id=");
                                      if (s!=-1) // command has id attribute
                                      {
                                        char c=tag[s+3];
                                        if (c=='\'' || c=='"') // valid start
                                        {
                                          int e=tag.find(c,s+4);
                                          if (e!=-1) // found matching end
                                          {
                                            g_secType = SectionType::Table;
                                            g_secLabel=tag.mid(s+4,e-s-4); // extract id
                                            processSection();
                                          }
                                        }
                                      }
                                    }
<St_Sections>{CMD}"anchor"{BLANK}+  {
                                      g_secType = SectionType::Anchor;
                                      BEGIN(St_SecLabel1);
                                    }
<St_Sections>{CMD}"section"{BLANK}+ {
                                      g_secType = SectionType::Section;
                                      BEGIN(St_SecLabel2);
                                    }
<St_Sections>{CMD}"subsection"{BLANK}+ {
                                      g_secType = SectionType::Subsection;
                                      BEGIN(St_SecLabel2);
                                    }
<St_Sections>{CMD}"subsubsection"{BLANK}+ {
                                      g_secType = SectionType::Subsubsection;
                                      BEGIN(St_SecLabel2);
                                    }
<St_Sections>{CMD}"paragraph"{BLANK}+ {
                                      g_secType = SectionType::Paragraph;
                                      BEGIN(St_SecLabel2);
                                    }
<St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9]  {
                                      g_endMarker="endverbatim";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
                                      g_endMarker="enddot";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
                                      g_endMarker="endmsc";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"startuml"/[^a-z_A-Z0-9] {
                                      g_endMarker="enduml";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
                                      g_endMarker="endhtmlonly";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
                                      g_endMarker="endlatexonly";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"manonly"/[^a-z_A-Z0-9] {
                                      g_endMarker="endmanonly";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"rtfonly"/[^a-z_A-Z0-9] {
                                      g_endMarker="endrtfonly";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
                                      g_endMarker="endxmlonly";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"docbookonly"/[^a-z_A-Z0-9] {
                                      g_endMarker="enddocbookonly";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
                                      g_endMarker="endcode";
                                      BEGIN(St_SecSkip);
                                    }
<St_Sections>"<!--"                 {
                                      g_endMarker="-->";
                                      BEGIN(St_SecSkip);
                                    }
<St_SecSkip>{CMD}{ID}               {
                                      if (g_endMarker==yytext+1)
                                      {
                                        BEGIN(St_Sections);
                                      }
                                    }
<St_SecSkip>"-->"                   {
                                      if (g_endMarker==yytext)
                                      {
                                        BEGIN(St_Sections);
                                      }
                                    }
<St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
<St_SecSkip>.
<St_SecSkip>(\n|"\\ilinebr")
<St_Sections>.
<St_Sections>(\n|"\\ilinebr")
<St_SecLabel1>{LABELID} {
                         lineCount(yytext,yyleng);
                         g_secLabel = yytext;
                         processSection();
                         BEGIN(St_Sections);
                       }
<St_SecLabel2>{LABELID}{BLANK}+ |
<St_SecLabel2>{LABELID}         {
                         g_secLabel = yytext;
                         g_secLabel = g_secLabel.stripWhiteSpace();
                         BEGIN(St_SecTitle);
                       }
<St_SecTitle>[^\n]+    |
<St_SecTitle>[^\n]*\n  {
                         lineCount(yytext,yyleng);
                         g_secTitle = yytext;
                         g_secTitle = g_secTitle.stripWhiteSpace();
                         if (g_secTitle.right(8)=="\\ilinebr")
                         {
                           g_secTitle.left(g_secTitle.length()-8);
                         }
                         processSection();
                         BEGIN(St_Sections);
                       }
<St_SecTitle,St_SecLabel1,St_SecLabel2>. {
                         warn(g_fileName,g_yyLineNr,"Unexpected character '%s' while looking for section label or title",yytext);
                       }

<St_Snippet>[^\\\n]+   {
                         g_token->name += yytext;
                       }
<St_Snippet>"\\"       {
                         g_token->name += yytext;
                       }
<St_Snippet>(\n|"\\ilinebr")  {
                         lineCount(yytext,yyleng);
                         g_token->name = g_token->name.stripWhiteSpace();
                         return TK_WORD;
                       }

     /* Generic rules that work for all states */
<*>\n                  {
                         lineCount(yytext,yyleng);
                         warn(g_fileName,g_yyLineNr,"Unexpected new line character");
                       }
<*>"\\ilinebr"         {
                       }
<*>[\\@<>&$#%~"=]      { /* unescaped special character */
                         //warn(g_fileName,g_yyLineNr,"Unexpected character '%s', assuming command \\%s was meant.",yytext,yytext);
                         g_token->name = yytext;
                         return TK_COMMAND_SEL();
                       }
<*>.                   {
                         warn(g_fileName,g_yyLineNr,"Unexpected character '%s'",yytext);
                       }
%%

//--------------------------------------------------------------------------

void doctokenizerYYFindSections(const QCString &input,const Definition *d,
                                const QCString &fileName)
{
  if (input.isEmpty()) return;
  printlex(yy_flex_debug, TRUE, __FILE__, qPrint(fileName));
  g_inputString = input.data();
  //printf("parsing --->'%s'<---\n",input);
  g_inputPos    = 0;
  g_definition  = d;
  g_fileName    = fileName;
  BEGIN(St_Sections);
  g_yyLineNr = 1;
  doctokenizerYYlex();
  printlex(yy_flex_debug, FALSE, __FILE__, qPrint(fileName));
}

void doctokenizerYYinit(const char *input,const QCString &fileName,bool markdownSupport)
{
  g_autoListLevel = 0;
  g_inputString = input;
  g_inputPos    = 0;
  g_fileName    = fileName;
  g_insidePre   = FALSE;
  g_markdownSupport = markdownSupport;
  BEGIN(St_Para);
}

void doctokenizerYYsetStatePara()
{
  BEGIN(St_Para);
}

void doctokenizerYYsetStateTitle()
{
  BEGIN(St_Title);
}

void doctokenizerYYsetStateTitleAttrValue()
{
  BEGIN(St_TitleV);
}

void doctokenizerYYsetStateCode()
{
  g_token->verb="";
  g_token->name="";
  BEGIN(St_CodeOpt);
}

void doctokenizerYYsetStateXmlCode()
{
  g_token->verb="";
  g_token->name="";
  BEGIN(St_XmlCode);
}

void doctokenizerYYsetStateHtmlOnly()
{
  g_token->verb="";
  g_token->name="";
  BEGIN(St_HtmlOnlyOption);
}

void doctokenizerYYsetStateManOnly()
{
  g_token->verb="";
  BEGIN(St_ManOnly);
}

void doctokenizerYYsetStateRtfOnly()
{
  g_token->verb="";
  BEGIN(St_RtfOnly);
}

void doctokenizerYYsetStateXmlOnly()
{
  g_token->verb="";
  BEGIN(St_XmlOnly);
}

void doctokenizerYYsetStateDbOnly()
{
  g_token->verb="";
  BEGIN(St_DbOnly);
}

void doctokenizerYYsetStateLatexOnly()
{
  g_token->verb="";
  BEGIN(St_LatexOnly);
}

void doctokenizerYYsetStateVerbatim()
{
  g_token->verb="";
  BEGIN(St_Verbatim);
}

void doctokenizerYYsetStateDot()
{
  g_token->verb="";
  BEGIN(St_Dot);
}

void doctokenizerYYsetStateMsc()
{
  g_token->verb="";
  BEGIN(St_Msc);
}

void doctokenizerYYsetStatePlantUMLOpt()
{
  g_token->verb="";
  g_token->sectionId="";
  BEGIN(St_PlantUMLOpt);
}

void doctokenizerYYsetStatePlantUML()
{
  g_token->verb="";
  BEGIN(St_PlantUML);
}

void doctokenizerYYsetStateParam()
{
  BEGIN(St_Param);
}

void doctokenizerYYsetStateXRefItem()
{
  BEGIN(St_XRefItem);
}

void doctokenizerYYsetStateFile()
{
  BEGIN(St_File);
}

void doctokenizerYYsetStatePattern()
{
  g_token->name = "";
  BEGIN(St_Pattern);
}

void doctokenizerYYsetStateLink()
{
  BEGIN(St_Link);
}

void doctokenizerYYsetStateCite()
{
  BEGIN(St_Cite);
}

void doctokenizerYYsetStateRef()
{
  BEGIN(St_Ref);
}

void doctokenizerYYsetStateInternalRef()
{
  BEGIN(St_IntRef);
}

void doctokenizerYYsetStateText()
{
  BEGIN(St_Text);
}

void doctokenizerYYsetStateSkipTitle()
{
  BEGIN(St_SkipTitle);
}

void doctokenizerYYsetStateAnchor()
{
  BEGIN(St_Anchor);
}

void doctokenizerYYsetStateSnippet()
{
  g_token->name="";
  BEGIN(St_Snippet);
}

void doctokenizerYYsetStateSetScope()
{
  BEGIN(St_SetScope);
}

void doctokenizerYYsetStateOptions()
{
  g_token->name="";
  BEGIN(St_Options);
}

void doctokenizerYYsetStateBlock()
{
  g_token->name="";
  BEGIN(St_Block);
}

void doctokenizerYYsetStateEmoji()
{
  g_token->name="";
  BEGIN(St_Emoji);
}

void doctokenizerYYcleanup()
{
  yy_delete_buffer( YY_CURRENT_BUFFER );
}

void doctokenizerYYsetInsidePre(bool b)
{
  g_insidePre = b;
}

void doctokenizerYYpushBackHtmlTag(const QCString &tag)
{
  QCString tagName = tag;
  int i,l = tagName.length();
  unput('>');
  for (i=l-1;i>=0;i--)
  {
    unput(tag[i]);
  }
  unput('<');
}

void doctokenizerYYstartAutoList()
{
  g_autoListLevel++;
}

void doctokenizerYYendAutoList()
{
  g_autoListLevel--;
}

//REAL_YY_DECL
//{
//  printlex(yy_flex_debug, TRUE, __FILE__, g_fileName);
//  int retval = LOCAL_YY_DECL;
//  printlex(yy_flex_debug, FALSE, __FILE__, g_fileName);
//  return retval;
//}

void setDoctokinizerLineNr(int lineno)
{
  g_yyLineNr = lineno;
}

int getDoctokinizerLineNr(void)
{
  return g_yyLineNr;
}


#if USE_STATE2STRING
#include "doctokenizer.l.h"
#endif