diff options
author | Dimitri van Heesch <doxygen@gmail.com> | 2020-08-01 11:27:31 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-08-01 11:27:31 (GMT) |
commit | 8d96d4594188e4bcdffe8051a097bab8e40596f8 (patch) | |
tree | 905dd9b59c9c8004de3506a0b3799450a23c26eb /src/doctokenizer.l | |
parent | 88e70b5242b2446f33ab700e6d10e7f25ae2c349 (diff) | |
parent | 4a1c9f98bc5f9c82e81a694160fecc5a22f0e02c (diff) | |
download | Doxygen-8d96d4594188e4bcdffe8051a097bab8e40596f8.zip Doxygen-8d96d4594188e4bcdffe8051a097bab8e40596f8.tar.gz Doxygen-8d96d4594188e4bcdffe8051a097bab8e40596f8.tar.bz2 |
Merge branch 'master' into feature/bug_xml_lang
Diffstat (limited to 'src/doctokenizer.l')
-rw-r--r-- | src/doctokenizer.l | 141 |
1 files changed, 89 insertions, 52 deletions
diff --git a/src/doctokenizer.l b/src/doctokenizer.l index 5317de5..ac5b6d2 100644 --- a/src/doctokenizer.l +++ b/src/doctokenizer.l @@ -60,6 +60,7 @@ static const char *g_inputString; static QCString g_fileName; static bool g_insidePre; static int g_sharpCount=0; +static bool g_markdownSupport=TRUE; // context for section finding phase static const Definition *g_definition; @@ -113,6 +114,20 @@ bool doctokenizerYYpopContext() return TRUE; } +QCString extractPartAfterNewLine(const QCString &text) +{ + int nl1 = text.findRev('\n'); + if (nl1!=-1) + { + return text.mid(nl1+1); + } + int nl2 = text.findRev("\\ilinebr"); + if (nl2!=-1) + { + return text.mid(nl2+8); + } + return text; +} //-------------------------------------------------------------------------- @@ -377,7 +392,7 @@ SPCMD2 {CMD}[\\@<>&$#%~".+=|-] SPCMD3 {CMD}_form#[0-9]+ SPCMD4 {CMD}"::" SPCMD5 {CMD}":" -INOUT "inout"|"in"|"out"|("in"{BLANK}*","{BLANK}*"out")|("out"{BLANK}*","{BLANK}*"in") +INOUT "in"|"out"|("in"{BLANK}*","?{BLANK}*"out")|("out"{BLANK}*","?{BLANK}*"in") PARAMIO {CMD}param{BLANK}*"["{BLANK}*{INOUT}{BLANK}*"]" VARARGS "..." TEMPCHAR [a-z_A-Z0-9.,: \t\*\&\(\)\[\]] @@ -487,7 +502,7 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} return TK_LISTITEM; } <St_Para>^{MLISTITEM} { /* list item */ - if (!Doxygen::markdownSupport || g_insidePre) + if (!g_markdownSupport || g_insidePre) { REJECT; } @@ -503,7 +518,7 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} } } <St_Para>^{OLISTITEM} { /* numbered list item */ - if (!Doxygen::markdownSupport || g_insidePre) + if (!g_markdownSupport || g_insidePre) { REJECT; } @@ -519,25 +534,23 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} return TK_LISTITEM; } } -<St_Para>{BLANK}*\n{LISTITEM} { /* list item on next line */ - QCString text=yytext; - text=text.right(text.length()-text.find('\n')-1); +<St_Para>{BLANK}*(\n|"\\ilinebr"){LISTITEM} { /* list item on next line */ + QCString text=extractPartAfterNewLine(yytext); int dashPos = text.findRev('-'); g_token->isEnumList = text.at(dashPos+1)=='#'; g_token->id = -1; g_token->indent = computeIndent(text,dashPos); return TK_LISTITEM; } -<St_Para>{BLANK}*\n{MLISTITEM} { /* list item on next line */ - if (!Doxygen::markdownSupport || g_insidePre) +<St_Para>{BLANK}*(\n|"\\ilinebr"){MLISTITEM} { /* list item on next line */ + if (!g_markdownSupport || g_insidePre) { REJECT; } else { - QCString text=yytext; + QCString text=extractPartAfterNewLine(yytext); static QRegExp re("[*+]"); - text=text.right(text.length()-text.find('\n')-1); int markPos = text.findRev(re); g_token->isEnumList = FALSE; g_token->id = -1; @@ -545,17 +558,14 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} return TK_LISTITEM; } } -<St_Para>{BLANK}*\n{OLISTITEM} { /* list item on next line */ - if (!Doxygen::markdownSupport || g_insidePre) +<St_Para>{BLANK}*(\n|"\\ilinebr"){OLISTITEM} { /* list item on next line */ + if (!g_markdownSupport || g_insidePre) { REJECT; } else { - QCString text=yytext; - int nl=text.findRev('\n'); - int len=text.length(); - text=text.right(len-nl-1); + QCString text=extractPartAfterNewLine(yytext); static QRegExp re("[1-9]"); int digitPos = text.find(re); int dotPos = text.find('.',digitPos); @@ -570,11 +580,10 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->indent = computeIndent(yytext,dotPos); return TK_ENDLIST; } -<St_Para>{BLANK}*\n{ENDLIST} { /* end list on next line */ - QCString text=yytext; - text=text.right(text.length()-text.find('\n')-1); +<St_Para>{BLANK}*(\n|"\\ilinebr"){ENDLIST} { /* end list on next line */ + QCString text=extractPartAfterNewLine(yytext); int dotPos = text.findRev('.'); - g_token->indent = computeIndent(text,dotPos); + g_token->indent = computeIndent(text,dotPos); return TK_ENDLIST; } <St_Para>"{"{BLANK}*"@link"/{BLANK}+ { @@ -585,9 +594,9 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->name = "inheritdoc"; return TK_COMMAND_AT; } -<St_Para>"@_fakenl" { // artificial new line - yylineno++; - } +<St_Para>"@_fakenl" { // artificial new line + yylineno++; + } <St_Para>{SPCMD3} { g_token->name = "_form"; bool ok; @@ -602,6 +611,8 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->paramDir=TokenInfo::Unspecified; return TK_COMMAND_SEL(); } +<St_Para>"\\ilinebr" { + } <St_Para>{SPCMD1} | <St_Para>{SPCMD2} | <St_Para>{SPCMD5} | @@ -768,13 +779,13 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} } } <St_Para>({BLANK}*\n)+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */ - if (!Doxygen::markdownSupport || g_insidePre || g_autoListLevel==0) + if (!g_markdownSupport || g_insidePre || g_autoListLevel==0) { REJECT; } } <St_Para>({BLANK}*\n)+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */ - if (!Doxygen::markdownSupport || g_insidePre || g_autoListLevel==0) + if (!g_markdownSupport || g_insidePre || g_autoListLevel==0) { REJECT; } @@ -799,9 +810,10 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->name = g_token->name.mid(i+1,g_token->name.length()-i-2); BEGIN(St_Code); } +<St_CodeOpt>"\\ilinebr" | <St_CodeOpt>\n | <St_CodeOpt>. { - unput(*yytext); + for (int i=yyleng-1;i>=0;i--) unput(yytext[i]); BEGIN(St_Code); } <St_Code>{WS}*{CMD}"endcode" { @@ -823,6 +835,10 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} unput(*yytext); BEGIN(St_HtmlOnly); } +<St_HtmlOnlyOption>"\\ilinebr" { + for (int i=yyleng-1;i>=0;i--) unput(yytext[i]); + BEGIN(St_HtmlOnly); + } <St_HtmlOnly>{CMD}"endhtmlonly" { return RetVal_OK; } @@ -914,10 +930,15 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->sectionId = QCString(yytext).stripWhiteSpace(); return RetVal_OK; } +<St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}*/"\\ilinebr" { // case 5: plain file name specified without title or attributes + g_token->sectionId = QCString(yytext).stripWhiteSpace(); + return RetVal_OK; + } +<St_PlantUMLOpt>"\\ilinebr" | <St_PlantUMLOpt>"\n" | <St_PlantUMLOpt>. { g_token->sectionId = ""; - unput(*yytext); + for (int i=yyleng-1;i>=0;i--) unput(yytext[i]); return RetVal_OK; } <St_PlantUML>{CMD}"enduml" { @@ -943,12 +964,20 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} unput(*yytext); return 0; } +<St_Title>"\\ilinebr" { + for (int i=yyleng-1;i>=0;i--) unput(yytext[i]); + return 0; + } <St_TitleN>"&"{ID}";" { /* symbol */ g_token->name = yytext; return TK_SYMBOL; } <St_TitleN>{HTMLTAG} { } +<St_TitleN>(\n|"\\ilinebr") { /* new line => end of title */ + for (int i=yyleng-1;i>=0;i--) unput(yytext[i]); + return 0; + } <St_TitleN>{SPCMD1} | <St_TitleN>{SPCMD2} { /* special command */ g_token->name = yytext+1; @@ -975,14 +1004,14 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->chars=yytext; return TK_WHITESPACE; } -<St_TitleN>\n { /* new line => end of title */ - unput(*yytext); - return 0; - } <St_TitleQ>"&"{ID}";" { /* symbol */ g_token->name = yytext; return TK_SYMBOL; } +<St_TitleQ>(\n|"\\ilinebr") { /* new line => end of title */ + for (int i=yyleng-1;i>=0;i--) unput(yytext[i]); + return 0; + } <St_TitleQ>{SPCMD1} | <St_TitleQ>{SPCMD2} { /* special command */ g_token->name = yytext+1; @@ -1002,10 +1031,6 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} BEGIN(St_TitleA); return 0; } -<St_TitleQ>\n { /* new line => end of title */ - unput(*yytext); - return 0; - } <St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute g_token->name = yytext; g_token->name = g_token->name.left(g_token->name.find('=')).stripWhiteSpace(); @@ -1020,8 +1045,8 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} unput(*yytext); return 0; } -<St_TitleV,St_TitleA>\n { - unput(*yytext); +<St_TitleV,St_TitleA>(\n|"\\ilinebr") { + for (int i=yyleng-1;i>=0;i--) unput(yytext[i]); return 0; } @@ -1049,8 +1074,8 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} unput(' '); return 0; } -<St_Cite>\n { // new line - unput(*yytext); +<St_Cite>(\n|"\\ilinebr") { // new line + for (int i=yyleng-1;i>=0;i--) unput(yytext[i]); return 0; } <St_Cite>. { // any other character @@ -1076,8 +1101,8 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} <St_Ref>{WS}+"\""{WS}* { // white space following by quoted string BEGIN(St_Ref2); } -<St_Ref>\n { // new line - unput(*yytext); +<St_Ref>(\n|"\\ilinebr") { // new line + for (int i=yyleng-1;i>=0;i--) unput(yytext[i]); return 0; } <St_Ref>. { // any other character @@ -1123,6 +1148,9 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->name = yytext; return TK_SYMBOL; } +<St_Ref2>"\""|\n|"\\ilinebr" { /* " or \n => end of title */ + return 0; + } <St_Ref2>{SPCMD1} | <St_Ref2>{SPCMD2} { /* special command */ g_token->name = yytext+1; @@ -1139,9 +1167,6 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->chars=yytext; return TK_WHITESPACE; } -<St_Ref2>"\""|\n { /* " or \n => end of title */ - return 0; - } <St_XRefItem>{LABELID} { g_token->name=yytext; } @@ -1235,7 +1260,7 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} /* State for skipping title (all chars until the end of the line) */ <St_SkipTitle>. -<St_SkipTitle>\n { return 0; } +<St_SkipTitle>(\n|"\\ilinebr") { return 0; } /* State for the pass used to find the anchors and sections */ @@ -1341,9 +1366,9 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} } <St_SecSkip>[^a-z_A-Z0-9\-\\\@]+ <St_SecSkip>. -<St_SecSkip>\n +<St_SecSkip>(\n|"\\ilinebr") <St_Sections>. -<St_Sections>\n +<St_Sections>(\n|"\\ilinebr") <St_SecLabel1>{LABELID} { g_secLabel = yytext; processSection(); @@ -1359,6 +1384,10 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} <St_SecTitle>[^\n]*\n { g_secTitle = yytext; g_secTitle = g_secTitle.stripWhiteSpace(); + if (g_secTitle.right(8)=="\\ilinebr") + { + g_secTitle.left(g_secTitle.length()-8); + } processSection(); BEGIN(St_Sections); } @@ -1366,17 +1395,23 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} warn(g_fileName,yylineno,"Unexpected character '%s' while looking for section label or title",yytext); } -<St_Snippet>[^\n]+ | -<St_Snippet>[^\n]*\n { - g_token->name = yytext; +<St_Snippet>[^\\\n]+ { + g_token->name += yytext; + } +<St_Snippet>"\\" { + g_token->name += yytext; + } +<St_Snippet>(\n|"\\ilinebr") { g_token->name = g_token->name.stripWhiteSpace(); - return TK_WORD; + return TK_WORD; } /* Generic rules that work for all states */ <*>\n { warn(g_fileName,yylineno,"Unexpected new line character"); } +<*>"\\ilinebr" { + } <*>[\\@<>&$#%~"=] { /* unescaped special character */ //warn(g_fileName,yylineno,"Unexpected character '%s', assuming command \\%s was meant.",yytext,yytext); g_token->name = yytext; @@ -1405,13 +1440,14 @@ void doctokenizerYYFindSections(const char *input,const Definition *d, printlex(yy_flex_debug, FALSE, __FILE__, fileName); } -void doctokenizerYYinit(const char *input,const char *fileName) +void doctokenizerYYinit(const char *input,const char *fileName,bool markdownSupport) { g_autoListLevel = 0; g_inputString = input; g_inputPos = 0; g_fileName = fileName; g_insidePre = FALSE; + g_markdownSupport = markdownSupport; BEGIN(St_Para); } @@ -1569,6 +1605,7 @@ void doctokenizerYYsetStateAnchor() void doctokenizerYYsetStateSnippet() { + g_token->name=""; BEGIN(St_Snippet); } |