From e922facbb92cda058eae33f58f7640be8d1fb5b8 Mon Sep 17 00:00:00 2001 From: Dimitri van Heesch Date: Mon, 15 Jun 2020 19:25:46 +0200 Subject: Refactor: modernize markdown and make it thread-safe --- addon/doxyapp/CMakeLists.txt | 1 + addon/doxyparse/CMakeLists.txt | 1 + src/classdef.cpp | 12 +- src/commentscan.h | 8 +- src/commentscan.l | 2095 ++++++++++++++++++++-------------------- src/context.cpp | 3 +- src/dirdef.cpp | 15 +- src/docparser.cpp | 11 +- src/docparser.h | 103 +- src/doctokenizer.h | 8 +- src/doctokenizer.l | 16 +- src/doxygen.cpp | 9 +- src/doxygen.h | 2 +- src/filedef.cpp | 9 +- src/fortranscanner.l | 8 +- src/ftvhelp.cpp | 3 +- src/groupdef.cpp | 24 +- src/growbuf.h | 1 + src/index.cpp | 25 +- src/markdown.cpp | 725 ++++++-------- src/markdown.h | 83 +- src/memberdef.cpp | 33 +- src/memberlist.cpp | 9 +- src/namespacedef.cpp | 12 +- src/outputlist.cpp | 5 +- src/outputlist.h | 5 +- src/pagedef.cpp | 60 +- src/perlmodgen.cpp | 3 +- src/pyscanner.l | 6 +- src/scanner.l | 12 +- src/sqlite3gen.cpp | 6 +- src/util.cpp | 6 +- src/vhdldocgen.cpp | 11 +- src/vhdljjparser.cpp | 16 +- src/xmlgen.cpp | 3 +- 35 files changed, 1688 insertions(+), 1661 deletions(-) diff --git a/addon/doxyapp/CMakeLists.txt b/addon/doxyapp/CMakeLists.txt index 9d74fbc..8ec4d2e 100644 --- a/addon/doxyapp/CMakeLists.txt +++ b/addon/doxyapp/CMakeLists.txt @@ -12,6 +12,7 @@ include_directories( add_executable(doxyapp doxyapp.cpp ) +add_sanitizers(doxyapp) if (use_libclang) set(CLANG_LIBS libclang clangTooling ${llvm_libs}) diff --git a/addon/doxyparse/CMakeLists.txt b/addon/doxyparse/CMakeLists.txt index 8e0ad49..6ddc113 100644 --- a/addon/doxyparse/CMakeLists.txt +++ b/addon/doxyparse/CMakeLists.txt @@ -12,6 +12,7 @@ include_directories( add_executable(doxyparse doxyparse.cpp ) +add_sanitizers(doxyparse) if (use_libclang) set(CLANG_LIBS libclang clangTooling ${llvm_libs}) diff --git a/src/classdef.cpp b/src/classdef.cpp index 04e057b..5dee0dc 100644 --- a/src/classdef.cpp +++ b/src/classdef.cpp @@ -1431,7 +1431,8 @@ void ClassDefImpl::writeBriefDescription(OutputList &ol,bool exampleFlag) const ol.writeString(" - "); ol.popGeneratorState(); ol.generateDoc(briefFile(),briefLine(),this,0, - briefDescription(),TRUE,FALSE,0,TRUE,FALSE); + briefDescription(),TRUE,FALSE,0, + TRUE,FALSE,Config_getBool(MARKDOWN_SUPPORT)); ol.pushGeneratorState(); ol.disable(OutputGenerator::RTF); ol.writeString(" \n"); @@ -1462,7 +1463,8 @@ void ClassDefImpl::writeDetailedDocumentationBody(OutputList &ol) const // repeat brief description if (!briefDescription().isEmpty() && repeatBrief) { - ol.generateDoc(briefFile(),briefLine(),this,0,briefDescription(),FALSE,FALSE); + ol.generateDoc(briefFile(),briefLine(),this,0,briefDescription(),FALSE,FALSE, + 0,FALSE,FALSE,Config_getBool(MARKDOWN_SUPPORT)); } if (!briefDescription().isEmpty() && repeatBrief && !documentation().isEmpty()) @@ -1475,7 +1477,8 @@ void ClassDefImpl::writeDetailedDocumentationBody(OutputList &ol) const // write documentation if (!documentation().isEmpty()) { - ol.generateDoc(docFile(),docLine(),this,0,documentation(),TRUE,FALSE); + ol.generateDoc(docFile(),docLine(),this,0,documentation(),TRUE,FALSE, + 0,FALSE,FALSE,Config_getBool(MARKDOWN_SUPPORT)); } // write type constraints writeTypeConstraints(ol,this,m_impl->typeConstraints); @@ -2572,7 +2575,8 @@ void ClassDefImpl::writeDeclarationLink(OutputList &ol,bool &found,const char *h if (!briefDescription().isEmpty() && Config_getBool(BRIEF_MEMBER_DESC)) { DocRoot *rootNode = validatingParseDoc(briefFile(),briefLine(),this,0, - briefDescription(),FALSE,FALSE,0,TRUE,FALSE); + briefDescription(),FALSE,FALSE, + 0,TRUE,FALSE,Config_getBool(MARKDOWN_SUPPORT)); if (rootNode && !rootNode->isEmpty()) { ol.startMemberDescription(anchor()); diff --git a/src/commentscan.h b/src/commentscan.h index be92920..a111352 100644 --- a/src/commentscan.h +++ b/src/commentscan.h @@ -3,8 +3,8 @@ * Copyright (C) 1997-2015 by Dimitri van Heesch. * * Permission to use, copy, modify, and distribute this software and its - * documentation under the terms of the GNU General Public License is hereby - * granted. No representations are made about the suitability of this software + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software * for any purpose. It is provided "as is" without express or implied warranty. * See the GNU General Public License for more details. * @@ -60,6 +60,7 @@ class CommentScanner * @param[out] newEntryNeeded Boolean that is TRUE if the comment block parser * finds that a the comment block finishes the entry and a new one * needs to be started. + * @param[in] markdownEnabled Indicates if markdown specific processing should be done. * @returns TRUE if the comment requires further processing. The * parameter \a newEntryNeeded will typically be true in this case and * \a position will indicate the offset inside the \a comment string @@ -76,7 +77,8 @@ class CommentScanner bool isInbody, Protection &prot, int &position, - bool &newEntryNeeded + bool &newEntryNeeded, + bool markdownEnabled ); void initGroupInfo(Entry *entry); void enterFile(const char *fileName,int lineNr); diff --git a/src/commentscan.l b/src/commentscan.l index 309a334..f521528 100644 --- a/src/commentscan.l +++ b/src/commentscan.l @@ -3,8 +3,8 @@ * Copyright (C) 1997-2020 by Dimitri van Heesch. * * Permission to use, copy, modify, and distribute this software and its - * documentation under the terms of the GNU General Public License is hereby - * granted. No representations are made about the suitability of this software + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software * for any purpose. It is provided "as is" without express or implied warranty. * See the GNU General Public License for more details. * @@ -24,7 +24,7 @@ %{ /* - * includes + * includes */ #include @@ -329,11 +329,11 @@ enum GuardType class GuardedSection { public: - GuardedSection(bool enabled,bool parentVisible) + GuardedSection(bool enabled,bool parentVisible) : m_enabled(enabled),m_parentVisible(parentVisible) {} bool isEnabled() const { return m_enabled; } bool parentVisible() const { return m_parentVisible; } - + private: bool m_enabled; bool m_parentVisible; @@ -341,7 +341,7 @@ class GuardedSection /* ----------------------------------------------------------------- * - * statics + * statics */ struct commentscanYY_state @@ -398,6 +398,7 @@ struct commentscanYY_state bool inInternalDocs = FALSE; int prevPosition = 0; DocGroup docGroup; + bool markdownSupport = TRUE; }; //----------------------------------------------------------------------------- @@ -428,20 +429,20 @@ static void addCite(yyscan_t yyscanner); //----------------------------------------------------------------------------- -#undef YY_INPUT -#define YY_INPUT(buf,result,max_size) result=yyread(yyscanner,buf,max_size); +#undef YY_INPUT +#define YY_INPUT(buf,result,max_size) result=yyread(yyscanner,buf,max_size); %} /* start command character */ -CMD ("\\"|"@") -XREFCMD {CMD}("bug"|"deprecated"|"test"|"todo"|"xrefitem") +CMD ("\\"|"@") +XREFCMD {CMD}("bug"|"deprecated"|"test"|"todo"|"xrefitem") PRE [pP][rR][eE] -TABLE [tT][aA][bB][lL][eE] -P [pP] +TABLE [tT][aA][bB][lL][eE] +P [pP] UL [uU][lL] -OL [oO][lL] -DL [dD][lL] +OL [oO][lL] +DL [dD][lL] IMG [iI][mM][gG] HR [hH][rR] PARA [pP][aA][rR][aA] @@ -452,13 +453,13 @@ DIV [dD][iI][vV] DETAILEDHTML {CENTER}|{DIV}|{PRE}|{UL}|{TABLE}|{OL}|{DL}|{P}|[Hh][1-6]|{IMG}|{HR}|{PARA} DETAILEDHTMLOPT {CODE} BN [ \t\n\r] -BL [ \t\r]*"\n" +BL [ \t\r]*"\n" B [ \t] BS ^(({B}*"//")?)(({B}*"*"+)?){B}* ATTR ({B}+[^>\n]*)? DOCNL "\n"|"\\_linebr" LC "\\"{B}*"\n" -NW [^a-z_A-Z0-9] +NW [^a-z_A-Z0-9] FILESCHAR [a-z_A-Z0-9\x80-\xFF\\:\\\/\-\+@&#] FILEECHAR [a-z_A-Z0-9\x80-\xFF\-\+@&#] FILE ({FILESCHAR}*{FILEECHAR}+("."{FILESCHAR}*{FILEECHAR}+)*)|("\""[^\n\"]*"\"") @@ -470,7 +471,7 @@ CITEID {CITESCHAR}{CITEECHAR}*("."{CITESCHAR}{CITEECHAR}*)*|"\""{CITESCHAR}{C SCOPEID {ID}({ID}*{BN}*"::"{BN}*)*({ID}?) SCOPENAME "$"?(({ID}?{BN}*("::"|"."){BN}*)*)((~{BN}*)?{ID}) TMPLSPEC "<"{BN}*[^>]+{BN}*">" -MAILADDR [a-z_A-Z0-9.+\-]+"@"[a-z_A-Z0-9\-]+("."[a-z_A-Z0-9\-]+)+[a-z_A-Z0-9\-]+ +MAILADDR [a-z_A-Z0-9.+\-]+"@"[a-z_A-Z0-9\-]+("."[a-z_A-Z0-9\-]+)+[a-z_A-Z0-9\-]+ RCSTAG "$"{ID}":"[^\n$]+"$" %option noyywrap @@ -489,34 +490,34 @@ RCSTAG "$"{ID}":"[^\n$]+"$" %x XRefItemParam3 %x FileDocArg1 %x ParamArg1 -%x EnumDocArg1 -%x NameSpaceDocArg1 -%x PackageDocArg1 -%x GroupDocArg1 -%x GroupDocArg2 -%x SectionLabel -%x SectionTitle -%x SubpageLabel -%x SubpageTitle -%x FormatBlock -%x LineParam -%x GuardParam -%x GuardParamEnd -%x SkipGuardedSection -%x SkipInternal +%x EnumDocArg1 +%x NameSpaceDocArg1 +%x PackageDocArg1 +%x GroupDocArg1 +%x GroupDocArg2 +%x SectionLabel +%x SectionTitle +%x SubpageLabel +%x SubpageTitle +%x FormatBlock +%x LineParam +%x GuardParam +%x GuardParamEnd +%x SkipGuardedSection +%x SkipInternal %x NameParam -%x InGroupParam -%x FnParam -%x OverloadParam -%x InheritParam -%x ExtendsParam +%x InGroupParam +%x FnParam +%x OverloadParam +%x InheritParam +%x ExtendsParam %x ReadFormulaShort -%x ReadFormulaLong -%x AnchorLabel +%x ReadFormulaLong +%x AnchorLabel %x HtmlComment %x SkipLang -%x CiteLabel -%x CopyDoc +%x CiteLabel +%x CopyDoc %x GuardExpr %x CdataSection %x Noop @@ -540,52 +541,52 @@ RCSTAG "$"{ID}":"[^\n$]+"$" * XML commands, */ -{CMD}{CMD}[a-z_A-Z]+{B}* { // escaped command - addOutput(yyscanner,yytext); - } -{CMD}{CMD}"~"[a-z_A-Z]* { // escaped command - addOutput(yyscanner,yytext); - } -{MAILADDR} { // mail address - addOutput(yyscanner,yytext); - } -"\""[^"\n]*"\"" { // quoted text +{CMD}{CMD}[a-z_A-Z]+{B}* { // escaped command + addOutput(yyscanner,yytext); + } +{CMD}{CMD}"~"[a-z_A-Z]* { // escaped command + addOutput(yyscanner,yytext); + } +{MAILADDR} { // mail address + addOutput(yyscanner,yytext); + } +"\""[^"\n]*"\"" { // quoted text addOutput(yyscanner,yytext); - } -("\\"[a-z_A-Z]+)+"\\" { // directory (or chain of commands!) - addOutput(yyscanner,yytext); - } -"<"{DETAILEDHTML}{ATTR}">" { // HTML command that ends a brief description - setOutput(yyscanner,OutputDoc); - // continue with the same input + } +("\\"[a-z_A-Z]+)+"\\" { // directory (or chain of commands!) + addOutput(yyscanner,yytext); + } +"<"{DETAILEDHTML}{ATTR}">" { // HTML command that ends a brief description + setOutput(yyscanner,OutputDoc); + // continue with the same input REJECT; - } -"<"{DETAILEDHTMLOPT}{ATTR}">" { // HTML command that ends a brief description - if (yyextra->current->lang==SrcLangExt_CSharp) + } +"<"{DETAILEDHTMLOPT}{ATTR}">" { // HTML command that ends a brief description + if (yyextra->current->lang==SrcLangExt_CSharp) { setOutput(yyscanner,OutputDoc); } - // continue with the same input + // continue with the same input REJECT; - } -"" { // start of a .NET XML style brief description - setOutput(yyscanner,OutputBrief); + } +"" { // start of a .NET XML style brief description + setOutput(yyscanner,OutputBrief); addOutput(yyscanner,yytext); - } -"" { // start of a .NET XML style detailed description - setOutput(yyscanner,OutputDoc); + } +"" { // start of a .NET XML style detailed description + setOutput(yyscanner,OutputDoc); addOutput(yyscanner,yytext); - } -"" { // start of a .NET XML style detailed description - setOutput(yyscanner,OutputBrief); + } +"" { // start of a .NET XML style detailed description + setOutput(yyscanner,OutputBrief); addOutput(yyscanner,yytext); - setOutput(yyscanner,OutputDoc); - } -"" { // end of a brief or detailed description - - setOutput(yyscanner,OutputDoc); + setOutput(yyscanner,OutputDoc); + } +"" { // end of a brief or detailed description + + setOutput(yyscanner,OutputDoc); addOutput(yyscanner,yytext); - } + } "<"{CAPTION}{ATTR}">" { QCString tag=yytext; int s=tag.find("id="); @@ -616,22 +617,22 @@ RCSTAG "$"{ID}":"[^\n$]+"$" setOutput(yyscanner,OutputDoc); REJECT; } -"