From 4575eef95039149eb19c4f5e5bf99e67e36afb5a Mon Sep 17 00:00:00 2001 From: albert-github Date: Tue, 3 Apr 2018 19:19:17 +0200 Subject: Create test possibilities for xhtml and pdf output The tests in the testing directory provided possibilities to do some regression tests based on the xml output. With the option --xhtml it is possible to see whether or not a test or set of tests is xhtml compliant. With the option --pdf it is possible to see whether or not a test or set of tests can be build to a pdf file (per test). --- testing/README.txt | 30 +- testing/dtd/xhtml-lat1.ent | 196 ++++++ testing/dtd/xhtml-special.ent | 80 +++ testing/dtd/xhtml-symbol.ent | 237 +++++++ testing/dtd/xhtml1-strict.dtd | 978 ++++++++++++++++++++++++++++ testing/dtd/xhtml1-transitional.dtd | 1201 +++++++++++++++++++++++++++++++++++ testing/runtests.py | 207 ++++-- 7 files changed, 2877 insertions(+), 52 deletions(-) create mode 100755 testing/dtd/xhtml-lat1.ent create mode 100755 testing/dtd/xhtml-special.ent create mode 100755 testing/dtd/xhtml-symbol.ent create mode 100755 testing/dtd/xhtml1-strict.dtd create mode 100755 testing/dtd/xhtml1-transitional.dtd diff --git a/testing/README.txt b/testing/README.txt index ee3de59..1aae023 100644 --- a/testing/README.txt +++ b/testing/README.txt @@ -7,17 +7,27 @@ has the same 3 digit number. The directory contains one or more reference files that are compared against the XML output produced by doxygen. If the result is the same, there is no regression and the test passes. If there is a difference the test fails and the difference (in diff -u format) will be shown. +It is also possible to see whether or not the test can be build to a xhtml set +of files (and tested against a DTD), it is also possible to create a pdf file +for each test to see if the LaTeX / pdf generation is possible. The runtest.py script responsible for running the tests takes a number of optional parameters: --id n: run test with number n only (the option may be specified - multiple times) default is to run all tests. --updateref: update the reference files. Should be used in combination - with -id to update the reference file(s) for the given test. --all: can be used in combination with -updateref to update the - reference files for all tests. --doxygen exe: run the specified doxygen executable. --xmllint exe: run the specified xmllint executable. + --updateref update the reference data for a test + --doxygen [DOXYGEN] path/name of the doxygen executable + --xmllint [XMLLINT] path/name of the xmllint executable + --id IDS [IDS ...] id of the test to perform + --all perform all tests + --inputdir [INPUTDIR] + input directory containing the tests + --outputdir [OUTPUTDIR] + output directory to write the doxygen output to + --noredir disable redirection of doxygen warnings + --xml create xml output and check + --xhtml create xhtml output and check with xmllint + --pdf create LaTeX output and create pdf from it + --keep keep result directories +In case neither --xml, --pdf or --xhtml is used the default is set to --xml. The runtest.pl has the following dependenies on 3rd party tools: - python to run the script @@ -44,5 +54,7 @@ Example to update the reference files for a test python runtest.py -updateref -id 10 There is also a CMakeLists.txt, which can be used from the build directory -to run all tests by simply invoking 'make tests'. +to run all tests by simply invoking 'make tests', to use the specific options use +the flag TEST_FLAGS with make + e.g. make tests TEST_FLAGS="--id=5 --id=10 --pdf --xhtml" diff --git a/testing/dtd/xhtml-lat1.ent b/testing/dtd/xhtml-lat1.ent new file mode 100755 index 0000000..ffee223 --- /dev/null +++ b/testing/dtd/xhtml-lat1.ent @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/dtd/xhtml-special.ent b/testing/dtd/xhtml-special.ent new file mode 100755 index 0000000..ca358b2 --- /dev/null +++ b/testing/dtd/xhtml-special.ent @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/dtd/xhtml-symbol.ent b/testing/dtd/xhtml-symbol.ent new file mode 100755 index 0000000..63c2abf --- /dev/null +++ b/testing/dtd/xhtml-symbol.ent @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/dtd/xhtml1-strict.dtd b/testing/dtd/xhtml1-strict.dtd new file mode 100755 index 0000000..2927b9e --- /dev/null +++ b/testing/dtd/xhtml1-strict.dtd @@ -0,0 +1,978 @@ + + + + + +%HTMLlat1; + + +%HTMLsymbol; + + +%HTMLspecial; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/dtd/xhtml1-transitional.dtd b/testing/dtd/xhtml1-transitional.dtd new file mode 100755 index 0000000..628f27a --- /dev/null +++ b/testing/dtd/xhtml1-transitional.dtd @@ -0,0 +1,1201 @@ + + + + + +%HTMLlat1; + + +%HTMLsymbol; + + +%HTMLspecial; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/runtests.py b/testing/runtests.py index 83647c6..d1ce5fd 100644 --- a/testing/runtests.py +++ b/testing/runtests.py @@ -30,6 +30,19 @@ class Tester: return (True,'Difference between generated output and reference:\n%s' % diff) return (False,'') + def cleanup_xmllint(self,errmsg): + msg = errmsg.split('\n') + rtnmsg = "" + for o in msg: + if (o): + if (o.startswith("I/O error : Attempt")): + pass + else: + if (rtnmsg): + rtnmsg += '\n' + rtnmsg += o + return rtnmsg + def get_config(self): config = {} with open(self.args.inputdir+'/'+self.test,'r') as f: @@ -50,13 +63,25 @@ class Tester: os.mkdir(self.test_out) shutil.copy(self.args.inputdir+'/Doxyfile',self.test_out) with open(self.test_out+'/Doxyfile','a') as f: - print('INPUT=%s/%s' % (self.args.inputdir,self.test), file=f) - print('STRIP_FROM_PATH=%s' % self.args.inputdir, file=f) - print('XML_OUTPUT=%s/out' % self.test_out, file=f) - print('EXAMPLE_PATH=%s' % self.args.inputdir, file=f) if 'config' in self.config: for option in self.config['config']: print(option, file=f) + print('INPUT=%s/%s' % (self.args.inputdir,self.test), file=f) + print('STRIP_FROM_PATH=%s' % self.args.inputdir, file=f) + if (self.args.xml): + print('GENERATE_XML=YES', file=f) + print('XML_OUTPUT=%s/out' % self.test_out, file=f) + else: + print('GENERATE_XML=NO', file=f) + if (self.args.xhtml): + print('GENERATE_HTML=YES', file=f) + # HTML_OUTPUT can also be set locally + print('HTML_OUTPUT=%s/html' % self.test_out, file=f) + print('HTML_FILE_EXTENSION=.xhtml', file=f) + if (self.args.pdf): + print('GENERATE_LATEX=YES', file=f) + print('LATEX_OUTPUT=%s/latex' % self.test_out, file=f) + print('EXAMPLE_PATH=%s' % self.args.inputdir, file=f) if 'check' not in self.config or not self.config['check']: print('Test doesn\'t specify any files to check') @@ -64,10 +89,14 @@ class Tester: # run doxygen if (sys.platform == 'win32'): - redir=' > nul:' + redir=' > nul:' else: - redir=' 2> /dev/null' - if os.system('%s %s/Doxyfile %s' % (self.args.doxygen,self.test_out, redir))!=0: + redir=' 2> /dev/null > /dev/null' + + if (self.args.noredir): + redir='' + + if os.system('%s %s/Doxyfile %s' % (self.args.doxygen,self.test_out,redir))!=0: print('Error: failed to run %s on %s/Doxyfile' % (self.args.doxygen,self.test_out)); sys.exit(1) @@ -98,32 +127,92 @@ class Tester: # check the relevant files of a doxygen run with the reference material def perform_test(self,testmgr): + if (sys.platform == 'win32'): + redir=' > nul:' + separ='&' + else: + redir=' 2> /dev/null' + separ=';' + + if (self.args.noredir): + redir='' + + failed_xml=False + failed_html=False + failed_latex=False + msg = () # look for files to check against the reference - if 'check' in self.config: - for check in self.config['check']: - check_file='%s/out/%s' % (self.test_out,check) - # check if the file we need to check is actually generated - if not os.path.isfile(check_file): - testmgr.ok(False,self.test_name,msg='Non-existing file %s after \'check:\' statement' % check_file) - return - # convert output to canonical form - data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read() - if data: - # strip version - data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n') - else: - testmgr.ok(False,self.test_name,msg='Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out)) - return - out_file='%s/%s' % (self.test_out,check) - with open(out_file,'w') as f: - print(data,file=f) - ref_file='%s/%s/%s' % (self.args.inputdir,self.test_id,check) - (failed,msg) = self.compare_ok(out_file,ref_file,self.test_name) - if failed: - testmgr.ok(False,self.test_name,msg) - return - shutil.rmtree(self.test_out,ignore_errors=True) + if self.args.xml: + failed_xml=True + if 'check' in self.config: + for check in self.config['check']: + check_file='%s/out/%s' % (self.test_out,check) + # check if the file we need to check is actually generated + if not os.path.isfile(check_file): + msg += ('Non-existing file %s after \'check:\' statement' % check_file,) + break + # convert output to canonical form + data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read() + if data: + # strip version + data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n') + else: + msg += ('Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out),) + break + out_file='%s/%s' % (self.test_out,check) + with open(out_file,'w') as f: + print(data,file=f) + ref_file='%s/%s/%s' % (self.args.inputdir,self.test_id,check) + (failed_xml,xml_msg) = self.compare_ok(out_file,ref_file,self.test_name) + if failed_xml: + msg+= (xml_msg,) + break + if not failed_xml and not self.args.keep: + xml_output='%s/out' % self.test_out + shutil.rmtree(xml_output,ignore_errors=True) + + if (self.args.xhtml): + html_output='%s/html' % self.test_out + if (sys.platform == 'win32'): + redirx=' 2> %s/temp >nul:'%html_output + else: + redirx='2>%s/temp >/dev/null'%html_output + exe_string = '%s --path dtd --nonet --postvalid %s/*xhtml %s %s ' % (self.args.xmllint,html_output,redirx,separ) + exe_string += 'more "%s/temp"' % (html_output) + failed_html=False + xmllint_out = os.popen(exe_string).read() + xmllint_out = self.cleanup_xmllint(xmllint_out) + if xmllint_out: + msg += (xmllint_out,) + failed_html=True + elif not self.args.keep: + shutil.rmtree(html_output,ignore_errors=True) + if (self.args.pdf): + failed_latex=False + latex_output='%s/latex' % self.test_out + if (sys.platform == 'win32'): + redirl='>nul: 2>temp' + else: + redirl='>/dev/null 2>temp' + exe_string = 'cd %s %s echo "q" | make %s %s' % (latex_output,separ,redirl,separ) + exe_string += 'more temp' + latex_out = os.popen(exe_string).read() + if latex_out.find("Error")!=-1: + msg += ("PDF generation failed\n For a description of the problem see 'refman.log' in the latex directory of this test",) + failed_html=True + elif open(latex_output + "/refman.log",'r').read().find("Emergency stop")!= -1: + msg += ("PDF generation failed\n For a description of the problem see 'refman.log' in the latex directory of this test",) + failed_html=True + elif not self.args.keep: + shutil.rmtree(latex_output,ignore_errors=True) + + if failed_xml or failed_html or failed_latex: + testmgr.ok(False,self.test_name,msg) + return + testmgr.ok(True,self.test_name) + if not self.args.keep: + shutil.rmtree(self.test_out,ignore_errors=True) def run(self,testmgr): if self.update: @@ -138,6 +227,8 @@ class TestManager: self.num_tests = len(tests) self.count=1 self.passed=0 + if self.args.xhtml: + self.prepare_dtd() print('1..%d' % self.num_tests) def ok(self,result,test_name,msg='Ok'): @@ -147,8 +238,9 @@ class TestManager: else: print('not ok %s - %s' % (self.count,test_name)) print('-------------------------------------') - print(msg) - print('-------------------------------------') + for o in msg: + print(o) + print('-------------------------------------') self.count = self.count + 1 def result(self): @@ -162,27 +254,56 @@ class TestManager: for test in self.tests: tester = Tester(self.args,test) tester.run(self) - return 0 if self.args.updateref else self.result() + res=self.result() + if self.args.xhtml and not res and not self.args.keep: + shutil.rmtree("dtd",ignore_errors=True) + return 0 if self.args.updateref else res + + def prepare_dtd(self): + shutil.rmtree("dtd",ignore_errors=True) + shutil.copytree(self.args.inputdir+"/dtd", "dtd") def main(): # argument handling parser = argparse.ArgumentParser(description='run doxygen tests') - parser.add_argument('--updateref',help='update the reference data for a test',action="store_true") - parser.add_argument('--doxygen',nargs='?',default='doxygen',help='path/name of the doxygen executable') - parser.add_argument('--xmllint',nargs='?',default='xmllint',help='path/name of the xmllint executable') - parser.add_argument('--id',nargs='+',dest='ids',action='append',type=int,help='id of the test to perform') - parser.add_argument('--all',help='perform all tests',action="store_true") - parser.add_argument('--inputdir',nargs='?',default='.',help='input directory containing the tests') - parser.add_argument('--outputdir',nargs='?',default='.',help='output directory to write the doxygen output to') + parser.add_argument('--updateref',help= + 'update the reference files. Should be used in combination with -id to ' + 'update the reference file(s) for the given test',action="store_true") + parser.add_argument('--doxygen',nargs='?',default='doxygen',help= + 'path/name of the doxygen executable') + parser.add_argument('--xmllint',nargs='?',default='xmllint',help= + 'path/name of the xmllint executable') + parser.add_argument('--id',nargs='+',dest='ids',action='append',type=int,help= + 'run test with number n only (the option may be specified run test with ' + 'number n only (the option may be specified') + parser.add_argument('--all',help= + 'can be used in combination with -updateref to update the reference files ' + 'for all tests.',action="store_true") + parser.add_argument('--inputdir',nargs='?',default='.',help= + 'input directory containing the tests') + parser.add_argument('--outputdir',nargs='?',default='.',help= + 'output directory to write the doxygen output to') + parser.add_argument('--noredir',help= + 'disable redirection of doxygen warnings',action="store_true") + parser.add_argument('--xml',help='create xml output and check', + action="store_true") + parser.add_argument('--xhtml',help= + 'create xhtml output and check with xmllint',action="store_true") + parser.add_argument('--pdf',help='create LaTeX output and create pdf from it', + action="store_true") + parser.add_argument('--keep',help='keep result directories', + action="store_true") args = parser.parse_args() - # sanity check + # sanity check + if (not args.xml) and (not args.pdf) and (not args.xhtml): + args.xml=True if (not args.updateref is None) and (args.ids is None) and (args.all is None): parser.error('--updateref requires either --id or --all') starting_directory = os.getcwd() os.chdir(args.inputdir) - # find the tests to run + # find the tests to run if args.ids: # test ids are given by user tests = [] for id in list(itertools.chain.from_iterable(args.ids)): -- cgit v0.12 From 95a6451107f8964b1c2290e2f4e9761a64be124b Mon Sep 17 00:00:00 2001 From: albert-github Date: Tue, 3 Apr 2018 19:48:15 +0200 Subject: Create test possibilities for xhtml and pdf output Regression of default output (INPUT setting was overruled). --- testing/runtests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/runtests.py b/testing/runtests.py index d1ce5fd..a714cd8 100644 --- a/testing/runtests.py +++ b/testing/runtests.py @@ -63,11 +63,12 @@ class Tester: os.mkdir(self.test_out) shutil.copy(self.args.inputdir+'/Doxyfile',self.test_out) with open(self.test_out+'/Doxyfile','a') as f: + print('INPUT=%s/%s' % (self.args.inputdir,self.test), file=f) + print('STRIP_FROM_PATH=%s' % self.args.inputdir, file=f) + print('EXAMPLE_PATH=%s' % self.args.inputdir, file=f) if 'config' in self.config: for option in self.config['config']: print(option, file=f) - print('INPUT=%s/%s' % (self.args.inputdir,self.test), file=f) - print('STRIP_FROM_PATH=%s' % self.args.inputdir, file=f) if (self.args.xml): print('GENERATE_XML=YES', file=f) print('XML_OUTPUT=%s/out' % self.test_out, file=f) @@ -81,7 +82,6 @@ class Tester: if (self.args.pdf): print('GENERATE_LATEX=YES', file=f) print('LATEX_OUTPUT=%s/latex' % self.test_out, file=f) - print('EXAMPLE_PATH=%s' % self.args.inputdir, file=f) if 'check' not in self.config or not self.config['check']: print('Test doesn\'t specify any files to check') -- cgit v0.12