diff options
| -rw-r--r-- | testing/README.txt | 9 | ||||
| -rw-r--r-- | testing/runtests.py | 117 |
2 files changed, 111 insertions, 15 deletions
diff --git a/testing/README.txt b/testing/README.txt index fd7536d..e66632f 100644 --- a/testing/README.txt +++ b/testing/README.txt @@ -7,7 +7,7 @@ has the same 3 digit number. The directory contains one or more reference files that are compared against the XML output produced by doxygen. If the result is the same, there is no regression and the test passes. If there is a difference the test fails and the difference (in diff -u format) will be shown. -It is also possible to see whether or not the test can be built to an xhtml set +It is also possible to see whether or not the test can be built to a xhtml set of files (and tested against a DTD), it is also possible to create a pdf file for each test to see if the LaTeX / pdf generation is possible. @@ -17,6 +17,8 @@ optional parameters: --doxygen [DOXYGEN] path/name of the doxygen executable --xmllint [XMLLINT] path/name of the xmllint executable --id IDS [IDS ...] id of the test to perform + --start_id START_ID run tests starting with number n + --end_id END_ID run tests ending with number n --all perform all tests --inputdir [INPUTDIR] input directory containing the tests @@ -24,10 +26,13 @@ optional parameters: output directory to write the doxygen output to --noredir disable redirection of doxygen warnings --xml create xml output and check + --rtf create rtf output + --docbook create docbook output and check with xmllint --xhtml create xhtml output and check with xmllint --pdf create LaTeX output and create pdf from it + --subdirs use the configuration parameter CREATE_SUBDIRS=YES --keep keep result directories -In case neither --xml, --pdf or --xhtml is used the default is set to --xml. +In case neither --xml, --pdf, --rtf, --docbook or --xhtml is used the default is set to --xml. The runtest.pl has the following dependencies on 3rd party tools: - python to run the script diff --git a/testing/runtests.py b/testing/runtests.py index 9330d23..be8aae9 100644 --- a/testing/runtests.py +++ b/testing/runtests.py @@ -43,6 +43,28 @@ class Tester: rtnmsg += o return rtnmsg + def cleanup_xmllint_docbook(self,errmsg): + # For future work, first get everything valid XML + msg = self.cleanup_xmllint(errmsg).split('\n') + rtnmsg = "" + cnt = 0 + for o in msg: + if (o): + if (cnt): + cnt -= 1 + pass + elif (o.endswith("does not validate")): + pass + elif (o.find("no DTD found!")!=-1): + pass + elif (o.find("is not an NCName")!=-1): + cnt = 2 + else: + if (rtnmsg): + rtnmsg += '\n' + rtnmsg += o + return rtnmsg + def get_config(self): config = {} with open(self.args.inputdir+'/'+self.test,'r') as f: @@ -53,7 +75,7 @@ class Tester: value = m.group('value') if (key=='config'): value = value.replace('$INPUTDIR',self.args.inputdir) - #print('key=%s value=%s' % (key,value)) + # print('key=%s value=%s' % (key,value)) config.setdefault(key, []).append(value) return config @@ -74,6 +96,16 @@ class Tester: print('XML_OUTPUT=%s/out' % self.test_out, file=f) else: print('GENERATE_XML=NO', file=f) + if (self.args.rtf): + print('GENERATE_RTF=YES', file=f) + print('RTF_OUTPUT=%s/rtf' % self.test_out, file=f) + else: + print('GENERATE_RTF=NO', file=f) + if (self.args.docbook): + print('GENERATE_DOCBOOK=YES', file=f) + print('DOCBOOK_OUTPUT=%s/docbook' % self.test_out, file=f) + else: + print('GENERATE_DOCBOOK=NO', file=f) if (self.args.xhtml): print('GENERATE_HTML=YES', file=f) # HTML_OUTPUT can also be set locally @@ -82,6 +114,8 @@ class Tester: if (self.args.pdf): print('GENERATE_LATEX=YES', file=f) print('LATEX_OUTPUT=%s/latex' % self.test_out, file=f) + if self.args.subdirs: + print('CREATE_SUBDIRS=YES', file=f) if 'check' not in self.config or not self.config['check']: print('Test doesn\'t specify any files to check') @@ -97,7 +131,7 @@ class Tester: redir='' if os.system('%s %s/Doxyfile %s' % (self.args.doxygen,self.test_out,redir))!=0: - print('Error: failed to run %s on %s/Doxyfile' % (self.args.doxygen,self.test_out)); + print('Error: failed to run %s on %s/Doxyfile' % (self.args.doxygen,self.test_out)) sys.exit(1) # update the reference data for this test @@ -140,6 +174,8 @@ class Tester: failed_xml=False failed_html=False failed_latex=False + failed_docbook=False + failed_rtf=False msg = () # look for files to check against the reference if self.args.xml: @@ -149,8 +185,14 @@ class Tester: check_file='%s/out/%s' % (self.test_out,check) # check if the file we need to check is actually generated if not os.path.isfile(check_file): - msg += ('Non-existing file %s after \'check:\' statement' % check_file,) - break + # try with sub dirs + check_file = glob.glob('%s/out/*/*/%s' % (self.test_out,check)) + if not check_file: + check_file='%s/out/%s' % (self.test_out,check) + msg += ('Non-existing file %s after \'check:\' statement' % check_file,) + break + else: + check_file = check_file[0] # convert output to canonical form data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read() if data: @@ -171,6 +213,34 @@ class Tester: xml_output='%s/out' % self.test_out shutil.rmtree(xml_output,ignore_errors=True) + if (self.args.rtf): + # no tests defined yet + pass + + if (self.args.docbook): + docbook_output='%s/docbook' % self.test_out + if (sys.platform == 'win32'): + redirx=' 2> %s/temp >nul:'%docbook_output + else: + redirx='2>%s/temp >/dev/null'%docbook_output + # For future work, first get everything valid XML + # exe_string = '%s --relaxng db/docbook.rng --nonet --postvalid %s/*xml %s % (self.args.xmllint,docbook_output,redirx) + tests = [] + tests.append(glob.glob('%s/*.xml' % (docbook_output))) + tests.append(glob.glob('%s/*/*/*.xml' % (docbook_output))) + tests = ' '.join(list(itertools.chain.from_iterable(tests))).replace(self.args.outputdir +'/','').replace('\\','/') + exe_string = '%s --nonet --postvalid %s %s' % (self.args.xmllint,tests,redirx) + exe_string += ' %s more "%s/temp"' % (separ,docbook_output) + + failed_docbook=False + xmllint_out = os.popen(exe_string).read() + xmllint_out = self.cleanup_xmllint_docbook(xmllint_out) + if xmllint_out: + msg += (xmllint_out,) + failed_docbook=True + elif not self.args.keep: + shutil.rmtree(docbook_output,ignore_errors=True) + if (self.args.xhtml): html_output='%s/html' % self.test_out if (sys.platform == 'win32'): @@ -206,7 +276,7 @@ class Tester: elif not self.args.keep: shutil.rmtree(latex_output,ignore_errors=True) - if failed_xml or failed_html or failed_latex: + if failed_xml or failed_html or failed_latex or failed_docbook or failed_rtf: testmgr.ok(False,self.test_name,msg) return @@ -268,14 +338,18 @@ def main(): parser = argparse.ArgumentParser(description='run doxygen tests') parser.add_argument('--updateref',help= 'update the reference files. Should be used in combination with -id to ' - 'update the reference file(s) for the given test',action="store_true") + 'update the reference file(s) for the given test',action="store_true") parser.add_argument('--doxygen',nargs='?',default='doxygen',help= 'path/name of the doxygen executable') parser.add_argument('--xmllint',nargs='?',default='xmllint',help= 'path/name of the xmllint executable') parser.add_argument('--id',nargs='+',dest='ids',action='append',type=int,help= - 'run test with number n only (the option may be specified run test with ' - 'number n only (the option may be specified') + 'run test with number n only (the option can be specified to run test with ' + 'number n only (the option can be specified multiple times') + parser.add_argument('--start_id',dest='start_id',type=int,help= + 'run tests starting with number n') + parser.add_argument('--end_id',dest='end_id',type=int,help= + 'run tests ending with number n') parser.add_argument('--all',help= 'can be used in combination with -updateref to update the reference files ' 'for all tests.',action="store_true") @@ -287,17 +361,23 @@ def main(): 'disable redirection of doxygen warnings',action="store_true") parser.add_argument('--xml',help='create xml output and check', action="store_true") + parser.add_argument('--rtf',help= + 'create rtf output',action="store_true") + parser.add_argument('--docbook',help= + 'create docbook output and check with xmllint',action="store_true") parser.add_argument('--xhtml',help= 'create xhtml output and check with xmllint',action="store_true") parser.add_argument('--pdf',help='create LaTeX output and create pdf from it', action="store_true") + parser.add_argument('--subdirs',help='use the configuration parameter CREATE_SUBDIRS=YES', + action="store_true") parser.add_argument('--keep',help='keep result directories', action="store_true") test_flags = os.getenv('TEST_FLAGS', default='').split() args = parser.parse_args(test_flags + sys.argv[1:]) # sanity check - if (not args.xml) and (not args.pdf) and (not args.xhtml): + if (not args.xml) and (not args.pdf) and (not args.xhtml) and (not args.docbook and (not args.rtf)): args.xml=True if (not args.updateref is None) and (args.ids is None) and (args.all is None): parser.error('--updateref requires either --id or --all') @@ -305,15 +385,26 @@ def main(): starting_directory = os.getcwd() os.chdir(args.inputdir) # find the tests to run - if args.ids: # test ids are given by user - tests = [] + tests = [] + if args.start_id: + if args.end_id: + for id in range(args.start_id, args.end_id + 1): + tests.append(glob.glob('%s_*'%id)) + tests.append(glob.glob('0%s_*'%id)) + tests.append(glob.glob('00%s_*'%id)) + else: + parser.error('--start_id requires --end_id') + elif args.end_id: + parser.error('--end_id requires --start_id') + if args.ids: # test ids are given by user for id in list(itertools.chain.from_iterable(args.ids)): tests.append(glob.glob('%s_*'%id)) tests.append(glob.glob('0%s_*'%id)) tests.append(glob.glob('00%s_*'%id)) - tests = list(itertools.chain.from_iterable(tests)) - else: # find all tests + if (not args.ids and not args.start_id): # find all tests tests = glob.glob('[0-9][0-9][0-9]_*') + else: + tests = list(itertools.chain.from_iterable(tests)) os.chdir(starting_directory) # create test manager to run the tests |