Making tests valid for python 3

Test 65 gave some problems with python3:
```
Traceback (most recent call last):
  File "D:/Programs/Doxygen/Doxygen-.git/doxygen/testing/runtests.py", line 487, in <module>
    main()
  File "D:/Programs/Doxygen/Doxygen-.git/doxygen/testing/runtests.py", line 484, in main
    sys.exit(testManager.perform_tests())
  File "D:/Programs/Doxygen/Doxygen-.git/doxygen/testing/runtests.py", line 388, in perform_tests
    tester = Tester(self.args,test)
  File "D:/Programs/Doxygen/Doxygen-.git/doxygen/testing/runtests.py", line 13, in __init__
    self.config    = self.get_config()
  File "D:/Programs/Doxygen/Doxygen-.git/doxygen/testing/runtests.py", line 71, in get_config
    for line in f.readlines():
  File "D:\Programs\Python\Python37\lib\encodings\cp1252.py", line 23, in decode
    return codecs.charmap_decode(input,self.errors,decoding_table)[0]
UnicodeDecodeError: 'charmap' codec can't decode byte 0x81 in position 236: character maps to <undefined>
```
this happened on multiple places and also with the `popen` command.

Created, analogous to `doc/translator.py`, special open functions so that the code works for Python 2 and Python 3.

1 files changed, 39 insertions, 13 deletions

diff --git a/testing/runtests.py b/testing/runtests.py
index fa3c186..3e099ee 100755
--- a/testing/runtests.py
+++ b/testing/runtests.py
@@ -2,9 +2,35 @@
 
 from __future__ import print_function
 import argparse, glob, itertools, re, shutil, os, sys
+import subprocess
 
 config_reg = re.compile('.*\/\/\s*(?P<name>\S+):\s*(?P<value>.*)$')
 
+
+def xopen(fname, mode='r', encoding='utf-8'):
+	'''Unified file opening for Python 2 an Python 3.
+
+	Python 2 does not have the encoding argument. Python 3 has one.
+	'''
+
+	if sys.version_info[0] == 2:
+		return open(fname, mode=mode) # Python 2 without encoding
+	else:
+		return open(fname, mode=mode, encoding=encoding) # Python 3 with encoding
+
+def xpopen(cmd, encoding='utf-8-sig'):
+	'''Unified file pipe opening for Python 2 an Python 3.
+
+	Python 2 does not have the encoding argument. Python 3 has one. and
+	'''
+
+	if sys.version_info[0] == 2:
+		return os.popen(cmd).read() # Python 2 without encoding
+	else:
+		proc = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,encoding=encoding) # Python 3 with encoding
+		proc.wait
+		return proc.stdout.read()
+
 class Tester:
 	def __init__(self,args,test):
 		self.args      = args
@@ -25,7 +51,7 @@ class Tester:
 		elif not os.path.isfile(expected_file):
 			return (True,'%s absent' % expected_file)
 		else:
-			diff = os.popen('diff -b -w -u %s %s' % (got_file,expected_file)).read()
+			diff = xpopen('diff -b -w -u %s %s' % (got_file,expected_file))
 			if diff and not diff.startswith("No differences"):
 				return (True,'Difference between generated output and reference:\n%s' % diff)
 		return (False,'')
@@ -67,7 +93,7 @@ class Tester:
 
 	def get_config(self):
 		config = {}
-		with open(self.args.inputdir+'/'+self.test,'r') as f:
+		with xopen(self.args.inputdir+'/'+self.test,'r') as f:
 			for line in f.readlines():
 				m = config_reg.match(line)
 				if m:
@@ -84,7 +110,7 @@ class Tester:
 		shutil.rmtree(self.test_out,ignore_errors=True)
 		os.mkdir(self.test_out)
 		shutil.copy(self.args.inputdir+'/Doxyfile',self.test_out)
-		with open(self.test_out+'/Doxyfile','a') as f:
+		with xopen(self.test_out+'/Doxyfile','a') as f:
 			print('INPUT=%s/%s' % (self.args.inputdir,self.test), file=f)
 			print('STRIP_FROM_PATH=%s' % self.args.inputdir, file=f)
 			print('EXAMPLE_PATH=%s' % self.args.inputdir, file=f)
@@ -154,7 +180,7 @@ class Tester:
 					print('Non-existing file %s after \'check:\' statement' % check_file)
 					return
 				# convert output to canonical form
-				data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read()
+				data = xpopen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read()
 				if data:
 					# strip version
 					data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
@@ -162,7 +188,7 @@ class Tester:
 					print('Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out))
 					return
 				out_file='%s/%s' % (self.test_out,check)
-				with open(out_file,'w') as f:
+				with xopen(out_file,'w') as f:
 					print(data,file=f)
 		shutil.rmtree(self.test_out+'/out',ignore_errors=True)
 		os.remove(self.test_out+'/Doxyfile')
@@ -204,7 +230,7 @@ class Tester:
 						else:
 							check_file = check_file[0]
 					# convert output to canonical form
-					data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read()
+					data = xpopen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file))
 					if data:
 						# strip version
 						data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
@@ -212,7 +238,7 @@ class Tester:
 						msg += ('Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out),)
 						break
 					out_file='%s/%s' % (self.test_out,check)
-					with open(out_file,'w') as f:
+					with xopen(out_file,'w') as f:
 						print(data,file=f)
 					ref_file='%s/%s/%s' % (self.args.inputdir,self.test_id,check)
 					(failed_xml,xml_msg) = self.compare_ok(out_file,ref_file,self.test_name)
@@ -238,7 +264,7 @@ class Tester:
 				exe_string = '%s --noout --schema %s %s %s' % (self.args.xmllint,index_xsd,index_xml,redirx)
 				exe_string += ' %s more "%s/temp"' % (separ,xmlxsd_output)
 
-				xmllint_out = os.popen(exe_string).read()
+				xmllint_out = xpopen(exe_string).read()
 				if xmllint_out:
 					xmllint_out = re.sub(r'.*validates','',xmllint_out).rstrip('\n')
 				else:
@@ -262,7 +288,7 @@ class Tester:
 				exe_string = '%s --noout --schema %s %s %s' % (self.args.xmllint,compound_xsd,compound_xml,redirx)
 				exe_string += ' %s more "%s/temp"' % (separ,xmlxsd_output)
 
-				xmllint_out = os.popen(exe_string).read()
+				xmllint_out = xpopen(exe_string).read()
 				if xmllint_out:
 					xmllint_out = re.sub(r'.*validates','',xmllint_out).rstrip('\n')
 				else:
@@ -296,7 +322,7 @@ class Tester:
 			exe_string += ' %s more "%s/temp"' % (separ,docbook_output)
 
 			failed_docbook=False
-			xmllint_out = os.popen(exe_string).read()
+			xmllint_out = xpopen(exe_string).read()
 			xmllint_out = self.cleanup_xmllint_docbook(xmllint_out)
 			if xmllint_out:
 				msg += (xmllint_out,)
@@ -313,7 +339,7 @@ class Tester:
 			exe_string = '%s --path dtd --nonet --postvalid %s/*xhtml %s %s ' % (self.args.xmllint,html_output,redirx,separ)
 			exe_string += 'more "%s/temp"' % (html_output)
 			failed_html=False
-			xmllint_out = os.popen(exe_string).read()
+			xmllint_out = xpopen(exe_string).read()
 			xmllint_out = self.cleanup_xmllint(xmllint_out)
 			if xmllint_out:
 				msg += (xmllint_out,)
@@ -329,11 +355,11 @@ class Tester:
 				redirl='>/dev/null 2>temp'
 			exe_string = 'cd %s %s echo "q" | make %s %s' % (latex_output,separ,redirl,separ)
 			exe_string += 'more temp'
-			latex_out = os.popen(exe_string).read()
+			latex_out = xpopen(exe_string).read()
 			if latex_out.find("Error")!=-1:
 				msg += ("PDF generation failed\n  For a description of the problem see 'refman.log' in the latex directory of this test",)
 				failed_html=True
-			elif open(latex_output + "/refman.log",'r').read().find("Emergency stop")!= -1:
+			elif xopen(latex_output + "/refman.log",'r').read().find("Emergency stop")!= -1:
 				msg += ("PDF generation failed\n  For a description of the problem see 'refman.log' in the latex directory of this test",)
 				failed_html=True
 			elif not self.args.keep: