Improve building of docs using Py3 [ci skip]

* context managers on file r/w + use shutil.copy where it makes sense.
* lxml wants (demands?) that xml files be processed as bytes
* for the phase where we gen the entity files, read as text anyway
* Need to solve a problem where the generated xml is putting the \n in
  literally, not evaluating it.
* Fix some examples broken for py3
* Fix more octal constant instances
* Cleanups suggested by PyCharm: staticmethods, two blanks
  before class definition, others.

This addresses issues called out in #3300, but is not a complete
solution because the actual doc build step still fails with the
epydoc failures (which aren't directly because of Py3; epydoc
build doesn't work any better on my system with Py3, even with
the forked version with patches).

Signed-off-by: Mats Wichmann <mats@linux.com>

4 files changed, 187 insertions, 131 deletions

diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py
index 5fd5f00..d200f8b 100644
--- a/bin/SConsDoc.py
+++ b/bin/SConsDoc.py
@@ -26,7 +26,8 @@
 #
 from __future__ import print_function
 
-__doc__ = """
+
+__doc__ = r"""
 This module parses home-brew XML files that document various things
 in SCons.  Right now, it handles Builders, functions, construction
 variables, and Tools, but we expect it to get extended in the future.
@@ -117,16 +118,18 @@ import re
 import sys
 import copy
 
+PY2 = sys.version_info[0] == 2
+
 # Do we have libxml2/libxslt/lxml?
 has_libxml2 = True
 try:
     import libxml2
     import libxslt
-except:
+except ImportError:
     has_libxml2 = False
     try:
         import lxml
-    except:
+    except ImportError:
         raise ImportError("Failed to import either libxml2/libxslt or lxml")
 
 has_etree = False
@@ -155,13 +158,16 @@ if not has_etree:
                 except ImportError:
                     raise ImportError("Failed to import ElementTree from any known place")
 
-re_entity = re.compile(r"\&([^;]+);")
+# patterns to help trim XML passed in as strings
+re_entity = re.compile(r"&([^;]+);")
 re_entity_header = re.compile(r"<!DOCTYPE\s+sconsdoc\s+[^\]]+\]>")
 
 # Namespace for the SCons Docbook XSD
-dbxsd="http://www.scons.org/dbxsd/v1.0"
+dbxsd = "http://www.scons.org/dbxsd/v1.0"
+# Namsespace pattern to help identify an scons-xml file read as  bytes
+dbxsdpat = b'xmlns="%s"' % dbxsd.encode('utf-8')
 # Namespace map identifier for the SCons Docbook XSD
-dbxid="dbx"
+dbxid = "dbx"
 # Namespace for schema instances
 xsi = "http://www.w3.org/2001/XMLSchema-instance"
 
@@ -170,19 +176,22 @@ copyright_comment = """
 __COPYRIGHT__
 
 This file is processed by the bin/SConsDoc.py module.
-See its __doc__ string for a discussion of the format.
+See its docstring for a discussion of the format.
 """
 
 def isSConsXml(fpath):
-    """ Check whether the given file is a SCons XML file, i.e. it
-        contains the default target namespace definition.
+    """ Check whether the given file is an SCons XML file.
+
+    It is SCons XML if it contains the default target namespace definition
+    described by dbxsdpat
+
     """
     try:
-        with open(fpath,'r') as f:
+        with open(fpath, 'rb') as f:
             content = f.read()
-        if content.find('xmlns="%s"' % dbxsd) >= 0:
+        if content.find(dbxsdpat) >= 0:
             return True
-    except:
+    except Exception:
         pass
 
     return False
@@ -195,10 +204,11 @@ def remove_entities(content):
 
     return content
 
-default_xsd = os.path.join('doc','xsd','scons.xsd')
+default_xsd = os.path.join('doc', 'xsd', 'scons.xsd')
 
 ARG = "dbscons"
 
+
 class Libxml2ValidityHandler:
 
     def __init__(self):
@@ -224,10 +234,11 @@ class DoctypeEntity:
     def getEntityString(self):
         txt = """    <!ENTITY %(perc)s %(name)s SYSTEM "%(uri)s">
     %(perc)s%(name)s;
-""" % {'perc' : perc, 'name' : self.name, 'uri' : self.uri}
+""" % {'perc': perc, 'name': self.name, 'uri': self.uri}
 
         return txt
 
+
 class DoctypeDeclaration:
     def __init__(self, name_=None):
         self.name = name_
@@ -253,14 +264,16 @@ class DoctypeDeclaration:
         return content
 
 if not has_libxml2:
-    class TreeFactory:
+    class TreeFactory(object):
         def __init__(self):
             pass
 
-        def newNode(self, tag):
+        @staticmethod
+        def newNode(tag):
             return etree.Element(tag)
 
-        def newEtreeNode(self, tag, init_ns=False):
+        @staticmethod
+        def newEtreeNode(tag, init_ns=False):
             if init_ns:
                 NSMAP = {None: dbxsd,
                          'xsi' : xsi}
@@ -268,47 +281,60 @@ if not has_libxml2:
 
             return etree.Element(tag)
 
-        def copyNode(self, node):
+        @staticmethod
+        def copyNode(node):
             return copy.deepcopy(node)
 
-        def appendNode(self, parent, child):
+        @staticmethod
+        def appendNode(parent, child):
             parent.append(child)
 
-        def hasAttribute(self, node, att):
+        @staticmethod
+        def hasAttribute(node, att):
             return att in node.attrib
 
-        def getAttribute(self, node, att):
+        @staticmethod
+        def getAttribute(node, att):
             return node.attrib[att]
 
-        def setAttribute(self, node, att, value):
+        @staticmethod
+        def setAttribute(node, att, value):
             node.attrib[att] = value
 
-        def getText(self, root):
+        @staticmethod
+        def getText(root):
             return root.text
 
-        def setText(self, root, txt):
+        @staticmethod
+        def setText(root, txt):
             root.text = txt
 
-        def writeGenTree(self, root, fp):
+        @staticmethod
+        def writeGenTree(root, fp):
             dt = DoctypeDeclaration()
-            fp.write(etree.tostring(root, xml_declaration=True,
-                                    encoding="UTF-8", pretty_print=True,
+            encfun = unicode if PY2 else str
+            fp.write(etree.tostring(root, encoding=encfun,
+                                    pretty_print=True,
                                     doctype=dt.createDoctype()))
 
-        def writeTree(self, root, fpath):
-            with open(fpath, 'w') as fp:
-                fp.write(etree.tostring(root, xml_declaration=True,
-                                        encoding="UTF-8", pretty_print=True))
+        @staticmethod
+        def writeTree(root, fpath):
+            encfun = unicode if PY2 else None
+            with open(fpath, 'wb') as fp:
+                fp.write(etree.tostring(root, encoding=encfun,
+                                        pretty_print=True))
 
-        def prettyPrintFile(self, fpath):
-            with open(fpath,'r') as fin:
+        @staticmethod
+        def prettyPrintFile(fpath):
+            with open(fpath,'rb') as fin:
                 tree = etree.parse(fin)
                 pretty_content = etree.tostring(tree, pretty_print=True)
 
-            with open(fpath,'w') as fout:
+            with open(fpath,'wb') as fout:
                 fout.write(pretty_content)
 
-        def decorateWithHeader(self, root):
+        @staticmethod
+        def decorateWithHeader(root):
             root.attrib["{"+xsi+"}schemaLocation"] = "%s %s/scons.xsd" % (dbxsd, dbxsd)
             return root
 
@@ -316,12 +342,12 @@ if not has_libxml2:
             """ Return a XML file tree with the correct namespaces set,
                 the element root as top entry and the given header comment.
             """
-            NSMAP = {None: dbxsd,
-                     'xsi' : xsi}
+            NSMAP = {None: dbxsd, 'xsi' : xsi}
             t = etree.Element(root, nsmap=NSMAP)
             return self.decorateWithHeader(t)
 
-        def validateXml(self, fpath, xmlschema_context):
+        @staticmethod
+        def validateXml(fpath, xmlschema_context):
             # Use lxml
             xmlschema = etree.XMLSchema(xmlschema_context)
             try:
@@ -339,19 +365,22 @@ if not has_libxml2:
                 return False
             return True
 
-        def findAll(self, root, tag, ns=None, xp_ctxt=None, nsmap=None):
+        @staticmethod
+        def findAll(root, tag, ns=None, xp_ctxt=None, nsmap=None):
             expression = ".//{%s}%s" % (nsmap[ns], tag)
             if not ns or not nsmap:
                 expression = ".//%s" % tag
             return root.findall(expression)
 
-        def findAllChildrenOf(self, root, tag, ns=None, xp_ctxt=None, nsmap=None):
+        @staticmethod
+        def findAllChildrenOf(root, tag, ns=None, xp_ctxt=None, nsmap=None):
             expression = "./{%s}%s/*" % (nsmap[ns], tag)
             if not ns or not nsmap:
                 expression = "./%s/*" % tag
             return root.findall(expression)
 
-        def convertElementTree(self, root):
+        @staticmethod
+        def convertElementTree(root):
             """ Convert the given tree of etree.Element
                 entries to a list of tree nodes for the
                 current XML toolkit.
@@ -359,53 +388,63 @@ if not has_libxml2:
             return [root]
 
 else:
-    class TreeFactory:
+    class TreeFactory(object):
         def __init__(self):
             pass
 
-        def newNode(self, tag):
+        @staticmethod
+        def newNode(tag):
             return libxml2.newNode(tag)
 
-        def newEtreeNode(self, tag, init_ns=False):
+        @staticmethod
+        def newEtreeNode(tag, init_ns=False):
             return etree.Element(tag)
 
-        def copyNode(self, node):
+        @staticmethod
+        def copyNode(node):
             return node.copyNode(1)
 
-        def appendNode(self, parent, child):
+        @staticmethod
+        def appendNode(parent, child):
             if hasattr(parent, 'addChild'):
                 parent.addChild(child)
             else:
                 parent.append(child)
 
-        def hasAttribute(self, node, att):
+        @staticmethod
+        def hasAttribute(node, att):
             if hasattr(node, 'hasProp'):
                 return node.hasProp(att)
             return att in node.attrib
 
-        def getAttribute(self, node, att):
+        @staticmethod
+        def getAttribute(node, att):
             if hasattr(node, 'prop'):
                 return node.prop(att)
             return node.attrib[att]
 
-        def setAttribute(self, node, att, value):
+        @staticmethod
+        def setAttribute(node, att, value):
             if hasattr(node, 'setProp'):
                 node.setProp(att, value)
             else:
                 node.attrib[att] = value
 
-        def getText(self, root):
+        @staticmethod
+        def getText(root):
             if hasattr(root, 'getContent'):
                 return root.getContent()
             return root.text
 
-        def setText(self, root, txt):
+        @staticmethod
+        def setText(root, txt):
             if hasattr(root, 'setContent'):
                 root.setContent(txt)
             else:
                 root.text = txt
 
-        def writeGenTree(self, root, fp):
+        @staticmethod
+        def writeGenTree(root, fp):
             doc = libxml2.newDoc('1.0')
             dtd = doc.newDtd("sconsdoc", None, None)
             doc.addChild(dtd)
@@ -420,23 +459,26 @@ else:
             fp.write(content)
             doc.freeDoc()
 
-        def writeTree(self, root, fpath):
-            with open(fpath, 'w') as fp:
+        @staticmethod
+        def writeTree(root, fpath):
+            with open(fpath, 'wb') as fp:
                 doc = libxml2.newDoc('1.0')
                 doc.setRootElement(root)
                 fp.write(doc.serialize("UTF-8", 1))
                 doc.freeDoc()
 
-        def prettyPrintFile(self, fpath):
+        @staticmethod
+        def prettyPrintFile(fpath):
             # Read file and resolve entities
             doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT)
-            with open(fpath, 'w') as fp:
+            with open(fpath, 'wb') as fp:
                 # Prettyprint
                 fp.write(doc.serialize("UTF-8", 1))
             # Cleanup
             doc.freeDoc()
 
-        def decorateWithHeader(self, root):
+        @staticmethod
+        def decorateWithHeader(root):
             # Register the namespaces
             ns = root.newNs(dbxsd, None)
             xi = root.newNs(xsi, 'xsi')
@@ -453,7 +495,8 @@ else:
             t = libxml2.newNode(root)
             return self.decorateWithHeader(t)
 
-        def validateXml(self, fpath, xmlschema_context):
+        @staticmethod
+        def validateXml(fpath, xmlschema_context):
             retval = True
 
             # Create validation context
@@ -479,7 +522,8 @@ else:
 
             return retval
 
-        def findAll(self, root, tag, ns=None, xpath_context=None, nsmap=None):
+        @staticmethod
+        def findAll(root, tag, ns=None, xpath_context=None, nsmap=None):
             if hasattr(root, 'xpathEval') and xpath_context:
                 # Use the xpath context
                 xpath_context.setContextNode(root)
@@ -493,7 +537,8 @@ else:
                     expression = ".//%s" % tag
                 return root.findall(expression)
 
-        def findAllChildrenOf(self, root, tag, ns=None, xpath_context=None, nsmap=None):
+        @staticmethod
+        def findAllChildrenOf(root, tag, ns=None, xpath_context=None, nsmap=None):
             if hasattr(root, 'xpathEval') and xpath_context:
                 # Use the xpath context
                 xpath_context.setContextNode(root)
@@ -562,20 +607,18 @@ else:
 tf = TreeFactory()
 
 
-class SConsDocTree:
+class SConsDocTree(object):
     def __init__(self):
-        self.nsmap = {'dbx' : dbxsd}
+        self.nsmap = {'dbx': dbxsd}
         self.doc = None
         self.root = None
         self.xpath_context = None
 
     def parseContent(self, content, include_entities=True):
-        """ Parses the given content as XML file. This method
-            is used when we generate the basic lists of entities
-            for the builders, tools and functions.
-            So we usually don't bother about namespaces and resolving
-            entities here...this is handled in parseXmlFile below
-            (step 2 of the overall process).
+        """ Parses the given text content as XML
+
+        This is the setup portion, called from parseContent in
+        an SConsDocHandler instance - see the notes there.
         """
         if not include_entities:
             content = remove_entities(content)
@@ -583,7 +626,6 @@ class SConsDocTree:
         self.root = etree.fromstring(content)
 
     def parseXmlFile(self, fpath):
-        nsmap = {'dbx' : dbxsd}
         if not has_libxml2:
             # Create domtree from file
             domtree = etree.parse(fpath)
@@ -604,7 +646,7 @@ class SConsDocTree:
         if self.xpath_context is not None:
             self.xpath_context.xpathFreeContext()
 
-perc="%"
+perc = "%"
 
 def validate_all_xml(dpaths, xsdfile=default_xsd):
     xmlschema_context = None
@@ -620,7 +662,7 @@ def validate_all_xml(dpaths, xsdfile=default_xsd):
     fpaths = []
     for dp in dpaths:
         if dp.endswith('.xml') and isSConsXml(dp):
-            path='.'
+            path = '.'
             fpaths.append(dp)
         else:
             for path, dirs, files in os.walk(dp):
@@ -633,8 +675,8 @@ def validate_all_xml(dpaths, xsdfile=default_xsd):
     fails = []
     for idx, fp in enumerate(fpaths):
         fpath = os.path.join(path, fp)
-        print("%.2f%s (%d/%d) %s" % (float(idx+1)*100.0/float(len(fpaths)),
-                                     perc, idx+1, len(fpaths),fp))
+        print("%.2f%s (%d/%d) %s" % (float(idx + 1) * 100.0 /float(len(fpaths)),
+                                     perc, idx + 1, len(fpaths), fp))
 
         if not tf.validateXml(fp, xmlschema_context):
             fails.append(fp)
@@ -649,6 +691,7 @@ def validate_all_xml(dpaths, xsdfile=default_xsd):
 
     return True
 
+
 class Item(object):
     def __init__(self, name):
         self.name = name
@@ -668,21 +711,25 @@ class Item(object):
     def __lt__(self, other):
         return self.sort_name < other.sort_name
 
+
 class Builder(Item):
     pass
 
+
 class Function(Item):
-    def __init__(self, name):
-        super(Function, self).__init__(name)
+    pass
+
 
 class Tool(Item):
     def __init__(self, name):
         Item.__init__(self, name)
         self.entity = self.name.replace('+', 'X')
 
+
 class ConstructionVariable(Item):
     pass
 
+
 class Arguments(object):
     def __init__(self, signature, body=None):
         if not body:
@@ -692,7 +739,7 @@ class Arguments(object):
     def __str__(self):
         s = ''.join(self.body).strip()
         result = []
-        for m in re.findall('([a-zA-Z/_]+|[^a-zA-Z/_]+)', s):
+        for m in re.findall(r'([a-zA-Z/_]+|[^a-zA-Z/_]+)', s):
             if ' ' in m:
                 m = '"%s"' % m
             result.append(m)
@@ -700,6 +747,7 @@ class Arguments(object):
     def append(self, data):
         self.body.append(data)
 
+
 class SConsDocHandler(object):
     def __init__(self):
         self.builders = {}
@@ -794,7 +842,7 @@ class SConsDocHandler(object):
         self.parseDomtree(t.root, t.xpath_context, t.nsmap)
 
 # lifted from Ka-Ping Yee's way cool pydoc module.
-if sys.version_info[0] == 2:
+if PY2:
     def importfile(path):
         """Import a Python source file or compiled file given its path."""
         import imp
@@ -817,11 +865,10 @@ if sys.version_info[0] == 2:
 else:  # PY3 version, from newer pydoc
     def importfile(path):
         """Import a Python source file or compiled file given its path."""
-        import importlib
-        from pydoc import ErrorDuringImport
-        magic = importlib.util.MAGIC_NUMBER
+        from importlib import MAGIC_NUMBER
+        import pydoc
         with open(path, 'rb') as ifp:
-            is_bytecode = magic == ifp.read(len(magic))
+            is_bytecode = MAGIC_NUMBER == ifp.read(len(MAGIC_NUMBER))
         filename = os.path.basename(path)
         name, ext = os.path.splitext(filename)
         if is_bytecode:
@@ -832,7 +879,7 @@ else:  # PY3 version, from newer pydoc
         spec = importlib.util.spec_from_file_location(name, path, loader=loader)
         try:
             return importlib._bootstrap._load(spec)
-        except:
+        except ImportError:
             raise ErrorDuringImport(path, sys.exc_info())
 
 # Local Variables:
diff --git a/bin/SConsExamples.py b/bin/SConsExamples.py
index 501169e..8a264e4 100644
--- a/bin/SConsExamples.py
+++ b/bin/SConsExamples.py
@@ -92,6 +92,7 @@ import os
 import re
 import sys
 import time
+import shutil
 
 import SConsDoc
 from SConsDoc import tf as stf
@@ -287,9 +288,8 @@ def ensureExampleOutputsExist(dpath):
                 fpath = os.path.join(generated_examples,
                                      key + '_' + r.name.replace("/", "_"))
                 # Write file
-                f = open(fpath, 'w')
-                f.write("%s\n" % content)
-                f.close()
+                with open(fpath, 'w') as f:
+                    f.write("%s\n" % content)
 
 perc = "%"
 
@@ -324,9 +324,8 @@ def createAllExampleOutputs(dpath):
                 fpath = os.path.join(generated_examples,
                                      key + '_' + r.name.replace("/", "_"))
                 # Write file
-                f = open(fpath, 'w')
-                f.write("%s\n" % content)
-                f.close()
+                with open(fpath, 'w') as f:
+                    f.write("%s\n" % content)
         idx += 1
 
 def collectSConsExampleNames(fpath):
@@ -458,6 +457,7 @@ import re
 import SCons.Action
 import SCons.Defaults
 import SCons.Node.FS
+import shutil
 
 platform = '%(osname)s'
 
@@ -540,30 +540,28 @@ def Null(target, source, env):
 
 def Cat(target, source, env):
     target = str(target[0])
-    f = open(target, "wb")
     for src in map(str, source):
-        f.write(open(src, "rb").read())
-    f.close()
+        shutil.copy(src, target)
 
 def CCCom(target, source, env):
-    target = str(target[0])
-    fp = open(target, "wb")
-    def process(source_file, fp=fp):
-        for line in open(source_file, "rb").readlines():
-            m = re.match(r'#include\s[<"]([^<"]+)[>"]', line)
-            if m:
-                include = m.group(1)
-                for d in [str(env.Dir('$CPPPATH')), '.']:
-                    f = os.path.join(d, include)
-                    if os.path.exists(f):
-                        process(f)
-                        break
-            elif line[:11] != "STRIP CCCOM":
-                fp.write(line)
-    for src in map(str, source):
-        process(src)
-        fp.write('debug = ' + ARGUMENTS.get('debug', '0') + '\\n')
-    fp.close()
+    def process(source_file, ofp):
+        with open(source_file, "r") as ifp:
+            for line in ifp.readlines():
+                m = re.match(r'#include\s[<"]([^<"]+)[>"]', line)
+                if m:
+                    include = m.group(1)
+                    for d in [str(env.Dir('$CPPPATH')), '.']:
+                        f = os.path.join(d, include)
+                        if os.path.exists(f):
+                            process(f, ofp)
+                            break
+                elif line[:11] != "STRIP CCCOM":
+                    ofp.write(line)
+
+    with open(str(target[0]), "w") as fp:
+        for src in map(str, source):
+            process(src, fp)
+            fp.write('debug = ' + ARGUMENTS.get('debug', '0') + '\\n')
 
 public_class_re = re.compile('^public class (\S+)', re.MULTILINE)
 
@@ -577,20 +575,23 @@ def JavaCCom(target, source, env):
     for t in tlist:
        not_copied[t] = 1
     for src in map(str, source):
-        contents = open(src, "rb").read()
+        with open(src, "r") as f:
+            contents = f.read()
         classes = public_class_re.findall(contents)
         for c in classes:
             for t in [x for x in tlist if x.find(c) != -1]:
-                open(t, "wb").write(contents)
+                with open(t, "w") as f:
+                    f.write(contents)
                 del not_copied[t]
     for t in not_copied.keys():
-        open(t, "wb").write("\\n")
+        with open(t, "w") as f:
+            f.write("\\n")
 
 def JavaHCom(target, source, env):
     tlist = map(str, target)
     slist = map(str, source)
     for t, s in zip(tlist, slist):
-        open(t, "wb").write(open(s, "rb").read())
+        shutil.copy(s, t)
 
 def JarCom(target, source, env):
     target = str(target[0])
@@ -599,10 +600,8 @@ def JarCom(target, source, env):
         for dirpath, dirnames, filenames in os.walk(src):
             class_files.extend([ os.path.join(dirpath, f)
                                  for f in filenames if f.endswith('.class') ])
-    f = open(target, "wb")
     for cf in class_files:
-        f.write(open(cf, "rb").read())
-    f.close()
+        shutil.copy(cf, target)
 
 # XXX Adding COLOR, COLORS and PACKAGE to the 'cc' varlist(s) by hand
 # here is bogus.  It's for the benefit of doc/user/command-line.in, which
@@ -721,7 +720,8 @@ def command_touch(args, command, test, values):
         if not os.path.isabs(file):
             file = os.path.join(test.workpath('WORK'), file)
         if not os.path.exists(file):
-            open(file, 'wb')
+            with open(file, 'w'):
+                pass
         os.utime(file, times)
     return []
 
@@ -735,8 +735,8 @@ def command_edit(args, c, test, values):
     for file in args:
         if not os.path.isabs(file):
             file = os.path.join(test.workpath('WORK'), file)
-        contents = open(file, 'rb').read()
-        open(file, 'wb').write(contents + add_string)
+        with open(file, 'a') as f:
+            f.write(add_string)
     return []
 
 def command_ls(args, c, test, values):
@@ -825,7 +825,7 @@ def create_scons_output(e):
             t.write(path, content)
             if hasattr(f, 'chmod'):
                 if len(f.chmod):
-                    os.chmod(path, int(f.chmod, 0))
+                    os.chmod(path, int(f.chmod, base=8))
 
         # Regular expressions for making the doc output consistent,
         # regardless of reported addresses or Python version.
diff --git a/bin/docs-update-generated.py b/bin/docs-update-generated.py
index c164baf..307f843 100644
--- a/bin/docs-update-generated.py
+++ b/bin/docs-update-generated.py
@@ -10,6 +10,8 @@ from __future__ import print_function
 
 import os
 import sys
+import subprocess
+
 import SConsDoc
 
 # Directory where all generated files are stored
@@ -17,8 +19,8 @@ gen_folder = os.path.join('doc','generated')
 
 def argpair(key):
     """ Return the argument pair *.gen,*.mod for the given key. """
-    arg = '%s,%s' % (os.path.join(gen_folder,'%s.gen' % key),
-                     os.path.join(gen_folder,'%s.mod' % key))
+    arg = '%s,%s' % (os.path.join(gen_folder, '%s.gen' % key),
+                     os.path.join(gen_folder, '%s.mod' % key))
     
     return arg
 
@@ -43,10 +45,13 @@ def generate_all():
                 print("Couldn't create destination folder %s! Exiting..." % gen_folder)
                 return
         # Call scons-proc.py
-        os.system('%s %s -b %s -f %s -t %s -v %s %s' %
-                  (sys.executable, os.path.join('bin','scons-proc.py'),
-                   argpair('builders'), argpair('functions'),
-                   argpair('tools'), argpair('variables'), ' '.join(flist)))
+        rv = subprocess.call([sys.executable,
+                              os.path.join('bin','scons-proc.py'),
+                              '-b', argpair('builders'),
+                              '-f', argpair('functions'),
+                              '-t', argpair('tools'),
+                              '-v', argpair('variables')] + flist,
+                             shell=False)
     
     
 if __name__ == "__main__":
diff --git a/bin/scons-proc.py b/bin/scons-proc.py
index e09c853..d8b37df 100644
--- a/bin/scons-proc.py
+++ b/bin/scons-proc.py
@@ -32,8 +32,8 @@ Options:
   -t file(s)        dump tool information to the specified file(s)
   -v file(s)        dump variable information to the specified file(s)
   
-  Regard that each -[btv] argument is a pair of
-  comma-separated .gen,.mod file names.
+  The "files" argument following a -[bftv] argument is expected to
+  be a comma-separated pair of names like: foo.gen,foo.mod
   
 """
 
@@ -70,7 +70,9 @@ def parse_docs(args, include_entities=True):
                 sys.stderr.write("error in %s\n" % f)
                 raise
         else:
-            content = open(f).read()
+            # mode we read (text/bytes) has to match handling in SConsDoc
+            with open(f, 'r') as fp:
+                content = fp.read()
             if content:
                 try:
                     h.parseContent(content, include_entities)
@@ -166,6 +168,7 @@ class SCons_XML(object):
         # Write file        
         f = self.fopen(filename)
         stf.writeGenTree(root, f)
+        f.close()
             
     def write_mod(self, filename):
         try:
@@ -212,6 +215,7 @@ class SCons_XML(object):
                              v.tag, v.entityfunc(), v.tag))
         f.write('\n')
         f.write(Warning)
+        f.close()
 
 class Proxy(object):
     def __init__(self, subject):
@@ -348,7 +352,7 @@ processor_class = SCons_XML
 
 # Step 1: Creating entity files for builders, functions,...
 print("Generating entity files...")
-h = parse_docs(args, False)
+h = parse_docs(args, include_entities=False)
 write_output_files(h, buildersfiles, functionsfiles, toolsfiles,
                    variablesfiles, SCons_XML.write_mod)
 
@@ -362,7 +366,7 @@ else:
 # Step 3: Creating actual documentation snippets, using the
 #         fully resolved and updated entities from the *.mod files.
 print("Updating documentation for builders, tools and functions...")
-h = parse_docs(args, True)
+h = parse_docs(args, include_entities=True)
 write_output_files(h, buildersfiles, functionsfiles, toolsfiles,
                    variablesfiles, SCons_XML.write)
 print("Done")