First set of changes, started to rip out libxslt2.

1 files changed, 155 insertions, 416 deletions

diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py
index edf38a9..63aa610 100644
--- a/bin/SConsDoc.py
+++ b/bin/SConsDoc.py
@@ -24,6 +24,8 @@
 #
 # Module for handling SCons documentation processing.
 #
+# TODO DB Check file encoding for unicode/utf-8
+
 __doc__ = r"""
 This module parses home-brew XML files that document various things
 in SCons.  Right now, it handles Builders, functions, construction
@@ -116,34 +118,19 @@ import sys
 import copy
 import importlib
 
-# Do we have libxml2/libxslt/lxml?
-has_libxml2 = True
+# Do we have lxml?
 try:
-    import libxml2
-    import libxslt
+    import lxml
 except ImportError:
-    has_libxml2 = False
-    try:
-        import lxml
-    except ImportError:
-        raise ImportError("Failed to import either libxml2/libxslt or lxml")
+    raise ImportError("Failed to import lxml")
 
-has_etree = False
-if not has_libxml2:
-    try:
-        from lxml import etree
-        has_etree = True
-    except ImportError:
-        pass
-if not has_etree:
+try:
+    from lxml import etree
+except ImportError:
     try:
-        # TODO: this is for Python 2.7, cElementTee is deprecated since Py3.3
-        import xml.etree.cElementTree as etree
+        import xml.etree.ElementTree as etree
     except ImportError:
-        try:
-            import xml.etree.ElementTree as etree
-        except ImportError:
-            raise ImportError("Failed to import ElementTree from any known place")
+        raise ImportError("Failed to import ElementTree from any known place")
 
 # patterns to help trim XML passed in as strings
 re_entity = re.compile(r"&([^;]+);")
@@ -250,374 +237,148 @@ class DoctypeDeclaration:
 
         return content
 
-if not has_libxml2:
-    class TreeFactory:
-        def __init__(self):
-            pass
-
-        @staticmethod
-        def newNode(tag, **kwargs):
-            return etree.Element(tag, **kwargs)
-
-        @staticmethod
-        def newSubNode(parent, tag, **kwargs):
-            return etree.SubElement(parent, tag, **kwargs)
-
-        @staticmethod
-        def newEtreeNode(tag, init_ns=False, **kwargs):
-            if init_ns:
-                NSMAP = {None: dbxsd,
-                         'xsi' : xsi}
-                return etree.Element(tag, nsmap=NSMAP, **kwargs)
-
-            return etree.Element(tag, **kwargs)
-
-        @staticmethod
-        def copyNode(node):
-            return copy.deepcopy(node)
-
-        @staticmethod
-        def appendNode(parent, child):
-            parent.append(child)
-
-        @staticmethod
-        def hasAttribute(node, att):
-            return att in node.attrib
-
-        @staticmethod
-        def getAttribute(node, att):
-            return node.attrib[att]
-
-        @staticmethod
-        def setAttribute(node, att, value):
-            node.attrib[att] = value
-
-        @staticmethod
-        def getText(root):
-            return root.text
-
-        @staticmethod
-        def setText(root, txt):
-            root.text = txt
-
-        @staticmethod
-        def getTail(root):
-            return root.tail
-
-        @staticmethod
-        def setTail(root, txt):
-            root.tail = txt
-
-        @staticmethod
-        def writeGenTree(root, fp):
-            dt = DoctypeDeclaration()
-            encfun = str
-            fp.write(etree.tostring(root, encoding=encfun,
-                                    pretty_print=True,
-                                    doctype=dt.createDoctype()))
-
-        @staticmethod
-        def writeTree(root, fpath):
-            encfun = "utf-8"
-            with open(fpath, 'wb') as fp:
-                fp.write(etree.tostring(root, encoding=encfun,
-                                        pretty_print=True))
-
-        @staticmethod
-        def prettyPrintFile(fpath):
-            with open(fpath,'rb') as fin:
-                tree = etree.parse(fin)
-                pretty_content = etree.tostring(tree, pretty_print=True)
-
-            with open(fpath,'wb') as fout:
-                fout.write(pretty_content)
-
-        @staticmethod
-        def decorateWithHeader(root):
-            root.attrib["{"+xsi+"}schemaLocation"] = "%s %s/scons.xsd" % (dbxsd, dbxsd)
-            return root
-
-        def newXmlTree(self, root):
-            """ Return a XML file tree with the correct namespaces set,
-                the element root as top entry and the given header comment.
-            """
-            NSMAP = {None: dbxsd, 'xsi' : xsi}
-            t = etree.Element(root, nsmap=NSMAP)
-            return self.decorateWithHeader(t)
-
-        # singleton to cache parsed xmlschema..
-        xmlschema = None
-
-        @staticmethod
-        def validateXml(fpath, xmlschema_context):
-
-            if TreeFactory.xmlschema is None:
-                TreeFactory.xmlschema = etree.XMLSchema(xmlschema_context)
-            try:
-                doc = etree.parse(fpath)
-            except Exception as e:
-                print("ERROR: %s fails to parse:"%fpath)
-                print(e)
-                return False
-            doc.xinclude()
-            try:
-                TreeFactory.xmlschema.assertValid(doc)
-            except Exception as e:
-                print("ERROR: %s fails to validate:" % fpath)
-                print(e)
-                return False
-            return True
+class TreeFactory:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def newNode(tag, **kwargs):
+        return etree.Element(tag, **kwargs)
+
+    @staticmethod
+    def newSubNode(parent, tag, **kwargs):
+        return etree.SubElement(parent, tag, **kwargs)
+
+    @staticmethod
+    def newEtreeNode(tag, init_ns=False, **kwargs):
+        if init_ns:
+            NSMAP = {None: dbxsd,
+                     'xsi' : xsi}
+            return etree.Element(tag, nsmap=NSMAP, **kwargs)
+
+        return etree.Element(tag, **kwargs)
+
+    @staticmethod
+    def copyNode(node):
+        return copy.deepcopy(node)
+
+    @staticmethod
+    def appendNode(parent, child):
+        parent.append(child)
+
+    @staticmethod
+    def hasAttribute(node, att):
+        return att in node.attrib
+
+    @staticmethod
+    def getAttribute(node, att):
+        return node.attrib[att]
+
+    @staticmethod
+    def setAttribute(node, att, value):
+        node.attrib[att] = value
+
+    @staticmethod
+    def getText(root):
+        return root.text
+
+    @staticmethod
+    def setText(root, txt):
+        root.text = txt
+
+    @staticmethod
+    def getTail(root):
+        return root.tail
+
+    @staticmethod
+    def setTail(root, txt):
+        root.tail = txt
+
+    @staticmethod
+    def writeGenTree(root, fp):
+        dt = DoctypeDeclaration()
+# TODO DB Check file encoding for unicode/utf-8
+        fp.write(etree.tostring(root, encoding="utf-8",
+                                pretty_print=True,
+                                doctype=dt.createDoctype()))
+
+    @staticmethod
+    def writeTree(root, fpath):
+# TODO DB Check file encoding for unicode/utf-8
+        with open(fpath, 'wb') as fp:
+# TODO DB Check file encoding for unicode/utf-8
+            fp.write(etree.tostring(root, encoding="utf-8",
+                                    pretty_print=True))
+
+    @staticmethod
+    def prettyPrintFile(fpath):
+# TODO DB Check file encoding for unicode/utf-8
+        with open(fpath,'rb') as fin:
+            tree = etree.parse(fin)
+# TODO DB Check file encoding for unicode/utf-8
+            pretty_content = etree.tostring(tree, encoding="utf-8", 
+                                            pretty_print=True)
+    
+        with open(fpath,'wb') as fout:
+            fout.write(pretty_content)
+
+    @staticmethod
+    def decorateWithHeader(root):
+        root.attrib["{"+xsi+"}schemaLocation"] = "%s %s/scons.xsd" % (dbxsd, dbxsd)
+        return root
+
+    def newXmlTree(self, root):
+        """ Return a XML file tree with the correct namespaces set,
+            the element root as top entry and the given header comment.
+        """
+        NSMAP = {None: dbxsd, 'xsi' : xsi}
+        t = etree.Element(root, nsmap=NSMAP)
+        return self.decorateWithHeader(t)
 
-        @staticmethod
-        def findAll(root, tag, ns=None, xp_ctxt=None, nsmap=None):
-            expression = ".//{%s}%s" % (nsmap[ns], tag)
-            if not ns or not nsmap:
-                expression = ".//%s" % tag
-            return root.findall(expression)
-
-        @staticmethod
-        def findAllChildrenOf(root, tag, ns=None, xp_ctxt=None, nsmap=None):
-            expression = "./{%s}%s/*" % (nsmap[ns], tag)
-            if not ns or not nsmap:
-                expression = "./%s/*" % tag
-            return root.findall(expression)
-
-        @staticmethod
-        def convertElementTree(root):
-            """ Convert the given tree of etree.Element
-                entries to a list of tree nodes for the
-                current XML toolkit.
-            """
-            return [root]
-
-else:
-    class TreeFactory:
-        def __init__(self):
-            pass
-
-        @staticmethod
-        def newNode(tag, **kwargs):
-            return etree.Element(tag, **kwargs)
-
-        @staticmethod
-        def newSubNode(parent, tag, **kwargs):
-            return etree.SubElement(parent, tag, **kwargs)
-
-        @staticmethod
-        def newEtreeNode(tag, init_ns=False, **kwargs):
-            return etree.Element(tag, **kwargs)
-
-        @staticmethod
-        def copyNode(node):
-            return node.copyNode(1)
-
-        @staticmethod
-        def appendNode(parent, child):
-            if hasattr(parent, 'addChild'):
-                parent.addChild(child)
-            else:
-                parent.append(child)
-
-        @staticmethod
-        def hasAttribute(node, att):
-            if hasattr(node, 'hasProp'):
-                return node.hasProp(att)
-            return att in node.attrib
-
-        @staticmethod
-        def getAttribute(node, att):
-            if hasattr(node, 'prop'):
-                return node.prop(att)
-            return node.attrib[att]
-
-        @staticmethod
-        def setAttribute(node, att, value):
-            if hasattr(node, 'setProp'):
-                node.setProp(att, value)
-            else:
-                node.attrib[att] = value
-
-        @staticmethod
-        def getText(root):
-            if hasattr(root, 'getContent'):
-                return root.getContent()
-            return root.text
-
-        @staticmethod
-        def setText(root, txt):
-            if hasattr(root, 'setContent'):
-                root.setContent(txt)
-            else:
-                root.text = txt
-
-        @staticmethod
-        def getTail(root):
-            return root.tail
-
-        @staticmethod
-        def setTail(root, txt):
-            root.tail = txt
-
-        @staticmethod
-        def writeGenTree(root, fp):
-            doc = libxml2.newDoc('1.0')
-            dtd = doc.newDtd("sconsdoc", None, None)
-            doc.addChild(dtd)
-            doc.setRootElement(root)
-            content = doc.serialize("UTF-8", 1)
-            dt = DoctypeDeclaration()
-            # This is clearly a hack, but unfortunately libxml2
-            # doesn't support writing PERs (Parsed Entity References).
-            # So, we simply replace the empty doctype with the
-            # text we need...
-            content = content.replace("<!DOCTYPE sconsdoc>", dt.createDoctype())
-            fp.write(content)
-            doc.freeDoc()
-
-        @staticmethod
-        def writeTree(root, fpath):
-            with open(fpath, 'wb') as fp:
-                doc = libxml2.newDoc('1.0')
-                doc.setRootElement(root)
-                fp.write(doc.serialize("UTF-8", 1))
-                doc.freeDoc()
-
-        @staticmethod
-        def prettyPrintFile(fpath):
-            # Read file and resolve entities
-            doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT)
-            with open(fpath, 'wb') as fp:
-                # Prettyprint
-                fp.write(doc.serialize("UTF-8", 1))
-            # Cleanup
-            doc.freeDoc()
-
-        @staticmethod
-        def decorateWithHeader(root):
-            # Register the namespaces
-            ns = root.newNs(dbxsd, None)
-            xi = root.newNs(xsi, 'xsi')
-            root.setNs(ns)  #put this node in the target namespace
-
-            root.setNsProp(xi, 'schemaLocation', "%s %s/scons.xsd" % (dbxsd, dbxsd))
-
-            return root
-
-        def newXmlTree(self, root):
-            """ Return a XML file tree with the correct namespaces set,
-                the element root as top entry and the given header comment.
-            """
-            t = libxml2.newNode(root)
-            return self.decorateWithHeader(t)
-
-        @staticmethod
-        def validateXml(fpath, xmlschema_context):
-            retval = True
-
-            # Create validation context
-            validation_context = xmlschema_context.schemaNewValidCtxt()
-            # Set error/warning handlers
-            eh = Libxml2ValidityHandler()
-            validation_context.setValidityErrorHandler(eh.error, eh.warning, ARG)
-            # Read file and resolve entities
-            doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT)
-            doc.xincludeProcessFlags(libxml2.XML_PARSE_NOENT)
-            err = validation_context.schemaValidateDoc(doc)
-
-            if err or eh.errors:
-                for e in eh.errors:
-                    print(e.rstrip("\n"))
-                # import pdb; pdb.set_trace()
-                print("%s fails to validate" % fpath)
-                retval = False
-
-            # Cleanup
-            doc.freeDoc()
-            del validation_context
-
-            return retval
-
-        @staticmethod
-        def findAll(root, tag, ns=None, xpath_context=None, nsmap=None):
-            if hasattr(root, 'xpathEval') and xpath_context:
-                # Use the xpath context
-                xpath_context.setContextNode(root)
-                expression = ".//%s" % tag
-                if ns:
-                    expression = ".//%s:%s" % (ns, tag)
-                return xpath_context.xpathEval(expression)
-            else:
-                expression = ".//{%s}%s" % (nsmap[ns], tag)
-                if not ns or not nsmap:
-                    expression = ".//%s" % tag
-                return root.findall(expression)
-
-        @staticmethod
-        def findAllChildrenOf(root, tag, ns=None, xpath_context=None, nsmap=None):
-            if hasattr(root, 'xpathEval') and xpath_context:
-                # Use the xpath context
-                xpath_context.setContextNode(root)
-                expression = "./%s/node()" % tag
-                if ns:
-                    expression = "./%s:%s/node()" % (ns, tag)
-
-                return xpath_context.xpathEval(expression)
-            else:
-                expression = "./{%s}%s/node()" % (nsmap[ns], tag)
-                if not ns or not nsmap:
-                    expression = "./%s/node()" % tag
-                return root.findall(expression)
-
-        def expandChildElements(self, child):
-            """ Helper function for convertElementTree,
-                converts a single child recursively.
-            """
-            nchild = self.newNode(child.tag)
-            # Copy attributes
-            for key, val in child.attrib:
-                self.setAttribute(nchild, key, val)
-            elements = []
-            # Add text
-            if child.text:
-                t = libxml2.newText(child.text)
-                self.appendNode(nchild, t)
-            # Add children
-            for c in child:
-                for n in self.expandChildElements(c):
-                    self.appendNode(nchild, n)
-            elements.append(nchild)
-            # Add tail
-            if child.tail:
-                tail = libxml2.newText(child.tail)
-                elements.append(tail)
-
-            return elements
-
-        def convertElementTree(self, root):
-            """ Convert the given tree of etree.Element
-                entries to a list of tree nodes for the
-                current XML toolkit.
-            """
-            nroot = self.newNode(root.tag)
-            # Copy attributes
-            for key, val in root.attrib:
-                self.setAttribute(nroot, key, val)
-            elements = []
-            # Add text
-            if root.text:
-                t = libxml2.newText(root.text)
-                self.appendNode(nroot, t)
-            # Add children
-            for c in root:
-                for n in self.expandChildElements(c):
-                    self.appendNode(nroot, n)
-            elements.append(nroot)
-            # Add tail
-            if root.tail:
-                tail = libxml2.newText(root.tail)
-                elements.append(tail)
-
-            return elements
+    # singleton to cache parsed xmlschema..
+    xmlschema = None
+
+    @staticmethod
+    def validateXml(fpath, xmlschema_context):
+
+        if TreeFactory.xmlschema is None:
+            TreeFactory.xmlschema = etree.XMLSchema(xmlschema_context)
+        try:
+            doc = etree.parse(fpath)
+        except Exception as e:
+            print("ERROR: %s fails to parse:"%fpath)
+            print(e)
+            return False
+        doc.xinclude()
+        try:
+            TreeFactory.xmlschema.assertValid(doc)
+        except Exception as e:
+            print("ERROR: %s fails to validate:" % fpath)
+            print(e)
+            return False
+        return True
+
+    @staticmethod
+    def findAll(root, tag, ns=None, xp_ctxt=None, nsmap=None):
+        expression = ".//{%s}%s" % (nsmap[ns], tag)
+        if not ns or not nsmap:
+            expression = ".//%s" % tag
+        return root.findall(expression)
+
+    @staticmethod
+    def findAllChildrenOf(root, tag, ns=None, xp_ctxt=None, nsmap=None):
+        expression = "./{%s}%s/*" % (nsmap[ns], tag)
+        if not ns or not nsmap:
+            expression = "./%s/*" % tag
+        return root.findall(expression)
+
+    @staticmethod
+    def convertElementTree(root):
+        """ Convert the given tree of etree.Element
+            entries to a list of tree nodes for the
+            current XML toolkit.
+        """
+        return [root]
 
 tf = TreeFactory()
 
@@ -641,19 +402,9 @@ class SConsDocTree:
         self.root = etree.fromstring(content)
 
     def parseXmlFile(self, fpath):
-        if not has_libxml2:
-            # Create domtree from file
-            domtree = etree.parse(fpath)
-            self.root = domtree.getroot()
-        else:
-            # Read file and resolve entities
-            self.doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT)
-            self.root = self.doc.getRootElement()
-            # Create xpath context
-            self.xpath_context = self.doc.xpathNewContext()
-            # Register namespaces
-            for key, val in self.nsmap.items():
-                self.xpath_context.xpathRegisterNs(key, val)
+        # Create domtree from file
+        domtree = etree.parse(fpath)
+        self.root = domtree.getroot()
 
     def __del__(self):
         if self.doc is not None:
@@ -664,15 +415,7 @@ class SConsDocTree:
 perc = "%"
 
 def validate_all_xml(dpaths, xsdfile=default_xsd):
-    xmlschema_context = None
-    if not has_libxml2:
-        # Use lxml
-        xmlschema_context = etree.parse(xsdfile)
-    else:
-        # Use libxml2 and prepare the schema validation context
-        ctxt = libxml2.schemaNewParserCtxt(xsdfile)
-        xmlschema_context = ctxt.schemaParse()
-        del ctxt
+    xmlschema_context = etree.parse(xsdfile)
 
     fpaths = []
     for dp in dpaths:
@@ -698,10 +441,6 @@ def validate_all_xml(dpaths, xsdfile=default_xsd):
             fails.append(fp)
             continue
 
-    if has_libxml2:
-        # Cleanup
-        del xmlschema_context
-
     if fails:
         return False