- corrected xincludes for MAN pages

- started to work on support for UserGuide examples
- added xslt for transforming SCons XSD to Docbook documents
- added SConstructs for all documents and titlepage for the UserGuide

5 files changed, 746 insertions, 417 deletions

diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py
index 7cd06f8..e3eaf0f 100644
--- a/bin/SConsDoc.py
+++ b/bin/SConsDoc.py
@@ -112,6 +112,7 @@ import imp
 import os.path
 import re
 import sys
+import copy
 
 # Do we have libxml2/libxslt/lxml?
 has_libxml2 = True
@@ -158,6 +159,8 @@ re_entity_header = re.compile("<!DOCTYPE\s+sconsdoc\s+[^\]]+\]>")
 
 # Namespace for the SCons Docbook XSD
 dbxsd="http://www.scons.org/dbxsd/v1.0"
+# Namespace map identifier for the SCons Docbook XSD
+dbxid="dbx"
 # Namespace for schema instances
 xsi = "http://www.w3.org/2001/XMLSchema-instance"
 
@@ -188,22 +191,6 @@ def isSConsXml(fpath):
     
     return False 
 
-def xml_tree(root, comment=generated_comment):
-    """ Return a XML file tree with the correct namespaces set,
-        the element root as top entry and the given header comment.
-    """
-    NSMAP = {None: dbxsd,
-             'xsi' : xsi}
-
-    t = etree.Element(root,
-                      nsmap = NSMAP,
-                      attrib = {"{"+xsi+"}schemaLocation" : "%s scons.xsd" % dbxsd})
-
-    c = etree.Comment(comment)
-    t.append(c)
-
-    return t
-
 def remove_entities(content):
     # Cut out entity inclusions
     content = re_entity_header.sub("", content, re.M)
@@ -232,59 +219,311 @@ class Libxml2ValidityHandler:
             raise Exception, "Warning handler did not receive correct argument"
         self.warnings.append(msg)
 
-def validate_xml(fpath, xmlschema_context):
-    if not has_libxml2:
-        # Use lxml
-        xmlschema = etree.XMLSchema(xmlschema_context)
-        doc = etree.parse(fpath)
-        doc.xinclude()
-        try:
-            xmlschema.assertValid(doc)
-        except Exception, e:
-            print e
-            print "%s fails to validate" % fpath
-            return False
-        return True
-
-    # Create validation context
-    validation_context = xmlschema_context.schemaNewValidCtxt()
-    # Set error/warning handlers
-    eh = Libxml2ValidityHandler()
-    validation_context.setValidityErrorHandler(eh.error, eh.warning, ARG)
-    # Read file and resolve entities
-    doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT)
-    doc.xincludeProcessFlags(libxml2.XML_PARSE_NOENT)
-    err = validation_context.schemaValidateDoc(doc)
-    # Cleanup
-    doc.freeDoc()
-    del validation_context
-
-    if err or eh.errors:
-        for e in eh.errors:
-            print e.rstrip("\n")
-        print "%s fails to validate" % fpath
-        return False
+
+class DoctypeEntity:
+    def __init__(self, name_, uri_):
+        self.name = name_
+        self.uri = uri_
         
-    return True
+    def getEntityString(self):
+        txt = """    <!ENTITY %(perc)s %(name)s SYSTEM "%(uri)s">
+    %(perc)s%(name)s;
+""" % {'perc' : perc, 'name' : self.name, 'uri' : self.uri}
 
-def prettyprint_xml(fpath):
-    if not has_libxml2:
-        # Use lxml
-        fin = open(fpath,'r')
-        tree = etree.parse(fin)
-        pretty_content = etree.tostring(tree, pretty_print=True)
-        fin.close()
-
-        fout = open(fpath,'w')
-        fout.write(pretty_content)
-        fout.close()
-
-    # Read file and resolve entities
-    doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT)
-    err = xmlschema_context.schemaValidateDoc(doc)
-    # Cleanup
-    doc.freeDoc()
+        return txt
+        
+class DoctypeDeclaration:
+    def __init__(self, name_=None):
+        self.name = name_
+        self.entries = []
+        if self.name is None:
+            # Add default entries
+            self.name = "sconsdoc"
+            self.addEntity("scons", "../scons.mod")
+            self.addEntity("builders-mod", "builders.mod")
+            self.addEntity("functions-mod", "functions.mod")
+            self.addEntity("tools-mod", "tools.mod")
+            self.addEntity("variables-mod", "variables.mod")
+        
+    def addEntity(self, name, uri):
+        self.entries.append(DoctypeEntity(name, uri))
+        
+    def createDoctype(self):
+        content = '<!DOCTYPE %s [\n' % self.name
+        for e in self.entries:
+            content += e.getEntityString()
+        content += ']>\n'
+        
+        return content
+
+if not has_libxml2:
+    class TreeFactory:
+        def __init__(self):
+            pass
+        
+        def newNode(self, tag):
+            return etree.Element(tag)
+        
+        def copyNode(self, node):
+            return copy.deepcopy(node)
+        
+        def appendNode(self, parent, child):
+            parent.append(child)
+
+        def hasAttribute(self, node, att):
+            return att in node.attrib
+        
+        def getAttribute(self, node, att):
+            return node.attrib[att]
+        
+        def setAttribute(self, node, att, value):
+            node.attrib[att] = value
+            
+        def getText(self, root):
+            return root.text
+
+        def setText(self, root, txt):
+            root.text = txt
+
+        def writeGenTree(self, root, fp):
+            dt = DoctypeDeclaration()
+            fp.write(etree.tostring(root, xml_declaration=True, 
+                                    encoding="UTF-8", pretty_print=True, 
+                                    doctype=dt.createDoctype()))
+
+        def prettyPrintFile(self, fpath):
+            fin = open(fpath,'r')
+            tree = etree.parse(fin)
+            pretty_content = etree.tostring(tree, pretty_print=True)
+            fin.close()
     
+            fout = open(fpath,'w')
+            fout.write(pretty_content)
+            fout.close()
+            
+        def newXmlTree(self, root, comment=generated_comment):
+            """ Return a XML file tree with the correct namespaces set,
+                the element root as top entry and the given header comment.
+            """
+            NSMAP = {None: dbxsd,
+                     'xsi' : xsi}
+        
+            t = etree.Element(root,
+                              nsmap = NSMAP,
+                              attrib = {"{"+xsi+"}schemaLocation" : "%s scons.xsd" % dbxsd})
+        
+            c = etree.Comment(comment)
+            t.append(c)
+        
+            return t
+        
+        def validateXml(self, fpath, xmlschema_context):
+            # Use lxml
+            xmlschema = etree.XMLSchema(xmlschema_context)
+            doc = etree.parse(fpath)
+            doc.xinclude()
+            try:
+                xmlschema.assertValid(doc)
+            except Exception, e:
+                print e
+                print "%s fails to validate" % fpath
+                return False
+            return True
+
+        def findAll(self, root, tag, ns=None, xp_ctxt=None, nsmap=None):
+            expression = ".//{%s}%s" % (nsmap[ns], tag)
+            if not ns or not nsmap:
+                expression = ".//%s" % tag
+            return root.findall(expression)
+
+        def findAllChildrenOf(self, root, tag, ns=None, xp_ctxt=None, nsmap=None):
+            expression = "./{%s}%s/*" % (nsmap[ns], tag)
+            if not ns or not nsmap:
+                expression = "./%s/*" % tag
+            return root.findall(expression)
+
+else:        
+    class TreeFactory:
+        def __init__(self):
+            pass
+        
+        def newNode(self, tag):
+            return libxml2.newNode(tag)
+        
+        def copyNode(self, node):
+            return node.copyNode(1)
+        
+        def appendNode(self, parent, child):
+            if hasattr(parent, 'addChild'):
+                parent.addChild(child)
+            else:
+                parent.append(child)
+
+        def hasAttribute(self, node, att):
+            if hasattr(node, 'hasProp'):
+                return node.hasProp(att)
+            return att in node.attrib
+        
+        def getAttribute(self, node, att):
+            if hasattr(node, 'prop'):
+                return node.prop(att)
+            return node.attrib[att]
+
+        def setAttribute(self, node, att, value):
+            if hasattr(node, 'setProp'):
+                node.setProp(att, value)
+            else:
+                node.attrib[att] = value
+                
+        def getText(self, root):
+            if hasattr(root, 'getContent'):
+                return root.getContent()
+            return root.text
+
+        def setText(self, root, txt):
+            if hasattr(root, 'setContent'):
+                root.setContent(txt)
+            else:
+                root.text = txt
+
+        def writeGenTree(self, root, fp):
+            doc = libxml2.newDoc('1.0')
+            dtd = doc.newDtd("sconsdoc", None, None)
+            doc.addChild(dtd)
+            doc.setRootElement(root)
+            content = doc.serialize("UTF-8", 1)
+            dt = DoctypeDeclaration()
+            # This is clearly a hack, but unfortunately libxml2
+            # doesn't support writing PERs (Parsed Entity References).
+            # So, we simply replace the empty doctype with the
+            # text we need...
+            content = content.replace("<!DOCTYPE sconsdoc>", dt.createDoctype())
+            fp.write(content)
+            doc.freeDoc()
+
+        def prettyPrintFile(self, fpath):
+            # Read file and resolve entities
+            doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT)
+            fp = open(fpath, 'w')
+            # Prettyprint
+            fp.write(doc.serialize("UTF-8", 1))
+            fp.close()
+            # Cleanup
+            doc.freeDoc()
+
+        def newXmlTree(self, root, comment=generated_comment):
+            """ Return a XML file tree with the correct namespaces set,
+                the element root as top entry and the given header comment.
+            """
+            t = libxml2.newNode(root)
+            # Register the namespaces
+            ns = t.newNs(dbxsd, None)
+            xi = t.newNs(xsi, 'xsi')
+            t.setNs(ns)  #put this node in the target namespace
+    
+            t.setNsProp(xi, 'schemaLocation', "%s scons.xsd" % dbxsd)
+        
+            c = libxml2.newComment(comment)
+            t.addChild(c)
+        
+            return t
+
+        def validateXml(self, fpath, xmlschema_context):
+            # Create validation context
+            validation_context = xmlschema_context.schemaNewValidCtxt()
+            # Set error/warning handlers
+            eh = Libxml2ValidityHandler()
+            validation_context.setValidityErrorHandler(eh.error, eh.warning, ARG)
+            # Read file and resolve entities
+            doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT)
+            doc.xincludeProcessFlags(libxml2.XML_PARSE_NOENT)
+            err = validation_context.schemaValidateDoc(doc)
+            # Cleanup
+            doc.freeDoc()
+            del validation_context
+        
+            if err or eh.errors:
+                for e in eh.errors:
+                    print e.rstrip("\n")
+                print "%s fails to validate" % fpath
+                return False
+                
+            return True
+
+        def findAll(self, root, tag, ns=None, xpath_context=None, nsmap=None):
+            if hasattr(root, 'xpathEval') and xpath_context:
+                # Use the xpath context
+                xpath_context.setContextNode(root)
+                expression = ".//%s" % tag
+                if ns:
+                    expression = ".//%s:%s" % (ns, tag)
+                return xpath_context.xpathEval(expression)
+            else:
+                expression = ".//{%s}%s" % (nsmap[ns], tag)
+                if not ns or not nsmap:
+                    expression = ".//%s" % tag
+                return root.findall(expression)
+
+        def findAllChildrenOf(self, root, tag, ns=None, xpath_context=None, nsmap=None):
+            if hasattr(root, 'xpathEval') and xpath_context:
+                # Use the xpath context
+                xpath_context.setContextNode(root)
+                expression = "./%s/node()" % tag
+                if ns:
+                    expression = "./%s:%s/node()" % (ns, tag)
+                
+                return xpath_context.xpathEval(expression)
+            else:
+                expression = "./{%s}%s/node()" % (nsmap[ns], tag)
+                if not ns or not nsmap:
+                    expression = "./%s/node()" % tag
+                return root.findall(expression)
+
+
+tf = TreeFactory()
+
+
+class SConsDocTree:
+    def __init__(self):
+        self.nsmap = {'dbx' : dbxsd}
+        self.doc = None
+        self.root = None
+        self.xpath_context = None
+
+    def parseContent(self, content, include_entities=True):
+        """ Parses the given content as XML file. This method
+            is used when we generate the basic lists of entities
+            for the builders, tools and functions.
+            So we usually don't bother about namespaces and resolving
+            entities here...this is handled in parseXmlFile below
+            (step 2 of the overall process).
+        """
+        if not include_entities:
+            content = remove_entities(content)
+        # Create domtree from given content string
+        self.root = etree.fromstring(content)
+
+    def parseXmlFile(self, fpath):
+        nsmap = {'dbx' : dbxsd}
+        if not has_libxml2:
+            # Create domtree from file
+            domtree = etree.parse(fpath)
+            self.root = domtree.getroot()
+        else:
+            # Read file and resolve entities
+            self.doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT)
+            self.root = self.doc.getRootElement()
+            # Create xpath context
+            self.xpath_context = self.doc.xpathNewContext()
+            # Register namespaces
+            for key, val in self.nsmap.iteritems():
+                self.xpath_context.xpathRegisterNs(key, val)
+            
+    def __del__(self):
+        if self.doc is not None:
+            self.doc.freeDoc()
+        if self.xpath_context is not None:
+            self.xpath_context.xpathFreeContext()
 
 perc="%"
 
@@ -314,7 +553,7 @@ def validate_all_xml(dpaths, xsdfile=default_xsd):
         print "%.2f%s (%d/%d) %s" % (float(idx+1)*100.0/float(len(fpaths)),
                                      perc, idx+1, len(fpaths),fp)
                                               
-        if not validate_xml(fp, xmlschema_context):
+        if not tf.validateXml(fp, xmlschema_context):
             fails.append(fp)
             continue
 
@@ -333,9 +572,10 @@ class Item(object):
         self.sort_name = name.lower()
         if self.sort_name[0] == '_':
             self.sort_name = self.sort_name[1:]
-        self.summary = []
         self.sets = []
         self.uses = []
+        self.summary = None
+        self.arguments = None
     def cmp_name(self, name):
         if name[0] == '_':
             name = name[1:]
@@ -358,18 +598,6 @@ class Tool(Item):
 class ConstructionVariable(Item):
     pass
 
-class Chunk(object):
-    def __init__(self, tag, body=None):
-        self.tag = tag
-        if not body:
-            body = []
-        self.body = body
-    def __str__(self):
-        body = ''.join(self.body)
-        return "<%s>%s</%s>\n" % (self.tag, body, self.tag)
-    def append(self, data):
-        self.body.append(data)
-
 class Arguments(object):
     def __init__(self, signature, body=None):
         if not body:
@@ -394,85 +622,92 @@ class SConsDocHandler(object):
         self.tools = {}
         self.cvars = {}
 
-    def parseText(self, root):
-        txt = ""
-        for e in root.childNodes:
-            if (e.nodeType == e.TEXT_NODE):
-                txt += e.data
-        return txt
-
-    def parseItems(self, domelem):
+    def parseItems(self, domelem, xpath_context, nsmap):
         items = []
 
-        for i in domelem.iterchildren(tag="item"):
-            items.append(self.parseText(i))
+        for i in tf.findAll(domelem, "item", dbxid, xpath_context, nsmap):
+            txt = tf.getText(i)
+            if txt is not None:
+                txt = txt.strip()
+                if len(txt):
+                    items.append(txt.strip())
 
         return items
 
-    def parseUsesSets(self, domelem):
+    def parseUsesSets(self, domelem, xpath_context, nsmap):
         uses = []
         sets = []
 
-        for u in domelem.iterchildren(tag="uses"):
-            uses.extend(self.parseItems(u))
-        for s in domelem.iterchildren(tag="sets"):
-            sets.extend(self.parseItems(s))
+        for u in tf.findAll(domelem, "uses", dbxid, xpath_context, nsmap):
+            uses.extend(self.parseItems(u, xpath_context, nsmap))
+        for s in tf.findAll(domelem, "sets", dbxid, xpath_context, nsmap):
+            sets.extend(self.parseItems(s, xpath_context, nsmap))
         
         return sorted(uses), sorted(sets)
 
-    def parseInstance(self, domelem, map, Class):
-        name = domelem.attrib.get('name','unknown')
+    def parseInstance(self, domelem, map, Class, 
+                        xpath_context, nsmap, include_entities=True):
+        name = 'unknown'
+        if tf.hasAttribute(domelem, 'name'):
+            name = tf.getAttribute(domelem, 'name')
         try:
             instance = map[name]
         except KeyError:
             instance = Class(name)
             map[name] = instance
-        uses, sets = self.parseUsesSets(domelem)
+        uses, sets = self.parseUsesSets(domelem, xpath_context, nsmap)
         instance.uses.extend(uses)
         instance.sets.extend(sets)
-        # Parse summary and function arguments
-        for s in domelem.iterchildren(tag="{%s}summary" % dbxsd):
-            if not hasattr(instance, 'summary'):
-                instance.summary = []
-            for c in s:
-                instance.summary.append(c)
-        for a in domelem.iterchildren(tag="{%s}arguments" % dbxsd):
-            if not hasattr(instance, 'arguments'):
-                instance.arguments = []
-            instance.arguments.append(a)
-
-    def parseDomtree(self, root):    
+        if include_entities:
+            # Parse summary and function arguments
+            for s in tf.findAllChildrenOf(domelem, "summary", dbxid, xpath_context, nsmap):
+                if instance.summary is None:
+                    instance.summary = []
+                instance.summary.append(tf.copyNode(s))
+            for a in tf.findAll(domelem, "arguments", dbxid, xpath_context, nsmap):
+                if instance.arguments is None:
+                    instance.arguments = []
+                instance.arguments.append(tf.copyNode(a))
+
+    def parseDomtree(self, root, xpath_context=None, nsmap=None, include_entities=True):    
         # Process Builders
-        for b in root.iterchildren(tag="{%s}builder" % dbxsd):
-            self.parseInstance(b, self.builders, Builder)
+        for b in tf.findAll(root, "builder", dbxid, xpath_context, nsmap):
+            self.parseInstance(b, self.builders, Builder, 
+                               xpath_context, nsmap, include_entities)
         # Process Functions
-        for f in root.iterchildren(tag="{%s}scons_function" % dbxsd):
-            self.parseInstance(f, self.functions, Function)
+        for f in tf.findAll(root, "scons_function", dbxid, xpath_context, nsmap):
+            self.parseInstance(f, self.functions, Function, 
+                               xpath_context, nsmap, include_entities)
         # Process Tools
-        for t in root.iterchildren(tag="{%s}tool" % dbxsd):
-            self.parseInstance(t, self.tools, Tool)
+        for t in tf.findAll(root, "tool", dbxid, xpath_context, nsmap):
+            self.parseInstance(t, self.tools, Tool, 
+                               xpath_context, nsmap, include_entities)
         # Process CVars
-        for c in root.iterchildren(tag="{%s}cvar" % dbxsd):
-            self.parseInstance(c, self.cvars, ConstructionVariable)
+        for c in tf.findAll(root, "cvar", dbxid, xpath_context, nsmap):
+            self.parseInstance(c, self.cvars, ConstructionVariable, 
+                               xpath_context, nsmap, include_entities)
         
     def parseContent(self, content, include_entities=True):
-        if not include_entities:
-            content = remove_entities(content)
-        # Create domtree from given content string
-        root = etree.fromstring(content)
+        """ Parses the given content as XML file. This method
+            is used when we generate the basic lists of entities
+            for the builders, tools and functions.
+            So we usually don't bother about namespaces and resolving
+            entities here...this is handled in parseXmlFile below
+            (step 2 of the overall process).
+        """
+        # Create doctree
+        t = SConsDocTree()
+        t.parseContent(content, include_entities)
         # Parse it
-        self.parseDomtree(root)
+        self.parseDomtree(t.root, t.xpath_context, t.nsmap, include_entities)
 
     def parseXmlFile(self, fpath):
-        # Create domtree from file
-        domtree = etree.parse(fpath)
+        # Create doctree
+        t = SConsDocTree()
+        t.parseXmlFile(fpath)
         # Parse it
-        self.parseDomtree(domtree.getroot())
-
-    def set_file_info(self, filename, preamble_lines):
-        self.filename = filename
-        self.preamble_lines = preamble_lines
-
+        self.parseDomtree(t.root, t.xpath_context, t.nsmap)
+        
 # lifted from Ka-Ping Yee's way cool pydoc module.
 def importfile(path):
     """Import a Python source file or compiled file given its path."""
diff --git a/bin/SConsExamples.py b/bin/SConsExamples.py
new file mode 100644
index 0000000..77f4041
--- /dev/null
+++ b/bin/SConsExamples.py
@@ -0,0 +1,295 @@
+#!/usr/bin/env python
+#
+# Module for handling SCons examples processing.
+#
+
+__doc__ = """
+"""
+
+import os
+import re
+import SConsDoc
+from SConsDoc import tf as stf
+
+#
+# The available types for ExampleFile entries
+#
+FT_FILE = 0     # a physical file (=<file>)
+FT_FILEREF = 1  # a reference     (=<scons_example_file>)
+
+class ExampleFile:
+    def __init__(self, type_=FT_FILE):
+        self.type = type_
+        self.name = ''
+        self.content = ''
+        self.chmod = ''
+        
+    def isFileRef(self):
+        return self.type == FT_FILEREF
+
+class ExampleFolder:
+    def __init__(self):
+        self.name = ''
+        self.chmod = ''
+
+class ExampleCommand:
+    def __init__(self):
+        self.edit = ''
+        self.environment = ''
+        self.output = ''
+        self.cmd = ''
+        self.suffix = ''
+
+class ExampleOutput:
+    def __init__(self):
+        self.name = ''
+        self.tools = ''
+        self.os = ''
+        self.commands = []
+        
+class ExampleInfo:
+    def __init__(self):
+        self.name = ''
+        self.files = []
+        self.folders = []
+        self.outputs = []
+        
+    def getFileContents(self, fname):
+        for f in self.files:
+            if fname == f.name and not f.isFileRef():
+                return f.content
+            
+        return ''
+
+def readExampleInfos(fpath, examples):
+    """ Add the example infos for the file fpath to the
+        global dictionary examples.
+    """
+
+    # Create doctree    
+    t = SConsDoc.SConsDocTree()
+    t.parseXmlFile(fpath)
+    
+    # Parse scons_examples
+    for e in stf.findAll(t.root, "scons_example", SConsDoc.dbxid, 
+                         t.xpath_context, t.nsmap):
+        n = ''
+        if stf.hasAttribute(e, 'name'):
+            n = stf.getAttribute(e, 'name')
+        if n and n not in examples:
+            i = ExampleInfo()
+            i.name = n
+            examples[n] = i
+            
+        # Parse file and directory entries
+        for f in stf.findAll(e, "file", SConsDoc.dbxid, 
+                             t.xpath_context, t.nsmap):
+            fi = ExampleFile()
+            if stf.hasAttribute(f, 'name'):
+                fi.name = stf.getAttribute(f, 'name')
+            if stf.hasAttribute(f, 'chmod'):
+                fi.chmod = stf.getAttribute(f, 'chmod')
+            fi.content = stf.getText(f)
+            examples[n].files.append(fi)
+        for d in stf.findAll(e, "directory", SConsDoc.dbxid, 
+                             t.xpath_context, t.nsmap):
+            di = ExampleFolder()
+            if stf.hasAttribute(d, 'name'):
+                di.name = stf.getAttribute(d, 'name')
+            if stf.hasAttribute(d, 'chmod'):
+                di.chmod = stf.getAttribute(d, 'chmod')
+            examples[n].folders.append(di)
+
+
+    # Parse scons_example_files
+    for f in stf.findAll(t.root, "scons_example_file", SConsDoc.dbxid, 
+                         t.xpath_context, t.nsmap):
+        if stf.hasAttribute(f, 'example'):
+            e = stf.getAttribute(f, 'example')
+        else:
+            continue
+        fi = ExampleFile(FT_FILEREF)
+        if stf.hasAttribute(f, 'name'):
+            fi.name = stf.getAttribute(f, 'name')
+        if stf.hasAttribute(f, 'chmod'):
+            fi.chmod = stf.getAttribute(f, 'chmod')
+        fi.content = stf.getText(f)
+        examples[e].files.append(fi)
+        
+    
+    # Parse scons_output
+    for o in stf.findAll(t.root, "scons_output", SConsDoc.dbxid, 
+                         t.xpath_context, t.nsmap):
+        if stf.hasAttribute(o, 'example'):
+            n = stf.getAttribute(o, 'example')
+        else:
+            continue
+
+        eout = ExampleOutput()
+        if stf.hasAttribute(o, 'name'):
+            eout.name = stf.getAttribute(o, 'name')
+        if stf.hasAttribute(o, 'tools'):
+            eout.tools = stf.getAttribute(o, 'tools')
+        if stf.hasAttribute(o, 'os'):
+            eout.os = stf.getAttribute(o, 'os')
+
+        for c in stf.findAll(o, "scons_output_command", SConsDoc.dbxid, 
+                         t.xpath_context, t.nsmap):
+            if stf.hasAttribute(c, 'suffix'):
+                s = stf.getAttribute(c, 'suffix')
+            else:
+                continue
+
+            oc = ExampleCommand()
+            oc.suffix = s
+            if stf.hasAttribute(c, 'edit'):
+                oc.edit = stf.getAttribute(c, 'edit')
+            if stf.hasAttribute(c, 'environment'):
+                oc.environment = stf.getAttribute(c, 'environment')
+            if stf.hasAttribute(c, 'output'):
+                oc.output = stf.getAttribute(c, 'output')
+            if stf.hasAttribute(c, 'cmd'):
+                oc.cmd = stf.getAttribute(c, 'cmd')
+
+            eout.commands.append(oc)
+
+        examples[n].outputs.append(eout)
+
+def readAllExampleInfos(dpath):
+    """ Scan for XML files in the given directory and 
+        collect together all relevant infos (files/folders,
+        output commands) in a map, which gets returned.
+    """
+    examples = {}
+    for path, dirs, files in os.walk(dpath):
+        for f in files:
+            if f.endswith('.xml'):
+                fpath = os.path.join(path, f)
+                if SConsDoc.isSConsXml(fpath):
+                    readExampleInfos(fpath, examples)
+                   
+    return examples
+
+generated_examples = os.path.join('doc','generated','examples')
+
+def ensureExampleOutputsExist(dpath):
+    """ Scan for XML files in the given directory and 
+        ensure that for every example output we have a
+        corresponding output file in the 'generated/examples'
+        folder.
+    """
+    # Ensure that the output folder exists
+    if not os.path.isdir(generated_examples):
+        os.mkdir(generated_examples)
+        
+    examples = readAllExampleInfos(dpath)
+    for key, value in examples.iteritems():
+        # Process all scons_output tags
+        for o in value.outputs:
+            for c in o.commands:
+                cpath = os.path.join(generated_examples, 
+                                     key+'_'+c.suffix+'.out')
+                if not os.path.isfile(cpath):
+                    content = c.output
+                    if not content:
+                        content = "NO OUTPUT YET! Run the script to generate/update all examples."
+
+                    f = open(cpath, 'w')
+                    f.write("%s\n" % content)
+                    f.close()
+        # Process all scons_example_file tags
+        for r in value.files:
+            if r.isFileRef():
+                # Get file's content
+                content = value.getFileContents(r.name)
+                fpath = os.path.join(generated_examples, 
+                                     key+'_'+r.name.replace("/","_"))
+                # Write file
+                f = open(fpath, 'w')
+                f.write("%s\n" % content)
+                f.close()
+
+def collectSConsExampleNames(fpath):
+    """ Return a set() of example names, used in the given file fpath.
+    """
+    names = set()
+    suffixes = {}
+    failed_suffixes = False
+
+    # Create doctree    
+    t = SConsDoc.SConsDocTree()
+    t.parseXmlFile(fpath)
+    
+    # Parse it
+    for e in stf.findAll(t.root, "scons_example", SConsDoc.dbxid, 
+                         t.xpath_context, t.nsmap):
+        n = ''
+        if stf.hasAttribute(e, 'name'):
+            n = stf.getAttribute(e, 'name')
+        if n:
+            names.add(n)
+            if n not in suffixes:
+                suffixes[n] = []
+        else:
+            print "Error: Example in file '%s' is missing a name!" % fpath
+            failed_suffixes = True
+    
+    for o in stf.findAll(t.root, "scons_output", SConsDoc.dbxid, 
+                         t.xpath_context, t.nsmap):
+        n = ''
+        if stf.hasAttribute(o, 'example'):
+            n = stf.getAttribute(o, 'example')
+        else:
+            print "Error: scons_output in file '%s' is missing an example name!" % fpath
+            failed_suffixes = True
+            
+        if n not in suffixes:
+            print "Error: scons_output in file '%s' is referencing non-existent example '%s'!" % (fpath, n)
+            failed_suffixes = True
+            continue
+            
+        for c in stf.findAll(o, "scons_output_command", SConsDoc.dbxid, 
+                         t.xpath_context, t.nsmap):
+            s = ''
+            if stf.hasAttribute(c, 'suffix'):
+                s = stf.getAttribute(c, 'suffix')
+            else:
+                print "Error: scons_output_command in file '%s' (example '%s') is missing a suffix!" % (fpath, n)
+                failed_suffixes = True
+            
+            if s not in suffixes[n]:
+                suffixes[n].append(s)
+            else:
+                print "Error: scons_output_command in file '%s' (example '%s') is using a duplicate suffix '%s'!" % (fpath, n, s)
+                failed_suffixes = True
+    
+    return names, failed_suffixes
+
+def exampleNamesAreUnique(dpath):
+    """ Scan for XML files in the given directory and 
+        check whether the scons_example names are unique.
+    """
+    unique = True
+    allnames = set()
+    for path, dirs, files in os.walk(dpath):
+        for f in files:
+            if f.endswith('.xml'):
+                fpath = os.path.join(path, f)
+                if SConsDoc.isSConsXml(fpath):
+                    names, failed_suffixes = collectSConsExampleNames(fpath)
+                    if failed_suffixes:
+                        unique = False
+                    i = allnames.intersection(names)
+                    if i:
+                        print "Not unique in %s are: %s" % (fpath, ', '.join(i))
+                        unique = False
+                    
+                    allnames |= names
+                   
+    return unique
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4:
diff --git a/bin/docs-check-unique-examples.py b/bin/docs-check-unique-examples.py
new file mode 100644
index 0000000..d9e5cf6
--- /dev/null
+++ b/bin/docs-check-unique-examples.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+#
+# Searches through the whole doc/user tree and verifies
+# that the names of the single examples are unique over
+# all *.xml files.
+# Additionally, the suffix entries have to be unique
+# within each scons_command_output.
+#
+
+import os
+import SConsExamples
+
+if __name__ == "__main__":
+    if SConsExamples.exampleNamesAreUnique(os.path.join('doc','user')):
+        print "OK"
+    else:
+        print "Not all example names and suffixes are unique! Please correct the errors listed above and try again."
diff --git a/bin/scons-doc.py b/bin/scons-doc.py
index cf5d5b2..ff06c04 100644
--- a/bin/scons-doc.py
+++ b/bin/scons-doc.py
@@ -580,109 +580,6 @@ class MySGML(sgmllib.SGMLParser):
         contents = contents.replace('>', '&gt;')
         return contents
 
-    def start_scons_example(self, attrs):
-        t = [t for t in attrs if t[0] == 'name']
-        if t:
-            name = t[0][1]
-            try:
-               e = self.examples[name]
-            except KeyError:
-               e = self.examples[name] = Example()
-        else:
-            e = Example()
-        for name, value in attrs:
-            setattr(e, name, value)
-        self.e = e
-        self.afunclist.append(e.afunc)
-
-    def end_scons_example(self):
-        e = self.e
-        files = [f for f in e.files if f.printme]
-        if files:
-            self.outfp.write('<programlisting>')
-            for f in files:
-                if f.printme:
-                    i = len(f.data) - 1
-                    while f.data[i] == ' ':
-                        i = i - 1
-                    output = self.for_display(f.data[:i+1])
-                    self.outfp.write(output)
-            if e.data and e.data[0] == '\n':
-                e.data = e.data[1:]
-            self.outfp.write(e.data + '</programlisting>')
-        delattr(self, 'e')
-        self.afunclist = self.afunclist[:-1]
-
-    def start_file(self, attrs):
-        try:
-            e = self.e
-        except AttributeError:
-            self.error("<file> tag outside of <scons_example>")
-        t = [t for t in attrs if t[0] == 'name']
-        if not t:
-            self.error("no <file> name attribute found")
-        try:
-            e.prefix
-        except AttributeError:
-            e.prefix = e.data
-            e.data = ""
-        f = File(t[0][1])
-        f.printme = None
-        for name, value in attrs:
-            setattr(f, name, value)
-        e.files.append(f)
-        self.afunclist.append(f.afunc)
-
-    def end_file(self):
-        self.e.data = ""
-        self.afunclist = self.afunclist[:-1]
-
-    def start_directory(self, attrs):
-        try:
-            e = self.e
-        except AttributeError:
-            self.error("<directory> tag outside of <scons_example>")
-        t = [t for t in attrs if t[0] == 'name']
-        if not t:
-            self.error("no <directory> name attribute found")
-        try:
-            e.prefix
-        except AttributeError:
-            e.prefix = e.data
-            e.data = ""
-        d = Directory(t[0][1])
-        for name, value in attrs:
-            setattr(d, name, value)
-        e.dirs.append(d)
-        self.afunclist.append(d.afunc)
-
-    def end_directory(self):
-        self.e.data = ""
-        self.afunclist = self.afunclist[:-1]
-
-    def start_scons_example_file(self, attrs):
-        t = [t for t in attrs if t[0] == 'example']
-        if not t:
-            self.error("no <scons_example_file> example attribute found")
-        exname = t[0][1]
-        try:
-            e = self.examples[exname]
-        except KeyError:
-            self.error("unknown example name '%s'" % exname)
-        fattrs = [t for t in attrs if t[0] == 'name']
-        if not fattrs:
-            self.error("no <scons_example_file> name attribute found")
-        fname = fattrs[0][1]
-        f = [f for f in e.files if f.name == fname]
-        if not f:
-            self.error("example '%s' does not have a file named '%s'" % (exname, fname))
-        self.f = f[0]
-
-    def end_scons_example_file(self):
-        f = self.f
-        self.outfp.write('<programlisting>')
-        self.outfp.write(f.data + '</programlisting>')
-        delattr(self, 'f')
 
     def start_scons_output(self, attrs):
         t = [t for t in attrs if t[0] == 'example']
@@ -816,119 +713,11 @@ class MySGML(sgmllib.SGMLParser):
         self.o.data = ""
         self.afunclist = self.afunclist[:-1]
 
-    def start_sconstruct(self, attrs):
-        f = File('')
-        self.f = f
-        self.afunclist.append(f.afunc)
-
-    def end_sconstruct(self):
-        f = self.f
-        self.outfp.write('<programlisting>')
-        output = self.for_display(f.data)
-        self.outfp.write(output + '</programlisting>')
-        delattr(self, 'f')
-        self.afunclist = self.afunclist[:-1]
 
-def process(filename, fout=sys.stdout):
-    if filename == '-':
-        f = sys.stdin
-    else:
-        try:
-            f = open(filename, 'r')
-        except EnvironmentError, e:
-            sys.stderr.write('%s: %s\n' % (filename, e))
-            return 1
-
-    data = f.read()
-    if f is not sys.stdin:
-        f.close()
-
-    if data.startswith('<?xml '):
-        first_line, data = data.split('\n', 1)
-        fout.write(first_line + '\n')
-
-    x = MySGML(fout)
-    for c in data:
-        x.feed(c)
-    x.close()
-
-    return 0
-
-def main(argv=None):
-    if argv is None:
-        argv = sys.argv
-
-    parser = optparse.OptionParser()
-    parser.add_option('-d', '--diff',
-                  action='store_true', dest='diff', default=False,
-                  help='create examples for the .in file and output a unified diff against the related .xml file')
-    parser.add_option('-r', '--run',
-                  action='store_true', dest='run', default=False,
-                  help='create examples for the .in file, but do not change any files')
-    parser.add_option('-s', '--simple_diff',
-                  action='store_true', dest='simple', default=False,
-                  help='use a simpler output for the diff mode (no unified diff!)')
-    parser.add_option('-u', '--update',
-                  action='store_true', dest='update', default=False,
-                  help='create examples for the .in file and update the related .xml file')
-
-    opts, args = parser.parse_args(argv[1:])
-
-    if opts.diff:
-        import StringIO
-        import difflib
-        
-        if not args:
-            args = glob.glob('doc/user/*.in')
-        for arg in sorted(args):
-            diff = None
-            s = StringIO.StringIO()
-            process(arg,s)
-            filename = arg[:-2]+'xml'
-            try:
-                fxml = open(filename, 'r')
-                xmlcontent = fxml.read()
-                fxml.close()
-                if opts.simple:
-                    diff = list(difflib.context_diff(xmlcontent.splitlines(),
-                                                     s.getvalue().splitlines(),
-                                                     fromfile=arg, tofile=filename))
-                else:
-                    diff = list(difflib.unified_diff(xmlcontent.splitlines(),
-                                                     s.getvalue().splitlines(),
-                                                     fromfile=arg, tofile=filename, 
-                                                     lineterm=''))
-            except EnvironmentError, e:
-                sys.stderr.write('%s: %s\n' % (filename, e))
-                
-            s.close()
-            if diff:
-                print "%s:" % arg
-                print '\n'.join(diff)
-    elif opts.run:
-        if not args:
-            args = glob.glob('doc/user/*.in')
-        for arg in sorted(args):
-            print "%s:" % arg
-            process(arg)
-    elif opts.update:
-        if not args:
-            args = glob.glob('doc/user/*.in')
-        for arg in sorted(args):
-            print "%s:" % arg
-            filename = arg[:-2]+'xml'
-            try:
-                fxml = open(filename, 'w')
-                process(arg, fxml)
-                fxml.close()
-            except EnvironmentError, e:
-                sys.stderr.write('%s: %s\n' % (filename, e))
-    else:
-        if not args:
-            args = ['-']
-    
-        for arg in args:
-            process(arg)
+
+def main():
+    argv = sys.argv
+
 
 if __name__ == "__main__":
     sys.exit(main())
diff --git a/bin/scons-proc.py b/bin/scons-proc.py
index d6bc1a2..36176f1 100644
--- a/bin/scons-proc.py
+++ b/bin/scons-proc.py
@@ -14,7 +14,6 @@ import os
 import re
 import string
 import sys
-import copy
 try:
     from io import StringIO     # usable as of 2.6; takes unicode only
 except ImportError:
@@ -22,28 +21,7 @@ except ImportError:
     exec('from cStringIO import StringIO')
 
 import SConsDoc
-
-try:
-  from lxml import etree
-except ImportError:
-  try:
-    # Python 2.5
-    import xml.etree.cElementTree as etree
-  except ImportError:
-    try:
-      # Python 2.5
-      import xml.etree.ElementTree as etree
-    except ImportError:
-      try:
-        # normal cElementTree install
-        import cElementTree as etree
-      except ImportError:
-        try:
-          # normal ElementTree install
-          import elementtree.ElementTree as etree
-        except ImportError:
-          print("Failed to import ElementTree from any known place")
-          sys.exit(1)
+from SConsDoc import tf as stf
 
 base_sys_path = [os.getcwd() + '/build/test-tar-gz/lib/scons'] + sys.path
 
@@ -150,33 +128,45 @@ class SCons_XML(object):
             filename = fl[0]
             
         # Start new XML file
-        root = SConsDoc.xml_tree("variablelist")
+        root = stf.newXmlTree("variablelist")
         
         for v in self.values:
-            ve = etree.Element("varlistentry",
-                               attrib = {'id' : '%s%s' % (v.prefix, v.idfunc())})
-            ve.append(v.xml_term())
-            vl = etree.Element("listitem")
-            if v.summary:
+            
+            ve = stf.newNode("varlistentry")
+            stf.setAttribute(ve, 'id', '%s%s' % (v.prefix, v.idfunc()))
+            stf.appendNode(ve, v.xml_term())
+            vl = stf.newNode("listitem")
+            added = False
+            if v.summary is not None:
                 for s in v.summary:
-                    vl.append(copy.deepcopy(s))
+                    added = True
+                    stf.appendNode(vl, stf.copyNode(s))
             
-            if v.sets:
-                vp = etree.Element("para")
+            if len(v.sets):
+                added = True
+                vp = stf.newNode("para")
                 s = ['&cv-link-%s;' % x for x in v.sets]
-                vp.text = 'Sets:  ' + ', '.join(s) + '.'
-                vl.append(vp)
-            if v.uses:
-                vp = etree.Element("para")
+                stf.setText(vp, 'Sets:  ' + ', '.join(s) + '.')
+                stf.appendNode(vl, vp)
+            if len(v.uses):
+                added = True
+                vp = stf.newNode("para")
                 u = ['&cv-link-%s;' % x for x in v.uses]
-                vp.text = 'Uses:  ' + ', '.join(u) + '.'
-                vl.append(vp)
-            ve.append(vl)
-            root.append(ve)
+                stf.setText(vp, 'Uses:  ' + ', '.join(u) + '.')
+                stf.appendNode(vl, vp)
+                
+            # Still nothing added to this list item?
+            if not added:
+                # Append an empty para
+                vp = stf.newNode("para")
+                stf.appendNode(vl, vp)
+                
+            stf.appendNode(ve, vl)
+            stf.appendNode(root, ve)
             
         # Write file        
         f = self.fopen(filename)
-        f.write(etree.tostring(root, xml_declaration=True, encoding="UTF-8", pretty_print=True))
+        stf.writeGenTree(root, f)
             
     def write_mod(self, filename):
         try:
@@ -248,8 +238,8 @@ class SConsThing(Proxy):
         return self.name
     
     def xml_term(self):
-        e = etree.Element("term")
-        e.text = self.name
+        e = stf.newNode("term")
+        stf.setText(e, self.name)
         return e
 
 class Builder(SConsThing):
@@ -258,17 +248,17 @@ class Builder(SConsThing):
     tag = 'function'
     
     def xml_term(self):
-        t = etree.Element("term")
-        s = etree.Element("synopsis")
-        b = etree.Element(self.tag)
-        b.text = self.name+'()'
-        s.append(b)
-        t.append(s)
-        s = etree.Element("synopsis")
-        b = etree.Element(self.tag)
-        b.text = 'env.'+self.name+'()'
-        s.append(b)
-        t.append(s)
+        t = stf.newNode("term")
+        s = stf.newNode("synopsis")
+        b = stf.newNode(self.tag)
+        stf.setText(b, self.name+'()')
+        stf.appendNode(s, b)
+        stf.appendNode(t, s)
+        s = stf.newNode("synopsis")
+        b = stf.newNode(self.tag)
+        stf.setText(b, 'env.'+self.name+'()')
+        stf.appendNode(s, b)
+        stf.appendNode(t, s)
         return t
             
     def entityfunc(self):
@@ -280,27 +270,26 @@ class Function(SConsThing):
     tag = 'function'
     
     def xml_term(self):
-        try:
-            arguments = self.arguments
-        except AttributeError:
-            a = etree.Element("arguments")
-            a.text = '()'
+        if self.arguments is None:
+            a = stf.newNode("arguments")
+            stf.setText(a, '()')
             arguments = [a]
-        t = etree.Element("term")
+        else:
+            arguments = self.arguments    
+        t = stf.newNode("term")
         for arg in arguments:
-            if 'signature' in arg.attrib:
-                signature = arg.attrib['signature']
-            else:
-                signature = "both"
-            s = arg.text.strip()
+            signature = 'both'
+            if stf.hasAttribute(arg, 'signature'):
+                signature = stf.getAttribute(arg, 'signature')
+            s = stf.getText(arg).strip()
             if signature in ('both', 'global'):
-                syn = etree.Element("synopsis")
-                syn.text = '%s%s' % (self.name, s)
-                t.append(syn)
+                syn = stf.newNode("synopsis")
+                stf.setText(syn, '%s%s' % (self.name, s))
+                stf.appendNode(t, syn)
             if signature in ('both', 'env'):
-                syn = etree.Element("synopsis")
-                syn.text = 'env.%s%s' % (self.name, s)
-                t.append(syn)
+                syn = stf.newNode("synopsis")
+                stf.setText(syn, 'env.%s%s' % (self.name, s))
+                stf.appendNode(t, syn)
 
         return t
     
@@ -351,11 +340,13 @@ def write_output_files(h, buildersfiles, functionsfiles,
 processor_class = SCons_XML
 
 # Step 1: Creating entity files for builders, functions,...
+print "Generating entity files..."
 h = parse_docs(args, False)
 write_output_files(h, buildersfiles, functionsfiles, toolsfiles,
                    variablesfiles, SCons_XML.write_mod)
 
 # Step 2: Patching the include paths for entity definitions in XML files
+print "Patching include paths..."
 os.system('python bin/docs-correct-mod-paths.py')
 
 # Step 3: Validating all input files
@@ -367,9 +358,11 @@ else:
 
 # Step 4: Creating actual documentation snippets, using the
 #         fully resolved and updated entities from the *.mod files.
+print "Updating documentation for builders, tools and functions..."
 h = parse_docs(args, True)
 write_output_files(h, buildersfiles, functionsfiles, toolsfiles,
                    variablesfiles, SCons_XML.write)
+print "Done"
 
 # Local Variables:
 # tab-width:4