diff options
Diffstat (limited to 'bin')
| -rw-r--r-- | bin/SConsDoc.py | 497 | ||||
| -rw-r--r-- | bin/SConsExamples.py | 295 | ||||
| -rw-r--r-- | bin/docs-check-unique-examples.py | 17 | ||||
| -rw-r--r-- | bin/scons-doc.py | 219 | ||||
| -rw-r--r-- | bin/scons-proc.py | 135 |
5 files changed, 746 insertions, 417 deletions
diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index 7cd06f8..e3eaf0f 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -112,6 +112,7 @@ import imp import os.path import re import sys +import copy # Do we have libxml2/libxslt/lxml? has_libxml2 = True @@ -158,6 +159,8 @@ re_entity_header = re.compile("<!DOCTYPE\s+sconsdoc\s+[^\]]+\]>") # Namespace for the SCons Docbook XSD dbxsd="http://www.scons.org/dbxsd/v1.0" +# Namespace map identifier for the SCons Docbook XSD +dbxid="dbx" # Namespace for schema instances xsi = "http://www.w3.org/2001/XMLSchema-instance" @@ -188,22 +191,6 @@ def isSConsXml(fpath): return False -def xml_tree(root, comment=generated_comment): - """ Return a XML file tree with the correct namespaces set, - the element root as top entry and the given header comment. - """ - NSMAP = {None: dbxsd, - 'xsi' : xsi} - - t = etree.Element(root, - nsmap = NSMAP, - attrib = {"{"+xsi+"}schemaLocation" : "%s scons.xsd" % dbxsd}) - - c = etree.Comment(comment) - t.append(c) - - return t - def remove_entities(content): # Cut out entity inclusions content = re_entity_header.sub("", content, re.M) @@ -232,59 +219,311 @@ class Libxml2ValidityHandler: raise Exception, "Warning handler did not receive correct argument" self.warnings.append(msg) -def validate_xml(fpath, xmlschema_context): - if not has_libxml2: - # Use lxml - xmlschema = etree.XMLSchema(xmlschema_context) - doc = etree.parse(fpath) - doc.xinclude() - try: - xmlschema.assertValid(doc) - except Exception, e: - print e - print "%s fails to validate" % fpath - return False - return True - - # Create validation context - validation_context = xmlschema_context.schemaNewValidCtxt() - # Set error/warning handlers - eh = Libxml2ValidityHandler() - validation_context.setValidityErrorHandler(eh.error, eh.warning, ARG) - # Read file and resolve entities - doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT) - doc.xincludeProcessFlags(libxml2.XML_PARSE_NOENT) - err = validation_context.schemaValidateDoc(doc) - # Cleanup - doc.freeDoc() - del validation_context - - if err or eh.errors: - for e in eh.errors: - print e.rstrip("\n") - print "%s fails to validate" % fpath - return False + +class DoctypeEntity: + def __init__(self, name_, uri_): + self.name = name_ + self.uri = uri_ - return True + def getEntityString(self): + txt = """ <!ENTITY %(perc)s %(name)s SYSTEM "%(uri)s"> + %(perc)s%(name)s; +""" % {'perc' : perc, 'name' : self.name, 'uri' : self.uri} -def prettyprint_xml(fpath): - if not has_libxml2: - # Use lxml - fin = open(fpath,'r') - tree = etree.parse(fin) - pretty_content = etree.tostring(tree, pretty_print=True) - fin.close() - - fout = open(fpath,'w') - fout.write(pretty_content) - fout.close() - - # Read file and resolve entities - doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT) - err = xmlschema_context.schemaValidateDoc(doc) - # Cleanup - doc.freeDoc() + return txt + +class DoctypeDeclaration: + def __init__(self, name_=None): + self.name = name_ + self.entries = [] + if self.name is None: + # Add default entries + self.name = "sconsdoc" + self.addEntity("scons", "../scons.mod") + self.addEntity("builders-mod", "builders.mod") + self.addEntity("functions-mod", "functions.mod") + self.addEntity("tools-mod", "tools.mod") + self.addEntity("variables-mod", "variables.mod") + + def addEntity(self, name, uri): + self.entries.append(DoctypeEntity(name, uri)) + + def createDoctype(self): + content = '<!DOCTYPE %s [\n' % self.name + for e in self.entries: + content += e.getEntityString() + content += ']>\n' + + return content + +if not has_libxml2: + class TreeFactory: + def __init__(self): + pass + + def newNode(self, tag): + return etree.Element(tag) + + def copyNode(self, node): + return copy.deepcopy(node) + + def appendNode(self, parent, child): + parent.append(child) + + def hasAttribute(self, node, att): + return att in node.attrib + + def getAttribute(self, node, att): + return node.attrib[att] + + def setAttribute(self, node, att, value): + node.attrib[att] = value + + def getText(self, root): + return root.text + + def setText(self, root, txt): + root.text = txt + + def writeGenTree(self, root, fp): + dt = DoctypeDeclaration() + fp.write(etree.tostring(root, xml_declaration=True, + encoding="UTF-8", pretty_print=True, + doctype=dt.createDoctype())) + + def prettyPrintFile(self, fpath): + fin = open(fpath,'r') + tree = etree.parse(fin) + pretty_content = etree.tostring(tree, pretty_print=True) + fin.close() + fout = open(fpath,'w') + fout.write(pretty_content) + fout.close() + + def newXmlTree(self, root, comment=generated_comment): + """ Return a XML file tree with the correct namespaces set, + the element root as top entry and the given header comment. + """ + NSMAP = {None: dbxsd, + 'xsi' : xsi} + + t = etree.Element(root, + nsmap = NSMAP, + attrib = {"{"+xsi+"}schemaLocation" : "%s scons.xsd" % dbxsd}) + + c = etree.Comment(comment) + t.append(c) + + return t + + def validateXml(self, fpath, xmlschema_context): + # Use lxml + xmlschema = etree.XMLSchema(xmlschema_context) + doc = etree.parse(fpath) + doc.xinclude() + try: + xmlschema.assertValid(doc) + except Exception, e: + print e + print "%s fails to validate" % fpath + return False + return True + + def findAll(self, root, tag, ns=None, xp_ctxt=None, nsmap=None): + expression = ".//{%s}%s" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = ".//%s" % tag + return root.findall(expression) + + def findAllChildrenOf(self, root, tag, ns=None, xp_ctxt=None, nsmap=None): + expression = "./{%s}%s/*" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = "./%s/*" % tag + return root.findall(expression) + +else: + class TreeFactory: + def __init__(self): + pass + + def newNode(self, tag): + return libxml2.newNode(tag) + + def copyNode(self, node): + return node.copyNode(1) + + def appendNode(self, parent, child): + if hasattr(parent, 'addChild'): + parent.addChild(child) + else: + parent.append(child) + + def hasAttribute(self, node, att): + if hasattr(node, 'hasProp'): + return node.hasProp(att) + return att in node.attrib + + def getAttribute(self, node, att): + if hasattr(node, 'prop'): + return node.prop(att) + return node.attrib[att] + + def setAttribute(self, node, att, value): + if hasattr(node, 'setProp'): + node.setProp(att, value) + else: + node.attrib[att] = value + + def getText(self, root): + if hasattr(root, 'getContent'): + return root.getContent() + return root.text + + def setText(self, root, txt): + if hasattr(root, 'setContent'): + root.setContent(txt) + else: + root.text = txt + + def writeGenTree(self, root, fp): + doc = libxml2.newDoc('1.0') + dtd = doc.newDtd("sconsdoc", None, None) + doc.addChild(dtd) + doc.setRootElement(root) + content = doc.serialize("UTF-8", 1) + dt = DoctypeDeclaration() + # This is clearly a hack, but unfortunately libxml2 + # doesn't support writing PERs (Parsed Entity References). + # So, we simply replace the empty doctype with the + # text we need... + content = content.replace("<!DOCTYPE sconsdoc>", dt.createDoctype()) + fp.write(content) + doc.freeDoc() + + def prettyPrintFile(self, fpath): + # Read file and resolve entities + doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT) + fp = open(fpath, 'w') + # Prettyprint + fp.write(doc.serialize("UTF-8", 1)) + fp.close() + # Cleanup + doc.freeDoc() + + def newXmlTree(self, root, comment=generated_comment): + """ Return a XML file tree with the correct namespaces set, + the element root as top entry and the given header comment. + """ + t = libxml2.newNode(root) + # Register the namespaces + ns = t.newNs(dbxsd, None) + xi = t.newNs(xsi, 'xsi') + t.setNs(ns) #put this node in the target namespace + + t.setNsProp(xi, 'schemaLocation', "%s scons.xsd" % dbxsd) + + c = libxml2.newComment(comment) + t.addChild(c) + + return t + + def validateXml(self, fpath, xmlschema_context): + # Create validation context + validation_context = xmlschema_context.schemaNewValidCtxt() + # Set error/warning handlers + eh = Libxml2ValidityHandler() + validation_context.setValidityErrorHandler(eh.error, eh.warning, ARG) + # Read file and resolve entities + doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT) + doc.xincludeProcessFlags(libxml2.XML_PARSE_NOENT) + err = validation_context.schemaValidateDoc(doc) + # Cleanup + doc.freeDoc() + del validation_context + + if err or eh.errors: + for e in eh.errors: + print e.rstrip("\n") + print "%s fails to validate" % fpath + return False + + return True + + def findAll(self, root, tag, ns=None, xpath_context=None, nsmap=None): + if hasattr(root, 'xpathEval') and xpath_context: + # Use the xpath context + xpath_context.setContextNode(root) + expression = ".//%s" % tag + if ns: + expression = ".//%s:%s" % (ns, tag) + return xpath_context.xpathEval(expression) + else: + expression = ".//{%s}%s" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = ".//%s" % tag + return root.findall(expression) + + def findAllChildrenOf(self, root, tag, ns=None, xpath_context=None, nsmap=None): + if hasattr(root, 'xpathEval') and xpath_context: + # Use the xpath context + xpath_context.setContextNode(root) + expression = "./%s/node()" % tag + if ns: + expression = "./%s:%s/node()" % (ns, tag) + + return xpath_context.xpathEval(expression) + else: + expression = "./{%s}%s/node()" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = "./%s/node()" % tag + return root.findall(expression) + + +tf = TreeFactory() + + +class SConsDocTree: + def __init__(self): + self.nsmap = {'dbx' : dbxsd} + self.doc = None + self.root = None + self.xpath_context = None + + def parseContent(self, content, include_entities=True): + """ Parses the given content as XML file. This method + is used when we generate the basic lists of entities + for the builders, tools and functions. + So we usually don't bother about namespaces and resolving + entities here...this is handled in parseXmlFile below + (step 2 of the overall process). + """ + if not include_entities: + content = remove_entities(content) + # Create domtree from given content string + self.root = etree.fromstring(content) + + def parseXmlFile(self, fpath): + nsmap = {'dbx' : dbxsd} + if not has_libxml2: + # Create domtree from file + domtree = etree.parse(fpath) + self.root = domtree.getroot() + else: + # Read file and resolve entities + self.doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT) + self.root = self.doc.getRootElement() + # Create xpath context + self.xpath_context = self.doc.xpathNewContext() + # Register namespaces + for key, val in self.nsmap.iteritems(): + self.xpath_context.xpathRegisterNs(key, val) + + def __del__(self): + if self.doc is not None: + self.doc.freeDoc() + if self.xpath_context is not None: + self.xpath_context.xpathFreeContext() perc="%" @@ -314,7 +553,7 @@ def validate_all_xml(dpaths, xsdfile=default_xsd): print "%.2f%s (%d/%d) %s" % (float(idx+1)*100.0/float(len(fpaths)), perc, idx+1, len(fpaths),fp) - if not validate_xml(fp, xmlschema_context): + if not tf.validateXml(fp, xmlschema_context): fails.append(fp) continue @@ -333,9 +572,10 @@ class Item(object): self.sort_name = name.lower() if self.sort_name[0] == '_': self.sort_name = self.sort_name[1:] - self.summary = [] self.sets = [] self.uses = [] + self.summary = None + self.arguments = None def cmp_name(self, name): if name[0] == '_': name = name[1:] @@ -358,18 +598,6 @@ class Tool(Item): class ConstructionVariable(Item): pass -class Chunk(object): - def __init__(self, tag, body=None): - self.tag = tag - if not body: - body = [] - self.body = body - def __str__(self): - body = ''.join(self.body) - return "<%s>%s</%s>\n" % (self.tag, body, self.tag) - def append(self, data): - self.body.append(data) - class Arguments(object): def __init__(self, signature, body=None): if not body: @@ -394,85 +622,92 @@ class SConsDocHandler(object): self.tools = {} self.cvars = {} - def parseText(self, root): - txt = "" - for e in root.childNodes: - if (e.nodeType == e.TEXT_NODE): - txt += e.data - return txt - - def parseItems(self, domelem): + def parseItems(self, domelem, xpath_context, nsmap): items = [] - for i in domelem.iterchildren(tag="item"): - items.append(self.parseText(i)) + for i in tf.findAll(domelem, "item", dbxid, xpath_context, nsmap): + txt = tf.getText(i) + if txt is not None: + txt = txt.strip() + if len(txt): + items.append(txt.strip()) return items - def parseUsesSets(self, domelem): + def parseUsesSets(self, domelem, xpath_context, nsmap): uses = [] sets = [] - for u in domelem.iterchildren(tag="uses"): - uses.extend(self.parseItems(u)) - for s in domelem.iterchildren(tag="sets"): - sets.extend(self.parseItems(s)) + for u in tf.findAll(domelem, "uses", dbxid, xpath_context, nsmap): + uses.extend(self.parseItems(u, xpath_context, nsmap)) + for s in tf.findAll(domelem, "sets", dbxid, xpath_context, nsmap): + sets.extend(self.parseItems(s, xpath_context, nsmap)) return sorted(uses), sorted(sets) - def parseInstance(self, domelem, map, Class): - name = domelem.attrib.get('name','unknown') + def parseInstance(self, domelem, map, Class, + xpath_context, nsmap, include_entities=True): + name = 'unknown' + if tf.hasAttribute(domelem, 'name'): + name = tf.getAttribute(domelem, 'name') try: instance = map[name] except KeyError: instance = Class(name) map[name] = instance - uses, sets = self.parseUsesSets(domelem) + uses, sets = self.parseUsesSets(domelem, xpath_context, nsmap) instance.uses.extend(uses) instance.sets.extend(sets) - # Parse summary and function arguments - for s in domelem.iterchildren(tag="{%s}summary" % dbxsd): - if not hasattr(instance, 'summary'): - instance.summary = [] - for c in s: - instance.summary.append(c) - for a in domelem.iterchildren(tag="{%s}arguments" % dbxsd): - if not hasattr(instance, 'arguments'): - instance.arguments = [] - instance.arguments.append(a) - - def parseDomtree(self, root): + if include_entities: + # Parse summary and function arguments + for s in tf.findAllChildrenOf(domelem, "summary", dbxid, xpath_context, nsmap): + if instance.summary is None: + instance.summary = [] + instance.summary.append(tf.copyNode(s)) + for a in tf.findAll(domelem, "arguments", dbxid, xpath_context, nsmap): + if instance.arguments is None: + instance.arguments = [] + instance.arguments.append(tf.copyNode(a)) + + def parseDomtree(self, root, xpath_context=None, nsmap=None, include_entities=True): # Process Builders - for b in root.iterchildren(tag="{%s}builder" % dbxsd): - self.parseInstance(b, self.builders, Builder) + for b in tf.findAll(root, "builder", dbxid, xpath_context, nsmap): + self.parseInstance(b, self.builders, Builder, + xpath_context, nsmap, include_entities) # Process Functions - for f in root.iterchildren(tag="{%s}scons_function" % dbxsd): - self.parseInstance(f, self.functions, Function) + for f in tf.findAll(root, "scons_function", dbxid, xpath_context, nsmap): + self.parseInstance(f, self.functions, Function, + xpath_context, nsmap, include_entities) # Process Tools - for t in root.iterchildren(tag="{%s}tool" % dbxsd): - self.parseInstance(t, self.tools, Tool) + for t in tf.findAll(root, "tool", dbxid, xpath_context, nsmap): + self.parseInstance(t, self.tools, Tool, + xpath_context, nsmap, include_entities) # Process CVars - for c in root.iterchildren(tag="{%s}cvar" % dbxsd): - self.parseInstance(c, self.cvars, ConstructionVariable) + for c in tf.findAll(root, "cvar", dbxid, xpath_context, nsmap): + self.parseInstance(c, self.cvars, ConstructionVariable, + xpath_context, nsmap, include_entities) def parseContent(self, content, include_entities=True): - if not include_entities: - content = remove_entities(content) - # Create domtree from given content string - root = etree.fromstring(content) + """ Parses the given content as XML file. This method + is used when we generate the basic lists of entities + for the builders, tools and functions. + So we usually don't bother about namespaces and resolving + entities here...this is handled in parseXmlFile below + (step 2 of the overall process). + """ + # Create doctree + t = SConsDocTree() + t.parseContent(content, include_entities) # Parse it - self.parseDomtree(root) + self.parseDomtree(t.root, t.xpath_context, t.nsmap, include_entities) def parseXmlFile(self, fpath): - # Create domtree from file - domtree = etree.parse(fpath) + # Create doctree + t = SConsDocTree() + t.parseXmlFile(fpath) # Parse it - self.parseDomtree(domtree.getroot()) - - def set_file_info(self, filename, preamble_lines): - self.filename = filename - self.preamble_lines = preamble_lines - + self.parseDomtree(t.root, t.xpath_context, t.nsmap) + # lifted from Ka-Ping Yee's way cool pydoc module. def importfile(path): """Import a Python source file or compiled file given its path.""" diff --git a/bin/SConsExamples.py b/bin/SConsExamples.py new file mode 100644 index 0000000..77f4041 --- /dev/null +++ b/bin/SConsExamples.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python +# +# Module for handling SCons examples processing. +# + +__doc__ = """ +""" + +import os +import re +import SConsDoc +from SConsDoc import tf as stf + +# +# The available types for ExampleFile entries +# +FT_FILE = 0 # a physical file (=<file>) +FT_FILEREF = 1 # a reference (=<scons_example_file>) + +class ExampleFile: + def __init__(self, type_=FT_FILE): + self.type = type_ + self.name = '' + self.content = '' + self.chmod = '' + + def isFileRef(self): + return self.type == FT_FILEREF + +class ExampleFolder: + def __init__(self): + self.name = '' + self.chmod = '' + +class ExampleCommand: + def __init__(self): + self.edit = '' + self.environment = '' + self.output = '' + self.cmd = '' + self.suffix = '' + +class ExampleOutput: + def __init__(self): + self.name = '' + self.tools = '' + self.os = '' + self.commands = [] + +class ExampleInfo: + def __init__(self): + self.name = '' + self.files = [] + self.folders = [] + self.outputs = [] + + def getFileContents(self, fname): + for f in self.files: + if fname == f.name and not f.isFileRef(): + return f.content + + return '' + +def readExampleInfos(fpath, examples): + """ Add the example infos for the file fpath to the + global dictionary examples. + """ + + # Create doctree + t = SConsDoc.SConsDocTree() + t.parseXmlFile(fpath) + + # Parse scons_examples + for e in stf.findAll(t.root, "scons_example", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + n = '' + if stf.hasAttribute(e, 'name'): + n = stf.getAttribute(e, 'name') + if n and n not in examples: + i = ExampleInfo() + i.name = n + examples[n] = i + + # Parse file and directory entries + for f in stf.findAll(e, "file", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + fi = ExampleFile() + if stf.hasAttribute(f, 'name'): + fi.name = stf.getAttribute(f, 'name') + if stf.hasAttribute(f, 'chmod'): + fi.chmod = stf.getAttribute(f, 'chmod') + fi.content = stf.getText(f) + examples[n].files.append(fi) + for d in stf.findAll(e, "directory", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + di = ExampleFolder() + if stf.hasAttribute(d, 'name'): + di.name = stf.getAttribute(d, 'name') + if stf.hasAttribute(d, 'chmod'): + di.chmod = stf.getAttribute(d, 'chmod') + examples[n].folders.append(di) + + + # Parse scons_example_files + for f in stf.findAll(t.root, "scons_example_file", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + if stf.hasAttribute(f, 'example'): + e = stf.getAttribute(f, 'example') + else: + continue + fi = ExampleFile(FT_FILEREF) + if stf.hasAttribute(f, 'name'): + fi.name = stf.getAttribute(f, 'name') + if stf.hasAttribute(f, 'chmod'): + fi.chmod = stf.getAttribute(f, 'chmod') + fi.content = stf.getText(f) + examples[e].files.append(fi) + + + # Parse scons_output + for o in stf.findAll(t.root, "scons_output", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + if stf.hasAttribute(o, 'example'): + n = stf.getAttribute(o, 'example') + else: + continue + + eout = ExampleOutput() + if stf.hasAttribute(o, 'name'): + eout.name = stf.getAttribute(o, 'name') + if stf.hasAttribute(o, 'tools'): + eout.tools = stf.getAttribute(o, 'tools') + if stf.hasAttribute(o, 'os'): + eout.os = stf.getAttribute(o, 'os') + + for c in stf.findAll(o, "scons_output_command", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + if stf.hasAttribute(c, 'suffix'): + s = stf.getAttribute(c, 'suffix') + else: + continue + + oc = ExampleCommand() + oc.suffix = s + if stf.hasAttribute(c, 'edit'): + oc.edit = stf.getAttribute(c, 'edit') + if stf.hasAttribute(c, 'environment'): + oc.environment = stf.getAttribute(c, 'environment') + if stf.hasAttribute(c, 'output'): + oc.output = stf.getAttribute(c, 'output') + if stf.hasAttribute(c, 'cmd'): + oc.cmd = stf.getAttribute(c, 'cmd') + + eout.commands.append(oc) + + examples[n].outputs.append(eout) + +def readAllExampleInfos(dpath): + """ Scan for XML files in the given directory and + collect together all relevant infos (files/folders, + output commands) in a map, which gets returned. + """ + examples = {} + for path, dirs, files in os.walk(dpath): + for f in files: + if f.endswith('.xml'): + fpath = os.path.join(path, f) + if SConsDoc.isSConsXml(fpath): + readExampleInfos(fpath, examples) + + return examples + +generated_examples = os.path.join('doc','generated','examples') + +def ensureExampleOutputsExist(dpath): + """ Scan for XML files in the given directory and + ensure that for every example output we have a + corresponding output file in the 'generated/examples' + folder. + """ + # Ensure that the output folder exists + if not os.path.isdir(generated_examples): + os.mkdir(generated_examples) + + examples = readAllExampleInfos(dpath) + for key, value in examples.iteritems(): + # Process all scons_output tags + for o in value.outputs: + for c in o.commands: + cpath = os.path.join(generated_examples, + key+'_'+c.suffix+'.out') + if not os.path.isfile(cpath): + content = c.output + if not content: + content = "NO OUTPUT YET! Run the script to generate/update all examples." + + f = open(cpath, 'w') + f.write("%s\n" % content) + f.close() + # Process all scons_example_file tags + for r in value.files: + if r.isFileRef(): + # Get file's content + content = value.getFileContents(r.name) + fpath = os.path.join(generated_examples, + key+'_'+r.name.replace("/","_")) + # Write file + f = open(fpath, 'w') + f.write("%s\n" % content) + f.close() + +def collectSConsExampleNames(fpath): + """ Return a set() of example names, used in the given file fpath. + """ + names = set() + suffixes = {} + failed_suffixes = False + + # Create doctree + t = SConsDoc.SConsDocTree() + t.parseXmlFile(fpath) + + # Parse it + for e in stf.findAll(t.root, "scons_example", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + n = '' + if stf.hasAttribute(e, 'name'): + n = stf.getAttribute(e, 'name') + if n: + names.add(n) + if n not in suffixes: + suffixes[n] = [] + else: + print "Error: Example in file '%s' is missing a name!" % fpath + failed_suffixes = True + + for o in stf.findAll(t.root, "scons_output", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + n = '' + if stf.hasAttribute(o, 'example'): + n = stf.getAttribute(o, 'example') + else: + print "Error: scons_output in file '%s' is missing an example name!" % fpath + failed_suffixes = True + + if n not in suffixes: + print "Error: scons_output in file '%s' is referencing non-existent example '%s'!" % (fpath, n) + failed_suffixes = True + continue + + for c in stf.findAll(o, "scons_output_command", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + s = '' + if stf.hasAttribute(c, 'suffix'): + s = stf.getAttribute(c, 'suffix') + else: + print "Error: scons_output_command in file '%s' (example '%s') is missing a suffix!" % (fpath, n) + failed_suffixes = True + + if s not in suffixes[n]: + suffixes[n].append(s) + else: + print "Error: scons_output_command in file '%s' (example '%s') is using a duplicate suffix '%s'!" % (fpath, n, s) + failed_suffixes = True + + return names, failed_suffixes + +def exampleNamesAreUnique(dpath): + """ Scan for XML files in the given directory and + check whether the scons_example names are unique. + """ + unique = True + allnames = set() + for path, dirs, files in os.walk(dpath): + for f in files: + if f.endswith('.xml'): + fpath = os.path.join(path, f) + if SConsDoc.isSConsXml(fpath): + names, failed_suffixes = collectSConsExampleNames(fpath) + if failed_suffixes: + unique = False + i = allnames.intersection(names) + if i: + print "Not unique in %s are: %s" % (fpath, ', '.join(i)) + unique = False + + allnames |= names + + return unique + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/docs-check-unique-examples.py b/bin/docs-check-unique-examples.py new file mode 100644 index 0000000..d9e5cf6 --- /dev/null +++ b/bin/docs-check-unique-examples.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# +# Searches through the whole doc/user tree and verifies +# that the names of the single examples are unique over +# all *.xml files. +# Additionally, the suffix entries have to be unique +# within each scons_command_output. +# + +import os +import SConsExamples + +if __name__ == "__main__": + if SConsExamples.exampleNamesAreUnique(os.path.join('doc','user')): + print "OK" + else: + print "Not all example names and suffixes are unique! Please correct the errors listed above and try again." diff --git a/bin/scons-doc.py b/bin/scons-doc.py index cf5d5b2..ff06c04 100644 --- a/bin/scons-doc.py +++ b/bin/scons-doc.py @@ -580,109 +580,6 @@ class MySGML(sgmllib.SGMLParser): contents = contents.replace('>', '>') return contents - def start_scons_example(self, attrs): - t = [t for t in attrs if t[0] == 'name'] - if t: - name = t[0][1] - try: - e = self.examples[name] - except KeyError: - e = self.examples[name] = Example() - else: - e = Example() - for name, value in attrs: - setattr(e, name, value) - self.e = e - self.afunclist.append(e.afunc) - - def end_scons_example(self): - e = self.e - files = [f for f in e.files if f.printme] - if files: - self.outfp.write('<programlisting>') - for f in files: - if f.printme: - i = len(f.data) - 1 - while f.data[i] == ' ': - i = i - 1 - output = self.for_display(f.data[:i+1]) - self.outfp.write(output) - if e.data and e.data[0] == '\n': - e.data = e.data[1:] - self.outfp.write(e.data + '</programlisting>') - delattr(self, 'e') - self.afunclist = self.afunclist[:-1] - - def start_file(self, attrs): - try: - e = self.e - except AttributeError: - self.error("<file> tag outside of <scons_example>") - t = [t for t in attrs if t[0] == 'name'] - if not t: - self.error("no <file> name attribute found") - try: - e.prefix - except AttributeError: - e.prefix = e.data - e.data = "" - f = File(t[0][1]) - f.printme = None - for name, value in attrs: - setattr(f, name, value) - e.files.append(f) - self.afunclist.append(f.afunc) - - def end_file(self): - self.e.data = "" - self.afunclist = self.afunclist[:-1] - - def start_directory(self, attrs): - try: - e = self.e - except AttributeError: - self.error("<directory> tag outside of <scons_example>") - t = [t for t in attrs if t[0] == 'name'] - if not t: - self.error("no <directory> name attribute found") - try: - e.prefix - except AttributeError: - e.prefix = e.data - e.data = "" - d = Directory(t[0][1]) - for name, value in attrs: - setattr(d, name, value) - e.dirs.append(d) - self.afunclist.append(d.afunc) - - def end_directory(self): - self.e.data = "" - self.afunclist = self.afunclist[:-1] - - def start_scons_example_file(self, attrs): - t = [t for t in attrs if t[0] == 'example'] - if not t: - self.error("no <scons_example_file> example attribute found") - exname = t[0][1] - try: - e = self.examples[exname] - except KeyError: - self.error("unknown example name '%s'" % exname) - fattrs = [t for t in attrs if t[0] == 'name'] - if not fattrs: - self.error("no <scons_example_file> name attribute found") - fname = fattrs[0][1] - f = [f for f in e.files if f.name == fname] - if not f: - self.error("example '%s' does not have a file named '%s'" % (exname, fname)) - self.f = f[0] - - def end_scons_example_file(self): - f = self.f - self.outfp.write('<programlisting>') - self.outfp.write(f.data + '</programlisting>') - delattr(self, 'f') def start_scons_output(self, attrs): t = [t for t in attrs if t[0] == 'example'] @@ -816,119 +713,11 @@ class MySGML(sgmllib.SGMLParser): self.o.data = "" self.afunclist = self.afunclist[:-1] - def start_sconstruct(self, attrs): - f = File('') - self.f = f - self.afunclist.append(f.afunc) - - def end_sconstruct(self): - f = self.f - self.outfp.write('<programlisting>') - output = self.for_display(f.data) - self.outfp.write(output + '</programlisting>') - delattr(self, 'f') - self.afunclist = self.afunclist[:-1] -def process(filename, fout=sys.stdout): - if filename == '-': - f = sys.stdin - else: - try: - f = open(filename, 'r') - except EnvironmentError, e: - sys.stderr.write('%s: %s\n' % (filename, e)) - return 1 - - data = f.read() - if f is not sys.stdin: - f.close() - - if data.startswith('<?xml '): - first_line, data = data.split('\n', 1) - fout.write(first_line + '\n') - - x = MySGML(fout) - for c in data: - x.feed(c) - x.close() - - return 0 - -def main(argv=None): - if argv is None: - argv = sys.argv - - parser = optparse.OptionParser() - parser.add_option('-d', '--diff', - action='store_true', dest='diff', default=False, - help='create examples for the .in file and output a unified diff against the related .xml file') - parser.add_option('-r', '--run', - action='store_true', dest='run', default=False, - help='create examples for the .in file, but do not change any files') - parser.add_option('-s', '--simple_diff', - action='store_true', dest='simple', default=False, - help='use a simpler output for the diff mode (no unified diff!)') - parser.add_option('-u', '--update', - action='store_true', dest='update', default=False, - help='create examples for the .in file and update the related .xml file') - - opts, args = parser.parse_args(argv[1:]) - - if opts.diff: - import StringIO - import difflib - - if not args: - args = glob.glob('doc/user/*.in') - for arg in sorted(args): - diff = None - s = StringIO.StringIO() - process(arg,s) - filename = arg[:-2]+'xml' - try: - fxml = open(filename, 'r') - xmlcontent = fxml.read() - fxml.close() - if opts.simple: - diff = list(difflib.context_diff(xmlcontent.splitlines(), - s.getvalue().splitlines(), - fromfile=arg, tofile=filename)) - else: - diff = list(difflib.unified_diff(xmlcontent.splitlines(), - s.getvalue().splitlines(), - fromfile=arg, tofile=filename, - lineterm='')) - except EnvironmentError, e: - sys.stderr.write('%s: %s\n' % (filename, e)) - - s.close() - if diff: - print "%s:" % arg - print '\n'.join(diff) - elif opts.run: - if not args: - args = glob.glob('doc/user/*.in') - for arg in sorted(args): - print "%s:" % arg - process(arg) - elif opts.update: - if not args: - args = glob.glob('doc/user/*.in') - for arg in sorted(args): - print "%s:" % arg - filename = arg[:-2]+'xml' - try: - fxml = open(filename, 'w') - process(arg, fxml) - fxml.close() - except EnvironmentError, e: - sys.stderr.write('%s: %s\n' % (filename, e)) - else: - if not args: - args = ['-'] - - for arg in args: - process(arg) + +def main(): + argv = sys.argv + if __name__ == "__main__": sys.exit(main()) diff --git a/bin/scons-proc.py b/bin/scons-proc.py index d6bc1a2..36176f1 100644 --- a/bin/scons-proc.py +++ b/bin/scons-proc.py @@ -14,7 +14,6 @@ import os import re import string import sys -import copy try: from io import StringIO # usable as of 2.6; takes unicode only except ImportError: @@ -22,28 +21,7 @@ except ImportError: exec('from cStringIO import StringIO') import SConsDoc - -try: - from lxml import etree -except ImportError: - try: - # Python 2.5 - import xml.etree.cElementTree as etree - except ImportError: - try: - # Python 2.5 - import xml.etree.ElementTree as etree - except ImportError: - try: - # normal cElementTree install - import cElementTree as etree - except ImportError: - try: - # normal ElementTree install - import elementtree.ElementTree as etree - except ImportError: - print("Failed to import ElementTree from any known place") - sys.exit(1) +from SConsDoc import tf as stf base_sys_path = [os.getcwd() + '/build/test-tar-gz/lib/scons'] + sys.path @@ -150,33 +128,45 @@ class SCons_XML(object): filename = fl[0] # Start new XML file - root = SConsDoc.xml_tree("variablelist") + root = stf.newXmlTree("variablelist") for v in self.values: - ve = etree.Element("varlistentry", - attrib = {'id' : '%s%s' % (v.prefix, v.idfunc())}) - ve.append(v.xml_term()) - vl = etree.Element("listitem") - if v.summary: + + ve = stf.newNode("varlistentry") + stf.setAttribute(ve, 'id', '%s%s' % (v.prefix, v.idfunc())) + stf.appendNode(ve, v.xml_term()) + vl = stf.newNode("listitem") + added = False + if v.summary is not None: for s in v.summary: - vl.append(copy.deepcopy(s)) + added = True + stf.appendNode(vl, stf.copyNode(s)) - if v.sets: - vp = etree.Element("para") + if len(v.sets): + added = True + vp = stf.newNode("para") s = ['&cv-link-%s;' % x for x in v.sets] - vp.text = 'Sets: ' + ', '.join(s) + '.' - vl.append(vp) - if v.uses: - vp = etree.Element("para") + stf.setText(vp, 'Sets: ' + ', '.join(s) + '.') + stf.appendNode(vl, vp) + if len(v.uses): + added = True + vp = stf.newNode("para") u = ['&cv-link-%s;' % x for x in v.uses] - vp.text = 'Uses: ' + ', '.join(u) + '.' - vl.append(vp) - ve.append(vl) - root.append(ve) + stf.setText(vp, 'Uses: ' + ', '.join(u) + '.') + stf.appendNode(vl, vp) + + # Still nothing added to this list item? + if not added: + # Append an empty para + vp = stf.newNode("para") + stf.appendNode(vl, vp) + + stf.appendNode(ve, vl) + stf.appendNode(root, ve) # Write file f = self.fopen(filename) - f.write(etree.tostring(root, xml_declaration=True, encoding="UTF-8", pretty_print=True)) + stf.writeGenTree(root, f) def write_mod(self, filename): try: @@ -248,8 +238,8 @@ class SConsThing(Proxy): return self.name def xml_term(self): - e = etree.Element("term") - e.text = self.name + e = stf.newNode("term") + stf.setText(e, self.name) return e class Builder(SConsThing): @@ -258,17 +248,17 @@ class Builder(SConsThing): tag = 'function' def xml_term(self): - t = etree.Element("term") - s = etree.Element("synopsis") - b = etree.Element(self.tag) - b.text = self.name+'()' - s.append(b) - t.append(s) - s = etree.Element("synopsis") - b = etree.Element(self.tag) - b.text = 'env.'+self.name+'()' - s.append(b) - t.append(s) + t = stf.newNode("term") + s = stf.newNode("synopsis") + b = stf.newNode(self.tag) + stf.setText(b, self.name+'()') + stf.appendNode(s, b) + stf.appendNode(t, s) + s = stf.newNode("synopsis") + b = stf.newNode(self.tag) + stf.setText(b, 'env.'+self.name+'()') + stf.appendNode(s, b) + stf.appendNode(t, s) return t def entityfunc(self): @@ -280,27 +270,26 @@ class Function(SConsThing): tag = 'function' def xml_term(self): - try: - arguments = self.arguments - except AttributeError: - a = etree.Element("arguments") - a.text = '()' + if self.arguments is None: + a = stf.newNode("arguments") + stf.setText(a, '()') arguments = [a] - t = etree.Element("term") + else: + arguments = self.arguments + t = stf.newNode("term") for arg in arguments: - if 'signature' in arg.attrib: - signature = arg.attrib['signature'] - else: - signature = "both" - s = arg.text.strip() + signature = 'both' + if stf.hasAttribute(arg, 'signature'): + signature = stf.getAttribute(arg, 'signature') + s = stf.getText(arg).strip() if signature in ('both', 'global'): - syn = etree.Element("synopsis") - syn.text = '%s%s' % (self.name, s) - t.append(syn) + syn = stf.newNode("synopsis") + stf.setText(syn, '%s%s' % (self.name, s)) + stf.appendNode(t, syn) if signature in ('both', 'env'): - syn = etree.Element("synopsis") - syn.text = 'env.%s%s' % (self.name, s) - t.append(syn) + syn = stf.newNode("synopsis") + stf.setText(syn, 'env.%s%s' % (self.name, s)) + stf.appendNode(t, syn) return t @@ -351,11 +340,13 @@ def write_output_files(h, buildersfiles, functionsfiles, processor_class = SCons_XML # Step 1: Creating entity files for builders, functions,... +print "Generating entity files..." h = parse_docs(args, False) write_output_files(h, buildersfiles, functionsfiles, toolsfiles, variablesfiles, SCons_XML.write_mod) # Step 2: Patching the include paths for entity definitions in XML files +print "Patching include paths..." os.system('python bin/docs-correct-mod-paths.py') # Step 3: Validating all input files @@ -367,9 +358,11 @@ else: # Step 4: Creating actual documentation snippets, using the # fully resolved and updated entities from the *.mod files. +print "Updating documentation for builders, tools and functions..." h = parse_docs(args, True) write_output_files(h, buildersfiles, functionsfiles, toolsfiles, variablesfiles, SCons_XML.write) +print "Done" # Local Variables: # tab-width:4 |