From d4eaa1986cfffdb11eee0aa5240fa44bef370fb1 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 25 Jun 2019 10:56:43 -0600 Subject: Improve building of docs using Py3 [ci skip] * context managers on file r/w + use shutil.copy where it makes sense. * lxml wants (demands?) that xml files be processed as bytes * for the phase where we gen the entity files, read as text anyway * Need to solve a problem where the generated xml is putting the \n in literally, not evaluating it. * Fix some examples broken for py3 * Fix more octal constant instances * Cleanups suggested by PyCharm: staticmethods, two blanks before class definition, others. This addresses issues called out in #3300, but is not a complete solution because the actual doc build step still fails with the epydoc failures (which aren't directly because of Py3; epydoc build doesn't work any better on my system with Py3, even with the forked version with patches). Signed-off-by: Mats Wichmann --- bin/SConsDoc.py | 211 ++++++++++++++++----------- bin/SConsExamples.py | 76 +++++----- bin/docs-update-generated.py | 17 ++- bin/scons-proc.py | 14 +- doc/generated/examples/factories_Chmod_1.xml | 2 +- doc/generated/functions.gen | 10 +- doc/man/scons.xml | 4 +- doc/user/README | 2 +- doc/user/builders-writing.xml | 12 +- doc/user/environments.xml | 2 +- doc/user/factories.xml | 10 +- doc/user/file-removal.xml | 2 +- doc/user/mergeflags.xml | 8 +- doc/user/parseconfig.xml | 4 +- doc/user/parseflags.xml | 12 +- doc/user/sideeffect.xml | 2 +- doc/user/troubleshoot.xml | 4 +- src/engine/SCons/Action.xml | 2 +- src/engine/SCons/Defaults.xml | 2 +- src/engine/SCons/Environment.xml | 8 +- src/engine/SCons/Platform/__init__.xml | 2 +- src/engine/SCons/Platform/posix.xml | 2 +- src/engine/SCons/Platform/sunos.xml | 2 +- src/engine/SCons/Platform/win32.xml | 2 +- src/engine/SCons/Scanner/__init__.xml | 2 +- src/engine/SCons/Script/Main.xml | 2 +- src/engine/SCons/Script/SConscript.xml | 2 +- src/engine/SCons/Subst.xml | 2 +- src/engine/SCons/Tool/386asm.xml | 2 +- src/engine/SCons/Tool/DCommon.xml | 2 +- src/engine/SCons/Tool/__init__.xml | 2 +- src/engine/SCons/Tool/aixc++.xml | 2 +- src/engine/SCons/Tool/aixcc.xml | 2 +- src/engine/SCons/Tool/aixf77.xml | 2 +- src/engine/SCons/Tool/aixlink.xml | 2 +- src/engine/SCons/Tool/applelink.xml | 2 +- src/engine/SCons/Tool/ar.xml | 2 +- src/engine/SCons/Tool/as.xml | 2 +- src/engine/SCons/Tool/bcc32.xml | 2 +- src/engine/SCons/Tool/c++.xml | 2 +- src/engine/SCons/Tool/cc.xml | 2 +- src/engine/SCons/Tool/clang.xml | 2 +- src/engine/SCons/Tool/clangxx.xml | 2 +- src/engine/SCons/Tool/cvf.xml | 2 +- src/engine/SCons/Tool/cyglink.xml | 2 +- src/engine/SCons/Tool/default.xml | 2 +- src/engine/SCons/Tool/dmd.xml | 2 +- src/engine/SCons/Tool/docbook/__init__.xml | 2 +- src/engine/SCons/Tool/dvi.xml | 2 +- src/engine/SCons/Tool/dvipdf.xml | 2 +- src/engine/SCons/Tool/dvips.xml | 2 +- src/engine/SCons/Tool/f03.xml | 2 +- src/engine/SCons/Tool/f08.xml | 2 +- src/engine/SCons/Tool/f77.xml | 2 +- src/engine/SCons/Tool/f90.xml | 2 +- src/engine/SCons/Tool/f95.xml | 2 +- src/engine/SCons/Tool/fortran.xml | 2 +- src/engine/SCons/Tool/g++.xml | 2 +- src/engine/SCons/Tool/g77.xml | 2 +- src/engine/SCons/Tool/gas.xml | 2 +- src/engine/SCons/Tool/gcc.xml | 2 +- src/engine/SCons/Tool/gdc.xml | 2 +- src/engine/SCons/Tool/gettext.xml | 2 +- src/engine/SCons/Tool/gfortran.xml | 2 +- src/engine/SCons/Tool/gnulink.xml | 2 +- src/engine/SCons/Tool/gs.xml | 2 +- src/engine/SCons/Tool/hpc++.xml | 2 +- src/engine/SCons/Tool/hpcc.xml | 2 +- src/engine/SCons/Tool/hplink.xml | 2 +- src/engine/SCons/Tool/icc.xml | 2 +- src/engine/SCons/Tool/icl.xml | 2 +- src/engine/SCons/Tool/ifl.xml | 2 +- src/engine/SCons/Tool/ifort.xml | 2 +- src/engine/SCons/Tool/ilink.xml | 2 +- src/engine/SCons/Tool/ilink32.xml | 2 +- src/engine/SCons/Tool/install.xml | 2 +- src/engine/SCons/Tool/intelc.xml | 2 +- src/engine/SCons/Tool/jar.xml | 2 +- src/engine/SCons/Tool/javac.xml | 2 +- src/engine/SCons/Tool/javah.xml | 2 +- src/engine/SCons/Tool/latex.xml | 2 +- src/engine/SCons/Tool/ldc.xml | 2 +- src/engine/SCons/Tool/lex.xml | 2 +- src/engine/SCons/Tool/link.xml | 2 +- src/engine/SCons/Tool/linkloc.xml | 2 +- src/engine/SCons/Tool/m4.xml | 2 +- src/engine/SCons/Tool/masm.xml | 2 +- src/engine/SCons/Tool/midl.xml | 2 +- src/engine/SCons/Tool/mingw.xml | 2 +- src/engine/SCons/Tool/msgfmt.xml | 2 +- src/engine/SCons/Tool/msginit.xml | 2 +- src/engine/SCons/Tool/msgmerge.xml | 2 +- src/engine/SCons/Tool/mslib.xml | 2 +- src/engine/SCons/Tool/mslink.xml | 2 +- src/engine/SCons/Tool/mssdk.xml | 2 +- src/engine/SCons/Tool/msvc.xml | 2 +- src/engine/SCons/Tool/msvs.xml | 2 +- src/engine/SCons/Tool/mwcc.xml | 2 +- src/engine/SCons/Tool/mwld.xml | 2 +- src/engine/SCons/Tool/nasm.xml | 2 +- src/engine/SCons/Tool/packaging.xml | 2 +- src/engine/SCons/Tool/packaging/__init__.xml | 6 +- src/engine/SCons/Tool/pdf.xml | 2 +- src/engine/SCons/Tool/pdflatex.xml | 2 +- src/engine/SCons/Tool/pdftex.xml | 2 +- src/engine/SCons/Tool/qt.xml | 2 +- src/engine/SCons/Tool/rmic.xml | 2 +- src/engine/SCons/Tool/rpcgen.xml | 2 +- src/engine/SCons/Tool/sgiar.xml | 2 +- src/engine/SCons/Tool/sgic++.xml | 2 +- src/engine/SCons/Tool/sgicc.xml | 2 +- src/engine/SCons/Tool/sgilink.xml | 2 +- src/engine/SCons/Tool/sunar.xml | 2 +- src/engine/SCons/Tool/sunc++.xml | 2 +- src/engine/SCons/Tool/suncc.xml | 2 +- src/engine/SCons/Tool/sunf77.xml | 2 +- src/engine/SCons/Tool/sunf90.xml | 2 +- src/engine/SCons/Tool/sunf95.xml | 2 +- src/engine/SCons/Tool/sunlink.xml | 2 +- src/engine/SCons/Tool/swig.xml | 2 +- src/engine/SCons/Tool/tar.xml | 2 +- src/engine/SCons/Tool/tex.xml | 2 +- src/engine/SCons/Tool/textfile.xml | 2 +- src/engine/SCons/Tool/tlib.xml | 2 +- src/engine/SCons/Tool/xgettext.xml | 2 +- src/engine/SCons/Tool/yacc.xml | 2 +- src/engine/SCons/Tool/zip.xml | 2 +- test/Fortran/FORTRANFILESUFFIXES2.py | 2 +- test/MSVC/query_vcbat.py | 2 +- test/packaging/rpm/tagging.py | 4 +- test/sconsign/script/Signatures.py | 2 +- test/sconsign/script/no-SConsignFile.py | 2 +- test/subdivide.py | 2 +- 133 files changed, 348 insertions(+), 288 deletions(-) diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index 5fd5f00..d200f8b 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -26,7 +26,8 @@ # from __future__ import print_function -__doc__ = """ + +__doc__ = r""" This module parses home-brew XML files that document various things in SCons. Right now, it handles Builders, functions, construction variables, and Tools, but we expect it to get extended in the future. @@ -117,16 +118,18 @@ import re import sys import copy +PY2 = sys.version_info[0] == 2 + # Do we have libxml2/libxslt/lxml? has_libxml2 = True try: import libxml2 import libxslt -except: +except ImportError: has_libxml2 = False try: import lxml - except: + except ImportError: raise ImportError("Failed to import either libxml2/libxslt or lxml") has_etree = False @@ -155,13 +158,16 @@ if not has_etree: except ImportError: raise ImportError("Failed to import ElementTree from any known place") -re_entity = re.compile(r"\&([^;]+);") +# patterns to help trim XML passed in as strings +re_entity = re.compile(r"&([^;]+);") re_entity_header = re.compile(r"") # Namespace for the SCons Docbook XSD -dbxsd="http://www.scons.org/dbxsd/v1.0" +dbxsd = "http://www.scons.org/dbxsd/v1.0" +# Namsespace pattern to help identify an scons-xml file read as bytes +dbxsdpat = b'xmlns="%s"' % dbxsd.encode('utf-8') # Namespace map identifier for the SCons Docbook XSD -dbxid="dbx" +dbxid = "dbx" # Namespace for schema instances xsi = "http://www.w3.org/2001/XMLSchema-instance" @@ -170,19 +176,22 @@ copyright_comment = """ __COPYRIGHT__ This file is processed by the bin/SConsDoc.py module. -See its __doc__ string for a discussion of the format. +See its docstring for a discussion of the format. """ def isSConsXml(fpath): - """ Check whether the given file is a SCons XML file, i.e. it - contains the default target namespace definition. + """ Check whether the given file is an SCons XML file. + + It is SCons XML if it contains the default target namespace definition + described by dbxsdpat + """ try: - with open(fpath,'r') as f: + with open(fpath, 'rb') as f: content = f.read() - if content.find('xmlns="%s"' % dbxsd) >= 0: + if content.find(dbxsdpat) >= 0: return True - except: + except Exception: pass return False @@ -195,10 +204,11 @@ def remove_entities(content): return content -default_xsd = os.path.join('doc','xsd','scons.xsd') +default_xsd = os.path.join('doc', 'xsd', 'scons.xsd') ARG = "dbscons" + class Libxml2ValidityHandler: def __init__(self): @@ -224,10 +234,11 @@ class DoctypeEntity: def getEntityString(self): txt = """ %(perc)s%(name)s; -""" % {'perc' : perc, 'name' : self.name, 'uri' : self.uri} +""" % {'perc': perc, 'name': self.name, 'uri': self.uri} return txt + class DoctypeDeclaration: def __init__(self, name_=None): self.name = name_ @@ -253,14 +264,16 @@ class DoctypeDeclaration: return content if not has_libxml2: - class TreeFactory: + class TreeFactory(object): def __init__(self): pass - def newNode(self, tag): + @staticmethod + def newNode(tag): return etree.Element(tag) - def newEtreeNode(self, tag, init_ns=False): + @staticmethod + def newEtreeNode(tag, init_ns=False): if init_ns: NSMAP = {None: dbxsd, 'xsi' : xsi} @@ -268,47 +281,60 @@ if not has_libxml2: return etree.Element(tag) - def copyNode(self, node): + @staticmethod + def copyNode(node): return copy.deepcopy(node) - def appendNode(self, parent, child): + @staticmethod + def appendNode(parent, child): parent.append(child) - def hasAttribute(self, node, att): + @staticmethod + def hasAttribute(node, att): return att in node.attrib - def getAttribute(self, node, att): + @staticmethod + def getAttribute(node, att): return node.attrib[att] - def setAttribute(self, node, att, value): + @staticmethod + def setAttribute(node, att, value): node.attrib[att] = value - def getText(self, root): + @staticmethod + def getText(root): return root.text - def setText(self, root, txt): + @staticmethod + def setText(root, txt): root.text = txt - def writeGenTree(self, root, fp): + @staticmethod + def writeGenTree(root, fp): dt = DoctypeDeclaration() - fp.write(etree.tostring(root, xml_declaration=True, - encoding="UTF-8", pretty_print=True, + encfun = unicode if PY2 else str + fp.write(etree.tostring(root, encoding=encfun, + pretty_print=True, doctype=dt.createDoctype())) - def writeTree(self, root, fpath): - with open(fpath, 'w') as fp: - fp.write(etree.tostring(root, xml_declaration=True, - encoding="UTF-8", pretty_print=True)) + @staticmethod + def writeTree(root, fpath): + encfun = unicode if PY2 else None + with open(fpath, 'wb') as fp: + fp.write(etree.tostring(root, encoding=encfun, + pretty_print=True)) - def prettyPrintFile(self, fpath): - with open(fpath,'r') as fin: + @staticmethod + def prettyPrintFile(fpath): + with open(fpath,'rb') as fin: tree = etree.parse(fin) pretty_content = etree.tostring(tree, pretty_print=True) - with open(fpath,'w') as fout: + with open(fpath,'wb') as fout: fout.write(pretty_content) - def decorateWithHeader(self, root): + @staticmethod + def decorateWithHeader(root): root.attrib["{"+xsi+"}schemaLocation"] = "%s %s/scons.xsd" % (dbxsd, dbxsd) return root @@ -316,12 +342,12 @@ if not has_libxml2: """ Return a XML file tree with the correct namespaces set, the element root as top entry and the given header comment. """ - NSMAP = {None: dbxsd, - 'xsi' : xsi} + NSMAP = {None: dbxsd, 'xsi' : xsi} t = etree.Element(root, nsmap=NSMAP) return self.decorateWithHeader(t) - def validateXml(self, fpath, xmlschema_context): + @staticmethod + def validateXml(fpath, xmlschema_context): # Use lxml xmlschema = etree.XMLSchema(xmlschema_context) try: @@ -339,19 +365,22 @@ if not has_libxml2: return False return True - def findAll(self, root, tag, ns=None, xp_ctxt=None, nsmap=None): + @staticmethod + def findAll(root, tag, ns=None, xp_ctxt=None, nsmap=None): expression = ".//{%s}%s" % (nsmap[ns], tag) if not ns or not nsmap: expression = ".//%s" % tag return root.findall(expression) - def findAllChildrenOf(self, root, tag, ns=None, xp_ctxt=None, nsmap=None): + @staticmethod + def findAllChildrenOf(root, tag, ns=None, xp_ctxt=None, nsmap=None): expression = "./{%s}%s/*" % (nsmap[ns], tag) if not ns or not nsmap: expression = "./%s/*" % tag return root.findall(expression) - def convertElementTree(self, root): + @staticmethod + def convertElementTree(root): """ Convert the given tree of etree.Element entries to a list of tree nodes for the current XML toolkit. @@ -359,53 +388,63 @@ if not has_libxml2: return [root] else: - class TreeFactory: + class TreeFactory(object): def __init__(self): pass - def newNode(self, tag): + @staticmethod + def newNode(tag): return libxml2.newNode(tag) - def newEtreeNode(self, tag, init_ns=False): + @staticmethod + def newEtreeNode(tag, init_ns=False): return etree.Element(tag) - def copyNode(self, node): + @staticmethod + def copyNode(node): return node.copyNode(1) - def appendNode(self, parent, child): + @staticmethod + def appendNode(parent, child): if hasattr(parent, 'addChild'): parent.addChild(child) else: parent.append(child) - def hasAttribute(self, node, att): + @staticmethod + def hasAttribute(node, att): if hasattr(node, 'hasProp'): return node.hasProp(att) return att in node.attrib - def getAttribute(self, node, att): + @staticmethod + def getAttribute(node, att): if hasattr(node, 'prop'): return node.prop(att) return node.attrib[att] - def setAttribute(self, node, att, value): + @staticmethod + def setAttribute(node, att, value): if hasattr(node, 'setProp'): node.setProp(att, value) else: node.attrib[att] = value - def getText(self, root): + @staticmethod + def getText(root): if hasattr(root, 'getContent'): return root.getContent() return root.text - def setText(self, root, txt): + @staticmethod + def setText(root, txt): if hasattr(root, 'setContent'): root.setContent(txt) else: root.text = txt - def writeGenTree(self, root, fp): + @staticmethod + def writeGenTree(root, fp): doc = libxml2.newDoc('1.0') dtd = doc.newDtd("sconsdoc", None, None) doc.addChild(dtd) @@ -420,23 +459,26 @@ else: fp.write(content) doc.freeDoc() - def writeTree(self, root, fpath): - with open(fpath, 'w') as fp: + @staticmethod + def writeTree(root, fpath): + with open(fpath, 'wb') as fp: doc = libxml2.newDoc('1.0') doc.setRootElement(root) fp.write(doc.serialize("UTF-8", 1)) doc.freeDoc() - def prettyPrintFile(self, fpath): + @staticmethod + def prettyPrintFile(fpath): # Read file and resolve entities doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT) - with open(fpath, 'w') as fp: + with open(fpath, 'wb') as fp: # Prettyprint fp.write(doc.serialize("UTF-8", 1)) # Cleanup doc.freeDoc() - def decorateWithHeader(self, root): + @staticmethod + def decorateWithHeader(root): # Register the namespaces ns = root.newNs(dbxsd, None) xi = root.newNs(xsi, 'xsi') @@ -453,7 +495,8 @@ else: t = libxml2.newNode(root) return self.decorateWithHeader(t) - def validateXml(self, fpath, xmlschema_context): + @staticmethod + def validateXml(fpath, xmlschema_context): retval = True # Create validation context @@ -479,7 +522,8 @@ else: return retval - def findAll(self, root, tag, ns=None, xpath_context=None, nsmap=None): + @staticmethod + def findAll(root, tag, ns=None, xpath_context=None, nsmap=None): if hasattr(root, 'xpathEval') and xpath_context: # Use the xpath context xpath_context.setContextNode(root) @@ -493,7 +537,8 @@ else: expression = ".//%s" % tag return root.findall(expression) - def findAllChildrenOf(self, root, tag, ns=None, xpath_context=None, nsmap=None): + @staticmethod + def findAllChildrenOf(root, tag, ns=None, xpath_context=None, nsmap=None): if hasattr(root, 'xpathEval') and xpath_context: # Use the xpath context xpath_context.setContextNode(root) @@ -562,20 +607,18 @@ else: tf = TreeFactory() -class SConsDocTree: +class SConsDocTree(object): def __init__(self): - self.nsmap = {'dbx' : dbxsd} + self.nsmap = {'dbx': dbxsd} self.doc = None self.root = None self.xpath_context = None def parseContent(self, content, include_entities=True): - """ Parses the given content as XML file. This method - is used when we generate the basic lists of entities - for the builders, tools and functions. - So we usually don't bother about namespaces and resolving - entities here...this is handled in parseXmlFile below - (step 2 of the overall process). + """ Parses the given text content as XML + + This is the setup portion, called from parseContent in + an SConsDocHandler instance - see the notes there. """ if not include_entities: content = remove_entities(content) @@ -583,7 +626,6 @@ class SConsDocTree: self.root = etree.fromstring(content) def parseXmlFile(self, fpath): - nsmap = {'dbx' : dbxsd} if not has_libxml2: # Create domtree from file domtree = etree.parse(fpath) @@ -604,7 +646,7 @@ class SConsDocTree: if self.xpath_context is not None: self.xpath_context.xpathFreeContext() -perc="%" +perc = "%" def validate_all_xml(dpaths, xsdfile=default_xsd): xmlschema_context = None @@ -620,7 +662,7 @@ def validate_all_xml(dpaths, xsdfile=default_xsd): fpaths = [] for dp in dpaths: if dp.endswith('.xml') and isSConsXml(dp): - path='.' + path = '.' fpaths.append(dp) else: for path, dirs, files in os.walk(dp): @@ -633,8 +675,8 @@ def validate_all_xml(dpaths, xsdfile=default_xsd): fails = [] for idx, fp in enumerate(fpaths): fpath = os.path.join(path, fp) - print("%.2f%s (%d/%d) %s" % (float(idx+1)*100.0/float(len(fpaths)), - perc, idx+1, len(fpaths),fp)) + print("%.2f%s (%d/%d) %s" % (float(idx + 1) * 100.0 /float(len(fpaths)), + perc, idx + 1, len(fpaths), fp)) if not tf.validateXml(fp, xmlschema_context): fails.append(fp) @@ -649,6 +691,7 @@ def validate_all_xml(dpaths, xsdfile=default_xsd): return True + class Item(object): def __init__(self, name): self.name = name @@ -668,21 +711,25 @@ class Item(object): def __lt__(self, other): return self.sort_name < other.sort_name + class Builder(Item): pass + class Function(Item): - def __init__(self, name): - super(Function, self).__init__(name) + pass + class Tool(Item): def __init__(self, name): Item.__init__(self, name) self.entity = self.name.replace('+', 'X') + class ConstructionVariable(Item): pass + class Arguments(object): def __init__(self, signature, body=None): if not body: @@ -692,7 +739,7 @@ class Arguments(object): def __str__(self): s = ''.join(self.body).strip() result = [] - for m in re.findall('([a-zA-Z/_]+|[^a-zA-Z/_]+)', s): + for m in re.findall(r'([a-zA-Z/_]+|[^a-zA-Z/_]+)', s): if ' ' in m: m = '"%s"' % m result.append(m) @@ -700,6 +747,7 @@ class Arguments(object): def append(self, data): self.body.append(data) + class SConsDocHandler(object): def __init__(self): self.builders = {} @@ -794,7 +842,7 @@ class SConsDocHandler(object): self.parseDomtree(t.root, t.xpath_context, t.nsmap) # lifted from Ka-Ping Yee's way cool pydoc module. -if sys.version_info[0] == 2: +if PY2: def importfile(path): """Import a Python source file or compiled file given its path.""" import imp @@ -817,11 +865,10 @@ if sys.version_info[0] == 2: else: # PY3 version, from newer pydoc def importfile(path): """Import a Python source file or compiled file given its path.""" - import importlib - from pydoc import ErrorDuringImport - magic = importlib.util.MAGIC_NUMBER + from importlib import MAGIC_NUMBER + import pydoc with open(path, 'rb') as ifp: - is_bytecode = magic == ifp.read(len(magic)) + is_bytecode = MAGIC_NUMBER == ifp.read(len(MAGIC_NUMBER)) filename = os.path.basename(path) name, ext = os.path.splitext(filename) if is_bytecode: @@ -832,7 +879,7 @@ else: # PY3 version, from newer pydoc spec = importlib.util.spec_from_file_location(name, path, loader=loader) try: return importlib._bootstrap._load(spec) - except: + except ImportError: raise ErrorDuringImport(path, sys.exc_info()) # Local Variables: diff --git a/bin/SConsExamples.py b/bin/SConsExamples.py index 501169e..8a264e4 100644 --- a/bin/SConsExamples.py +++ b/bin/SConsExamples.py @@ -92,6 +92,7 @@ import os import re import sys import time +import shutil import SConsDoc from SConsDoc import tf as stf @@ -287,9 +288,8 @@ def ensureExampleOutputsExist(dpath): fpath = os.path.join(generated_examples, key + '_' + r.name.replace("/", "_")) # Write file - f = open(fpath, 'w') - f.write("%s\n" % content) - f.close() + with open(fpath, 'w') as f: + f.write("%s\n" % content) perc = "%" @@ -324,9 +324,8 @@ def createAllExampleOutputs(dpath): fpath = os.path.join(generated_examples, key + '_' + r.name.replace("/", "_")) # Write file - f = open(fpath, 'w') - f.write("%s\n" % content) - f.close() + with open(fpath, 'w') as f: + f.write("%s\n" % content) idx += 1 def collectSConsExampleNames(fpath): @@ -458,6 +457,7 @@ import re import SCons.Action import SCons.Defaults import SCons.Node.FS +import shutil platform = '%(osname)s' @@ -540,30 +540,28 @@ def Null(target, source, env): def Cat(target, source, env): target = str(target[0]) - f = open(target, "wb") for src in map(str, source): - f.write(open(src, "rb").read()) - f.close() + shutil.copy(src, target) def CCCom(target, source, env): - target = str(target[0]) - fp = open(target, "wb") - def process(source_file, fp=fp): - for line in open(source_file, "rb").readlines(): - m = re.match(r'#include\s[<"]([^<"]+)[>"]', line) - if m: - include = m.group(1) - for d in [str(env.Dir('$CPPPATH')), '.']: - f = os.path.join(d, include) - if os.path.exists(f): - process(f) - break - elif line[:11] != "STRIP CCCOM": - fp.write(line) - for src in map(str, source): - process(src) - fp.write('debug = ' + ARGUMENTS.get('debug', '0') + '\\n') - fp.close() + def process(source_file, ofp): + with open(source_file, "r") as ifp: + for line in ifp.readlines(): + m = re.match(r'#include\s[<"]([^<"]+)[>"]', line) + if m: + include = m.group(1) + for d in [str(env.Dir('$CPPPATH')), '.']: + f = os.path.join(d, include) + if os.path.exists(f): + process(f, ofp) + break + elif line[:11] != "STRIP CCCOM": + ofp.write(line) + + with open(str(target[0]), "w") as fp: + for src in map(str, source): + process(src, fp) + fp.write('debug = ' + ARGUMENTS.get('debug', '0') + '\\n') public_class_re = re.compile('^public class (\S+)', re.MULTILINE) @@ -577,20 +575,23 @@ def JavaCCom(target, source, env): for t in tlist: not_copied[t] = 1 for src in map(str, source): - contents = open(src, "rb").read() + with open(src, "r") as f: + contents = f.read() classes = public_class_re.findall(contents) for c in classes: for t in [x for x in tlist if x.find(c) != -1]: - open(t, "wb").write(contents) + with open(t, "w") as f: + f.write(contents) del not_copied[t] for t in not_copied.keys(): - open(t, "wb").write("\\n") + with open(t, "w") as f: + f.write("\\n") def JavaHCom(target, source, env): tlist = map(str, target) slist = map(str, source) for t, s in zip(tlist, slist): - open(t, "wb").write(open(s, "rb").read()) + shutil.copy(s, t) def JarCom(target, source, env): target = str(target[0]) @@ -599,10 +600,8 @@ def JarCom(target, source, env): for dirpath, dirnames, filenames in os.walk(src): class_files.extend([ os.path.join(dirpath, f) for f in filenames if f.endswith('.class') ]) - f = open(target, "wb") for cf in class_files: - f.write(open(cf, "rb").read()) - f.close() + shutil.copy(cf, target) # XXX Adding COLOR, COLORS and PACKAGE to the 'cc' varlist(s) by hand # here is bogus. It's for the benefit of doc/user/command-line.in, which @@ -721,7 +720,8 @@ def command_touch(args, command, test, values): if not os.path.isabs(file): file = os.path.join(test.workpath('WORK'), file) if not os.path.exists(file): - open(file, 'wb') + with open(file, 'w'): + pass os.utime(file, times) return [] @@ -735,8 +735,8 @@ def command_edit(args, c, test, values): for file in args: if not os.path.isabs(file): file = os.path.join(test.workpath('WORK'), file) - contents = open(file, 'rb').read() - open(file, 'wb').write(contents + add_string) + with open(file, 'a') as f: + f.write(add_string) return [] def command_ls(args, c, test, values): @@ -825,7 +825,7 @@ def create_scons_output(e): t.write(path, content) if hasattr(f, 'chmod'): if len(f.chmod): - os.chmod(path, int(f.chmod, 0)) + os.chmod(path, int(f.chmod, base=8)) # Regular expressions for making the doc output consistent, # regardless of reported addresses or Python version. diff --git a/bin/docs-update-generated.py b/bin/docs-update-generated.py index c164baf..307f843 100644 --- a/bin/docs-update-generated.py +++ b/bin/docs-update-generated.py @@ -10,6 +10,8 @@ from __future__ import print_function import os import sys +import subprocess + import SConsDoc # Directory where all generated files are stored @@ -17,8 +19,8 @@ gen_folder = os.path.join('doc','generated') def argpair(key): """ Return the argument pair *.gen,*.mod for the given key. """ - arg = '%s,%s' % (os.path.join(gen_folder,'%s.gen' % key), - os.path.join(gen_folder,'%s.mod' % key)) + arg = '%s,%s' % (os.path.join(gen_folder, '%s.gen' % key), + os.path.join(gen_folder, '%s.mod' % key)) return arg @@ -43,10 +45,13 @@ def generate_all(): print("Couldn't create destination folder %s! Exiting..." % gen_folder) return # Call scons-proc.py - os.system('%s %s -b %s -f %s -t %s -v %s %s' % - (sys.executable, os.path.join('bin','scons-proc.py'), - argpair('builders'), argpair('functions'), - argpair('tools'), argpair('variables'), ' '.join(flist))) + rv = subprocess.call([sys.executable, + os.path.join('bin','scons-proc.py'), + '-b', argpair('builders'), + '-f', argpair('functions'), + '-t', argpair('tools'), + '-v', argpair('variables')] + flist, + shell=False) if __name__ == "__main__": diff --git a/bin/scons-proc.py b/bin/scons-proc.py index e09c853..d8b37df 100644 --- a/bin/scons-proc.py +++ b/bin/scons-proc.py @@ -32,8 +32,8 @@ Options: -t file(s) dump tool information to the specified file(s) -v file(s) dump variable information to the specified file(s) - Regard that each -[btv] argument is a pair of - comma-separated .gen,.mod file names. + The "files" argument following a -[bftv] argument is expected to + be a comma-separated pair of names like: foo.gen,foo.mod """ @@ -70,7 +70,9 @@ def parse_docs(args, include_entities=True): sys.stderr.write("error in %s\n" % f) raise else: - content = open(f).read() + # mode we read (text/bytes) has to match handling in SConsDoc + with open(f, 'r') as fp: + content = fp.read() if content: try: h.parseContent(content, include_entities) @@ -166,6 +168,7 @@ class SCons_XML(object): # Write file f = self.fopen(filename) stf.writeGenTree(root, f) + f.close() def write_mod(self, filename): try: @@ -212,6 +215,7 @@ class SCons_XML(object): v.tag, v.entityfunc(), v.tag)) f.write('\n') f.write(Warning) + f.close() class Proxy(object): def __init__(self, subject): @@ -348,7 +352,7 @@ processor_class = SCons_XML # Step 1: Creating entity files for builders, functions,... print("Generating entity files...") -h = parse_docs(args, False) +h = parse_docs(args, include_entities=False) write_output_files(h, buildersfiles, functionsfiles, toolsfiles, variablesfiles, SCons_XML.write_mod) @@ -362,7 +366,7 @@ else: # Step 3: Creating actual documentation snippets, using the # fully resolved and updated entities from the *.mod files. print("Updating documentation for builders, tools and functions...") -h = parse_docs(args, True) +h = parse_docs(args, include_entities=True) write_output_files(h, buildersfiles, functionsfiles, toolsfiles, variablesfiles, SCons_XML.write) print("Done") diff --git a/doc/generated/examples/factories_Chmod_1.xml b/doc/generated/examples/factories_Chmod_1.xml index a324ed4..59bba55 100644 --- a/doc/generated/examples/factories_Chmod_1.xml +++ b/doc/generated/examples/factories_Chmod_1.xml @@ -1,5 +1,5 @@ % scons -Q Copy("file.out", "file.in") -Chmod("file.out", 0755) +Chmod("file.out", 0o755) diff --git a/doc/generated/functions.gen b/doc/generated/functions.gen index f3f2a0c..d24eac3 100644 --- a/doc/generated/functions.gen +++ b/doc/generated/functions.gen @@ -1408,7 +1408,7 @@ This SConstruct: env=Environment() -print env.Dump('CCCOM') +print(env.Dump('CCCOM')) @@ -1425,7 +1425,7 @@ While this SConstruct: env=Environment() -print env.Dump() +print(env.Dump()) @@ -4463,7 +4463,7 @@ Example: -print env.subst("The C compiler is: $CC") +print(env.subst("The C compiler is: $CC")) def compile(target, source, env): sourceDir = env.subst("${SOURCE.srcdir}", @@ -4493,9 +4493,9 @@ Examples: -# makes sure the built library will be installed with 0644 file +# makes sure the built library will be installed with 0o644 file # access mode -Tag( Library( 'lib.c' ), UNIX_ATTR="0644" ) +Tag( Library( 'lib.c' ), UNIX_ATTR="0o644" ) # marks file2.txt to be a documentation file Tag( 'file2.txt', DOC ) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index ae54e2e..5aea45a 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -5721,11 +5721,11 @@ which can be octal or string, similar to the bash command. Examples: -Execute(Chmod('file', 0755)) +Execute(Chmod('file', 0o755)) env.Command('foo.out', 'foo.in', [Copy('$TARGET', '$SOURCE'), - Chmod('$TARGET', 0755)]) + Chmod('$TARGET', 0o755)]) Execute(Chmod('file', "ugo+w")) diff --git a/doc/user/README b/doc/user/README index 2d05359..f94632d 100644 --- a/doc/user/README +++ b/doc/user/README @@ -12,7 +12,7 @@ Writing examples: here's a simple template. env = Environment() - print env.Dump("CC") + print(env.Dump("CC")) diff --git a/doc/user/builders-writing.xml b/doc/user/builders-writing.xml index e20e99b..a906df8 100644 --- a/doc/user/builders-writing.xml +++ b/doc/user/builders-writing.xml @@ -196,7 +196,7 @@ env.Foo('file.foo', 'file.input') file.input - + cat @@ -309,7 +309,7 @@ file.input hello.c - + cat @@ -389,7 +389,7 @@ file1.input file2.input - + cat @@ -682,7 +682,7 @@ env.Foo('file') file.input - + cat @@ -766,7 +766,7 @@ file.input new_source - + cat @@ -840,7 +840,7 @@ modify1.input modify2.input - + cat diff --git a/doc/user/environments.xml b/doc/user/environments.xml index 0b246d2..43503a6 100644 --- a/doc/user/environments.xml +++ b/doc/user/environments.xml @@ -1652,7 +1652,7 @@ env['ENV']['PATH'] = '/usr/local/bin:/bin:/usr/bin' env = Environment() env.Command('foo', [], '__ROOT__/usr/bin/printenv.py') - + #!/usr/bin/env python import os import sys diff --git a/doc/user/factories.xml b/doc/user/factories.xml index c8480db..b68dfd9 100644 --- a/doc/user/factories.xml +++ b/doc/user/factories.xml @@ -161,7 +161,7 @@ env['ENV']['PATH'] = env['ENV']['PATH'] + os.pathsep + os.getcwd() SConscript('S') file.in - + touch $* @@ -231,7 +231,7 @@ env['ENV']['PATH'] = env['ENV']['PATH'] + os.pathsep + os.getcwd() SConscript('S') file.in - + touch $* @@ -325,7 +325,7 @@ env['ENV']['PATH'] = env['ENV']['PATH'] + os.pathsep + os.getcwd() SConscript('S') file.in - + touch $* @@ -416,7 +416,7 @@ env['ENV']['PATH'] = env['ENV']['PATH'] + os.pathsep + os.getcwd() SConscript('S') file.in - + touch $* @@ -452,7 +452,7 @@ touch $* Command("file.out", "file.in", [ Copy("$TARGET", "$SOURCE"), - Chmod("$TARGET", 0755), + Chmod("$TARGET", 0o755), ]) file.in diff --git a/doc/user/file-removal.xml b/doc/user/file-removal.xml index c6c695f..6c7c71a 100644 --- a/doc/user/file-removal.xml +++ b/doc/user/file-removal.xml @@ -214,7 +214,7 @@ foo.in foo.log - + cat $3 > $2 diff --git a/doc/user/mergeflags.xml b/doc/user/mergeflags.xml index 280eb82..879a7b2 100644 --- a/doc/user/mergeflags.xml +++ b/doc/user/mergeflags.xml @@ -77,7 +77,7 @@ env = Environment() env.Append(CCFLAGS = '-option -O3 -O1') flags = { 'CCFLAGS' : '-whatever -O3' } env.MergeFlags(flags) -print env['CCFLAGS'] +print(env['CCFLAGS']) @@ -104,7 +104,7 @@ env = Environment() env.Append(CPPPATH = ['/include', '/usr/local/include', '/usr/include']) flags = { 'CPPPATH' : ['/usr/opt/include', '/usr/local/include'] } env.MergeFlags(flags) -print env['CPPPATH'] +print(env['CPPPATH']) @@ -138,8 +138,8 @@ env = Environment() env.Append(CCFLAGS = '-option -O3 -O1') env.Append(CPPPATH = ['/include', '/usr/local/include', '/usr/include']) env.MergeFlags('-whatever -I/usr/opt/include -O3 -I/usr/local/include') -print env['CCFLAGS'] -print env['CPPPATH'] +print(env['CCFLAGS']) +print(env['CPPPATH']) diff --git a/doc/user/parseconfig.xml b/doc/user/parseconfig.xml index 1ddd42d..a07201a 100644 --- a/doc/user/parseconfig.xml +++ b/doc/user/parseconfig.xml @@ -87,7 +87,7 @@ env = Environment() env['CPPPATH'] = ['/lib/compat'] env.ParseConfig("pkg-config x11 --cflags --libs") -print env['CPPPATH'] +print(env['CPPPATH']) @@ -139,7 +139,7 @@ scons: `.' is up to date. env = Environment() env.ParseConfig("pkg-config x11 --cflags --libs") env.ParseConfig("pkg-config x11 --cflags --libs") -print env['CPPPATH'] +print(env['CPPPATH']) diff --git a/doc/user/parseflags.xml b/doc/user/parseflags.xml index 46d6866..836b7f2 100644 --- a/doc/user/parseflags.xml +++ b/doc/user/parseflags.xml @@ -80,11 +80,12 @@ +from __future__ import print_function env = Environment() d = env.ParseFlags("-I/opt/include -L/opt/lib -lfoo") for k,v in sorted(d.items()): if v: - print k, v + print(k, v) env.MergeFlags(d) env.Program('f1.c') @@ -119,11 +120,12 @@ int main() { return 0; } +from __future__ import print_function env = Environment() d = env.ParseFlags("-whatever") for k,v in sorted(d.items()): if v: - print k, v + print(k, v) env.MergeFlags(d) env.Program('f1.c') @@ -145,11 +147,12 @@ env.Program('f1.c') +from __future__ import print_function env = Environment() d = env.ParseFlags(["-I/opt/include", ["-L/opt/lib", "-lfoo"]]) for k,v in sorted(d.items()): if v: - print k, v + print(k, v) env.MergeFlags(d) env.Program('f1.c') @@ -172,11 +175,12 @@ int main() { return 0; } +from __future__ import print_function env = Environment() d = env.ParseFlags(["!echo -I/opt/include", "!echo -L/opt/lib", "-lfoo"]) for k,v in sorted(d.items()): if v: - print k, v + print(k, v) env.MergeFlags(d) env.Program('f1.c') diff --git a/doc/user/sideeffect.xml b/doc/user/sideeffect.xml index ffbfde7..6a10c3b 100644 --- a/doc/user/sideeffect.xml +++ b/doc/user/sideeffect.xml @@ -164,7 +164,7 @@ env.SideEffect('logfile.txt', f1 + f2) file1.in file2.in - + cat diff --git a/doc/user/troubleshoot.xml b/doc/user/troubleshoot.xml index feac970..80e0e24 100644 --- a/doc/user/troubleshoot.xml +++ b/doc/user/troubleshoot.xml @@ -289,7 +289,7 @@ file3.c env = Environment() -print env.Dump() +print(env.Dump()) @@ -349,7 +349,7 @@ print env.Dump() env = Environment() -print env.Dump('ENV') +print(env.Dump('ENV')) diff --git a/src/engine/SCons/Action.xml b/src/engine/SCons/Action.xml index ab42958..7a8194e 100644 --- a/src/engine/SCons/Action.xml +++ b/src/engine/SCons/Action.xml @@ -1,4 +1,4 @@ - +